Merge pull request #9528 from jumpserver/pr@dev@perf_audit_log_and_platform

perf: 优化 audit log

apps/assets/api/asset/host.py

@@ -1,8 +1,8 @@
-from assets.models import Host, Asset
-from assets.serializers import HostSerializer, HostInfoSerializer
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
+from assets.models import Host, Asset
+from assets.serializers import HostSerializer, HostInfoSerializer
 from .asset import AssetViewSet
 
 __all__ = ['HostViewSet']
@@ -22,4 +22,3 @@ class HostViewSet(AssetViewSet):
     def info(self, *args, **kwargs):
         asset = super().get_object()
         return Response(asset.info)
-

apps/assets/const/database.py

@@ -63,7 +63,20 @@ class DatabaseTypes(BaseType):
             cls.SQLSERVER: [{'name': 'SQLServer'}],
             cls.CLICKHOUSE: [{'name': 'ClickHouse'}],
             cls.MONGODB: [{'name': 'MongoDB'}],
-            cls.REDIS: [{'name': 'Redis'}],
+            cls.REDIS: [
+                {
+                    'name': 'Redis',
+                    'protocols_setting': {
+                        'redis': {'auth_username': False}
+                    }
+                },
+                {
+                    'name': 'Redis6+',
+                    'protocols_setting': {
+                        'redis': {'auth_username': True}
+                    }
+                }
+            ]
         }
 
     @classmethod

apps/assets/const/protocol.py

@@ -96,6 +96,9 @@ class Protocol(ChoicesMixin, models.TextChoices):
                 'port': 6379,
                 'required': True,
                 'secret_types': ['password'],
+                'setting': {
+                    'auth_username': True,
+                }
             },
         }
 

apps/assets/const/types.py

@@ -304,7 +304,7 @@ class AllTypes(ChoicesMixin):
                         setting = _protocols_setting.get(p['name'], {})
                         p['required'] = p.pop('required', False)
                         p['default'] = p.pop('default', False)
-                        p['setting'] = {**setting, **p.get('setting', {})}
+                        p['setting'] = {**p.get('setting', {}), **setting}
 
                     platform_data = {
                         **default_platform_data, **d,

apps/assets/serializers/platform.py

@@ -38,6 +38,9 @@ class ProtocolSettingSerializer(serializers.Serializer):
     )
     script = serializers.JSONField(default=list, label=_("Script"))
 
+    # Redis
+    auth_username = serializers.BooleanField(default=False, label=_("Auth with username"))
+
 
 class PlatformAutomationSerializer(serializers.ModelSerializer):
     class Meta:

apps/audits/signal_handlers/activity_log.py

@@ -4,18 +4,18 @@ from celery import signals
 from django.db.models.signals import post_save
 from django.utils.translation import ugettext_lazy as _
 
-from audits.models import ActivityLog
-from assets.models import Asset, Node
 from accounts.const import AutomationTypes
 from accounts.models import AccountBackupAutomation
+from assets.models import Asset, Node
+from audits.models import ActivityLog
 from common.utils import get_object_or_none
+from jumpserver.utils import current_request
 from ops.celery import app
 from orgs.utils import tmp_to_root_org
 from terminal.models import Session
 from users.models import User
-from jumpserver.utils import current_request
-
 from ..const import ActivityChoices
+from ..models import UserLoginLog
 
 
 class ActivityLogHandler(object):
@@ -118,7 +118,7 @@ class ActivityLogHandler(object):
     @staticmethod
     def login_log_for_activity(obj):
         login_status = _('Success') if obj.status else _('Failed')
-        detail = _('User {} login into this service.[{}]').format(
+        detail = _('User {} login this system {}').format(
             obj.username, login_status
         )
         user_id = User.objects.filter(username=obj.username).values('id').first()
@@ -164,12 +164,11 @@ def on_celery_task_pre_run_for_activity_log(task_id='', **kwargs):
         activities.append(
             ActivityLog(id=activity_id, detail_id=task_id)
         )
-    ActivityLog.objects.bulk_update(activities, ('detail_id', ))
+    ActivityLog.objects.bulk_update(activities, ('detail_id',))
 
 
-@post_save.connect
-def on_object_created(
-        sender, instance=None, created=False, update_fields=None, **kwargs
+def on_session_or_login_log_created(
+        sender, instance=None, created=False, **kwargs
 ):
     handler_mapping = {
         'Session': activity_handler.session_for_activity,
@@ -179,12 +178,13 @@ def on_object_created(
     if not created or model_name not in handler_mapping:
         return
 
-    resource_id, detail, a_type = handler_mapping[model_name](instance)
+    resource_id, detail, act_type = handler_mapping[model_name](instance)
 
     ActivityLog.objects.create(
-        resource_id=resource_id, type=a_type,
+        resource_id=resource_id, type=act_type,
         detail=detail, detail_id=instance.id
     )
 
 
-
+for sender in [Session, UserLoginLog]:
+    post_save.connect(on_session_or_login_log_created, sender=sender)

apps/locale/ja/LC_MESSAGES/django.po

@@ -1802,6 +1802,20 @@ msgid "No"
 msgstr "否"
 
 #: audits/models.py:34 audits/models.py:61 audits/models.py:130
+#: audits/handler.py:140
+msgid "{} used account[{}], login method[{}] login the asset."
+msgstr ""
+"{} トムはアカウント[{}]、ログイン方法[{}]を使ってこの資産を登録しました"
+
+#: audits/handler.py:155
+msgid "User {} has executed change auth plan for this account.({})"
+msgstr "ユーザー {} はこのアカウントのために改密計画を実行しました。({})"
+
+#: audits/handler.py:168
+msgid "User [{}] login this system.[{}]"
+msgstr "ユーザー {} がサービスにログインしました。[{}]"
+
+#: audits/models.py:32 audits/models.py:59 audits/models.py:101
 #: terminal/models/session/session.py:37 terminal/models/session/sharing.py:95
 msgid "Remote addr"
 msgstr "リモートaddr"

apps/locale/zh/LC_MESSAGES/django.po

@@ -1789,7 +1789,19 @@ msgstr "是"
 msgid "No"
 msgstr "否"
 
-#: audits/models.py:34 audits/models.py:61 audits/models.py:130
+#: audits/handler.py:140
+msgid "{} used account[{}], login method[{}] login the asset."
+msgstr "{} 使用账户[{}], 登录方式[{}]登录了这个资产."
+
+#: audits/handler.py:155
+msgid "User {} has executed change auth plan for this account.({})"
+msgstr "用户 {} 为这个账号执行了改密计划.({})"
+
+#: audits/handler.py:168
+msgid "User [{}] login this system.[{}]"
+msgstr "用户 {} 登录了服务.[{}]"
+
+#: audits/models.py:32 audits/models.py:59 audits/models.py:101
 #: terminal/models/session/session.py:37 terminal/models/session/sharing.py:95
 msgid "Remote addr"
 msgstr "远端地址"

apps/orgs/signal_handlers/common.py

@@ -4,12 +4,13 @@ from collections import defaultdict
 from functools import partial
 
 from django.conf import settings
-from django.db.models.signals import post_save, pre_delete, m2m_changed
+from django.db.models.signals import post_save, pre_delete, m2m_changed, post_delete
 from django.db.utils import ProgrammingError, OperationalError
 from django.dispatch import receiver
 from django.utils.functional import LazyObject
 
 from common.const.signals import PRE_REMOVE, POST_REMOVE
+from common.decorators import delay_run
 from common.decorators import on_transaction_commit
 from common.signals import django_ready
 from common.utils import get_logger
@@ -51,9 +52,21 @@ def subscribe_orgs_mapping_expire(sender, **kwargs):
     )
 
 
+@delay_run(ttl=5)
+def expire_user_orgs(*args):
+    User.expire_users_rbac_perms_cache()
+
+
+@receiver(post_save, sender=Organization)
+def on_org_create(sender, instance, created=False, **kwargs):
+    if created:
+        return
+    expire_user_orgs()
+
+
 # 创建对应的root
 @receiver(post_save, sender=Organization)
-def on_org_create_or_update(sender, instance, created=False, **kwargs):
+def on_org_create_or_update(sender, instance, **kwargs):
     # 必须放到最开始, 因为下面调用Node.save方法时会获取当前组织的org_id(即instance.org_id), 如果不过期会找不到
     expire_orgs_mapping_for_memory(instance.id)
 
@@ -77,7 +90,26 @@ def on_org_delete(sender, instance, **kwargs):
             root_node.delete()
 
 
-def _remove_users(model, users, org, user_field_name='users'):
+@receiver(post_delete, sender=Organization)
+def on_org_delete(sender, instance, **kwargs):
+    expire_user_orgs()
+
+
+@receiver(post_save, sender=User)
+@on_transaction_commit
+def on_user_created_set_default_org(sender, instance, created, **kwargs):
+    if not instance.id:
+        # 用户已被手动删除，instance.orgs 时会使用 id 进行查找报错，所以判断不存在id时不做处理
+        return
+    if not created:
+        return
+    if instance.orgs.count() > 0:
+        return
+    with tmp_to_org(Organization.default()):
+        Organization.default().add_member(instance)
+
+
+def _remove_user_resource(model, users, org, user_field_name='users'):
     with tmp_to_org(org):
         if not isinstance(users, (tuple, list, set)):
             users = (users,)
@@ -103,8 +135,8 @@ def _remove_users(model, users, org, user_field_name='users'):
 
         objects = model.objects.filter(id__in=object_id_user_ids_map.keys())
         send_m2m_change_signal = partial(
-            m2m_changed.send,
-            sender=m2m_model, reverse=reverse, model=User, using=model.objects.db
+            m2m_changed.send, sender=m2m_model, reverse=reverse,
+            model=User, using=model.objects.db
         )
 
         for obj in objects:
@@ -126,7 +158,7 @@ def _remove_users(model, users, org, user_field_name='users'):
 
 def _clear_users_from_org(org, users):
     """
-    清理用户在该组织下的相关数据
+    清理用户在该组织下的相关数据, 包括用户组, 资产授权, 用户授权
     """
     if not users:
         return
@@ -134,23 +166,7 @@ def _clear_users_from_org(org, users):
     models = (AssetPermission, UserGroup)
 
     for m in models:
-        _remove_users(m, users, org)
-
-    # _remove_users(CommandFilterRule, users, org, user_field_name='reviewers')
-
-
-@receiver(post_save, sender=User)
-@on_transaction_commit
-def on_user_created_set_default_org(sender, instance, created, **kwargs):
-    if not instance.id:
-        # 用户已被手动删除，instance.orgs 时会使用 id 进行查找报错，所以判断不存在id时不做处理
-        return
-    if not created:
-        return
-    if instance.orgs.count() > 0:
-        return
-    with tmp_to_org(Organization.default()):
-        Organization.default().add_member(instance)
+        _remove_user_resource(m, users, org)
 
 
 @receiver(post_user_leave_org)