perf: 优化操作日志，activity日志都存入操作日志中

apps/audits/api.py

@@ -106,7 +106,7 @@ class OperateLogViewSet(RetrieveModelMixin, ListModelMixin, OrgGenericViewSet):
         return super().get_serializer_class()
 
     def get_queryset(self):
-        qs = OperateLog.objects.filter(is_activity=False)
+        qs = OperateLog.objects.all()
         es_config = settings.OPERATE_LOG_ELASTICSEARCH_CONFIG
         if es_config:
             engine_mod = import_module(TYPE_ENGINE_MAPPING['es'])

apps/audits/backends/db.py

@@ -5,9 +5,12 @@ from audits.models import OperateLog
 
 
 class OperateLogStore(object):
+    # 用不可见字符分割前后数据，节省存储-> diff: {'key': 'before\0after'}
+    SEP = '\0'
+
     def __init__(self, config):
         self.model = OperateLog
-        self.max_length = 1024
+        self.max_length = 2048
         self.max_length_tip_msg = _(
             'The text content is too long. Use Elasticsearch to store operation logs'
         )
@@ -16,27 +19,62 @@ class OperateLogStore(object):
     def ping(timeout=None):
         return True
 
+    @classmethod
+    def convert_before_after_to_diff(cls, before, after):
+        if not isinstance(before, dict):
+            before = dict()
+        if not isinstance(after, dict):
+            after = dict()
+
+        diff = dict()
+        keys = set(before.keys()) | set(after.keys())
+        for k in keys:
+            before_value = before.get(k, '')
+            after_value = after.get(k, '')
+            diff[k] = '%s%s%s' % (before_value, cls.SEP, after_value)
+        return diff
+
+    @classmethod
+    def convert_diff_to_before_after(cls, diff):
+        before, after = dict(), dict()
+        if not diff:
+            return before, after
+
+        for k, v in diff.items():
+            before_value, after_value = v.split(cls.SEP, 1)
+            before[k], after[k] = before_value, after_value
+        return before, after
+
+    @classmethod
+    def convert_diff_friendly(cls, raw_diff):
+        diff_list = list()
+        for k, v in raw_diff.items():
+            before, after = v.split(cls.SEP, 1)
+            diff_list.append({
+                'field': k,
+                'before': before if before else _('empty'),
+                'after': after if after else _('empty'),
+            })
+        return diff_list
+
     def save(self, **kwargs):
-        before_limit, after_limit = None, None
         log_id = kwargs.get('id', '')
-        before = kwargs.get('before') or {}
-        after = kwargs.get('after') or {}
-        if len(str(before)) > self.max_length:
-            before_limit = {str(_('Tips')): self.max_length_tip_msg}
-        if len(str(after)) > self.max_length:
-            after_limit = {str(_('Tips')): self.max_length_tip_msg}
+        before = kwargs.pop('before') or {}
+        after = kwargs.pop('after') or {}
 
         op_log = self.model.objects.filter(pk=log_id).first()
         if op_log is not None:
-            op_log_before = op_log.before or {}
-            op_log_after = op_log.after or {}
-            if not before_limit:
-                before.update(op_log_before)
-            if not after_limit:
-                after.update(op_log_after)
+            op_log_diff = op_log.diff or {}
+            op_before, op_after = self.convert_diff_to_before_after(op_log_diff)
+            before.update(op_before)
+            after.update(op_after)
         else:
             op_log = self.model(**kwargs)
 
-        op_log.before = before_limit if before_limit else before
-        op_log.after = after_limit if after_limit else after
+        diff = self.convert_before_after_to_diff(before, after)
+        if len(str(diff)) > self.max_length:
+            limit = {str(_('Tips')): self.max_length_tip_msg}
+            diff = self.convert_before_after_to_diff(limit, limit)
+
+        op_log.diff = diff
         op_log.save()

apps/audits/handler.py

@@ -11,7 +11,6 @@ from common.utils.encode import Singleton
 from common.local import encrypted_field_set
 from settings.serializers import SettingsSerializer
 from jumpserver.utils import current_request
-from audits.models import OperateLog
 from orgs.utils import get_current_org_id
 
 from .backends import get_operate_log_storage
@@ -21,25 +20,6 @@ from .const import ActionChoices
 logger = get_logger(__name__)
 
 
-class ModelClient:
-    @staticmethod
-    def save(**kwargs):
-        log_id = kwargs.get('id', '')
-        op_log = OperateLog.objects.filter(pk=log_id).first()
-        if op_log is not None:
-            raw_after = op_log.after or {}
-            raw_before = op_log.before or {}
-            cur_before = kwargs.get('before') or {}
-            cur_after = kwargs.get('after') or {}
-            raw_before.update(cur_before)
-            raw_after.update(cur_after)
-            op_log.before = raw_before
-            op_log.after = raw_after
-            op_log.save()
-        else:
-            OperateLog.objects.create(**kwargs)
-
-
 class OperatorLogHandler(metaclass=Singleton):
     CACHE_KEY = 'OPERATOR_LOG_CACHE_KEY'
 
@@ -156,28 +136,42 @@ class OperatorLogHandler(metaclass=Singleton):
         # 否则会话结束，录像文件结束操作的会话记录都会体现出来
         params = {}
         action = kwargs.get('data', {}).get('action', 'create')
+        detail = _(
+            '{} used account[{}], login method[{}] login the asset.'
+        ).format(
+            resource.user, resource.account, resource.login_from_display
+        )
         if action == ActionChoices.create:
             params = {
                 'action': ActionChoices.connect,
                 'resource_id': str(resource.asset_id),
-                'user': resource.user
+                'user': resource.user, 'detail': detail
             }
         return params
 
     @staticmethod
     def _get_ChangeSecretRecord_params(resource, **kwargs):
+        detail = _(
+            'User {} has executed change auth plan for this account.({})'
+        ).format(
+            resource.created_by, _(resource.status.title())
+        )
         return {
-            'action': ActionChoices.change_auth,
+            'action': ActionChoices.change_auth, 'detail': detail,
             'resource_id': str(resource.account_id),
         }
 
     @staticmethod
     def _get_UserLoginLog_params(resource, **kwargs):
         username = resource.username
+        login_status = _('Success') if resource.status else _('Failed')
+        detail = _('User {} login into this service.[{}]').format(
+            resource.username, login_status
+        )
         user_id = User.objects.filter(username=username).\
             values_list('id', flat=True)[0]
         return {
-            'action': ActionChoices.login,
+            'action': ActionChoices.login, 'detail': detail,
             'resource_id': str(user_id),
         }
 
@@ -185,7 +179,6 @@ class OperatorLogHandler(metaclass=Singleton):
         param_func = getattr(self, '_get_%s_params' % object_name, None)
         if param_func is not None:
             params = param_func(resource, data=data)
-            data['is_activity'] = True
             data.update(params)
         return data
 
@@ -228,6 +221,7 @@ class OperatorLogHandler(metaclass=Singleton):
 
 
 op_handler = OperatorLogHandler()
+# 理论上操作日志的唯一入口
 create_or_update_operate_log = op_handler.create_or_update_operate_log
 cache_instance_before_data = op_handler.cache_instance_before_data
 get_instance_current_with_cache_diff = op_handler.get_instance_current_with_cache_diff

apps/audits/migrations/0020_auto_20230112_1034.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.2.14 on 2023-01-12 02:34
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('audits', '0019_alter_operatelog_options'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='operatelog',
-            name='is_activity',
-            field=models.BooleanField(default=False, verbose_name='Is Activity'),
-        ),
-    ]

apps/audits/migrations/0020_auto_20230117_1004.py

@@ -0,0 +1,50 @@
+# Generated by Django 3.2.14 on 2023-01-17 02:04
+
+import common.db.encoder
+from django.db import migrations, models
+
+from audits.backends.db import OperateLogStore
+
+
+def migrate_operate_log_after_before(apps, schema_editor):
+    operate_log_model = apps.get_model("audits", "OperateLog")
+    db_alias = schema_editor.connection.alias
+    count, batch_size = 0, 1000
+
+    while True:
+        operate_logs = []
+        queryset = operate_log_model.objects.using(db_alias).all()[count:count + batch_size]
+        if not queryset:
+            break
+        count += len(queryset)
+        for inst in queryset:
+            before, after, diff = inst.before, inst.after, dict()
+            if not any([before, after]):
+                continue
+            diff = OperateLogStore.convert_before_after_to_diff(before, after)
+            inst.diff = diff
+            operate_logs.append(inst)
+        operate_log_model.objects.bulk_update(operate_logs, ['diff'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0019_alter_operatelog_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='operatelog',
+            name='diff',
+            field=models.JSONField(default=dict, encoder=common.db.encoder.ModelJSONFieldEncoder, null=True),
+        ),
+        migrations.AddField(
+            model_name='operatelog',
+            name='detail',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Detail'),
+        ),
+        migrations.RunPython(migrate_operate_log_after_before),
+        migrations.RemoveField(model_name='operatelog', name='after', ),
+        migrations.RemoveField(model_name='operatelog', name='before', ),
+    ]

apps/audits/models.py

@@ -58,9 +58,8 @@ class OperateLog(OrgModelMixin):
     )
     remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     datetime = models.DateTimeField(auto_now=True, verbose_name=_('Datetime'), db_index=True)
-    before = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True)
-    after = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True)
-    is_activity = models.BooleanField(default=False, verbose_name=(_('Is Activity')))
+    diff = models.JSONField(default=dict, encoder=ModelJSONFieldEncoder, null=True)
+    detail = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Detail'))
 
     def __str__(self):
         return "<{}> {} <{}>".format(self.user, self.action, self.resource)
@@ -139,6 +138,9 @@ class UserLoginLog(models.Model):
         max_length=32, default="", verbose_name=_("Authentication backend")
     )
 
+    def __str__(self):
+        return '%s(%s)' % (self.username, self.city)
+
     @property
     def backend_display(self):
         return gettext(self.backend)

apps/audits/serializers.py

@@ -3,6 +3,7 @@
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
+from audits.backends.db import OperateLogStore
 from common.serializers.fields import LabeledChoiceField
 from common.utils.timezone import as_current_tz
 from ops.models.job import JobAuditLog
@@ -68,7 +69,13 @@ class UserLoginLogSerializer(serializers.ModelSerializer):
 class OperateLogActionDetailSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.OperateLog
-        fields = ('before', 'after')
+        fields = ('diff',)
+
+    def to_representation(self, instance):
+        data = super().to_representation(instance)
+        diff = OperateLogStore.convert_diff_friendly(data['diff'])
+        data['diff'] = diff
+        return data
 
 
 class OperateLogSerializer(serializers.ModelSerializer):
@@ -109,5 +116,8 @@ class ActivitiesOperatorLogSerializer(serializers.Serializer):
     @staticmethod
     def get_content(obj):
         action = obj.action.replace('_', ' ').capitalize()
-        ctn = _('User {} {} this resource.').format(obj.user, _(action))
+        if not obj.detail:
+            ctn = _('User {} {} this resource.').format(obj.user, _(action))
+        else:
+            ctn = obj.detail
         return ctn

apps/audits/signal_handlers.py

@@ -285,11 +285,11 @@ def on_user_auth_failed(sender, username, request, reason='', **kwargs):
 
 @receiver(django_ready)
 def on_django_start_set_operate_log_monitor_models(sender, **kwargs):
-    exclude_label = {
+    exclude_apps = {
         'django_cas_ng', 'captcha', 'admin', 'jms_oidc_rp',
         'django_celery_beat', 'contenttypes', 'sessions', 'auth'
     }
-    exclude_object_name = {
+    exclude_models = {
         'UserPasswordHistory', 'ContentType',
         'SiteMessage', 'SiteMessageUsers',
         'PlatformAutomation', 'PlatformProtocol', 'Protocol',
@@ -305,10 +305,10 @@ def on_django_start_set_operate_log_monitor_models(sender, **kwargs):
         'FTPLog', 'OperateLog', 'PasswordChangeLog'
     }
     for i, app in enumerate(apps.get_models(), 1):
-        app_label = app._meta.app_label
-        app_object_name = app._meta.object_name
-        if app_label in exclude_label or \
-                app_object_name in exclude_object_name or \
-                app_object_name.endswith('Execution'):
+        app_name = app._meta.app_label
+        model_name = app._meta.object_name
+        if app_name in exclude_apps or \
+                model_name in exclude_models or \
+                model_name.endswith('Execution'):
             continue
-        MODELS_NEED_RECORD.add(app_object_name)
+        MODELS_NEED_RECORD.add(model_name)