Merge pull request #9593 from jumpserver/pr@dev@perf_log_clean

perf: 修改日志清理

apps/accounts/api/account/account.py

@@ -14,7 +14,8 @@ from common.views.mixins import RecordViewLogMixin
 from orgs.mixins.api import OrgBulkModelViewSet
 
 __all__ = [
-    'AccountViewSet', 'AccountSecretsViewSet', 'AccountTaskCreateAPI', 'AccountHistoriesSecretAPI'
+    'AccountViewSet', 'AccountSecretsViewSet',
+    'AccountTaskCreateAPI', 'AccountHistoriesSecretAPI'
 ]
 
 from rbac.permissions import RBACPermission

apps/audits/tasks.py

@@ -1,16 +1,25 @@
 # -*- coding: utf-8 -*-
 #
 import datetime
-from django.utils import timezone
+import os
-from celery import shared_task
+import subprocess
 
-from ops.celery.decorator import (
+from celery import shared_task
-    register_as_period_task
+from django.conf import settings
-)
+from django.core.files.storage import default_storage
-from .models import UserLoginLog, OperateLog, FTPLog, ActivityLog
+from django.utils import timezone
-from common.utils import get_log_keep_day
 from django.utils.translation import gettext_lazy as _
 
+from common.utils import get_log_keep_day, get_logger
+from ops.celery.decorator import (
+    register_as_period_task, after_app_shutdown_clean_periodic
+)
+from ops.models import CeleryTaskExecution
+from terminal.models import Session, Command
+from .models import UserLoginLog, OperateLog, FTPLog, ActivityLog
+
+logger = get_logger(__name__)
+
 
 def clean_login_log_period():
     now = timezone.now()
@@ -40,9 +49,52 @@ def clean_ftp_log_period():
     FTPLog.objects.filter(date_start__lt=expired_day).delete()
 
 
-@register_as_period_task(interval=3600 * 24)
+def clean_celery_tasks_period():
-@shared_task(verbose_name=_('Clean audits log'))
+    logger.debug("Start clean celery task history")
+    expire_days = get_log_keep_day('TASK_LOG_KEEP_DAYS')
+    days_ago = timezone.now() - timezone.timedelta(days=expire_days)
+    tasks = CeleryTaskExecution.objects.filter(date_start__lt=days_ago)
+    tasks.delete()
+    tasks = CeleryTaskExecution.objects.filter(date_start__isnull=True)
+    tasks.delete()
+    command = "find %s -mtime +%s -name '*.log' -type f -exec rm -f {} \\;" % (
+        settings.CELERY_LOG_DIR, expire_days
+    )
+    subprocess.call(command, shell=True)
+    command = "echo > {}".format(os.path.join(settings.LOG_DIR, 'celery.log'))
+    subprocess.call(command, shell=True)
+
+
+def clean_expired_session_period():
+    logger.info("Start clean expired session record, commands and replay")
+    days = get_log_keep_day('TERMINAL_SESSION_KEEP_DURATION')
+    expire_date = timezone.now() - timezone.timedelta(days=days)
+    expired_sessions = Session.objects.filter(date_start__lt=expire_date)
+    timestamp = expire_date.timestamp()
+    expired_commands = Command.objects.filter(timestamp__lt=timestamp)
+    replay_dir = os.path.join(default_storage.base_location, 'replay')
+
+    expired_sessions.delete()
+    logger.info("Clean session item done")
+    expired_commands.delete()
+    logger.info("Clean session command done")
+    command = "find %s -mtime +%s \\( -name '*.json' -o -name '*.tar' -o -name '*.gz' \\) -exec rm -f {} \\;" % (
+        replay_dir, days
+    )
+    subprocess.call(command, shell=True)
+    command = "find %s -type d -empty -delete;" % replay_dir
+    subprocess.call(command, shell=True)
+    logger.info("Clean session replay done")
+
+
+@shared_task(verbose_name=_('Clean audits session task log'))
+@register_as_period_task(crontab='0 2 * * *')
+@after_app_shutdown_clean_periodic
 def clean_audits_log_period():
+    print("Start clean audit session task log")
     clean_login_log_period()
     clean_operation_log_period()
     clean_ftp_log_period()
+    clean_activity_log_period()
+    clean_celery_tasks_period()
+    clean_expired_session_period()

apps/locale/ja/LC_MESSAGES/django.mo

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4190f4708c72e943e4dbe743c035270b7cfe0bf231383f09cb7b926459c9d74c
+oid sha256:6bbad43c6fd75da933f86aa609c54641cfbf45099564b07e5090d007287af984
-size 135279
+size 135540

apps/locale/ja/LC_MESSAGES/django.po

@@ -4526,10 +4526,8 @@ msgstr ""
 "タベースストレージにのみ影響します。ossなどは影響しません」影響を受ける)"
 
 #: settings/serializers/cleaning.py:36
-#, fuzzy
-#| msgid "Login log keep days"
 msgid "Activity log keep days"
-msgstr "ログインログは日数を保持します"
+msgstr "活動ログは日数を保持します"
 
 #: settings/serializers/email.py:21
 msgid "SMTP host"

apps/locale/zh/LC_MESSAGES/django.mo

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c33582a24048531069df755d224c8d2f27bc9246603b93131aa667e5e1f5e492
+oid sha256:910ef049b67560a279f8e0e702d74c1343f36c53d95ef84a32782c08c92a184a
-size 111766
+size 111866

apps/locale/zh/LC_MESSAGES/django.po

@@ -4469,7 +4469,7 @@ msgstr "云同步记录"
 
 #: settings/serializers/cleaning.py:31
 msgid "Session keep duration"
-msgstr "会话日志保存时间"
+msgstr "资产会话日志"
 
 #: settings/serializers/cleaning.py:32
 msgid ""
@@ -4480,10 +4480,8 @@ msgstr ""
 "受影响)"
 
 #: settings/serializers/cleaning.py:36
-#, fuzzy
-#| msgid "Login log keep days"
 msgid "Activity log keep days"
-msgstr "登录日志"
+msgstr "活动记录"
 
 #: settings/serializers/email.py:21
 msgid "SMTP host"

apps/ops/tasks.py

@@ -1,24 +1,19 @@
 # coding: utf-8
-import os
-import subprocess
 
 from celery import shared_task
 from celery.exceptions import SoftTimeLimitExceeded
-from django.conf import settings
-from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
-from common.utils import get_logger, get_object_or_none, get_log_keep_day
+from common.utils import get_logger, get_object_or_none
 from orgs.utils import tmp_to_org
 from .celery.decorator import (
-    register_as_period_task, after_app_shutdown_clean_periodic,
+    register_as_period_task, after_app_ready_start
-    after_app_ready_start
 )
 from .celery.utils import (
     create_or_update_celery_periodic_tasks, get_celery_periodic_task,
     disable_celery_periodic_task, delete_celery_periodic_task
 )
-from .models import CeleryTaskExecution, Job, JobExecution
+from .models import Job, JobExecution
 from .notifications import ServerPerformanceCheckUtil
 
 logger = get_logger(__file__)
@@ -55,25 +50,6 @@ def run_ops_job_execution(execution_id, **kwargs):
         logger.error("Start adhoc execution error: {}".format(e))
 
 
-@shared_task(verbose_name=_('Periodic clear celery tasks'))
-@after_app_shutdown_clean_periodic
-@register_as_period_task(interval=3600 * 24, description=_("Clean celery log period"))
-def clean_celery_tasks_period():
-    logger.debug("Start clean celery task history")
-    expire_days = get_log_keep_day('TASK_LOG_KEEP_DAYS')
-    days_ago = timezone.now() - timezone.timedelta(days=expire_days)
-    tasks = CeleryTaskExecution.objects.filter(date_start__lt=days_ago)
-    tasks.delete()
-    tasks = CeleryTaskExecution.objects.filter(date_start__isnull=True)
-    tasks.delete()
-    command = "find %s -mtime +%s -name '*.log' -type f -exec rm -f {} \\;" % (
-        settings.CELERY_LOG_DIR, expire_days
-    )
-    subprocess.call(command, shell=True)
-    command = "echo > {}".format(os.path.join(settings.LOG_DIR, 'celery.log'))
-    subprocess.call(command, shell=True)
-
-
 @shared_task(verbose_name=_('Clear celery periodic tasks'))
 @after_app_ready_start
 def clean_celery_periodic_tasks():

apps/settings/api/settings.py

@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 #
 
-from rest_framework import generics
 from django.conf import settings
+from rest_framework import generics
 
+from common.utils import get_logger
 from jumpserver.conf import Config
 from rbac.permissions import RBACPermission
-from common.utils import get_logger
 from .. import serializers
 from ..models import Setting
+from ..signals import category_setting_updated
 
 logger = get_logger(__file__)
 
@@ -115,10 +116,15 @@ class SettingsApi(generics.RetrieveUpdateAPIView):
             })
         return data
 
+    def send_signal(self, serializer):
+        category = self.request.query_params.get('category', '')
+        category_setting_updated.send(sender=self.__class__, category=category, serializer=serializer)
+
     def perform_update(self, serializer):
         post_data_names = list(self.request.data.keys())
         settings_items = self.parse_serializer_data(serializer)
         serializer_data = getattr(serializer, 'data', {})
+
         for item in settings_items:
             if item['name'] not in post_data_names:
                 continue
@@ -126,6 +132,8 @@ class SettingsApi(generics.RetrieveUpdateAPIView):
             if not changed:
                 continue
             serializer_data[setting.name] = setting.cleaned_value
+
         setattr(serializer, '_data', serializer_data)
         if hasattr(serializer, 'post_save'):
             serializer.post_save()
+        self.send_signal(serializer)

apps/settings/signals.py

@@ -0,0 +1,3 @@
+from django.dispatch import Signal
+
+category_setting_updated = Signal(providing_args=('category', 'serializer'))

apps/settings/ws.py

@@ -38,7 +38,7 @@ class ToolsWebsocket(JsonWebsocketConsumer):
                 self.send_json({'msg': msg})
                 break
             if delay is None:
-                msg = msg.format(dest_addr, 'failed. (timeout within {}sec.)'.format(timeout))
+                msg = msg.format(dest_addr, 'failed. (timeout within {}s)'.format(timeout))
             else:
                 delay = delay * 1000
                 msg = msg.format(dest_addr, 'get ping in %0.4fms' % delay)

apps/terminal/tasks.py

@@ -2,15 +2,13 @@
 #
 
 import datetime
-import os
-import subprocess
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
 from django.core.files.storage import default_storage
 from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
 
-from common.utils import get_log_keep_day
 from ops.celery.decorator import (
     register_as_period_task, after_app_ready_start,
     after_app_shutdown_clean_periodic
@@ -18,10 +16,9 @@ from ops.celery.decorator import (
 from orgs.utils import tmp_to_builtin_org
 from .backends import server_replay_storage
 from .models import (
-    Status, Session, Command, Task, AppletHostDeployment
+    Status, Session, Task, AppletHostDeployment
 )
 from .utils import find_session_replay_local
-from django.utils.translation import gettext_lazy as _
 
 CACHE_REFRESH_INTERVAL = 10
 RUNNING = False
@@ -56,32 +53,6 @@ def clean_orphan_session():
         session.save()
 
 
-@shared_task(verbose_name=_('Periodic clean expired session'))
-@register_as_period_task(interval=3600 * 24)
-@after_app_ready_start
-@after_app_shutdown_clean_periodic
-def clean_expired_session_period():
-    logger.info("Start clean expired session record, commands and replay")
-    days = get_log_keep_day('TERMINAL_SESSION_KEEP_DURATION')
-    expire_date = timezone.now() - timezone.timedelta(days=days)
-    expired_sessions = Session.objects.filter(date_start__lt=expire_date)
-    timestamp = expire_date.timestamp()
-    expired_commands = Command.objects.filter(timestamp__lt=timestamp)
-    replay_dir = os.path.join(default_storage.base_location, 'replay')
-
-    expired_sessions.delete()
-    logger.info("Clean session item done")
-    expired_commands.delete()
-    logger.info("Clean session command done")
-    command = "find %s -mtime +%s \\( -name '*.json' -o -name '*.tar' -o -name '*.gz' \\) -exec rm -f {} \\;" % (
-        replay_dir, days
-    )
-    subprocess.call(command, shell=True)
-    command = "find %s -type d -empty -delete;" % replay_dir
-    subprocess.call(command, shell=True)
-    logger.info("Clean session replay done")
-
-
 @shared_task(verbose_name=_('Upload session replay to external storage'))
 def upload_session_replay_to_external_storage(session_id):
     logger.info(f'Start upload session to external storage: {session_id}')