feat: 录像存储server类型，可以设置如何存储了 (#4699)

feat: Server 类型的录像存储可以上传到 oss等上面

apps/jumpserver/conf.py

@@ -273,6 +273,7 @@ class Config(dict):
         'SESSION_COOKIE_SECURE': False,
         'CSRF_COOKIE_SECURE': False,
         'REFERER_CHECK_ENABLED': False,
+        'SERVER_REPLAY_STORAGE': {}
     }
 
     def compatible_auth_openid_of_key(self):

apps/jumpserver/settings/custom.py

@@ -12,6 +12,17 @@ DEFAULT_TERMINAL_COMMAND_STORAGE = {
     },
 }
 TERMINAL_COMMAND_STORAGE = DYNAMIC.TERMINAL_COMMAND_STORAGE or {}
+
+# Server 类型的录像存储
+SERVER_REPLAY_STORAGE = CONFIG.SERVER_REPLAY_STORAGE
+# SERVER_REPLAY_STORAGE = {
+#     'TYPE': 's3',
+#     'BUCKET': '',
+#     'ACCESS_KEY': '',
+#     'SECRET_KEY': '',
+#     'ENDPOINT': ''
+# }
+
 DEFAULT_TERMINAL_REPLAY_STORAGE = {
     "default": {
         "TYPE": "server",

apps/terminal/api/session.py

@@ -125,7 +125,7 @@ class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet):
 
         if serializer.is_valid():
             file = serializer.validated_data['file']
-            name, err = session.save_to_storage(file)
+            name, err = session.save_replay_to_storage(file)
             if not name:
                 msg = "Failed save replay `{}`: {}".format(session_id, err)
                 logger.error(msg)
@@ -155,6 +155,7 @@ class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet):
         return data
 
     def is_need_async(self):
+        return False
         if self.action != 'retrieve':
             return False
         return True

apps/terminal/backends/__init__.py

@@ -1,5 +1,7 @@
 from importlib import import_module
 from django.conf import settings
+from django.utils.functional import LazyObject
+
 from .command.serializers import SessionCommandSerializer
 from ..const import COMMAND_STORAGE_TYPE_SERVER
 
@@ -17,6 +19,13 @@ def get_command_storage():
     return storage
 
 
+def get_server_replay_storage():
+    from jms_storage import get_object_storage
+    config = settings.SERVER_REPLAY_STORAGE
+    storage = get_object_storage(config)
+    return storage
+
+
 def get_terminal_command_storages():
     from ..models import CommandStorage
     storage_list = {}
@@ -40,3 +49,15 @@ def get_multi_command_storage():
     return storage
 
 
+class ServerReplayStorage(LazyObject):
+    def _setup(self):
+        self._wrapped = get_server_replay_storage()
+
+
+class ServerCommandStorage(LazyObject):
+    def _setup(self):
+        self._wrapped = get_command_storage()
+
+
+server_command_storage = ServerCommandStorage()
+server_replay_storage = ServerReplayStorage()

apps/terminal/backends/replay/__init__.py

@@ -0,0 +1,3 @@
+"""
+Replay 的 backend 已移动到 jms_storage 模块中
+"""

apps/terminal/models.py

@@ -253,14 +253,18 @@ class Session(OrgModelMixin):
             return False
         return True
 
-    def save_to_storage(self, f):
+    def save_replay_to_storage(self, f):
         local_path = self.get_local_path()
         try:
             name = default_storage.save(local_path, f)
-            return name, None
         except OSError as e:
             return None, e
 
+        if settings.SERVER_REPLAY_STORAGE:
+            from .tasks import upload_session_replay_to_external_storage
+            upload_session_replay_to_external_storage.delay(str(self.id))
+        return name, None
+
     @classmethod
     def set_sessions_active(cls, sessions_id):
         data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in sessions_id}
@@ -456,4 +460,3 @@ class ReplayStorage(CommonModelMixin):
 
     def can_delete(self):
         return not self.in_defaults()
-

apps/terminal/tasks.py

@@ -9,11 +9,12 @@ from django.utils import timezone
 from django.conf import settings
 from django.core.files.storage import default_storage
 
-
 from ops.celery.decorator import (
     register_as_period_task, after_app_ready_start, after_app_shutdown_clean_periodic
 )
 from .models import Status, Session, Command
+from .backends import server_replay_storage
+from .utils import find_session_replay_local
 
 
 CACHE_REFRESH_INTERVAL = 10
@@ -68,3 +69,26 @@ def clean_expired_session_period():
         # 删除session记录
         session.delete()
 
+
+@shared_task
+def upload_session_replay_to_external_storage(session_id):
+    logger.info(f'Start upload session to external storage: {session_id}')
+    session = Session.objects.filter(id=session_id).first()
+    if not session:
+        logger.error(f'Session db item not found: {session_id}')
+        return
+    local_path, foobar = find_session_replay_local(session)
+    if not local_path:
+        logger.error(f'Session replay not found, may be upload error: {local_path}')
+        return
+    abs_path = default_storage.path(local_path)
+    remote_path = session.get_rel_replay_path()
+    ok, err = server_replay_storage.upload(abs_path, remote_path)
+    if not ok:
+        logger.error(f'Session replay upload to external error: {err}')
+        return
+    try:
+        default_storage.delete(local_path)
+    except:
+        pass
+    return

apps/terminal/utils.py

@@ -2,43 +2,18 @@
 #
 import os
 
-from django.core.cache import cache
+from django.conf import settings
 from django.core.files.storage import default_storage
 import jms_storage
 
-from assets.models import Asset, SystemUser
-from users.models import User
 from common.utils import get_logger
-from .const import USERS_CACHE_KEY, ASSETS_CACHE_KEY, SYSTEM_USER_CACHE_KEY
+
+from .backends import server_replay_storage
 from .models import ReplayStorage
 
 logger = get_logger(__name__)
 
 
-def get_session_asset_list():
-    return Asset.objects.values_list('hostname', flat=True)
-
-
-def get_session_user_list():
-    return User.objects.exclude(role=User.ROLE.APP).values_list('username', flat=True)
-
-
-def get_session_system_user_list():
-    return SystemUser.objects.values_list('username', flat=True)
-
-
-def get_user_list_from_cache():
-    return cache.get(USERS_CACHE_KEY)
-
-
-def get_asset_list_from_cache():
-    return cache.get(ASSETS_CACHE_KEY)
-
-
-def get_system_user_list_from_cache():
-    return cache.get(SYSTEM_USER_CACHE_KEY)
-
-
 def find_session_replay_local(session):
     # 新版本和老版本的文件后缀不同
     session_path = session.get_rel_replay_path()  # 存在外部存储上的路径
@@ -62,6 +37,8 @@ def download_session_replay(session):
         for storage in replay_storages
         if not storage.in_defaults()
     }
+    if settings.SERVER_REPLAY_STORAGE:
+        configs['SERVER_REPLAY_STORAGE'] = settings.SERVER_REPLAY_STORAGE
     if not configs:
         msg = "Not found replay file, and not remote storage set"
         return None, msg