feat: 命令及录像存储可连接性定时检查 (#10594)

Co-authored-by: feng <1304903146@qq.com>

apps/notifications/api/notifications.py

@@ -1,12 +1,12 @@
 from rest_framework.mixins import ListModelMixin, UpdateModelMixin, RetrieveModelMixin
-from rest_framework.views import APIView
 from rest_framework.response import Response
+from rest_framework.views import APIView
 
 from common.api import JMSGenericViewSet
 from common.permissions import IsValidUser
-from notifications.notifications import system_msgs
-from notifications.models import SystemMsgSubscription, UserMsgSubscription
 from notifications.backends import BACKEND
+from notifications.models import SystemMsgSubscription, UserMsgSubscription
+from notifications.notifications import system_msgs
 from notifications.serializers import (
     SystemMsgSubscriptionSerializer, SystemMsgSubscriptionByCategorySerializer,
     UserMsgSubscriptionSerializer,
@@ -32,9 +32,9 @@ class BackendListView(APIView):
         return Response(data=data)
 
 
-class SystemMsgSubscriptionViewSet(ListModelMixin,
-                                   UpdateModelMixin,
-                                   JMSGenericViewSet):
+class SystemMsgSubscriptionViewSet(
+    ListModelMixin, UpdateModelMixin, JMSGenericViewSet
+):
     lookup_field = 'message_type'
     queryset = SystemMsgSubscription.objects.all()
     serializer_classes = {

apps/notifications/notifications.py

@@ -1,17 +1,17 @@
-import traceback
-from html2text import HTML2Text
-from typing import Iterable
-from itertools import chain
 import textwrap
+import traceback
+from itertools import chain
+from typing import Iterable
 
 from celery import shared_task
 from django.utils.translation import gettext_lazy as _
+from html2text import HTML2Text
 
-from common.utils.timezone import local_now
 from common.utils import lazyproperty
+from common.utils.timezone import local_now
+from notifications.backends import BACKEND
 from settings.utils import get_login_title
 from users.models import User
-from notifications.backends import BACKEND
 from .models import SystemMsgSubscription, UserMsgSubscription
 
 __all__ = ('SystemMessage', 'UserMessage', 'system_msgs', 'Message')

apps/ops/celery/decorator.py

@@ -36,7 +36,7 @@ def register_as_period_task(
         args=(), kwargs=None,
         description=''):
     """
-    Warning: Task must be have not any args and kwargs
+    Warning: Task must have not any args and kwargs
     :param crontab:  "* * * * *"
     :param interval:  60*60*60
     :param args: ()

apps/terminal/notifications.py

@@ -15,7 +15,7 @@ from users.models import User
 
 logger = get_logger(__name__)
 
-__all__ = ('CommandAlertMessage', 'CommandExecutionAlert')
+__all__ = ('CommandAlertMessage', 'CommandExecutionAlert', 'StorageConnectivityMessage')
 
 CATEGORY = 'terminal'
 CATEGORY_LABEL = _('Sessions')
@@ -156,3 +156,41 @@ class CommandExecutionAlert(CommandAlertMixin, SystemMessage):
             'subject': self.subject,
             'message': message
         }
+
+
+class StorageConnectivityMessage(SystemMessage):
+    category = 'storage'
+    category_label = _('Command and replay storage')
+    message_type_label = _('Connectivity alarm')
+
+    def __init__(self, errors):
+        self.errors = errors
+
+    @classmethod
+    def post_insert_to_db(cls, subscription: SystemMsgSubscription):
+        subscription.receive_backends = [b for b in BACKEND if b.is_enable]
+        subscription.save()
+
+    @classmethod
+    def gen_test_msg(cls):
+        from terminal.models import ReplayStorage
+        replay = ReplayStorage.objects.first()
+        errors = [{
+            'msg': str(_("Test failure: Account invalid")),
+            'type': replay.get_type_display(),
+            'name': replay.name
+        }]
+        return cls(errors)
+
+    def get_html_msg(self) -> dict:
+        context = {
+            'items': self.errors,
+        }
+        subject = str(_("Invalid storage"))
+        message = render_to_string(
+            'terminal/_msg_check_command_replay_storage_connectivity.html', context
+        )
+        return {
+            'subject': subject,
+            'message': message
+        }

apps/terminal/tasks.py

@@ -2,6 +2,7 @@
 #
 
 import datetime
+from itertools import chain
 
 from celery import shared_task
 from celery.utils.log import get_task_logger
@@ -14,10 +15,14 @@ from ops.celery.decorator import (
     after_app_shutdown_clean_periodic
 )
 from orgs.utils import tmp_to_builtin_org
+from orgs.utils import tmp_to_root_org
 from .backends import server_replay_storage
+from .const import ReplayStorageType, CommandStorageType
 from .models import (
-    Status, Session, Task, AppletHostDeployment, AppletHost
+    Status, Session, Task, AppletHostDeployment,
+    AppletHost, ReplayStorage, CommandStorage
 )
+from .notifications import StorageConnectivityMessage
 from .utils import find_session_replay_local
 
 CACHE_REFRESH_INTERVAL = 10
@@ -111,3 +116,36 @@ def applet_host_generate_accounts(host_id):
 
     with tmp_to_builtin_org(system=1):
         applet_host.generate_accounts()
+
+
+@shared_task(verbose_name=_('Check command replay storage connectivity'))
+@register_as_period_task(crontab='0 0 * * *')
+@tmp_to_root_org()
+def check_command_replay_storage_connectivity():
+    errors = []
+    replays = ReplayStorage.objects.exclude(
+        type__in=[ReplayStorageType.server, ReplayStorageType.null]
+    )
+    commands = CommandStorage.objects.exclude(
+        type__in=[CommandStorageType.server, CommandStorageType.null]
+    )
+
+    for instance in chain(replays, commands):
+        msg = None
+        try:
+            is_valid = instance.is_valid()
+        except Exception as e:
+            is_valid = False
+            msg = _("Test failure: {}".format(str(e)))
+        if is_valid:
+            continue
+        errors.append({
+            'msg': msg or _("Test failure: Account invalid"),
+            'type': instance.get_type_display(),
+            'name': instance.name
+        })
+
+    if not errors:
+        return
+
+    StorageConnectivityMessage(errors).publish_async()

apps/terminal/templates/terminal/_msg_check_command_replay_storage_connectivity.html

@@ -0,0 +1,13 @@
+{% load i18n %}
+
+<p>
+    {% blocktranslate %}
+        Invalid storage
+    {% endblocktranslate %}
+</p>
+
+<ul>
+    {% for item in items %}
+        <li>{{ item.name }}({{ item.type }}): {{ item.msg }}</li>
+    {% endfor %}
+</ul>