Merge pull request #5256 from jumpserver/dev

chore: Merge master from dev

apps/assets/models/node.py

@@ -494,7 +494,8 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin):
         for node in nodes_sorted:
             parent = nodes_mapper.get(node.parent_key)
             if not parent:
-                logger.error(f'Node parent node in mapper: {node.parent_key} {node.value}')
+                if node.parent_key:
+                    logger.error(f'Node parent node in mapper: {node.parent_key} {node.value}')
                 continue
             node.full_value = parent.full_value + '/' + node.value
         self.__class__.objects.bulk_update(nodes, ['full_value'])

apps/audits/migrations/0011_userloginlog_backend.py

@@ -13,6 +13,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='userloginlog',
             name='backend',
-            field=models.CharField(default='', max_length=32, verbose_name='Login backend'),
+            field=models.CharField(default='', max_length=32, verbose_name='Authentication backend'),
         ),
     ]

apps/jumpserver/urls.py

@@ -38,11 +38,7 @@ app_view_patterns = [
     re_path(r'flower/(?P<path>.*)', views.celery_flower_view, name='flower-view'),
 ]
 
-
 if settings.XPACK_ENABLED:
-    app_view_patterns.append(
-        path('xpack/', include('xpack.urls.view_urls', namespace='xpack'))
-    )
     api_v1.append(
         path('xpack/', include('xpack.urls.api_urls', namespace='api-xpack'))
     )
@@ -54,7 +50,6 @@ apps = [
     'flower', 'luna', 'koko', 'ws', 'docs', 'redocs',
 ]
 
-
 urlpatterns = [
     path('', views.IndexView.as_view(), name='index'),
     path('api/v1/', include(api_v1)),
@@ -67,29 +62,27 @@ urlpatterns = [
     path('ui/', views.UIView.as_view()),
 ]
 
+# 静态文件处理路由
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \
             + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
 
-js_i18n_patterns = [
+# js i18n 路由文件
+urlpatterns += [
     path('core/jsi18n/', JavaScriptCatalog.as_view(), name='javascript-catalog'),
 ]
-urlpatterns += js_i18n_patterns
 
-handler404 = 'jumpserver.views.handler404'
-handler500 = 'jumpserver.views.handler500'
+# docs 路由
+urlpatterns += [
+    re_path('^api/swagger(?P<format>\.json|\.yaml)$',
+            views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
+    re_path('api/docs/?', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"),
+    re_path('api/redoc/?', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'),
 
-if settings.DEBUG:
-    urlpatterns += [
-        re_path('^api/swagger(?P<format>\.json|\.yaml)$',
-                views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
-        re_path('api/docs/?', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"),
-        re_path('api/redoc/?', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'),
-
-        re_path('^api/v2/swagger(?P<format>\.json|\.yaml)$',
-                views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
-        path('api/docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"),
-        path('api/redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'),
-    ]
+    re_path('^api/v2/swagger(?P<format>\.json|\.yaml)$',
+            views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
+    path('api/docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"),
+    path('api/redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'),
+]
 
 
 # 兼容之前的
@@ -98,3 +91,5 @@ old_app_pattern = r'^{}'.format(old_app_pattern)
 urlpatterns += [re_path(old_app_pattern, views.redirect_old_apps_view)]
 
 
+handler404 = 'jumpserver.views.handler404'
+handler500 = 'jumpserver.views.handler500'

apps/jumpserver/views/swagger.py

@@ -76,9 +76,8 @@ def get_swagger_view(version='v1'):
         patterns = api_v1_patterns
     schema_view = get_schema_view(
         api_info,
-        public=True,
         patterns=patterns,
-        permission_classes=(permissions.AllowAny,),
+        permission_classes=(permissions.IsAuthenticated,),
     )
     return schema_view
 

apps/orgs/signals_handler.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 #
+from collections import defaultdict
+from functools import partial
 
 from django.db.models.signals import m2m_changed
 from django.db.models.signals import post_save
@@ -7,10 +9,10 @@ from django.dispatch import receiver
 
 from orgs.utils import tmp_to_org
 from .models import Organization, OrganizationMember
-from .hands import set_current_org, current_org, Node, get_current_org
-from perms.models import (AssetPermission, DatabaseAppPermission,
-                          K8sAppPermission, RemoteAppPermission)
-from users.models import UserGroup
+from .hands import set_current_org, Node, get_current_org
+from perms.models import (AssetPermission, ApplicationPermission)
+from users.models import UserGroup, User
+from common.const.signals import PRE_REMOVE, POST_REMOVE
 
 
 @receiver(post_save, sender=Organization)
@@ -34,11 +36,44 @@ def _remove_users(model, users, org):
             users = (users, )
 
         m2m_model = model.users.through
-        if model.users.reverse:
+        reverse = model.users.reverse
+        if reverse:
             m2m_field_name = model.users.field.m2m_reverse_field_name()
         else:
             m2m_field_name = model.users.field.m2m_field_name()
-        m2m_model.objects.filter(**{'user__in': users, f'{m2m_field_name}__org_id': org.id}).delete()
+        relations = m2m_model.objects.filter(**{
+            'user__in': users,
+            f'{m2m_field_name}__org_id': org.id
+        })
+
+        object_id_users_id_map = defaultdict(set)
+
+        m2m_field_attr_name = f'{m2m_field_name}_id'
+        for relation in relations:
+            object_id = getattr(relation, m2m_field_attr_name)
+            object_id_users_id_map[object_id].add(relation.user_id)
+
+        objects = model.objects.filter(id__in=object_id_users_id_map.keys())
+        send_m2m_change_signal = partial(
+            m2m_changed.send,
+            sender=m2m_model, reverse=reverse, model=User, using=model.objects.db
+        )
+
+        for obj in objects:
+            send_m2m_change_signal(
+                instance=obj,
+                pk_set=object_id_users_id_map[obj.id],
+                action=PRE_REMOVE
+            )
+
+        relations.delete()
+
+        for obj in objects:
+            send_m2m_change_signal(
+                instance=obj,
+                pk_set=object_id_users_id_map[obj.id],
+                action=POST_REMOVE
+            )
 
 
 def _clear_users_from_org(org, users):
@@ -48,8 +83,7 @@ def _clear_users_from_org(org, users):
     if not users:
         return
 
-    models = (AssetPermission, DatabaseAppPermission,
-              RemoteAppPermission, K8sAppPermission, UserGroup)
+    models = (AssetPermission, ApplicationPermission, UserGroup)
 
     for m in models:
         _remove_users(m, users, org)

apps/terminal/api/component.py

@@ -28,7 +28,7 @@ class ComponentsMetricsAPIView(generics.GenericAPIView):
     permission_classes = (IsSuperUser,)
 
     def get(self, request, *args, **kwargs):
-        component_type = request.query_params.get('type')
-        util = ComponentsMetricsUtil(component_type)
-        metrics = util.get_metrics()
+        tp = request.query_params.get('type')
+        util = ComponentsMetricsUtil()
+        metrics = util.get_metrics(tp)
         return Response(metrics, status=status.HTTP_200_OK)

apps/terminal/migrations/0030_terminal_type.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.1 on 2020-12-10 07:05
+# Generated by Django 3.1 on 2020-12-15 04:52
 
 from django.db import migrations, models
 
@@ -36,7 +36,6 @@ class Migration(migrations.Migration):
             model_name='terminal',
             name='type',
             field=models.CharField(choices=[('koko', 'KoKo'), ('guacamole', 'Guacamole'), ('omnidb', 'OmniDB')], default='koko', max_length=64, verbose_name='type'),
-            preserve_default=False,
         ),
         migrations.RunPython(migrate_terminal_type)
     ]

apps/terminal/models/terminal.py

@@ -129,7 +129,10 @@ class TerminalStatusMixin(TerminalStateMixin):
 class Terminal(TerminalStatusMixin, models.Model):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
-    type = models.CharField(choices=const.TerminalTypeChoices.choices, max_length=64, verbose_name=_('type'))
+    type = models.CharField(
+        choices=const.TerminalTypeChoices.choices, default=const.TerminalTypeChoices.koko.value,
+        max_length=64, verbose_name=_('type')
+    )
     remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address'))
     ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222)
     http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000)

apps/terminal/serializers_v2/terminal.py

@@ -18,7 +18,7 @@ class TerminalSerializer(serializers.ModelSerializer):
     class Meta:
         model = Terminal
         fields = [
-            'id', 'name', 'remote_addr', 'command_storage',
+            'id', 'name', 'type', 'remote_addr', 'command_storage',
             'replay_storage', 'user', 'is_accepted', 'is_deleted',
             'date_created', 'comment'
         ]
@@ -55,5 +55,6 @@ class TerminalSerializer(serializers.ModelSerializer):
 
 class TerminalRegistrationSerializer(serializers.Serializer):
     name = serializers.CharField(max_length=128)
-    comment = serializers.CharField(max_length=128, )
+    comment = serializers.CharField(max_length=128)
+    type = serializers.CharField(max_length=64)
     service_account = ServiceAccountSerializer(read_only=True)

apps/terminal/utils.py

@@ -106,46 +106,43 @@ def send_command_alert_mail(command):
 
 class ComponentsMetricsUtil(object):
 
-    def __init__(self, component_type=None):
-        self.type = component_type
-        self.components = []
-        self.initial_components()
-
-    def initial_components(self):
+    @staticmethod
+    def get_components(tp=None):
         from .models import Terminal
-        terminals = Terminal.objects.all().order_by('type')
-        if self.type:
-            terminals = terminals.filter(type=self.type)
-        self.components = list(terminals)
+        components = Terminal.objects.all().order_by('type')
+        if tp:
+            components = components.filter(type=tp)
+        return components
 
-    def get_metrics(self):
+    def get_metrics(self, tp=None):
+        components = self.get_components(tp)
         total_count = normal_count = high_count = critical_count = session_active_total = 0
-        for component in self.components:
+        for component in components:
             total_count += 1
-            if not component.is_alive:
-                critical_count += 1
-                continue
-            session_active_total += component.state.get('session_active_count', 0)
-            if component.is_normal:
-                normal_count += 1
-            elif component.is_high:
-                high_count += 1
+            if component.is_alive:
+                if component.is_normal:
+                    normal_count += 1
+                elif component.is_high:
+                    high_count += 1
+                else:
+                    # critical
+                    critical_count += 1
+                session_active_total += component.state.get('session_active_count', 0)
             else:
                 critical_count += 1
-        metrics = {
+        return {
             'total': total_count,
             'normal': normal_count,
             'high': high_count,
             'critical': critical_count,
             'session_active': session_active_total
         }
-        return metrics
 
 
 class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil):
 
     @staticmethod
-    def get_status_metrics(metrics):
+    def convert_status_metrics(metrics):
         return {
             'any': metrics['total'],
             'normal': metrics['normal'],
@@ -154,50 +151,47 @@ class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil):
         }
 
     def get_prometheus_metrics_text(self):
-        prometheus_metrics = []
-        prometheus_metrics.append('# JumpServer 各组件状态个数汇总')
-        base_status_metric_text = 'jumpserver_components_status_total{component_type="%s", status="%s"} %s'
-        for component in self.components:
-            component_type = component.type
-            base_metrics = self.get_metrics()
+        prometheus_metrics = list()
 
-            prometheus_metrics.append(f'## 组件: {component_type}')
-            status_metrics = self.get_status_metrics(base_metrics)
+        # 各组件状态个数汇总
+        prometheus_metrics.append('# JumpServer 各组件状态个数汇总')
+        status_metric_text = 'jumpserver_components_status_total{component_type="%s", status="%s"} %s'
+        for tp in const.TerminalTypeChoices.types():
+            prometheus_metrics.append(f'## 组件: {tp}')
+            metrics_tp = self.get_metrics(tp)
+            status_metrics = self.convert_status_metrics(metrics_tp)
             for status, value in status_metrics.items():
-                metric_text = base_status_metric_text % (component_type, status, value)
+                metric_text = status_metric_text % (tp, status, value)
                 prometheus_metrics.append(metric_text)
 
         prometheus_metrics.append('\n')
+
+        # 各组件在线会话数汇总
         prometheus_metrics.append('# JumpServer 各组件在线会话数汇总')
-        base_session_active_metric_text = 'jumpserver_components_session_active_total{component_type="%s"} %s'
-        for component in self.components:
-            component_type = component.type
-            prometheus_metrics.append(f'## 组件: {component_type}')
-            base_metrics = self.get_metrics()
-            metric_text = base_session_active_metric_text % (
-                component_type,
-                base_metrics['session_active']
-            )
+        session_active_metric_text = 'jumpserver_components_session_active_total{component_type="%s"} %s'
+        for tp in const.TerminalTypeChoices.types():
+            prometheus_metrics.append(f'## 组件: {tp}')
+            metrics_tp = self.get_metrics(tp)
+            metric_text = session_active_metric_text % (tp, metrics_tp['session_active'])
             prometheus_metrics.append(metric_text)
 
         prometheus_metrics.append('\n')
-        prometheus_metrics.append('# JumpServer 各组件节点一些指标')
-        base_system_state_metric_text = 'jumpserver_components_%s{component_type="%s", component="%s"} %s'
-        system_states_name = [
+
+        # 各组件节点指标
+        prometheus_metrics.append('# JumpServer 各组件一些指标')
+        state_metric_text = 'jumpserver_components_%s{component_type="%s", component="%s"} %s'
+        states = [
             'system_cpu_load_1', 'system_memory_used_percent',
             'system_disk_used_percent', 'session_active_count'
         ]
-        for system_state_name in system_states_name:
-            prometheus_metrics.append(f'## 指标: {system_state_name}')
-            for component in self.components:
+        for state in states:
+            prometheus_metrics.append(f'## 指标: {state}')
+            components = self.get_components()
+            for component in components:
                 if not component.is_alive:
                     continue
-                component_type = component.type
-                metric_text = base_system_state_metric_text % (
-                    system_state_name,
-                    component_type,
-                    component.name,
-                    component.state.get(system_state_name)
+                metric_text = state_metric_text % (
+                    state, component.type, component.name, component.state.get(state)
                 )
                 prometheus_metrics.append(metric_text)
 

apps/tickets/serializers/request_asset_perm.py

@@ -1,5 +1,3 @@
-from itertools import chain
-
 from rest_framework import serializers
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
@@ -9,7 +7,7 @@ from django.db.models import Q
 from common.utils.timezone import dt_parser, dt_formater
 from orgs.utils import tmp_to_root_org
 from orgs.models import Organization, ROLE as ORG_ROLE
-from assets.models.asset import Asset
+from assets.models import Asset, SystemUser
 from users.models.user import User
 from perms.serializers import ActionsField
 from perms.models import Action
@@ -130,12 +128,23 @@ class RequestAssetPermTicketSerializer(serializers.ModelSerializer):
             if hostname:
                 q |= Q(hostname__icontains=hostname)
 
-            data['confirmed_assets'] = list(
-                map(lambda x: str(x), chain(*Asset.objects.filter(q)[0: limit].values_list('id'))))
+            recomand_assets_id = Asset.objects.filter(q)[:limit].values_list('id', flat=True)
+            data['confirmed_assets'] = [str(i) for i in recomand_assets_id]
+
+    def _recommend_system_users(self, data, instance):
+        confirmed_system_users = data.get('confirmed_system_users')
+        if not confirmed_system_users and self._is_assignee(instance):
+            system_user = data.get('system_user')
+
+            recomand_system_users_id = SystemUser.objects.filter(
+                name__icontains=system_user
+            )[:3].values_list('id', flat=True)
+            data['confirmed_system_users'] = [str(i) for i in recomand_system_users_id]
 
     def to_representation(self, instance):
         data = super().to_representation(instance)
         self._recommend_assets(data, instance)
+        self._recommend_system_users(data, instance)
         return data
 
     def _create_body(self, validated_data):