Merge pull request #5213 from jumpserver/dev

chore(merge): 合并dev到master

Dockerfile

@@ -1,5 +1,6 @@
-FROM registry.fit2cloud.com/public/python:v3 as stage-build
-MAINTAINER Jumpserver Team <ibuler@qq.com>
+# 编译代码
+FROM python:3.8.6-slim as stage-build
+MAINTAINER JumpServer Team <ibuler@qq.com>
 ARG VERSION
 ENV VERSION=$VERSION
 
@@ -8,33 +9,38 @@ ADD . .
 RUN cd utils && bash -ixeu build.sh
 
 
-FROM registry.fit2cloud.com/public/python:v3
+# 构建运行时环境
+FROM python:3.8.6-slim
 ARG PIP_MIRROR=https://pypi.douban.com/simple
 ENV PIP_MIRROR=$PIP_MIRROR
-ARG MYSQL_MIRROR=https://mirrors.tuna.tsinghua.edu.cn/mysql/yum/mysql57-community-el6/
-ENV MYSQL_MIRROR=$MYSQL_MIRROR
+ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
+ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR
 
 WORKDIR /opt/jumpserver
 
-COPY ./requirements ./requirements
-RUN useradd jumpserver
-RUN yum -y install epel-release && \
-      echo -e "[mysql]\nname=mysql\nbaseurl=${MYSQL_MIRROR}\ngpgcheck=0\nenabled=1" > /etc/yum.repos.d/mysql.repo
-RUN yum -y install $(cat requirements/rpm_requirements.txt)
-RUN pip install --upgrade pip setuptools==49.6.0 wheel -i ${PIP_MIRROR} && \
-    pip config set global.index-url ${PIP_MIRROR}
-RUN pip install $(grep 'jms' requirements/requirements.txt) -i https://pypi.org/simple
-RUN pip install -r requirements/requirements.txt
+COPY ./requirements/deb_buster_requirements.txt ./requirements/deb_buster_requirements.txt
+RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list \
+    && sed -i 's/security.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list \
+    && apt update \
+    && grep -v '^#' ./requirements/deb_buster_requirements.txt | xargs apt -y install \
+    && localedef -c -f UTF-8 -i zh_CN zh_CN.UTF-8 \
+    && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+COPY ./requirements/requirements.txt ./requirements/requirements.txt
+RUN pip install --upgrade pip==20.2.4 setuptools==49.6.0 wheel==0.34.2 -i ${PIP_MIRROR} \
+    && pip config set global.index-url ${PIP_MIRROR} \
+    && pip install --no-cache-dir $(grep 'jms' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
+    && pip install --no-cache-dir -r requirements/requirements.txt
 
 COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
-RUN mkdir -p /root/.ssh/ && echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config
+RUN mkdir -p /root/.ssh/ \
+    && echo -e "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config
 
 RUN echo > config.yml
 VOLUME /opt/jumpserver/data
 VOLUME /opt/jumpserver/logs
 
 ENV LANG=zh_CN.UTF-8
-ENV LC_ALL=zh_CN.UTF-8
 
 EXPOSE 8070
 EXPOSE 8080

apps/applications/api/mixin.py

@@ -1,4 +1,5 @@
 from common.exceptions import JMSException
+from orgs.models import Organization
 from .. import models
 
 
@@ -85,11 +86,46 @@ class SerializeApplicationToTreeNodeMixin:
             'meta': {'type': 'k8s_app'}
         }
 
-    def _serialize(self, application):
+    def _serialize_application(self, application):
         method_name = f'_serialize_{application.category}'
         data = getattr(self, method_name)(application)
+        data.update({
+            'pId': application.org.id,
+            'org_name': application.org_name
+        })
         return data
 
     def serialize_applications(self, applications):
-        data = [self._serialize(application) for application in applications]
+        data = [self._serialize_application(application) for application in applications]
+        return data
+
+    @staticmethod
+    def _serialize_organization(org):
+        return {
+            'id': org.id,
+            'name': org.name,
+            'title': org.name,
+            'pId': '',
+            'open': True,
+            'isParent': True,
+            'meta': {
+                'type': 'node'
+            }
+        }
+
+    def serialize_organizations(self, organizations):
+        data = [self._serialize_organization(org) for org in organizations]
+        return data
+
+    @staticmethod
+    def filter_organizations(applications):
+        organizations_id = set(applications.values_list('org_id', flat=True))
+        organizations = [Organization.get_instance(org_id) for org_id in organizations_id]
+        return organizations
+
+    def serialize_applications_with_org(self, applications):
+        organizations = self.filter_organizations(applications)
+        data_organizations = self.serialize_organizations(organizations)
+        data_applications = self.serialize_applications(applications)
+        data = data_organizations + data_applications
         return data

apps/applications/migrations/0007_auto_20201119_1110.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1 on 2020-11-19 03:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0006_application'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='application',
+            name='attrs',
+            field=models.JSONField(),
+        ),
+    ]

apps/applications/models/application.py

@@ -2,7 +2,6 @@ from itertools import chain
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from django_mysql.models import JSONField, QuerySet
 
 from orgs.mixins.models import OrgModelMixin
 from common.mixins import CommonModelMixin
@@ -123,7 +122,7 @@ class Application(CommonModelMixin, OrgModelMixin):
     domain = models.ForeignKey('assets.Domain', null=True, blank=True, related_name='applications', verbose_name=_("Domain"), on_delete=models.SET_NULL)
     category = models.CharField(max_length=16, choices=Category.choices, verbose_name=_('Category'))
     type = models.CharField(max_length=16, choices=Category.get_all_type_choices(), verbose_name=_('Type'))
-    attrs = JSONField()
+    attrs = models.JSONField()
     comment = models.TextField(
         max_length=128, default='', blank=True, verbose_name=_('Comment')
     )

apps/applications/serializers/application.py

@@ -27,10 +27,8 @@ class ApplicationSerializer(BulkOrgResourceModelSerializer):
         ]
 
     def create(self, validated_data):
-        attrs = validated_data.pop('attrs', {})
+        validated_data['attrs'] = validated_data.pop('attrs', {})
         instance = super().create(validated_data)
-        instance.attrs = attrs
-        instance.save()
         return instance
 
     def update(self, instance, validated_data):

apps/applications/serializers/database_app.py

@@ -12,9 +12,8 @@ from .. import models
 class DBAttrsSerializer(serializers.Serializer):
     host = serializers.CharField(max_length=128, label=_('Host'))
     port = serializers.IntegerField(label=_('Port'))
-    database = serializers.CharField(
-        max_length=128, required=False, allow_blank=True, allow_null=True, label=_('Database')
-    )
+    # 添加allow_null=True，兼容之前数据库中database字段为None的情况
+    database = serializers.CharField(max_length=128, required=True, allow_null=True, label=_('Database'))
 
 
 class MySQLAttrsSerializer(DBAttrsSerializer):

apps/assets/api/mixin.py

@@ -69,6 +69,7 @@ class SerializeToTreeNodeMixin:
                         'ip': asset.ip,
                         'protocols': asset.protocols_as_list,
                         'platform': asset.platform_base,
+                        'org_name': asset.org_name
                     },
                 }
             }

apps/assets/api/node.py

@@ -5,11 +5,13 @@ from collections import namedtuple, defaultdict
 from rest_framework import status
 from rest_framework.serializers import ValidationError
 from rest_framework.response import Response
+from rest_framework.decorators import action
 from django.utils.translation import ugettext_lazy as _
 from django.shortcuts import get_object_or_404, Http404
 from django.utils.decorators import method_decorator
 from django.db.models.signals import m2m_changed
 
+from common.const.http import POST
 from common.exceptions import SomeoneIsDoingThis
 from common.const.signals import PRE_REMOVE, POST_REMOVE
 from assets.models import Asset
@@ -19,6 +21,8 @@ from common.const.distributed_lock_key import UPDATE_NODE_TREE_LOCK_KEY
 from orgs.mixins.api import OrgModelViewSet
 from orgs.mixins import generics
 from orgs.lock import org_level_transaction_lock
+from orgs.utils import current_org
+from assets.tasks import check_node_assets_amount_task
 from ..hands import IsOrgAdmin
 from ..models import Node
 from ..tasks import (
@@ -46,6 +50,11 @@ class NodeViewSet(OrgModelViewSet):
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.NodeSerializer
 
+    @action(methods=[POST], detail=False, url_name='launch-check-assets-amount-task')
+    def launch_check_assets_amount_task(self, request):
+        task = check_node_assets_amount_task.delay(current_org.id)
+        return Response(data={'task': task.id})
+
     # 仅支持根节点指直接创建，子节点下的节点需要通过children接口创建
     def perform_create(self, serializer):
         child_key = Node.org_root().get_next_child_key()
@@ -61,6 +70,9 @@ class NodeViewSet(OrgModelViewSet):
 
     def destroy(self, request, *args, **kwargs):
         node = self.get_object()
+        if node.is_org_root():
+            error = _("You can't delete the root node ({})".format(node.value))
+            return Response(data={'error': error}, status=status.HTTP_403_FORBIDDEN)
         if node.has_children_or_has_assets():
             error = _("Deletion failed and the node contains children or assets")
             return Response(data={'error': error}, status=status.HTTP_403_FORBIDDEN)
@@ -173,7 +185,7 @@ class NodeChildrenAsTreeApi(SerializeToTreeNodeMixin, NodeChildrenApi):
             return []
         assets = self.instance.get_assets().only(
             "id", "hostname", "ip", "os",
-            "org_id", "protocols",
+            "org_id", "protocols", "is_active"
         )
         return self.serialize_assets(assets, self.instance.key)
 
@@ -201,10 +213,8 @@ class NodeAddChildrenApi(generics.UpdateAPIView):
     def put(self, request, *args, **kwargs):
         instance = self.get_object()
         nodes_id = request.data.get("nodes")
-        children = [get_object_or_none(Node, id=pk) for pk in nodes_id]
+        children = Node.objects.filter(id__in=nodes_id)
         for node in children:
-            if not node:
-                continue
             node.parent = instance
         return Response("OK")
 

apps/assets/api/system_user.py

@@ -3,7 +3,8 @@ from django.shortcuts import get_object_or_404
 from rest_framework.response import Response
 
 from common.utils import get_logger
-from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser, IsAppUser
+from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
+from common.drf.filters import CustomFilter
 from orgs.mixins.api import OrgBulkModelViewSet
 from orgs.mixins import generics
 from orgs.utils import tmp_to_org
@@ -12,7 +13,7 @@ from .. import serializers
 from ..serializers import SystemUserWithAuthInfoSerializer
 from ..tasks import (
     push_system_user_to_assets_manual, test_system_user_connectivity_manual,
-    push_system_user_a_asset_manual,
+    push_system_user_to_assets
 )
 
 
@@ -82,18 +83,18 @@ class SystemUserTaskApi(generics.CreateAPIView):
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.SystemUserTaskSerializer
 
-    def do_push(self, system_user, asset=None):
-        if asset is None:
+    def do_push(self, system_user, assets_id=None):
+        if assets_id is None:
             task = push_system_user_to_assets_manual.delay(system_user)
         else:
             username = self.request.query_params.get('username')
-            task = push_system_user_a_asset_manual.delay(
-                system_user, asset, username=username
+            task = push_system_user_to_assets.delay(
+                system_user.id, assets_id, username=username
             )
         return task
 
     @staticmethod
-    def do_test(system_user, asset=None):
+    def do_test(system_user):
         task = test_system_user_connectivity_manual.delay(system_user)
         return task
 
@@ -104,11 +105,16 @@ class SystemUserTaskApi(generics.CreateAPIView):
     def perform_create(self, serializer):
         action = serializer.validated_data["action"]
         asset = serializer.validated_data.get('asset')
+        assets = serializer.validated_data.get('assets') or []
+
         system_user = self.get_object()
         if action == 'push':
-            task = self.do_push(system_user, asset)
+            assets = [asset] if asset else assets
+            assets_id = [asset.id for asset in assets]
+            assets_id = assets_id if assets_id else None
+            task = self.do_push(system_user, assets_id)
         else:
-            task = self.do_test(system_user, asset)
+            task = self.do_test(system_user)
         data = getattr(serializer, '_data', {})
         data["task"] = task.id
         setattr(serializer, '_data', data)

apps/assets/migrations/0063_migrate_default_node_key.py

@@ -0,0 +1,72 @@
+# Generated by Jiangjie.Bai on 2020-12-01 10:47
+
+from django.db import migrations
+from django.db.models import Q
+
+default_node_value = 'Default'  # Always
+old_default_node_key = '0'  # Version <= 1.4.3
+new_default_node_key = '1'  # Version >= 1.4.4
+
+
+def compute_parent_key(key):
+    try:
+        return key[:key.rindex(':')]
+    except ValueError:
+        return ''
+
+
+def migrate_default_node_key(apps, schema_editor):
+    """ 将已经存在的Default节点的key从0修改为1 """
+    # 1.4.3版本中Default节点的key为0
+    print('')
+    Node = apps.get_model('assets', 'Node')
+    Asset = apps.get_model('assets', 'Asset')
+
+    # key为0的节点
+    old_default_node = Node.objects.filter(key=old_default_node_key, value=default_node_value).first()
+    if not old_default_node:
+        print(f'Check old default node `key={old_default_node_key} value={default_node_value}` not exists')
+        return
+    print(f'Check old default node `key={old_default_node_key} value={default_node_value}` exists')
+    # key为1的节点
+    new_default_node = Node.objects.filter(key=new_default_node_key, value=default_node_value).first()
+    if new_default_node:
+        print(f'Check new default node `key={new_default_node_key} value={default_node_value}` exists')
+        all_assets = Asset.objects.filter(
+            Q(nodes__key__startswith=f'{new_default_node_key}:') | Q(nodes__key=new_default_node_key)
+        ).distinct()
+        if all_assets:
+            print(f'Check new default node has assets (count: {len(all_assets)})')
+            return
+        all_children = Node.objects.filter(key__startswith=f'{new_default_node_key}:')
+        if all_children:
+            print(f'Check new default node has children nodes (count: {len(all_children)})')
+            return
+        print(f'Check new default node not has assets and children nodes, delete it.')
+        new_default_node.delete()
+    # 执行修改
+    print(f'Modify old default node `key` from `{old_default_node_key}` to `{new_default_node_key}`')
+    nodes = Node.objects.filter(
+        Q(key__istartswith=f'{old_default_node_key}:') | Q(key=old_default_node_key)
+    )
+    for node in nodes:
+        old_key = node.key
+        key_list = old_key.split(':', maxsplit=1)
+        key_list[0] = new_default_node_key
+        new_key = ':'.join(key_list)
+        node.key = new_key
+        node.parent_key = compute_parent_key(node.key)
+    # 批量更新
+    print(f'Bulk update nodes `key` and `parent_key`, (count: {len(nodes)})')
+    Node.objects.bulk_update(nodes, ['key', 'parent_key'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0062_auto_20201117_1938'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_default_node_key)
+    ]

apps/assets/migrations/0064_auto_20201203_1100.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.1 on 2020-12-03 03:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0063_migrate_default_node_key'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='node',
+            options={'ordering': ['parent_key', 'value'], 'verbose_name': 'Node'},
+        ),
+    ]

apps/assets/models/node.py

@@ -103,7 +103,7 @@ class FamilyMixin:
             if value is None:
                 value = child_key
             child = self.__class__.objects.create(
-                id=_id, key=child_key, value=value, parent_key=self.key,
+                id=_id, key=child_key, value=value
             )
             return child
 
@@ -354,7 +354,8 @@ class SomeNodesMixin:
     def org_root(cls):
         root = cls.objects.filter(parent_key='')\
             .filter(key__regex=r'^[0-9]+$')\
-            .exclude(key__startswith='-')
+            .exclude(key__startswith='-')\
+            .order_by('key')
         if root:
             return root[0]
         else:
@@ -411,7 +412,7 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin):
 
     class Meta:
         verbose_name = _("Node")
-        ordering = ['value']
+        ordering = ['parent_key', 'value']
 
     def __str__(self):
         return self.full_value

apps/assets/serializers/asset.py

@@ -98,9 +98,6 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
         fields_as = list(annotates_fields.keys())
         fields = fields_small + fields_fk + fields_m2m + fields_as
         read_only_fields = [
-            'vendor', 'model', 'sn', 'cpu_model', 'cpu_count',
-            'cpu_cores', 'cpu_vcpus', 'memory', 'disk_total', 'disk_info',
-            'os', 'os_version', 'os_arch', 'hostname_raw',
             'created_by', 'date_created',
         ] + fields_as
 

apps/assets/serializers/system_user.py

@@ -257,4 +257,8 @@ class SystemUserTaskSerializer(serializers.Serializer):
     asset = serializers.PrimaryKeyRelatedField(
         queryset=Asset.objects, allow_null=True, required=False, write_only=True
     )
+    assets = serializers.PrimaryKeyRelatedField(
+        queryset=Asset.objects, allow_null=True, required=False, write_only=True,
+        many=True
+    )
     task = serializers.CharField(read_only=True)

apps/assets/signals_handler.py

@@ -4,7 +4,7 @@ from operator import add, sub
 
 from assets.utils import is_asset_exists_in_node
 from django.db.models.signals import (
-    post_save, m2m_changed, pre_delete, post_delete
+    post_save, m2m_changed, pre_delete, post_delete, pre_save
 )
 from django.db.models import Q, F
 from django.dispatch import receiver
@@ -37,6 +37,11 @@ def test_asset_conn_on_created(asset):
     test_asset_connectivity_util.delay([asset])
 
 
+@receiver(pre_save, sender=Node)
+def on_node_pre_save(sender, instance: Node, **kwargs):
+    instance.parent_key = instance.compute_parent_key()
+
+
 @receiver(post_save, sender=Asset)
 @on_transaction_commit
 def on_asset_created_or_update(sender, instance=None, created=False, **kwargs):
@@ -73,6 +78,7 @@ def on_system_user_update(instance: SystemUser, created, **kwargs):
 
 
 @receiver(m2m_changed, sender=SystemUser.assets.through)
+@on_transaction_commit
 def on_system_user_assets_change(instance, action, model, pk_set, **kwargs):
     """
     当系统用户和资产关系发生变化时，应该重新推送系统用户到新添加的资产中
@@ -91,25 +97,29 @@ def on_system_user_assets_change(instance, action, model, pk_set, **kwargs):
 
 
 @receiver(m2m_changed, sender=SystemUser.users.through)
-def on_system_user_users_change(sender, instance=None, action='', model=None, pk_set=None, **kwargs):
+@on_transaction_commit
+def on_system_user_users_change(sender, instance: SystemUser, action, model, pk_set, reverse, **kwargs):
     """
     当系统用户和用户关系发生变化时，应该重新推送系统用户资产中
     """
     if action != POST_ADD:
         return
+
+    if reverse:
+        raise M2MReverseNotAllowed
+
     if not instance.username_same_with_user:
         return
+
     logger.debug("System user users change signal recv: {}".format(instance))
-    queryset = model.objects.filter(pk__in=pk_set)
-    if model == SystemUser:
-        system_users = queryset
-    else:
-        system_users = [instance]
-    for s in system_users:
-        push_system_user_to_assets_manual.delay(s)
+    usernames = model.objects.filter(pk__in=pk_set).values_list('username', flat=True)
+
+    for username in usernames:
+        push_system_user_to_assets_manual.delay(instance, username)
 
 
 @receiver(m2m_changed, sender=SystemUser.nodes.through)
+@on_transaction_commit
 def on_system_user_nodes_change(sender, instance=None, action=None, model=None, pk_set=None, **kwargs):
     """
     当系统用户和节点关系发生变化时，应该将节点下资产关联到新的系统用户上

apps/assets/tasks/nodes_amount.py

@@ -1,14 +1,27 @@
 from celery import shared_task
+from django.utils.translation import gettext_lazy as _
 
+from orgs.models import Organization
+from orgs.utils import tmp_to_org
+from ops.celery.decorator import register_as_period_task
 from assets.utils import check_node_assets_amount
+
+from common.utils.lock import AcquireFailed
 from common.utils import get_logger
-from common.utils.timezone import now
 
 logger = get_logger(__file__)
 
 
-@shared_task()
-def check_node_assets_amount_celery_task():
-    logger.info(f'>>> {now()} begin check_node_assets_amount_celery_task ...')
-    check_node_assets_amount()
-    logger.info(f'>>> {now()} end check_node_assets_amount_celery_task ...')
+@shared_task(queue='celery_heavy_tasks')
+def check_node_assets_amount_task(org_id=Organization.ROOT_ID):
+    try:
+        with tmp_to_org(Organization.get_instance(org_id)):
+            check_node_assets_amount()
+    except AcquireFailed:
+        logger.error(_('The task of self-checking is already running and cannot be started repeatedly'))
+
+
+@register_as_period_task(crontab='0 2 * * *')
+@shared_task(queue='celery_heavy_tasks')
+def check_node_assets_amount_period_task():
+    check_node_assets_amount_task()

apps/assets/tasks/push_system_user.py

@@ -2,13 +2,13 @@
 
 from itertools import groupby
 from celery import shared_task
-from common.db.utils import get_object_if_need, get_objects_if_need, get_objects
+from common.db.utils import get_object_if_need, get_objects
 from django.utils.translation import ugettext as _
 from django.db.models import Empty
 
 from common.utils import encrypt_password, get_logger
-from assets.models import SystemUser, Asset
-from orgs.utils import org_aware_func
+from assets.models import SystemUser, Asset, AuthBook
+from orgs.utils import org_aware_func, tmp_to_root_org
 from . import const
 from .utils import clean_ansible_task_hosts, group_asset_by_platform
 
@@ -190,15 +190,12 @@ def get_push_system_user_tasks(system_user, platform="unixlike", username=None):
 @org_aware_func("system_user")
 def push_system_user_util(system_user, assets, task_name, username=None):
     from ops.utils import update_or_create_ansible_task
-    hosts = clean_ansible_task_hosts(assets, system_user=system_user)
-    if not hosts:
+    assets = clean_ansible_task_hosts(assets, system_user=system_user)
+    if not assets:
         return {}
 
-    platform_hosts_map = {}
-    hosts_sorted = sorted(hosts, key=group_asset_by_platform)
-    platform_hosts = groupby(hosts_sorted, key=group_asset_by_platform)
-    for i in platform_hosts:
-        platform_hosts_map[i[0]] = list(i[1])
+    assets_sorted = sorted(assets, key=group_asset_by_platform)
+    platform_hosts = groupby(assets_sorted, key=group_asset_by_platform)
 
     def run_task(_tasks, _hosts):
         if not _tasks:
@@ -209,27 +206,59 @@ def push_system_user_util(system_user, assets, task_name, username=None):
         )
         task.run()
 
-    for platform, _hosts in platform_hosts_map.items():
-        if not _hosts:
+    if system_user.username_same_with_user:
+        if username is None:
+            # 动态系统用户，但是没有指定 username
+            usernames = list(system_user.users.all().values_list('username', flat=True).distinct())
+        else:
+            usernames = [username]
+    else:
+        # 非动态系统用户指定 username 无效
+        assert username is None, 'Only Dynamic user can assign `username`'
+        usernames = [system_user.username]
+
+    for platform, _assets in platform_hosts:
+        _assets = list(_assets)
+        if not _assets:
             continue
         print(_("Start push system user for platform: [{}]").format(platform))
-        print(_("Hosts count: {}").format(len(_hosts)))
+        print(_("Hosts count: {}").format(len(_assets)))
 
-        # 如果没有特殊密码设置，就不需要单独推送某台机器了
-        if not system_user.has_special_auth(username=username):
-            logger.debug("System user not has special auth")
-            tasks = get_push_system_user_tasks(system_user, platform, username=username)
-            run_task(tasks, _hosts)
-            continue
+        id_asset_map = {_asset.id: _asset for _asset in _assets}
+        assets_id = id_asset_map.keys()
+        no_special_auth = []
+        special_auth_set = set()
 
-        for _host in _hosts:
-            system_user.load_asset_special_auth(_host, username=username)
-            tasks = get_push_system_user_tasks(system_user, platform, username=username)
-            run_task(tasks, [_host])
+        auth_books = AuthBook.objects.filter(username__in=usernames, asset_id__in=assets_id)
+
+        for auth_book in auth_books:
+            special_auth_set.add((auth_book.username, auth_book.asset_id))
+
+        for _username in usernames:
+            no_special_assets = []
+            for asset_id in assets_id:
+                if (_username, asset_id) not in special_auth_set:
+                    no_special_assets.append(id_asset_map[asset_id])
+            if no_special_assets:
+                no_special_auth.append((_username, no_special_assets))
+
+        for _username, no_special_assets in no_special_auth:
+            tasks = get_push_system_user_tasks(system_user, platform, username=_username)
+            run_task(tasks, no_special_assets)
+
+        for auth_book in auth_books:
+            system_user._merge_auth(auth_book)
+            tasks = get_push_system_user_tasks(system_user, platform, username=auth_book.username)
+            asset = id_asset_map[auth_book.asset_id]
+            run_task(tasks, [asset])
 
 
 @shared_task(queue="ansible")
+@tmp_to_root_org()
 def push_system_user_to_assets_manual(system_user, username=None):
+    """
+    将系统用户推送到与它关联的所有资产上
+    """
     system_user = get_object_if_need(SystemUser, system_user)
     assets = system_user.get_related_assets()
     task_name = _("Push system users to assets: {}").format(system_user.name)
@@ -237,7 +266,11 @@ def push_system_user_to_assets_manual(system_user, username=None):
 
 
 @shared_task(queue="ansible")
+@tmp_to_root_org()
 def push_system_user_a_asset_manual(system_user, asset, username=None):
+    """
+    将系统用户推送到一个资产上
+    """
     if username is None:
         username = system_user.username
     task_name = _("Push system users to asset: {}({}) => {}").format(
@@ -247,10 +280,15 @@ def push_system_user_a_asset_manual(system_user, asset, username=None):
 
 
 @shared_task(queue="ansible")
+@tmp_to_root_org()
 def push_system_user_to_assets(system_user_id, assets_id, username=None):
+    """
+    推送系统用户到指定的若干资产上
+    """
     system_user = SystemUser.objects.get(id=system_user_id)
     assets = get_objects(Asset, assets_id)
     task_name = _("Push system users to assets: {}").format(system_user.name)
+
     return push_system_user_util(system_user, assets, task_name, username=username)
 
 # @shared_task

apps/assets/utils.py

@@ -1,8 +1,11 @@
 # ~*~ coding: utf-8 ~*~
 #
+import time
+
 from django.db.models import Q
 
 from common.utils import get_logger, dict_get_any, is_uuid, get_object_or_none
+from common.utils.lock import DistributedLock
 from common.http import is_true
 from .models import Asset, Node
 
@@ -10,17 +13,21 @@ from .models import Asset, Node
 logger = get_logger(__file__)
 
 
+@DistributedLock(name="assets.node.check_node_assets_amount", blocking=False)
 def check_node_assets_amount():
     for node in Node.objects.all():
+        logger.info(f'Check node assets amount: {node}')
         assets_amount = Asset.objects.filter(
             Q(nodes__key__istartswith=f'{node.key}:') | Q(nodes=node)
         ).distinct().count()
 
         if node.assets_amount != assets_amount:
-            print(f'>>> <Node:{node.key}> wrong assets amount '
-                  f'{node.assets_amount} right is {assets_amount}')
+            logger.warn(f'Node wrong assets amount <Node:{node.key}> '
+                        f'{node.assets_amount} right is {assets_amount}')
             node.assets_amount = assets_amount
             node.save()
+        # 防止自检程序给数据库的压力太大
+        time.sleep(0.1)
 
 
 def is_asset_exists_in_node(asset_pk, node_key):

apps/audits/migrations/0011_userloginlog_backend.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1 on 2020-12-09 03:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0010_auto_20200811_1122'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userloginlog',
+            name='backend',
+            field=models.CharField(default='', max_length=32, verbose_name='Login backend'),
+        ),
+    ]

apps/audits/models.py

@@ -105,6 +105,7 @@ class UserLoginLog(models.Model):
     reason = models.CharField(default='', max_length=128, blank=True, verbose_name=_('Reason'))
     status = models.BooleanField(max_length=2, default=True, choices=STATUS_CHOICE, verbose_name=_('Status'))
     datetime = models.DateTimeField(default=timezone.now, verbose_name=_('Date login'))
+    backend = models.CharField(max_length=32, default='', verbose_name=_('Login backend'))
 
     @classmethod
     def get_login_logs(cls, date_from=None, date_to=None, user=None, keyword=None):

apps/audits/serializers.py

@@ -31,7 +31,8 @@ class UserLoginLogSerializer(serializers.ModelSerializer):
         model = models.UserLoginLog
         fields = (
             'id', 'username', 'type', 'type_display', 'ip', 'city', 'user_agent',
-            'mfa', 'reason', 'status', 'status_display', 'datetime', 'mfa_display'
+            'mfa', 'reason', 'status', 'status_display', 'datetime', 'mfa_display',
+            'backend'
         )
         extra_kwargs = {
             "user_agent": {'label': _('User agent')}

apps/audits/signals_handler.py

@@ -5,6 +5,8 @@ from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
 from django.db import transaction
 from django.utils import timezone
+from django.contrib.auth import BACKEND_SESSION_KEY
+from django.utils.translation import ugettext_lazy as _
 from rest_framework.renderers import JSONRenderer
 from rest_framework.request import Request
 
@@ -32,6 +34,19 @@ MODELS_NEED_RECORD = (
 )
 
 
+LOGIN_BACKEND = {
+    'PublicKeyAuthBackend': _('SSH Key'),
+    'RadiusBackend': User.Source.radius.label,
+    'RadiusRealmBackend': User.Source.radius.label,
+    'LDAPAuthorizationBackend': User.Source.ldap.label,
+    'ModelBackend': _('Password'),
+    'SSOAuthentication': _('SSO'),
+    'CASBackend': User.Source.cas.label,
+    'OIDCAuthCodeBackend': User.Source.openid.label,
+    'OIDCAuthPasswordBackend': User.Source.openid.label,
+}
+
+
 def create_operate_log(action, sender, resource):
     user = current_request.user if current_request else None
     if not user or not user.is_authenticated:
@@ -109,6 +124,12 @@ def on_audits_log_create(sender, instance=None, **kwargs):
     sys_logger.info(msg)
 
 
+def get_login_backend(request):
+    backend = request.session.get(BACKEND_SESSION_KEY, '')
+    backend = backend.rsplit('.', maxsplit=1)[-1]
+    return LOGIN_BACKEND.get(backend, '')
+
+
 def generate_data(username, request):
     user_agent = request.META.get('HTTP_USER_AGENT', '')
     login_ip = get_request_ip(request) or '0.0.0.0'
@@ -122,7 +143,8 @@ def generate_data(username, request):
         'ip': login_ip,
         'type': login_type,
         'user_agent': user_agent,
-        'datetime': timezone.now()
+        'datetime': timezone.now(),
+        'backend': get_login_backend(request)
     }
     return data
 

apps/authentication/backends/api.py

@@ -6,7 +6,7 @@ import time
 
 from django.core.cache import cache
 from django.utils.translation import ugettext as _
-from django.utils.six import text_type
+from six import text_type
 from django.contrib.auth import get_user_model
 from django.contrib.auth.backends import ModelBackend
 from rest_framework import HTTP_HEADER_ENCODING

apps/authentication/backends/radius.py

@@ -23,7 +23,7 @@ class CreateUserMixin:
                 email_suffix = settings.EMAIL_SUFFIX
                 email = '{}@{}'.format(username, email_suffix)
             user = User(username=username, name=username, email=email)
-            user.source = user.SOURCE_RADIUS
+            user.source = user.Source.radius.value
             user.save()
         return user
 

apps/authentication/errors.py

@@ -218,5 +218,14 @@ class PasswdTooSimple(JMSException):
     default_detail = _('Your password is too simple, please change it for security')
 
     def __init__(self, url, *args, **kwargs):
-        super(PasswdTooSimple, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
+        self.url = url
+
+
+class PasswordRequireResetError(JMSException):
+    default_code = 'passwd_has_expired'
+    default_detail = _('Your password has expired, please reset before logging in')
+
+    def __init__(self, url, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self.url = url

apps/authentication/forms.py

@@ -4,7 +4,7 @@
 from django import forms
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
-from captcha.fields import CaptchaField
+from captcha.fields import CaptchaField, CaptchaTextInput
 
 
 class UserLoginForm(forms.Form):
@@ -26,8 +26,12 @@ class UserCheckOtpCodeForm(forms.Form):
     otp_code = forms.CharField(label=_('MFA code'), max_length=6)
 
 
+class CustomCaptchaTextInput(CaptchaTextInput):
+    template_name = 'authentication/_captcha_field.html'
+
+
 class CaptchaMixin(forms.Form):
-    captcha = CaptchaField()
+    captcha = CaptchaField(widget=CustomCaptchaTextInput)
 
 
 class ChallengeMixin(forms.Form):

apps/authentication/mixins.py

@@ -110,9 +110,8 @@ class AuthMixin:
             raise CredentialError(error=errors.reason_user_inactive)
         elif not user.is_active:
             raise CredentialError(error=errors.reason_user_inactive)
-        elif user.password_has_expired:
-            raise CredentialError(error=errors.reason_password_expired)
 
+        self._check_password_require_reset_or_not(user)
         self._check_passwd_is_too_simple(user, password)
 
         clean_failed_count(username, ip)
@@ -123,20 +122,34 @@ class AuthMixin:
         return user
 
     @classmethod
-    def _check_passwd_is_too_simple(cls, user, password):
+    def generate_reset_password_url_with_flash_msg(cls, user: User, flash_view_name):
+        reset_passwd_url = reverse('authentication:reset-password')
+        query_str = urlencode({
+            'token': user.generate_reset_token()
+        })
+        reset_passwd_url = f'{reset_passwd_url}?{query_str}'
+
+        flash_page_url = reverse(flash_view_name)
+        query_str = urlencode({
+            'redirect_url': reset_passwd_url
+        })
+        return f'{flash_page_url}?{query_str}'
+
+    @classmethod
+    def _check_passwd_is_too_simple(cls, user: User, password):
         if user.is_superuser and password == 'admin':
-            reset_passwd_url = reverse('authentication:reset-password')
-            query_str = urlencode({
-                'token': user.generate_reset_token()
-            })
-            reset_passwd_url = f'{reset_passwd_url}?{query_str}'
+            url = cls.generate_reset_password_url_with_flash_msg(
+                user, 'authentication:passwd-too-simple-flash-msg'
+            )
+            raise errors.PasswdTooSimple(url)
 
-            flash_page_url = reverse('authentication:passwd-too-simple-flash-msg')
-            query_str = urlencode({
-                'redirect_url': reset_passwd_url
-            })
-
-            raise errors.PasswdTooSimple(f'{flash_page_url}?{query_str}')
+    @classmethod
+    def _check_password_require_reset_or_not(cls, user: User):
+        if user.password_has_expired:
+            url = cls.generate_reset_password_url_with_flash_msg(
+                user, 'authentication:passwd-has-expired-flash-msg'
+            )
+            raise errors.PasswordRequireResetError(url)
 
     def check_user_auth_if_need(self, decrypt_passwd=False):
         request = self.request

apps/authentication/models.py

@@ -1,11 +1,9 @@
 import uuid
-from functools import partial
 
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _, ugettext as __
 from rest_framework.authtoken.models import Token
 from django.conf import settings
-from django.utils.crypto import get_random_string
 
 from common.db import models
 from common.mixins.models import CommonModelMixin

apps/authentication/templates/authentication/_captcha_field.html

@@ -0,0 +1,29 @@
+{% load i18n %}
+{% spaceless %}
+    <img src="{{ image }}" alt="captcha" class="captcha" />
+    <div class="row" style="padding-bottom: 10px">
+        <div class="col-sm-6">
+            <div class="input-group-prepend">
+                {% if audio %}
+                    <a title="{% trans "Play CAPTCHA as audio file" %}" href="{{ audio }}">
+                {% endif %}
+            </div>
+            {% include "django/forms/widgets/multiwidget.html" %}
+         </div>
+    </div>
+    <script>
+        var placeholder = '{% trans "Captcha" %}'
+        function refresh_captcha() {
+            $.getJSON("{% url "captcha-refresh" %}",
+                function (result) {
+                    $('.captcha').attr('src', result['image_url']);
+                    $('#id_captcha_0').val(result['key'])
+                })
+        }
+        $(document).ready(function () {
+            $('.captcha').click(refresh_captcha)
+            $('#id_captcha_1').addClass('form-control').attr('placeholder', placeholder)
+        })
+    </script>
+
+{% endspaceless %}

apps/authentication/urls/view_urls.py

@@ -22,6 +22,7 @@ urlpatterns = [
          name='forgot-password-sendmail-success'),
     path('password/reset/', users_view.UserResetPasswordView.as_view(), name='reset-password'),
     path('password/too-simple-flash-msg/', views.FlashPasswdTooSimpleMsgView.as_view(), name='passwd-too-simple-flash-msg'),
+    path('password/has-expired-msg/', views.FlashPasswdHasExpiredMsgView.as_view(), name='passwd-has-expired-flash-msg'),
     path('password/reset/success/', users_view.UserResetPasswordSuccessView.as_view(), name='reset-password-success'),
     path('password/verify/', users_view.UserVerifyPasswordView.as_view(), name='user-verify-password'),
 

apps/authentication/views/login.py

@@ -32,7 +32,7 @@ from ..forms import get_user_login_form_cls
 __all__ = [
     'UserLoginView', 'UserLogoutView',
     'UserLoginGuardView', 'UserLoginWaitConfirmView',
-    'FlashPasswdTooSimpleMsgView',
+    'FlashPasswdTooSimpleMsgView', 'FlashPasswdHasExpiredMsgView'
 ]
 
 
@@ -96,7 +96,7 @@ class UserLoginView(mixins.AuthMixin, FormView):
             new_form._errors = form.errors
             context = self.get_context_data(form=new_form)
             return self.render_to_response(context)
-        except errors.PasswdTooSimple as e:
+        except (errors.PasswdTooSimple, errors.PasswordRequireResetError) as e:
             return redirect(e.url)
         self.clear_rsa_key()
         return self.redirect_to_guard_view()
@@ -250,3 +250,18 @@ class FlashPasswdTooSimpleMsgView(TemplateView):
             'auto_redirect': True,
         }
         return self.render_to_response(context)
+
+
+@method_decorator(never_cache, name='dispatch')
+class FlashPasswdHasExpiredMsgView(TemplateView):
+    template_name = 'flash_message_standalone.html'
+
+    def get(self, request, *args, **kwargs):
+        context = {
+            'title': _('Please change your password'),
+            'messages': _('Your password has expired, please reset before logging in'),
+            'interval': 5,
+            'redirect_url': request.GET.get('redirect_url'),
+            'auto_redirect': True,
+        }
+        return self.render_to_response(context)

apps/common/drf/parsers/__init__.py

@@ -1 +1,2 @@
-from .csv import *
+from .csv import *
+from .excel import *

apps/common/drf/parsers/base.py

@@ -0,0 +1,132 @@
+import abc
+import json
+import codecs
+from django.utils.translation import ugettext_lazy as _
+from rest_framework.parsers import BaseParser
+from rest_framework import status
+from rest_framework.exceptions import ParseError, APIException
+from common.utils import get_logger
+
+logger = get_logger(__file__)
+
+
+class FileContentOverflowedError(APIException):
+    status_code = status.HTTP_400_BAD_REQUEST
+    default_code = 'file_content_overflowed'
+    default_detail = _('The file content overflowed (The maximum length `{}` bytes)')
+
+
+class BaseFileParser(BaseParser):
+
+    FILE_CONTENT_MAX_LENGTH = 1024 * 1024 * 10
+
+    serializer_cls = None
+
+    def check_content_length(self, meta):
+        content_length = int(meta.get('CONTENT_LENGTH', meta.get('HTTP_CONTENT_LENGTH', 0)))
+        if content_length > self.FILE_CONTENT_MAX_LENGTH:
+            msg = FileContentOverflowedError.default_detail.format(self.FILE_CONTENT_MAX_LENGTH)
+            logger.error(msg)
+            raise FileContentOverflowedError(msg)
+
+    @staticmethod
+    def get_stream_data(stream):
+        stream_data = stream.read()
+        stream_data = stream_data.strip(codecs.BOM_UTF8)
+        return stream_data
+
+    @abc.abstractmethod
+    def generate_rows(self, stream_data):
+        raise NotImplemented
+
+    def get_column_titles(self, rows):
+        return next(rows)
+
+    def convert_to_field_names(self, column_titles):
+        fields_map = {}
+        fields = self.serializer_cls().fields
+        fields_map.update({v.label: k for k, v in fields.items()})
+        fields_map.update({k: k for k, _ in fields.items()})
+        field_names = [
+            fields_map.get(column_title.strip('*'), '')
+            for column_title in column_titles
+        ]
+        return field_names
+
+    @staticmethod
+    def _replace_chinese_quote(s):
+        trans_table = str.maketrans({
+            '“': '"',
+            '”': '"',
+            '‘': '"',
+            '’': '"',
+            '\'': '"'
+        })
+        return s.translate(trans_table)
+
+    @classmethod
+    def process_row(cls, row):
+        """
+        构建json数据前的行处理
+        """
+        new_row = []
+        for col in row:
+            # 转换中文引号
+            col = cls._replace_chinese_quote(col)
+            # 列表/字典转换
+            if isinstance(col, str) and (
+                    (col.startswith('[') and col.endswith(']'))
+                    or
+                    (col.startswith("{") and col.endswith("}"))
+            ):
+                col = json.loads(col)
+            new_row.append(col)
+        return new_row
+
+    @staticmethod
+    def process_row_data(row_data):
+        """
+        构建json数据后的行数据处理
+        """
+        new_row_data = {}
+        for k, v in row_data.items():
+            if isinstance(v, list) or isinstance(v, dict) or isinstance(v, str) and k.strip() and v.strip():
+                new_row_data[k] = v
+        return new_row_data
+
+    def generate_data(self, fields_name, rows):
+        data = []
+        for row in rows:
+            # 空行不处理
+            if not any(row):
+                continue
+            row = self.process_row(row)
+            row_data = dict(zip(fields_name, row))
+            row_data = self.process_row_data(row_data)
+            data.append(row_data)
+        return data
+
+    def parse(self, stream, media_type=None, parser_context=None):
+        parser_context = parser_context or {}
+
+        try:
+            view = parser_context['view']
+            meta = view.request.META
+            self.serializer_cls = view.get_serializer_class()
+        except Exception as e:
+            logger.debug(e, exc_info=True)
+            raise ParseError('The resource does not support imports!')
+
+        self.check_content_length(meta)
+
+        try:
+            stream_data = self.get_stream_data(stream)
+            rows = self.generate_rows(stream_data)
+            column_titles = self.get_column_titles(rows)
+            field_names = self.convert_to_field_names(column_titles)
+            data = self.generate_data(field_names, rows)
+            return data
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise ParseError('Parse error! ({})'.format(self.media_type))
+

apps/common/drf/parsers/csv.py

@@ -1,32 +1,13 @@
 # ~*~ coding: utf-8 ~*~
 #
 
-import json
 import chardet
-import codecs
 import unicodecsv
 
-from django.utils.translation import ugettext as _
-from rest_framework.parsers import BaseParser
-from rest_framework.exceptions import ParseError, APIException
-from rest_framework import status
-
-from common.utils import get_logger
-
-logger = get_logger(__file__)
+from .base import BaseFileParser
 
 
-class CsvDataTooBig(APIException):
-    status_code = status.HTTP_400_BAD_REQUEST
-    default_code = 'csv_data_too_big'
-    default_detail = _('The max size of CSV is %d bytes')
-
-
-class JMSCSVParser(BaseParser):
-    """
-    Parses CSV file to serializer data
-    """
-    CSV_UPLOAD_MAX_SIZE = 1024 * 1024 * 10
+class CSVFileParser(BaseFileParser):
 
     media_type = 'text/csv'
 
@@ -38,99 +19,10 @@ class JMSCSVParser(BaseParser):
         for line in stream.splitlines():
             yield line
 
-    @staticmethod
-    def _gen_rows(csv_data, charset='utf-8', **kwargs):
-        csv_reader = unicodecsv.reader(csv_data, encoding=charset, **kwargs)
+    def generate_rows(self, stream_data):
+        detect_result = chardet.detect(stream_data)
+        encoding = detect_result.get("encoding", "utf-8")
+        lines = self._universal_newlines(stream_data)
+        csv_reader = unicodecsv.reader(lines, encoding=encoding)
         for row in csv_reader:
-            if not any(row):  # 空行
-                continue
             yield row
-
-    @staticmethod
-    def _get_fields_map(serializer_cls):
-        fields_map = {}
-        fields = serializer_cls().fields
-        fields_map.update({v.label: k for k, v in fields.items()})
-        fields_map.update({k: k for k, _ in fields.items()})
-        return fields_map
-
-    @staticmethod
-    def _replace_chinese_quot(str_):
-        trans_table = str.maketrans({
-            '“': '"',
-            '”': '"',
-            '‘': '"',
-            '’': '"',
-            '\'': '"'
-        })
-        return str_.translate(trans_table)
-
-    @classmethod
-    def _process_row(cls, row):
-        """
-        构建json数据前的行处理
-        """
-        _row = []
-
-        for col in row:
-            # 列表转换
-            if isinstance(col, str) and col.startswith('[') and col.endswith(']'):
-                col = cls._replace_chinese_quot(col)
-                col = json.loads(col)
-            # 字典转换
-            if isinstance(col, str) and col.startswith("{") and col.endswith("}"):
-                col = cls._replace_chinese_quot(col)
-                col = json.loads(col)
-            _row.append(col)
-        return _row
-
-    @staticmethod
-    def _process_row_data(row_data):
-        """
-        构建json数据后的行数据处理
-        """
-        _row_data = {}
-        for k, v in row_data.items():
-            if isinstance(v, list) or isinstance(v, dict)\
-                    or isinstance(v, str) and k.strip() and v.strip():
-                _row_data[k] = v
-        return _row_data
-
-    def parse(self, stream, media_type=None, parser_context=None):
-        parser_context = parser_context or {}
-        try:
-            view = parser_context['view']
-            meta = view.request.META
-            serializer_cls = view.get_serializer_class()
-        except Exception as e:
-            logger.debug(e, exc_info=True)
-            raise ParseError('The resource does not support imports!')
-
-        content_length = int(meta.get('CONTENT_LENGTH', meta.get('HTTP_CONTENT_LENGTH', 0)))
-        if content_length > self.CSV_UPLOAD_MAX_SIZE:
-            msg = CsvDataTooBig.default_detail % self.CSV_UPLOAD_MAX_SIZE
-            logger.error(msg)
-            raise CsvDataTooBig(msg)
-
-        try:
-            stream_data = stream.read()
-            stream_data = stream_data.strip(codecs.BOM_UTF8)
-            detect_result = chardet.detect(stream_data)
-            encoding = detect_result.get("encoding", "utf-8")
-            binary = self._universal_newlines(stream_data)
-            rows = self._gen_rows(binary, charset=encoding)
-
-            header = next(rows)
-            fields_map = self._get_fields_map(serializer_cls)
-            header = [fields_map.get(name.strip('*'), '') for name in header]
-
-            data = []
-            for row in rows:
-                row = self._process_row(row)
-                row_data = dict(zip(header, row))
-                row_data = self._process_row_data(row_data)
-                data.append(row_data)
-            return data
-        except Exception as e:
-            logger.error(e, exc_info=True)
-            raise ParseError('CSV parse error!')

apps/common/drf/parsers/excel.py

@@ -0,0 +1,14 @@
+import pyexcel
+from .base import BaseFileParser
+
+
+class ExcelFileParser(BaseFileParser):
+
+    media_type = 'text/xlsx'
+
+    def generate_rows(self, stream_data):
+        workbook = pyexcel.get_book(file_type='xlsx', file_content=stream_data)
+        # 默认获取第一个工作表sheet
+        sheet = workbook.sheet_by_index(0)
+        rows = sheet.rows()
+        return rows

apps/common/drf/renders/__init__.py

@@ -1,6 +1,7 @@
 from rest_framework import renderers
 
 from .csv import *
+from .excel import *
 
 
 class PassthroughRenderer(renderers.BaseRenderer):

apps/common/drf/renders/base.py

@@ -0,0 +1,132 @@
+import abc
+from datetime import datetime
+from rest_framework.renderers import BaseRenderer
+from rest_framework.utils import encoders, json
+
+from common.utils import get_logger
+
+logger = get_logger(__file__)
+
+
+class BaseFileRenderer(BaseRenderer):
+    # 渲染模版标识, 导入、导出、更新模版: ['import', 'update', 'export']
+    template = 'export'
+    serializer = None
+
+    @staticmethod
+    def _check_validation_data(data):
+        detail_key = "detail"
+        if detail_key in data:
+            return False
+        return True
+
+    @staticmethod
+    def _json_format_response(response_data):
+        return json.dumps(response_data)
+
+    def set_response_disposition(self, response):
+        serializer = self.serializer
+        if response and hasattr(serializer, 'Meta') and hasattr(serializer.Meta, "model"):
+            model_name = serializer.Meta.model.__name__.lower()
+            now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+            filename = "{}_{}.{}".format(model_name, now, self.format)
+            disposition = 'attachment; filename="{}"'.format(filename)
+            response['Content-Disposition'] = disposition
+
+    def get_rendered_fields(self):
+        fields = self.serializer.fields
+        if self.template == 'import':
+            return [v for k, v in fields.items() if not v.read_only and k != "org_id" and k != 'id']
+        elif self.template == 'update':
+            return [v for k, v in fields.items() if not v.read_only and k != "org_id"]
+        else:
+            return [v for k, v in fields.items() if not v.write_only and k != "org_id"]
+
+    @staticmethod
+    def get_column_titles(render_fields):
+        return [
+            '*{}'.format(field.label) if field.required else str(field.label)
+            for field in render_fields
+        ]
+
+    def process_data(self, data):
+        results = data['results'] if 'results' in data else data
+
+        if isinstance(results, dict):
+            results = [results]
+
+        if self.template == 'import':
+            results = [results[0]] if results else results
+
+        else:
+            # 限制数据数量
+            results = results[:10000]
+        # 会将一些 UUID 字段转化为 string
+        results = json.loads(json.dumps(results, cls=encoders.JSONEncoder))
+        return results
+
+    @staticmethod
+    def generate_rows(data, render_fields):
+        for item in data:
+            row = []
+            for field in render_fields:
+                value = item.get(field.field_name)
+                value = str(value) if value else ''
+                row.append(value)
+            yield row
+
+    @abc.abstractmethod
+    def initial_writer(self):
+        raise NotImplementedError
+
+    def write_column_titles(self, column_titles):
+        self.write_row(column_titles)
+
+    def write_rows(self, rows):
+        for row in rows:
+            self.write_row(row)
+
+    @abc.abstractmethod
+    def write_row(self, row):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def get_rendered_value(self):
+        raise NotImplementedError
+
+    def render(self, data, accepted_media_type=None, renderer_context=None):
+        if data is None:
+            return bytes()
+
+        if not self._check_validation_data(data):
+            return self._json_format_response(data)
+
+        try:
+            renderer_context = renderer_context or {}
+            request = renderer_context['request']
+            response = renderer_context['response']
+            view = renderer_context['view']
+            self.template = request.query_params.get('template', 'export')
+            self.serializer = view.get_serializer()
+            self.set_response_disposition(response)
+        except Exception as e:
+            logger.debug(e, exc_info=True)
+            value = 'The resource not support export!'.encode('utf-8')
+            return value
+
+        try:
+            rendered_fields = self.get_rendered_fields()
+            column_titles = self.get_column_titles(rendered_fields)
+            data = self.process_data(data)
+            rows = self.generate_rows(data, rendered_fields)
+            self.initial_writer()
+            self.write_column_titles(column_titles)
+            self.write_rows(rows)
+            value = self.get_rendered_value()
+        except Exception as e:
+            logger.debug(e, exc_info=True)
+            value = 'Render error! ({})'.format(self.media_type).encode('utf-8')
+            return value
+
+        return value
+

apps/common/drf/renders/csv.py

@@ -1,83 +1,30 @@
 # ~*~ coding: utf-8 ~*~
 #
 
-import unicodecsv
 import codecs
-from datetime import datetime
-
+import unicodecsv
 from six import BytesIO
-from rest_framework.renderers import BaseRenderer
-from rest_framework.utils import encoders, json
 
-from common.utils import get_logger
-
-logger = get_logger(__file__)
+from .base import BaseFileRenderer
 
 
-class JMSCSVRender(BaseRenderer):
-
+class CSVFileRenderer(BaseFileRenderer):
     media_type = 'text/csv'
     format = 'csv'
 
-    @staticmethod
-    def _get_show_fields(fields, template):
-        if template == 'import':
-            return [v for k, v in fields.items() if not v.read_only and k != "org_id" and k != 'id']
-        elif template == 'update':
-            return [v for k, v in fields.items() if not v.read_only and k != "org_id"]
-        else:
-            return [v for k, v in fields.items() if not v.write_only and k != "org_id"]
+    writer = None
+    buffer = None
 
-    @staticmethod
-    def _gen_table(data, fields):
-        data = data[:10000]
-        yield ['*{}'.format(f.label) if f.required else f.label for f in fields]
+    def initial_writer(self):
+        csv_buffer = BytesIO()
+        csv_buffer.write(codecs.BOM_UTF8)
+        csv_writer = unicodecsv.writer(csv_buffer, encoding='utf-8')
+        self.buffer = csv_buffer
+        self.writer = csv_writer
 
-        for item in data:
-            row = [item.get(f.field_name) for f in fields]
-            yield row
-
-    def set_response_disposition(self, serializer, context):
-        response = context.get('response')
-        if response and hasattr(serializer, 'Meta') and \
-                hasattr(serializer.Meta, "model"):
-            model_name = serializer.Meta.model.__name__.lower()
-            now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-            filename = "{}_{}.csv".format(model_name, now)
-            disposition = 'attachment; filename="{}"'.format(filename)
-            response['Content-Disposition'] = disposition
-
-    def render(self, data, media_type=None, renderer_context=None):
-        renderer_context = renderer_context or {}
-        request = renderer_context['request']
-        template = request.query_params.get('template', 'export')
-        view = renderer_context['view']
-
-        if isinstance(data, dict):
-            data = data.get("results", [])
-
-        if template == 'import':
-            data = [data[0]] if data else data
-
-        data = json.loads(json.dumps(data, cls=encoders.JSONEncoder))
-
-        try:
-            serializer = view.get_serializer()
-            self.set_response_disposition(serializer, renderer_context)
-        except Exception as e:
-            logger.debug(e, exc_info=True)
-            value = 'The resource not support export!'.encode('utf-8')
-        else:
-            fields = serializer.fields
-            show_fields = self._get_show_fields(fields, template)
-            table = self._gen_table(data, show_fields)
-
-            csv_buffer = BytesIO()
-            csv_buffer.write(codecs.BOM_UTF8)
-            csv_writer = unicodecsv.writer(csv_buffer, encoding='utf-8')
-            for row in table:
-                csv_writer.writerow(row)
-
-            value = csv_buffer.getvalue()
+    def write_row(self, row):
+        self.writer.writerow(row)
 
+    def get_rendered_value(self):
+        value = self.buffer.getvalue()
         return value

apps/common/drf/renders/excel.py

@@ -0,0 +1,28 @@
+from openpyxl import Workbook
+from openpyxl.writer.excel import save_virtual_workbook
+
+from .base import BaseFileRenderer
+
+
+class ExcelFileRenderer(BaseFileRenderer):
+    media_type = "application/xlsx"
+    format = "xlsx"
+
+    wb = None
+    ws = None
+    row_count = 0
+
+    def initial_writer(self):
+        self.wb = Workbook()
+        self.ws = self.wb.active
+
+    def write_row(self, row):
+        self.row_count += 1
+        column_count = 0
+        for cell_value in row:
+            column_count += 1
+            self.ws.cell(row=self.row_count, column=column_count, value=cell_value)
+
+    def get_rendered_value(self):
+        value = save_virtual_workbook(self.wb)
+        return value

apps/common/fields/form.py

@@ -3,7 +3,7 @@
 import json
 
 from django import forms
-from django.utils import six
+import six
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext as _
 from ..utils import signer

apps/common/fields/model.py

@@ -31,7 +31,7 @@ class JsonMixin:
     def json_encode(data):
         return json.dumps(data)
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection, context=None):
         if value is None:
             return value
         return self.json_decode(value)
@@ -54,7 +54,7 @@ class JsonMixin:
 class JsonTypeMixin(JsonMixin):
     tp = dict
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection, context=None):
         value = super().from_db_value(value, expression, connection, context)
         if not isinstance(value, self.tp):
             value = self.tp()
@@ -116,7 +116,7 @@ class EncryptMixin:
     def decrypt_from_signer(self, value):
         return signer.unsign(value) or ''
 
-    def from_db_value(self, value, expression, connection, context):
+    def from_db_value(self, value, expression, connection, context=None):
         if value is None:
             return value
         value = force_text(value)

apps/common/fields/serializer.py

@@ -2,7 +2,7 @@
 #
 
 from rest_framework import serializers
-from django.utils import six
+import six
 
 
 __all__ = [

apps/common/utils/common.py

@@ -41,7 +41,7 @@ def timesince(dt, since='', default="just now"):
     3 days, 5 hours.
     """
 
-    if since is '':
+    if not since:
         since = datetime.datetime.utcnow()
 
     if since is None:

apps/common/utils/inspect.py

@@ -0,0 +1,10 @@
+import inspect
+
+
+def copy_function_args(func, locals_dict: dict):
+    signature = inspect.signature(func)
+    keys = signature.parameters.keys()
+    kwargs = {}
+    for k in keys:
+        kwargs[k] = locals_dict.get(k)
+    return kwargs

apps/common/utils/lock.py

@@ -0,0 +1,55 @@
+from functools import wraps
+
+from redis_lock import Lock as RedisLock
+from redis import Redis
+
+from common.utils import get_logger
+from common.utils.inspect import copy_function_args
+from apps.jumpserver.const import CONFIG
+
+logger = get_logger(__file__)
+
+
+class AcquireFailed(RuntimeError):
+    pass
+
+
+class DistributedLock(RedisLock):
+    def __init__(self, name, blocking=True, expire=60*2, auto_renewal=True):
+        """
+        使用 redis 构造的分布式锁
+
+        :param name:
+            锁的名字，要全局唯一
+        :param blocking:
+            该参数只在锁作为装饰器或者 `with` 时有效。
+        :param expire:
+            锁的过期时间，注意不一定是锁到这个时间就释放了，分两种情况
+            当 `auto_renewal=False` 时，锁会释放
+            当 `auto_renewal=True` 时，如果过期之前程序还没释放锁，我们会延长锁的存活时间。
+            这里的作用是防止程序意外终止没有释放锁，导致死锁。
+        """
+        self.kwargs_copy = copy_function_args(self.__init__, locals())
+        redis = Redis(host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, password=CONFIG.REDIS_PASSWORD)
+        super().__init__(redis_client=redis, name=name, expire=expire, auto_renewal=auto_renewal)
+        self._blocking = blocking
+
+    def __enter__(self):
+        acquired = self.acquire(blocking=self._blocking)
+        if self._blocking and not acquired:
+            raise EnvironmentError("Lock wasn't acquired, but blocking=True")
+        if not acquired:
+            raise AcquireFailed
+        return self
+
+    def __exit__(self, exc_type=None, exc_value=None, traceback=None):
+        self.release()
+
+    def __call__(self, func):
+        @wraps(func)
+        def inner(*args, **kwds):
+            # 要创建一个新的锁对象
+            with self.__class__(**self.kwargs_copy):
+                return func(*args, **kwds)
+
+        return inner

apps/common/validators.py

@@ -14,9 +14,9 @@ alphanumeric = RegexValidator(r'^[0-9a-zA-Z_@\-\.]*$', _('Special char not allow
 
 
 class ProjectUniqueValidator(UniqueTogetherValidator):
-    def __call__(self, attrs):
+    def __call__(self, attrs, serializer):
         try:
-            super().__call__(attrs)
+            super().__call__(attrs, serializer)
         except ValidationError as e:
             errors = {}
             for field in self.fields:

apps/jumpserver/api.py

@@ -2,13 +2,14 @@ from django.core.cache import cache
 from django.utils import timezone
 from django.utils.timesince import timesince
 from django.db.models import Count, Max
-from django.http.response import JsonResponse
+from django.http.response import JsonResponse, HttpResponse
 from rest_framework.views import APIView
 from collections import Counter
 
 from users.models import User
 from assets.models import Asset
 from terminal.models import Session
+from terminal.utils import ComponentsPrometheusMetricsUtil
 from orgs.utils import current_org
 from common.permissions import IsOrgAdmin, IsOrgAuditor
 from common.utils import lazyproperty
@@ -305,3 +306,11 @@ class IndexApi(TotalCountMixin, DatesLoginMetricMixin, APIView):
         return JsonResponse(data, status=200)
 
 
+class PrometheusMetricsApi(APIView):
+    permission_classes = ()
+
+    def get(self, request, *args, **kwargs):
+        util = ComponentsPrometheusMetricsUtil()
+        metrics_text = util.get_prometheus_metrics_text()
+        return HttpResponse(metrics_text, content_type='text/plain; version=0.0.4; charset=utf-8')
+

apps/jumpserver/conf.py

@@ -16,7 +16,7 @@ import json
 import yaml
 from importlib import import_module
 from django.urls import reverse_lazy
-from django.contrib.staticfiles.templatetags.staticfiles import static
+from django.templatetags.static import static
 from urllib.parse import urljoin, urlparse
 from django.utils.translation import ugettext_lazy as _
 

apps/jumpserver/context_processor.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-from django.contrib.staticfiles.templatetags.staticfiles import static
+from django.templatetags.static import static
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 

apps/jumpserver/settings/base.py

@@ -64,6 +64,7 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django.forms',
 ]
 
 

apps/jumpserver/settings/libs.py

@@ -11,14 +11,17 @@ REST_FRAMEWORK = {
     ),
     'DEFAULT_RENDERER_CLASSES': (
         'rest_framework.renderers.JSONRenderer',
-        'rest_framework.renderers.BrowsableAPIRenderer',
-        'common.drf.renders.JMSCSVRender',
+        # 'rest_framework.renderers.BrowsableAPIRenderer',
+        'common.drf.renders.CSVFileRenderer',
+        'common.drf.renders.ExcelFileRenderer',
+
     ),
     'DEFAULT_PARSER_CLASSES': (
         'rest_framework.parsers.JSONParser',
         'rest_framework.parsers.FormParser',
         'rest_framework.parsers.MultiPartParser',
-        'common.drf.parsers.JMSCSVParser',
+        'common.drf.parsers.CSVFileParser',
+        'common.drf.parsers.ExcelFileParser',
         'rest_framework.parsers.FileUploadParser',
     ),
     'DEFAULT_AUTHENTICATION_CLASSES': (
@@ -61,10 +64,10 @@ SWAGGER_SETTINGS = {
 
 
 # Captcha settings, more see https://django-simple-captcha.readthedocs.io/en/latest/advanced.html
-CAPTCHA_IMAGE_SIZE = (80, 33)
+CAPTCHA_IMAGE_SIZE = (140, 34)
 CAPTCHA_FOREGROUND_COLOR = '#001100'
 CAPTCHA_NOISE_FUNCTIONS = ('captcha.helpers.noise_dots',)
-CAPTCHA_TEST_MODE = CONFIG.CAPTCHA_TEST_MODE
+CAPTCHA_CHALLENGE_FUNCT = 'captcha.helpers.math_challenge'
 
 # Django bootstrap3 setting, more see http://django-bootstrap3.readthedocs.io/en/latest/settings.html
 BOOTSTRAP3 = {

apps/jumpserver/urls.py

@@ -23,6 +23,7 @@ api_v1 = [
     path('common/', include('common.urls.api_urls', namespace='api-common')),
     path('applications/', include('applications.urls.api_urls', namespace='api-applications')),
     path('tickets/', include('tickets.urls.api_urls', namespace='api-tickets')),
+    path('prometheus/metrics/', api.PrometheusMetricsApi.as_view())
 ]
 
 api_v2 = [
@@ -30,7 +31,6 @@ api_v2 = [
     path('users/', include('users.urls.api_urls_v2', namespace='api-users-v2')),
 ]
 
-
 app_view_patterns = [
     path('auth/', include('authentication.urls.view_urls'), name='auth'),
     path('ops/', include('ops.urls.view_urls'), name='ops'),
@@ -63,7 +63,7 @@ urlpatterns = [
     # External apps url
     path('core/auth/captcha/', include('captcha.urls')),
     path('core/', include(app_view_patterns)),
-    path('ui/', views.UIView.as_view())
+    path('ui/', views.UIView.as_view()),
 ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \

apps/ops/celery/__init__.py

@@ -29,16 +29,3 @@ configs["CELERY_ROUTES"] = {
 app.namespace = 'CELERY'
 app.conf.update(configs)
 app.autodiscover_tasks(lambda: [app_config.split('.')[0] for app_config in settings.INSTALLED_APPS])
-
-app.conf.beat_schedule = {
-    'check-asset-permission-expired': {
-        'task': 'perms.tasks.check_asset_permission_expired',
-        'schedule': settings.PERM_EXPIRED_CHECK_PERIODIC,
-        'args': ()
-    },
-    'check-node-assets-amount': {
-        'task': 'assets.tasks.nodes_amount.check_node_assets_amount_celery_task',
-        'schedule': crontab(minute=0, hour=0),
-        'args': ()
-    },
-}

apps/ops/celery/utils.py

@@ -3,6 +3,8 @@
 import json
 import os
 
+import redis_lock
+import redis
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
 from django.db.utils import ProgrammingError, OperationalError
@@ -105,3 +107,27 @@ def get_celery_task_log_path(task_id):
     path = os.path.join(settings.CELERY_LOG_DIR, rel_path)
     os.makedirs(os.path.dirname(path), exist_ok=True)
     return path
+
+
+def get_celery_status():
+    from . import app
+    i = app.control.inspect()
+    ping_data = i.ping() or {}
+    active_nodes = [k for k, v in ping_data.items() if v.get('ok') == 'pong']
+    active_queue_worker = set([n.split('@')[0] for n in active_nodes if n])
+    if len(active_queue_worker) < 5:
+        print("Not all celery worker worked")
+        return False
+    else:
+        return True
+
+
+def get_beat_status():
+    CONFIG = settings.CONFIG
+    r = redis.Redis(host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, password=CONFIG.REDIS_PASSWORD)
+    lock = redis_lock.Lock(r, name="beat-distribute-start-lock")
+    try:
+        locked = lock.locked()
+        return locked
+    except redis.ConnectionError:
+        return False

apps/ops/management/commands/check_celery.py

@@ -0,0 +1,20 @@
+from django.core.management.base import BaseCommand, CommandError
+
+
+class Command(BaseCommand):
+    help = 'Ops manage commands'
+
+    def add_arguments(self, parser):
+        parser.add_argument('check_celery', nargs='?', help='Check celery health')
+
+    def handle(self, *args, **options):
+        from ops.celery.utils import get_celery_status, get_beat_status
+
+        ok = get_celery_status()
+        if not ok:
+            raise CommandError('Celery worker unhealthy')
+
+        ok = get_beat_status()
+        if not ok:
+            raise CommandError('Beat unhealthy')
+

apps/orgs/api.py

@@ -92,6 +92,7 @@ class OrgMemberAdminRelationBulkViewSet(JMSBulkRelationModelViewSet):
     serializer_class = OrgMemberAdminSerializer
     filterset_class = OrgMemberRelationFilterSet
     search_fields = ('user__name', 'user__username', 'org__name')
+    lookup_field = 'user_id'
 
     def get_queryset(self):
         queryset = super().get_queryset()
@@ -116,6 +117,7 @@ class OrgMemberUserRelationBulkViewSet(JMSBulkRelationModelViewSet):
     serializer_class = OrgMemberUserSerializer
     filterset_class = OrgMemberRelationFilterSet
     search_fields = ('user__name', 'user__username', 'org__name')
+    lookup_field = 'user_id'
 
     def get_queryset(self):
         queryset = super().get_queryset()

apps/orgs/middleware.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 #
 
-from .models import Organization
 from .utils import get_org_from_request, set_current_org
 
 

apps/orgs/mixins/models.py

@@ -8,7 +8,7 @@ from django.core.exceptions import ValidationError
 from common.utils import get_logger
 from ..utils import (
     set_current_org, get_current_org, current_org,
-    filter_org_queryset
+    filter_org_queryset, get_org_by_id, get_org_name_by_id
 )
 from ..models import Organization
 
@@ -70,13 +70,11 @@ class OrgModelMixin(models.Model):
 
     @property
     def org(self):
-        from orgs.models import Organization
-        org = Organization.get_instance(self.org_id)
-        return org
+        return get_org_by_id(self.org_id)
 
     @property
     def org_name(self):
-        return self.org.name
+        return get_org_name_by_id(self.org_id)
 
     @property
     def fullname(self, attr=None):

apps/orgs/serializers.py

@@ -74,26 +74,29 @@ class OrgMemberSerializer(BulkModelSerializer):
         ).distinct()
 
 
-class OrgMemberAdminSerializer(BulkModelSerializer):
+class OrgMemberOldBaseSerializer(BulkModelSerializer):
+    organization = serializers.PrimaryKeyRelatedField(
+        label=_('Organization'), queryset=Organization.objects.all(), required=True, source='org'
+    )
+
+    def to_internal_value(self, data):
+        view = self.context['view']
+        org_id = view.kwargs.get('org_id')
+        if org_id:
+            data['organization'] = org_id
+        return super().to_internal_value(data)
+
+    class Meta:
+        model = OrganizationMember
+        fields = ('id', 'organization', 'user', 'role')
+
+
+class OrgMemberAdminSerializer(OrgMemberOldBaseSerializer):
     role = serializers.HiddenField(default=ROLE.ADMIN)
-    organization = serializers.PrimaryKeyRelatedField(
-        label=_('Organization'), queryset=Organization.objects.all(), required=True, source='org'
-    )
-
-    class Meta:
-        model = OrganizationMember
-        fields = ('id', 'organization', 'user', 'role')
 
 
-class OrgMemberUserSerializer(BulkModelSerializer):
+class OrgMemberUserSerializer(OrgMemberOldBaseSerializer):
     role = serializers.HiddenField(default=ROLE.USER)
-    organization = serializers.PrimaryKeyRelatedField(
-        label=_('Organization'), queryset=Organization.objects.all(), required=True, source='org'
-    )
-
-    class Meta:
-        model = OrganizationMember
-        fields = ('id', 'organization', 'user', 'role')
 
 
 class OrgRetrieveSerializer(OrgReadSerializer):

apps/orgs/utils.py

@@ -65,6 +65,47 @@ def get_current_org_id():
     return org_id
 
 
+def construct_org_mapper():
+    orgs = Organization.objects.all()
+    org_mapper = {str(org.id): org for org in orgs}
+    default_org = Organization.default()
+    org_mapper.update({
+        '': default_org,
+        Organization.DEFAULT_ID: default_org,
+        Organization.ROOT_ID: Organization.root(),
+        Organization.SYSTEM_ID: Organization.system()
+    })
+    return org_mapper
+
+
+def set_org_mapper(org_mapper):
+    setattr(thread_local, 'org_mapper', org_mapper)
+
+
+def get_org_mapper():
+    org_mapper = _find('org_mapper')
+    if org_mapper is None:
+        org_mapper = construct_org_mapper()
+        set_org_mapper(org_mapper)
+    return org_mapper
+
+
+def get_org_by_id(org_id):
+    org_id = str(org_id)
+    org_mapper = get_org_mapper()
+    org = org_mapper.get(org_id)
+    return org
+
+
+def get_org_name_by_id(org_id):
+    org = get_org_by_id(org_id)
+    if org:
+        org_name = org.name
+    else:
+        org_name = 'Not Found'
+    return org_name
+
+
 def get_current_org_id_for_serializer():
     org_id = get_current_org_id()
     if org_id == Organization.DEFAULT_ID:

apps/perms/api/application/user_permission/user_permission_applications.py

@@ -48,7 +48,7 @@ class ApplicationsAsTreeMixin(SerializeApplicationToTreeNodeMixin):
 
     def list(self, request, *args, **kwargs):
         queryset = self.filter_queryset(self.get_queryset())
-        data = self.serialize_applications(queryset)
+        data = self.serialize_applications_with_org(queryset)
         return Response(data=data)
 
 

apps/perms/api/asset/user_group_permission.py

@@ -32,9 +32,6 @@ class UserGroupMixin:
 
 
 class UserGroupGrantedAssetsApi(ListAPIView):
-    """
-    获取用户组直接授权的资产
-    """
     permission_classes = (IsOrgAdminOrAppUser,)
     serializer_class = serializers.AssetGrantedSerializer
     only_fields = serializers.AssetGrantedSerializer.Meta.only_fields
@@ -44,11 +41,27 @@ class UserGroupGrantedAssetsApi(ListAPIView):
     def get_queryset(self):
         user_group_id = self.kwargs.get('pk', '')
 
-        return Asset.objects.filter(
-            Q(granted_by_permissions__user_groups__id=user_group_id)
+        asset_perms_id = list(AssetPermission.objects.valid().filter(
+            user_groups__id=user_group_id
+        ).distinct().values_list('id', flat=True))
+
+        granted_node_keys = Node.objects.filter(
+            granted_by_permissions__id__in=asset_perms_id,
+        ).distinct().values_list('key', flat=True)
+
+        granted_q = Q()
+        for _key in granted_node_keys:
+            granted_q |= Q(nodes__key__startswith=f'{_key}:')
+            granted_q |= Q(nodes__key=_key)
+
+        granted_q |= Q(granted_by_permissions__id__in=asset_perms_id)
+
+        assets = Asset.objects.filter(
+            granted_q
         ).distinct().only(
             *self.only_fields
         )
+        return assets
 
 
 class UserGroupGrantedNodeAssetsApi(ListAPIView):
@@ -66,7 +79,7 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView):
         granted = AssetPermission.objects.filter(
             user_groups__id=user_group_id,
             nodes__id=node_id
-        ).exists()
+        ).valid().exists()
         if granted:
             assets = Asset.objects.filter(
                 Q(nodes__key__startswith=f'{node.key}:') |
@@ -74,8 +87,12 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView):
             )
             return assets
         else:
+            asset_perms_id = list(AssetPermission.objects.valid().filter(
+                user_groups__id=user_group_id
+            ).distinct().values_list('id', flat=True))
+
             granted_node_keys = Node.objects.filter(
-                granted_by_permissions__user_groups__id=user_group_id,
+                granted_by_permissions__id__in=asset_perms_id,
                 key__startswith=f'{node.key}:'
             ).distinct().values_list('key', flat=True)
 
@@ -85,7 +102,7 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView):
                 granted_node_q |= Q(nodes__key=_key)
 
             granted_asset_q = (
-                Q(granted_by_permissions__user_groups__id=user_group_id) &
+                Q(granted_by_permissions__id__in=asset_perms_id) &
                 (
                     Q(nodes__key__startswith=f'{node.key}:') |
                     Q(nodes__key=node.key)
@@ -129,12 +146,16 @@ class UserGroupGrantedNodeChildrenAsTreeApi(SerializeToTreeNodeMixin, ListAPIVie
         group_id = self.kwargs.get('pk')
         node_key = self.request.query_params.get('key', None)
 
+        asset_perms_id = list(AssetPermission.objects.valid().filter(
+            user_groups__id=group_id
+        ).distinct().values_list('id', flat=True))
+
         granted_keys = Node.objects.filter(
-            granted_by_permissions__user_groups__id=group_id
+            granted_by_permissions__id__in=asset_perms_id
         ).values_list('key', flat=True)
 
         asset_granted_keys = Node.objects.filter(
-            assets__granted_by_permissions__user_groups__id=group_id
+            assets__granted_by_permissions__id__in=asset_perms_id
         ).values_list('key', flat=True)
 
         if node_key is None:

apps/perms/api/asset/user_permission/user_permission_assets.py

@@ -3,6 +3,7 @@
 from perms.api.asset.user_permission.mixin import UserNodeGrantStatusDispatchMixin
 from rest_framework.generics import ListAPIView
 from rest_framework.response import Response
+from rest_framework.request import Request
 from django.conf import settings
 
 from assets.api.mixin import SerializeToTreeNodeMixin
@@ -55,8 +56,12 @@ class AssetsAsTreeMixin(SerializeToTreeNodeMixin):
     """
     将 资产 序列化成树的结构返回
     """
-    def list(self, request, *args, **kwargs):
+    def list(self, request: Request, *args, **kwargs):
         queryset = self.filter_queryset(self.get_queryset())
+        if request.query_params.get('search'):
+            # 如果用户搜索的条件不精准，会导致返回大量的无意义数据。
+            # 这里限制一下返回数据的最大条数
+            queryset = queryset[:999]
         data = self.serialize_assets(queryset, None)
         return Response(data=data)
 

apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py

@@ -139,11 +139,13 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView):
         return Response(data=data)
 
 
-class UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi(ForAdminMixin, UserNodeGrantStatusDispatchMixin,
-                                                         SerializeToTreeNodeMixin, ListAPIView):
+class GrantedNodeChildrenWithAssetsAsTreeApiMixin(UserNodeGrantStatusDispatchMixin,
+                                                  SerializeToTreeNodeMixin,
+                                                  ListAPIView):
     """
     带资产的授权树
     """
+    user: None
 
     def get_data_on_node_direct_granted(self, key):
         nodes = Node.objects.filter(parent_key=key)
@@ -203,5 +205,9 @@ class UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi(ForAdminMixin, UserNode
         return Response(data=[*tree_nodes, *tree_assets])
 
 
-class MyGrantedNodeChildrenWithAssetsAsTreeApi(ForUserMixin, UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi):
+class UserGrantedNodeChildrenWithAssetsAsTreeApi(ForAdminMixin, GrantedNodeChildrenWithAssetsAsTreeApiMixin):
+    pass
+
+
+class MyGrantedNodeChildrenWithAssetsAsTreeApi(ForUserMixin, GrantedNodeChildrenWithAssetsAsTreeApiMixin):
     pass

apps/perms/signals_handler.py

@@ -6,7 +6,7 @@ from django.dispatch import receiver
 from perms.tasks import create_rebuild_user_tree_task, \
     create_rebuild_user_tree_task_by_related_nodes_or_assets
 from users.models import User, UserGroup
-from assets.models import Asset
+from assets.models import Asset, SystemUser
 from common.utils import get_logger
 from common.exceptions import M2MReverseNotAllowed
 from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR
@@ -16,6 +16,42 @@ from .models import AssetPermission, RemoteAppPermission
 logger = get_logger(__file__)
 
 
+def handle_rebuild_user_tree(instance, action, reverse, pk_set, **kwargs):
+    if action.startswith('post'):
+        if reverse:
+            create_rebuild_user_tree_task(pk_set)
+        else:
+            create_rebuild_user_tree_task([instance.id])
+
+
+def handle_bind_groups_systemuser(instance, action, reverse, pk_set, **kwargs):
+    """
+    UserGroup 增加 User 时，增加的 User 需要与 UserGroup 关联的动态系统用户相关联
+    """
+    user: User
+
+    if action != POST_ADD:
+        return
+
+    if not reverse:
+        # 一个用户添加了多个用户组
+        users_id = [instance.id]
+        system_users = SystemUser.objects.filter(groups__id__in=pk_set).distinct()
+    else:
+        # 一个用户组添加了多个用户
+        users_id = pk_set
+        system_users = SystemUser.objects.filter(groups__id=instance.pk).distinct()
+
+    for system_user in system_users:
+        system_user.users.add(*users_id)
+
+
+@receiver(m2m_changed, sender=User.groups.through)
+def on_user_groups_change(**kwargs):
+    handle_rebuild_user_tree(**kwargs)
+    handle_bind_groups_systemuser(**kwargs)
+
+
 @receiver([pre_save], sender=AssetPermission)
 def on_asset_perm_deactive(instance: AssetPermission, **kwargs):
     try:

apps/perms/tasks.py

@@ -5,10 +5,12 @@ from datetime import timedelta
 from django.db import transaction
 from django.db.models import Q
 from django.db.transaction import atomic
+from django.conf import settings
 from celery import shared_task
 from common.utils import get_logger
 from common.utils.timezone import now, dt_formater, dt_parser
 from users.models import User
+from ops.celery.decorator import register_as_period_task
 from assets.models import Node
 from perms.models import RebuildUserTreeTask, AssetPermission
 from perms.utils.asset.user_permission import rebuild_user_mapping_nodes_if_need_with_lock, lock
@@ -33,7 +35,8 @@ def dispatch_mapping_node_tasks():
         rebuild_user_mapping_nodes_celery_task.delay(id)
 
 
-@shared_task(queue='check_asset_perm_expired')
+@register_as_period_task(interval=settings.PERM_EXPIRED_CHECK_PERIODIC)
+@shared_task(queue='celery_check_asset_perm_expired')
 @atomic()
 def check_asset_permission_expired():
     """

apps/perms/urls/asset_permission.py

@@ -21,7 +21,7 @@ user_permission_urlpatterns = [
     # ---------------------------------------------------------
     # 以 serializer 格式返回
     path('<uuid:pk>/assets/', api.UserAllGrantedAssetsApi.as_view(), name='user-assets'),
-    path('assets/', api.MyAllAssetsAsTreeApi.as_view(), name='my-assets'),
+    path('assets/', api.MyAllGrantedAssetsApi.as_view(), name='my-assets'),
 
     # Tree Node 的数据格式返回
     path('<uuid:pk>/assets/tree/', api.UserDirectGrantedAssetsAsTreeForAdminApi.as_view(), name='user-assets-as-tree'),
@@ -56,7 +56,7 @@ user_permission_urlpatterns = [
     path('nodes-with-assets/tree/', api.MyGrantedNodesWithAssetsAsTreeApi.as_view(), name='my-nodes-with-assets-as-tree'),
 
     # 主要用于 luna 页面，带资产的节点树
-    path('<uuid:pk>/nodes/children-with-assets/tree/', api.UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi.as_view(), name='user-nodes-children-with-assets-as-tree'),
+    path('<uuid:pk>/nodes/children-with-assets/tree/', api.UserGrantedNodeChildrenWithAssetsAsTreeApi.as_view(), name='user-nodes-children-with-assets-as-tree'),
     path('nodes/children-with-assets/tree/', api.MyGrantedNodeChildrenWithAssetsAsTreeApi.as_view(), name='my-nodes-children-with-assets-as-tree'),
 
     # 查询授权树上某个节点的所有资产

apps/perms/utils/asset/user_permission.py

@@ -34,27 +34,6 @@ TMP_ASSET_GRANTED_FIELD = '_asset_granted'
 TMP_GRANTED_ASSETS_AMOUNT_FIELD = '_granted_assets_amount'
 
 
-# 使用场景
-# Asset.objects.filter(get_user_resources_q_granted_by_permissions(user))
-def get_user_resources_q_granted_by_permissions(user: User):
-    """
-    获取用户关联的 asset permission 或者 用户组关联的 asset permission 获取规则,
-    前提 AssetPermission 对象中的 related_name 为 granted_by_permissions
-    :param user:
-    :return:
-    """
-    _now = now()
-    return reduce(and_, (
-        Q(granted_by_permissions__date_start__lt=_now),
-        Q(granted_by_permissions__date_expired__gt=_now),
-        Q(granted_by_permissions__is_active=True),
-        (
-            Q(granted_by_permissions__users=user) |
-            Q(granted_by_permissions__user_groups__users=user)
-        )
-    ))
-
-
 # 使用场景
 # `Node.objects.annotate(**node_annotate_mapping_node)`
 node_annotate_mapping_node = {
@@ -215,7 +194,7 @@ def compute_tmp_mapping_node_from_perm(user: User, asset_perms_id=None):
     return [*leaf_nodes, *ancestors]
 
 
-def create_mapping_nodes(user, nodes, clear=True):
+def create_mapping_nodes(user, nodes):
     to_create = []
     for node in nodes:
         _granted = getattr(node, TMP_GRANTED_FIELD, False)
@@ -231,8 +210,6 @@ def create_mapping_nodes(user, nodes, clear=True):
             assets_amount=_granted_assets_amount,
         ))
 
-    if clear:
-        UserGrantedMappingNode.objects.filter(user=user).delete()
     UserGrantedMappingNode.objects.bulk_create(to_create)
 
 
@@ -254,6 +231,9 @@ def set_node_granted_assets_amount(user, node, asset_perms_id=None):
 @tmp_to_root_org()
 def rebuild_user_mapping_nodes(user):
     logger.info(f'>>> {dt_formater(now())} start rebuild {user} mapping nodes')
+
+    # 先删除旧的授权树🌲
+    UserGrantedMappingNode.objects.filter(user=user).delete()
     asset_perms_id = get_user_all_assetpermissions_id(user)
     if not asset_perms_id:
         # 没有授权直接返回
@@ -384,7 +364,8 @@ def get_node_all_granted_assets(user: User, key):
 
     if only_asset_granted_nodes_qs:
         only_asset_granted_nodes_q = reduce(or_, only_asset_granted_nodes_qs)
-        only_asset_granted_nodes_q &= get_user_resources_q_granted_by_permissions(user)
+        asset_perms_id = get_user_all_assetpermissions_id(user)
+        only_asset_granted_nodes_q &= Q(granted_by_permissions__id__in=list(asset_perms_id))
         q.append(only_asset_granted_nodes_q)
 
     if q:
@@ -484,6 +465,9 @@ def get_user_all_assetpermissions_id(user: User):
     asset_perms_id = AssetPermission.objects.valid().filter(
         Q(users=user) | Q(user_groups__users=user)
     ).distinct().values_list('id', flat=True)
+
+    # !!! 这个很重要，必须转换成 list，避免 Django 生成嵌套子查询
+    asset_perms_id = list(asset_perms_id)
     return asset_perms_id
 
 

apps/settings/utils/ldap.py

@@ -333,7 +333,7 @@ class LDAPImportUtil(object):
     def update_or_create(self, user):
         user['email'] = self.get_user_email(user)
         if user['username'] not in ['admin']:
-            user['source'] = User.SOURCE_LDAP
+            user['source'] = User.Source.ldap.value
         obj, created = User.objects.update_or_create(
             username=user['username'], defaults=user
         )

apps/templates/captcha/field.html

@@ -1,12 +0,0 @@
-{{image}}{{hidden_field}}{{text_field}}
-
-<script>
-    function refresh_captcha() {
-        $.getJSON("{% url "captcha-refresh" %}",
-                function (result) {
-                    $('.captcha').attr('src', result['image_url']);
-                    $('#id_captcha_0').val(result['key'])
-                })
-    }
-    $('.captcha').click(refresh_captcha)
-</script>

apps/templates/captcha/hidden_field.html

@@ -1 +0,0 @@
-<input id="{{id}}_0" name="{{name}}_0" type="hidden" value="{{key}}" />

apps/templates/captcha/image.html

@@ -1,4 +0,0 @@
-{% load i18n %}
-{% spaceless %}
-    {% if audio %}<a title="{% trans "Play CAPTCHA as audio file" %}" href="{{audio}}">{% endif %}<img src="{{image}}" alt="captcha" class="captcha" />{% if audio %}</a>{% endif %}
-{% endspaceless %}

apps/templates/captcha/text_field.html

@@ -1,7 +0,0 @@
-{% load i18n %}
-<div class="row">
-    <div class="col-sm-6">
-        <input autocomplete="off" id="{{id}}_1" class="form-control" name="{{name}}_1" placeholder="{% trans 'Captcha' %}" type="text" />
-    </div>
-</div>
-</br>

apps/terminal/api/__init__.py

@@ -5,3 +5,4 @@ from .session import *
 from .command import *
 from .task import *
 from .storage import *
+from .component import *

apps/terminal/api/component.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+#
+
+import logging
+from rest_framework import generics, status
+from rest_framework.views import Response
+
+from .. import serializers
+from ..utils import ComponentsMetricsUtil
+from common.permissions import IsAppUser, IsSuperUser
+
+logger = logging.getLogger(__file__)
+
+
+__all__ = [
+    'ComponentsStateAPIView', 'ComponentsMetricsAPIView',
+]
+
+
+class ComponentsStateAPIView(generics.CreateAPIView):
+    """ koko, guacamole, omnidb 上报状态 """
+    permission_classes = (IsAppUser,)
+    serializer_class = serializers.ComponentsStateSerializer
+
+
+class ComponentsMetricsAPIView(generics.GenericAPIView):
+    """ 返回汇总组件指标数据 """
+    permission_classes = (IsSuperUser,)
+
+    def get(self, request, *args, **kwargs):
+        component_type = request.query_params.get('type')
+        util = ComponentsMetricsUtil(component_type)
+        metrics = util.get_metrics()
+        return Response(metrics, status=status.HTTP_200_OK)

apps/terminal/api/terminal.py

@@ -27,7 +27,7 @@ class TerminalViewSet(JMSBulkModelViewSet):
     queryset = Terminal.objects.filter(is_deleted=False)
     serializer_class = serializers.TerminalSerializer
     permission_classes = (IsSuperUser,)
-    filter_fields = ['name', 'remote_addr']
+    filter_fields = ['name', 'remote_addr', 'type']
 
     def create(self, request, *args, **kwargs):
         if isinstance(request.data, list):
@@ -60,6 +60,15 @@ class TerminalViewSet(JMSBulkModelViewSet):
             logger.error("Register terminal error: {}".format(data))
             return Response(data, status=400)
 
+    def filter_queryset(self, queryset):
+        queryset = super().filter_queryset(queryset)
+        status = self.request.query_params.get('status')
+        if not status:
+            return queryset
+        filtered_queryset_id = [str(q.id) for q in queryset if q.status == status]
+        queryset = queryset.filter(id__in=filtered_queryset_id)
+        return queryset
+
     def get_permissions(self):
         if self.action == "create":
             self.permission_classes = (AllowAny,)
@@ -104,15 +113,11 @@ class StatusViewSet(viewsets.ModelViewSet):
     task_serializer_class = serializers.TaskSerializer
 
     def create(self, request, *args, **kwargs):
-        self.handle_status(request)
         self.handle_sessions()
         tasks = self.request.user.terminal.task_set.filter(is_finished=False)
         serializer = self.task_serializer_class(tasks, many=True)
         return Response(serializer.data, status=201)
 
-    def handle_status(self, request):
-        request.user.terminal.is_alive = True
-
     def handle_sessions(self):
         sessions_id = self.request.data.get('sessions', [])
         # guacamole 上报的 session 是字符串

apps/terminal/const.py

@@ -108,3 +108,27 @@ COMMAND_STORAGE_TYPE_CHOICES_EXTENDS = [
 COMMAND_STORAGE_TYPE_CHOICES = COMMAND_STORAGE_TYPE_CHOICES_DEFAULT + \
                                COMMAND_STORAGE_TYPE_CHOICES_EXTENDS
 
+
+from django.db.models import TextChoices
+from django.utils.translation import ugettext_lazy as _
+
+
+class ComponentStatusChoices(TextChoices):
+    critical = 'critical', _('Critical')
+    high = 'high', _('High')
+    normal = 'normal', _('Normal')
+
+    @classmethod
+    def status(cls):
+        return set(dict(cls.choices).keys())
+
+
+class TerminalTypeChoices(TextChoices):
+    koko = 'koko', 'KoKo'
+    guacamole = 'guacamole', 'Guacamole'
+    omnidb = 'omnidb', 'OmniDB'
+
+    @classmethod
+    def types(cls):
+        return set(dict(cls.choices).keys())
+

apps/terminal/migrations/0030_terminal_type.py

@@ -0,0 +1,42 @@
+# Generated by Django 3.1 on 2020-12-10 07:05
+
+from django.db import migrations, models
+
+TERMINAL_TYPE_KOKO = 'koko'
+TERMINAL_TYPE_GUACAMOLE = 'guacamole'
+TERMINAL_TYPE_OMNIDB = 'omnidb'
+
+
+def migrate_terminal_type(apps, schema_editor):
+    terminal_model = apps.get_model("terminal", "Terminal")
+    db_alias = schema_editor.connection.alias
+    terminals = terminal_model.objects.using(db_alias).all()
+    for terminal in terminals:
+        name = terminal.name.lower()
+        if 'koko' in name:
+            _type = TERMINAL_TYPE_KOKO
+        elif 'gua' in name:
+            _type = TERMINAL_TYPE_GUACAMOLE
+        elif 'omnidb' in name:
+            _type = TERMINAL_TYPE_OMNIDB
+        else:
+            _type = TERMINAL_TYPE_KOKO
+        terminal.type = _type
+    terminal_model.objects.bulk_update(terminals, ['type'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('terminal', '0029_auto_20201116_1757'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='terminal',
+            name='type',
+            field=models.CharField(choices=[('koko', 'KoKo'), ('guacamole', 'Guacamole'), ('omnidb', 'OmniDB')], default='koko', max_length=64, verbose_name='type'),
+            preserve_default=False,
+        ),
+        migrations.RunPython(migrate_terminal_type)
+    ]

apps/terminal/models.py

@@ -1,486 +0,0 @@
-from __future__ import unicode_literals
-
-import os
-import uuid
-import jms_storage
-
-from django.db import models
-from django.db.models.signals import post_save
-from django.utils.translation import ugettext_lazy as _
-from django.utils import timezone
-from django.conf import settings
-from django.core.files.storage import default_storage
-from django.core.cache import cache
-
-from assets.models import Asset
-from users.models import User
-from orgs.mixins.models import OrgModelMixin
-from common.mixins import CommonModelMixin
-from common.fields.model import EncryptJsonDictTextField
-from common.db.models import ChoiceSet
-from .backends import get_multi_command_storage
-from .backends.command.models import AbstractSessionCommand
-from . import const
-
-
-class Terminal(models.Model):
-    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    name = models.CharField(max_length=128, verbose_name=_('Name'))
-    remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address'))
-    ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222)
-    http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000)
-    command_storage = models.CharField(max_length=128, verbose_name=_("Command storage"), default='default')
-    replay_storage = models.CharField(max_length=128, verbose_name=_("Replay storage"), default='default')
-    user = models.OneToOneField(User, related_name='terminal', verbose_name='Application User', null=True, on_delete=models.CASCADE)
-    is_accepted = models.BooleanField(default=False, verbose_name='Is Accepted')
-    is_deleted = models.BooleanField(default=False)
-    date_created = models.DateTimeField(auto_now_add=True)
-    comment = models.TextField(blank=True, verbose_name=_('Comment'))
-    STATUS_KEY_PREFIX = 'terminal_status_'
-
-    @property
-    def is_alive(self):
-        key = self.STATUS_KEY_PREFIX + str(self.id)
-        return bool(cache.get(key))
-
-    @is_alive.setter
-    def is_alive(self, value):
-        key = self.STATUS_KEY_PREFIX + str(self.id)
-        cache.set(key, value, 60)
-
-    @property
-    def is_active(self):
-        if self.user and self.user.is_active:
-            return True
-        return False
-
-    @is_active.setter
-    def is_active(self, active):
-        if self.user:
-            self.user.is_active = active
-            self.user.save()
-
-    def get_command_storage(self):
-        storage = CommandStorage.objects.filter(name=self.command_storage).first()
-        return storage
-
-    def get_command_storage_config(self):
-        s = self.get_command_storage()
-        if s:
-            config = s.config
-        else:
-            config = settings.DEFAULT_TERMINAL_COMMAND_STORAGE
-        return config
-
-    def get_command_storage_setting(self):
-        config = self.get_command_storage_config()
-        return {"TERMINAL_COMMAND_STORAGE": config}
-
-    def get_replay_storage(self):
-        storage = ReplayStorage.objects.filter(name=self.replay_storage).first()
-        return storage
-
-    def get_replay_storage_config(self):
-        s = self.get_replay_storage()
-        if s:
-            config = s.config
-        else:
-            config = settings.DEFAULT_TERMINAL_REPLAY_STORAGE
-        return config
-
-    def get_replay_storage_setting(self):
-        config = self.get_replay_storage_config()
-        return {"TERMINAL_REPLAY_STORAGE": config}
-
-    @staticmethod
-    def get_login_title_setting():
-        login_title = None
-        if settings.XPACK_ENABLED:
-            from xpack.plugins.interface.models import Interface
-            login_title = Interface.get_login_title()
-        return {'TERMINAL_HEADER_TITLE': login_title}
-
-    @property
-    def config(self):
-        configs = {}
-        for k in dir(settings):
-            if not k.startswith('TERMINAL'):
-                continue
-            configs[k] = getattr(settings, k)
-        configs.update(self.get_command_storage_setting())
-        configs.update(self.get_replay_storage_setting())
-        configs.update(self.get_login_title_setting())
-        configs.update({
-            'SECURITY_MAX_IDLE_TIME': settings.SECURITY_MAX_IDLE_TIME
-        })
-        return configs
-
-    @property
-    def service_account(self):
-        return self.user
-
-    def create_app_user(self):
-        random = uuid.uuid4().hex[:6]
-        user, access_key = User.create_app_user(
-            name="{}-{}".format(self.name, random), comment=self.comment
-        )
-        self.user = user
-        self.save()
-        return user, access_key
-
-    def delete(self, using=None, keep_parents=False):
-        if self.user:
-            self.user.delete()
-        self.user = None
-        self.is_deleted = True
-        self.save()
-        return
-
-    def __str__(self):
-        status = "Active"
-        if not self.is_accepted:
-            status = "NotAccept"
-        elif self.is_deleted:
-            status = "Deleted"
-        elif not self.is_active:
-            status = "Disable"
-        return '%s: %s' % (self.name, status)
-
-    class Meta:
-        ordering = ('is_accepted',)
-        db_table = "terminal"
-
-
-class Status(models.Model):
-    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    session_online = models.IntegerField(verbose_name=_("Session Online"), default=0)
-    cpu_used = models.FloatField(verbose_name=_("CPU Usage"))
-    memory_used = models.FloatField(verbose_name=_("Memory Used"))
-    connections = models.IntegerField(verbose_name=_("Connections"))
-    threads = models.IntegerField(verbose_name=_("Threads"))
-    boot_time = models.FloatField(verbose_name=_("Boot Time"))
-    terminal = models.ForeignKey(Terminal, null=True, on_delete=models.CASCADE)
-    date_created = models.DateTimeField(auto_now_add=True)
-
-    class Meta:
-        db_table = 'terminal_status'
-        get_latest_by = 'date_created'
-
-    def __str__(self):
-        return self.date_created.strftime("%Y-%m-%d %H:%M:%S")
-
-
-class Session(OrgModelMixin):
-    class LOGIN_FROM(ChoiceSet):
-        ST = 'ST', 'SSH Terminal'
-        WT = 'WT', 'Web Terminal'
-
-    class PROTOCOL(ChoiceSet):
-        SSH = 'ssh', 'ssh'
-        RDP = 'rdp', 'rdp'
-        VNC = 'vnc', 'vnc'
-        TELNET = 'telnet', 'telnet'
-        MYSQL = 'mysql', 'mysql'
-        ORACLE = 'oracle', 'oracle'
-        MARIADB = 'mariadb', 'mariadb'
-        POSTGRESQL = 'postgresql', 'postgresql'
-        K8S = 'k8s', 'kubernetes'
-
-    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    user = models.CharField(max_length=128, verbose_name=_("User"), db_index=True)
-    user_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
-    asset = models.CharField(max_length=128, verbose_name=_("Asset"), db_index=True)
-    asset_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
-    system_user = models.CharField(max_length=128, verbose_name=_("System user"), db_index=True)
-    system_user_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
-    login_from = models.CharField(max_length=2, choices=LOGIN_FROM.choices, default="ST", verbose_name=_("Login from"))
-    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
-    is_success = models.BooleanField(default=True, db_index=True)
-    is_finished = models.BooleanField(default=False, db_index=True)
-    has_replay = models.BooleanField(default=False, verbose_name=_("Replay"))
-    has_command = models.BooleanField(default=False, verbose_name=_("Command"))
-    terminal = models.ForeignKey(Terminal, null=True, on_delete=models.DO_NOTHING, db_constraint=False)
-    protocol = models.CharField(choices=PROTOCOL.choices, default='ssh', max_length=16, db_index=True)
-    date_start = models.DateTimeField(verbose_name=_("Date start"), db_index=True, default=timezone.now)
-    date_end = models.DateTimeField(verbose_name=_("Date end"), null=True)
-
-    upload_to = 'replay'
-    ACTIVE_CACHE_KEY_PREFIX = 'SESSION_ACTIVE_{}'
-    _DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = None
-
-    def get_rel_replay_path(self, version=2):
-        """
-        获取session日志的文件路径
-        :param version: 原来后缀是 .gz，为了统一新版本改为 .replay.gz
-        :return:
-        """
-        suffix = '.replay.gz'
-        if version == 1:
-            suffix = '.gz'
-        date = self.date_start.strftime('%Y-%m-%d')
-        return os.path.join(date, str(self.id) + suffix)
-
-    def get_local_path(self, version=2):
-        rel_path = self.get_rel_replay_path(version=version)
-        if version == 2:
-            local_path = os.path.join(self.upload_to, rel_path)
-        else:
-            local_path = rel_path
-        return local_path
-
-    @property
-    def asset_obj(self):
-        return Asset.objects.get(id=self.asset_id)
-
-    @property
-    def _date_start_first_has_replay_rdp_session(self):
-        if self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION is None:
-            instance = self.__class__.objects.filter(
-                protocol='rdp', has_replay=True
-            ).order_by('date_start').first()
-            if not instance:
-                date_start = timezone.now() - timezone.timedelta(days=365)
-            else:
-                date_start = instance.date_start
-            self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = date_start
-        return self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION
-
-    def can_replay(self):
-        if self.has_replay:
-            return True
-        if self.date_start < self._date_start_first_has_replay_rdp_session:
-            return True
-        return False
-
-    @property
-    def can_join(self):
-        _PROTOCOL = self.PROTOCOL
-        if self.is_finished:
-            return False
-        if self.protocol in [_PROTOCOL.SSH, _PROTOCOL.TELNET, _PROTOCOL.K8S]:
-            return True
-        else:
-            return False
-
-    @property
-    def db_protocols(self):
-        _PROTOCOL = self.PROTOCOL
-        return [_PROTOCOL.MYSQL, _PROTOCOL.MARIADB, _PROTOCOL.ORACLE, _PROTOCOL.POSTGRESQL]
-
-    @property
-    def can_terminate(self):
-        _PROTOCOL = self.PROTOCOL
-        if self.is_finished:
-            return False
-        if self.protocol in self.db_protocols:
-            return False
-        else:
-            return True
-
-    def save_replay_to_storage(self, f):
-        local_path = self.get_local_path()
-        try:
-            name = default_storage.save(local_path, f)
-        except OSError as e:
-            return None, e
-
-        if settings.SERVER_REPLAY_STORAGE:
-            from .tasks import upload_session_replay_to_external_storage
-            upload_session_replay_to_external_storage.delay(str(self.id))
-        return name, None
-
-    @classmethod
-    def set_sessions_active(cls, sessions_id):
-        data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in sessions_id}
-        cache.set_many(data, timeout=5*60)
-
-    @classmethod
-    def get_active_sessions(cls):
-        return cls.objects.filter(is_finished=False)
-
-    def is_active(self):
-        if self.protocol in ['ssh', 'telnet', 'rdp', 'mysql']:
-            key = self.ACTIVE_CACHE_KEY_PREFIX.format(self.id)
-            return bool(cache.get(key))
-        return True
-
-    @property
-    def command_amount(self):
-        command_store = get_multi_command_storage()
-        return command_store.count(session=str(self.id))
-
-    @property
-    def login_from_display(self):
-        return self.get_login_from_display()
-
-    @classmethod
-    def generate_fake(cls, count=100, is_finished=True):
-        import random
-        from orgs.models import Organization
-        from users.models import User
-        from assets.models import Asset, SystemUser
-        from orgs.utils import get_current_org
-        from common.utils.random import random_datetime, random_ip
-
-        org = get_current_org()
-        if not org or not org.is_real():
-            Organization.default().change_to()
-        i = 0
-        users = User.objects.all()[:100]
-        assets = Asset.objects.all()[:100]
-        system_users = SystemUser.objects.all()[:100]
-        while i < count:
-            user_random = random.choices(users, k=10)
-            assets_random = random.choices(assets, k=10)
-            system_users = random.choices(system_users, k=10)
-
-            ziped = zip(user_random, assets_random, system_users)
-            sessions = []
-            now = timezone.now()
-            month_ago = now - timezone.timedelta(days=30)
-            for user, asset, system_user in ziped:
-                ip = random_ip()
-                date_start = random_datetime(month_ago, now)
-                date_end = random_datetime(date_start, date_start+timezone.timedelta(hours=2))
-                data = dict(
-                    user=str(user), user_id=user.id,
-                    asset=str(asset), asset_id=asset.id,
-                    system_user=str(system_user), system_user_id=system_user.id,
-                    remote_addr=ip,
-                    date_start=date_start,
-                    date_end=date_end,
-                    is_finished=is_finished,
-                )
-                sessions.append(Session(**data))
-            cls.objects.bulk_create(sessions)
-            i += 10
-
-    class Meta:
-        db_table = "terminal_session"
-        ordering = ["-date_start"]
-
-    def __str__(self):
-        return "{0.id} of {0.user} to {0.asset}".format(self)
-
-
-class Task(models.Model):
-    NAME_CHOICES = (
-        ("kill_session", "Kill Session"),
-    )
-
-    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    name = models.CharField(max_length=128, choices=NAME_CHOICES, verbose_name=_("Name"))
-    args = models.CharField(max_length=1024, verbose_name=_("Args"))
-    terminal = models.ForeignKey(Terminal, null=True, on_delete=models.SET_NULL)
-    is_finished = models.BooleanField(default=False)
-    date_created = models.DateTimeField(auto_now_add=True)
-    date_finished = models.DateTimeField(null=True)
-
-    class Meta:
-        db_table = "terminal_task"
-
-
-class CommandManager(models.Manager):
-    def bulk_create(self, objs, **kwargs):
-        resp = super().bulk_create(objs, **kwargs)
-        for i in objs:
-            post_save.send(i.__class__, instance=i, created=True)
-        return resp
-
-
-class Command(AbstractSessionCommand):
-    objects = CommandManager()
-
-    class Meta:
-        db_table = "terminal_command"
-        ordering = ('-timestamp',)
-
-
-class CommandStorage(CommonModelMixin):
-    TYPE_CHOICES = const.COMMAND_STORAGE_TYPE_CHOICES
-    TYPE_DEFAULTS = dict(const.REPLAY_STORAGE_TYPE_CHOICES_DEFAULT).keys()
-    TYPE_SERVER = const.COMMAND_STORAGE_TYPE_SERVER
-
-    name = models.CharField(max_length=128, verbose_name=_("Name"), unique=True)
-    type = models.CharField(
-        max_length=16, choices=TYPE_CHOICES, verbose_name=_('Type'),
-        default=TYPE_SERVER
-    )
-    meta = EncryptJsonDictTextField(default={})
-    comment = models.TextField(
-        max_length=128, default='', blank=True, verbose_name=_('Comment')
-    )
-
-    def __str__(self):
-        return self.name
-
-    @property
-    def config(self):
-        config = self.meta
-        config.update({'TYPE': self.type})
-        return config
-
-    def in_defaults(self):
-        return self.type in self.TYPE_DEFAULTS
-
-    def is_valid(self):
-        if self.in_defaults():
-            return True
-        storage = jms_storage.get_log_storage(self.config)
-        return storage.ping()
-
-    def is_using(self):
-        return Terminal.objects.filter(command_storage=self.name).exists()
-
-
-class ReplayStorage(CommonModelMixin):
-    TYPE_CHOICES = const.REPLAY_STORAGE_TYPE_CHOICES
-    TYPE_SERVER = const.REPLAY_STORAGE_TYPE_SERVER
-    TYPE_DEFAULTS = dict(const.REPLAY_STORAGE_TYPE_CHOICES_DEFAULT).keys()
-
-    name = models.CharField(max_length=128, verbose_name=_("Name"), unique=True)
-    type = models.CharField(
-        max_length=16, choices=TYPE_CHOICES, verbose_name=_('Type'),
-        default=TYPE_SERVER
-    )
-    meta = EncryptJsonDictTextField(default={})
-    comment = models.TextField(
-        max_length=128, default='', blank=True, verbose_name=_('Comment')
-    )
-
-    def __str__(self):
-        return self.name
-
-    def convert_type(self):
-        s3_type_list = [const.REPLAY_STORAGE_TYPE_CEPH]
-        tp = self.type
-        if tp in s3_type_list:
-            tp = const.REPLAY_STORAGE_TYPE_S3
-        return tp
-
-    def get_extra_config(self):
-        extra_config = {'TYPE': self.convert_type()}
-        if self.type == const.REPLAY_STORAGE_TYPE_SWIFT:
-            extra_config.update({'signer': 'S3SignerType'})
-        return extra_config
-
-    @property
-    def config(self):
-        config = self.meta
-        extra_config = self.get_extra_config()
-        config.update(extra_config)
-        return config
-
-    def in_defaults(self):
-        return self.type in self.TYPE_DEFAULTS
-
-    def is_valid(self):
-        if self.in_defaults():
-            return True
-        storage = jms_storage.get_object_storage(self.config)
-        target = 'tests.py'
-        src = os.path.join(settings.BASE_DIR, 'common', target)
-        return storage.is_valid(src, target)
-
-    def is_using(self):
-        return Terminal.objects.filter(replay_storage=self.name).exists()

apps/terminal/models/__init__.py

@@ -0,0 +1,6 @@
+from .command import *
+from .session import *
+from .status import *
+from .storage import *
+from .task import *
+from .terminal import *

apps/terminal/models/command.py

@@ -0,0 +1,21 @@
+from __future__ import unicode_literals
+
+from django.db import models
+from django.db.models.signals import post_save
+from ..backends.command.models import AbstractSessionCommand
+
+
+class CommandManager(models.Manager):
+    def bulk_create(self, objs, **kwargs):
+        resp = super().bulk_create(objs, **kwargs)
+        for i in objs:
+            post_save.send(i.__class__, instance=i, created=True)
+        return resp
+
+
+class Command(AbstractSessionCommand):
+    objects = CommandManager()
+
+    class Meta:
+        db_table = "terminal_command"
+        ordering = ('-timestamp',)

apps/terminal/models/session.py

@@ -0,0 +1,210 @@
+from __future__ import unicode_literals
+
+import os
+import uuid
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from django.utils import timezone
+from django.conf import settings
+from django.core.files.storage import default_storage
+from django.core.cache import cache
+
+from assets.models import Asset
+from orgs.mixins.models import OrgModelMixin
+from common.db.models import ChoiceSet
+from ..backends import get_multi_command_storage
+from .terminal import Terminal
+
+
+class Session(OrgModelMixin):
+    class LOGIN_FROM(ChoiceSet):
+        ST = 'ST', 'SSH Terminal'
+        WT = 'WT', 'Web Terminal'
+
+    class PROTOCOL(ChoiceSet):
+        SSH = 'ssh', 'ssh'
+        RDP = 'rdp', 'rdp'
+        VNC = 'vnc', 'vnc'
+        TELNET = 'telnet', 'telnet'
+        MYSQL = 'mysql', 'mysql'
+        ORACLE = 'oracle', 'oracle'
+        MARIADB = 'mariadb', 'mariadb'
+        POSTGRESQL = 'postgresql', 'postgresql'
+        K8S = 'k8s', 'kubernetes'
+
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    user = models.CharField(max_length=128, verbose_name=_("User"), db_index=True)
+    user_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
+    asset = models.CharField(max_length=128, verbose_name=_("Asset"), db_index=True)
+    asset_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
+    system_user = models.CharField(max_length=128, verbose_name=_("System user"), db_index=True)
+    system_user_id = models.CharField(blank=True, default='', max_length=36, db_index=True)
+    login_from = models.CharField(max_length=2, choices=LOGIN_FROM.choices, default="ST", verbose_name=_("Login from"))
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
+    is_success = models.BooleanField(default=True, db_index=True)
+    is_finished = models.BooleanField(default=False, db_index=True)
+    has_replay = models.BooleanField(default=False, verbose_name=_("Replay"))
+    has_command = models.BooleanField(default=False, verbose_name=_("Command"))
+    terminal = models.ForeignKey(Terminal, null=True, on_delete=models.DO_NOTHING, db_constraint=False)
+    protocol = models.CharField(choices=PROTOCOL.choices, default='ssh', max_length=16, db_index=True)
+    date_start = models.DateTimeField(verbose_name=_("Date start"), db_index=True, default=timezone.now)
+    date_end = models.DateTimeField(verbose_name=_("Date end"), null=True)
+
+    upload_to = 'replay'
+    ACTIVE_CACHE_KEY_PREFIX = 'SESSION_ACTIVE_{}'
+    _DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = None
+
+    def get_rel_replay_path(self, version=2):
+        """
+        获取session日志的文件路径
+        :param version: 原来后缀是 .gz，为了统一新版本改为 .replay.gz
+        :return:
+        """
+        suffix = '.replay.gz'
+        if version == 1:
+            suffix = '.gz'
+        date = self.date_start.strftime('%Y-%m-%d')
+        return os.path.join(date, str(self.id) + suffix)
+
+    def get_local_path(self, version=2):
+        rel_path = self.get_rel_replay_path(version=version)
+        if version == 2:
+            local_path = os.path.join(self.upload_to, rel_path)
+        else:
+            local_path = rel_path
+        return local_path
+
+    @property
+    def asset_obj(self):
+        return Asset.objects.get(id=self.asset_id)
+
+    @property
+    def _date_start_first_has_replay_rdp_session(self):
+        if self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION is None:
+            instance = self.__class__.objects.filter(
+                protocol='rdp', has_replay=True
+            ).order_by('date_start').first()
+            if not instance:
+                date_start = timezone.now() - timezone.timedelta(days=365)
+            else:
+                date_start = instance.date_start
+            self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION = date_start
+        return self.__class__._DATE_START_FIRST_HAS_REPLAY_RDP_SESSION
+
+    def can_replay(self):
+        if self.has_replay:
+            return True
+        if self.date_start < self._date_start_first_has_replay_rdp_session:
+            return True
+        return False
+
+    @property
+    def can_join(self):
+        _PROTOCOL = self.PROTOCOL
+        if self.is_finished:
+            return False
+        if self.protocol in [_PROTOCOL.SSH, _PROTOCOL.TELNET, _PROTOCOL.K8S]:
+            return True
+        else:
+            return False
+
+    @property
+    def db_protocols(self):
+        _PROTOCOL = self.PROTOCOL
+        return [_PROTOCOL.MYSQL, _PROTOCOL.MARIADB, _PROTOCOL.ORACLE, _PROTOCOL.POSTGRESQL]
+
+    @property
+    def can_terminate(self):
+        _PROTOCOL = self.PROTOCOL
+        if self.is_finished:
+            return False
+        if self.protocol in self.db_protocols:
+            return False
+        else:
+            return True
+
+    def save_replay_to_storage(self, f):
+        local_path = self.get_local_path()
+        try:
+            name = default_storage.save(local_path, f)
+        except OSError as e:
+            return None, e
+
+        if settings.SERVER_REPLAY_STORAGE:
+            from .tasks import upload_session_replay_to_external_storage
+            upload_session_replay_to_external_storage.delay(str(self.id))
+        return name, None
+
+    @classmethod
+    def set_sessions_active(cls, sessions_id):
+        data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in sessions_id}
+        cache.set_many(data, timeout=5*60)
+
+    @classmethod
+    def get_active_sessions(cls):
+        return cls.objects.filter(is_finished=False)
+
+    def is_active(self):
+        if self.protocol in ['ssh', 'telnet', 'rdp', 'mysql']:
+            key = self.ACTIVE_CACHE_KEY_PREFIX.format(self.id)
+            return bool(cache.get(key))
+        return True
+
+    @property
+    def command_amount(self):
+        command_store = get_multi_command_storage()
+        return command_store.count(session=str(self.id))
+
+    @property
+    def login_from_display(self):
+        return self.get_login_from_display()
+
+    @classmethod
+    def generate_fake(cls, count=100, is_finished=True):
+        import random
+        from orgs.models import Organization
+        from users.models import User
+        from assets.models import Asset, SystemUser
+        from orgs.utils import get_current_org
+        from common.utils.random import random_datetime, random_ip
+
+        org = get_current_org()
+        if not org or not org.is_real():
+            Organization.default().change_to()
+        i = 0
+        users = User.objects.all()[:100]
+        assets = Asset.objects.all()[:100]
+        system_users = SystemUser.objects.all()[:100]
+        while i < count:
+            user_random = random.choices(users, k=10)
+            assets_random = random.choices(assets, k=10)
+            system_users = random.choices(system_users, k=10)
+
+            ziped = zip(user_random, assets_random, system_users)
+            sessions = []
+            now = timezone.now()
+            month_ago = now - timezone.timedelta(days=30)
+            for user, asset, system_user in ziped:
+                ip = random_ip()
+                date_start = random_datetime(month_ago, now)
+                date_end = random_datetime(date_start, date_start+timezone.timedelta(hours=2))
+                data = dict(
+                    user=str(user), user_id=user.id,
+                    asset=str(asset), asset_id=asset.id,
+                    system_user=str(system_user), system_user_id=system_user.id,
+                    remote_addr=ip,
+                    date_start=date_start,
+                    date_end=date_end,
+                    is_finished=is_finished,
+                )
+                sessions.append(Session(**data))
+            cls.objects.bulk_create(sessions)
+            i += 10
+
+    class Meta:
+        db_table = "terminal_session"
+        ordering = ["-date_start"]
+
+    def __str__(self):
+        return "{0.id} of {0.user} to {0.asset}".format(self)

apps/terminal/models/status.py

@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+import uuid
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from .terminal import Terminal
+
+
+class Status(models.Model):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    session_online = models.IntegerField(verbose_name=_("Session Online"), default=0)
+    cpu_used = models.FloatField(verbose_name=_("CPU Usage"))
+    memory_used = models.FloatField(verbose_name=_("Memory Used"))
+    connections = models.IntegerField(verbose_name=_("Connections"))
+    threads = models.IntegerField(verbose_name=_("Threads"))
+    boot_time = models.FloatField(verbose_name=_("Boot Time"))
+    terminal = models.ForeignKey(Terminal, null=True, on_delete=models.CASCADE)
+    date_created = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        db_table = 'terminal_status'
+        get_latest_by = 'date_created'
+
+    def __str__(self):
+        return self.date_created.strftime("%Y-%m-%d %H:%M:%S")
+

apps/terminal/models/storage.py

@@ -0,0 +1,103 @@
+from __future__ import unicode_literals
+
+import os
+import jms_storage
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from django.conf import settings
+
+from common.mixins import CommonModelMixin
+from common.fields.model import EncryptJsonDictTextField
+from .. import const
+from .terminal import Terminal
+
+
+class CommandStorage(CommonModelMixin):
+    TYPE_CHOICES = const.COMMAND_STORAGE_TYPE_CHOICES
+    TYPE_DEFAULTS = dict(const.REPLAY_STORAGE_TYPE_CHOICES_DEFAULT).keys()
+    TYPE_SERVER = const.COMMAND_STORAGE_TYPE_SERVER
+
+    name = models.CharField(max_length=128, verbose_name=_("Name"), unique=True)
+    type = models.CharField(
+        max_length=16, choices=TYPE_CHOICES, verbose_name=_('Type'),
+        default=TYPE_SERVER
+    )
+    meta = EncryptJsonDictTextField(default={})
+    comment = models.TextField(
+        max_length=128, default='', blank=True, verbose_name=_('Comment')
+    )
+
+    def __str__(self):
+        return self.name
+
+    @property
+    def config(self):
+        config = self.meta
+        config.update({'TYPE': self.type})
+        return config
+
+    def in_defaults(self):
+        return self.type in self.TYPE_DEFAULTS
+
+    def is_valid(self):
+        if self.in_defaults():
+            return True
+        storage = jms_storage.get_log_storage(self.config)
+        return storage.ping()
+
+    def is_using(self):
+        return Terminal.objects.filter(command_storage=self.name).exists()
+
+
+class ReplayStorage(CommonModelMixin):
+    TYPE_CHOICES = const.REPLAY_STORAGE_TYPE_CHOICES
+    TYPE_SERVER = const.REPLAY_STORAGE_TYPE_SERVER
+    TYPE_DEFAULTS = dict(const.REPLAY_STORAGE_TYPE_CHOICES_DEFAULT).keys()
+
+    name = models.CharField(max_length=128, verbose_name=_("Name"), unique=True)
+    type = models.CharField(
+        max_length=16, choices=TYPE_CHOICES, verbose_name=_('Type'),
+        default=TYPE_SERVER
+    )
+    meta = EncryptJsonDictTextField(default={})
+    comment = models.TextField(
+        max_length=128, default='', blank=True, verbose_name=_('Comment')
+    )
+
+    def __str__(self):
+        return self.name
+
+    def convert_type(self):
+        s3_type_list = [const.REPLAY_STORAGE_TYPE_CEPH]
+        tp = self.type
+        if tp in s3_type_list:
+            tp = const.REPLAY_STORAGE_TYPE_S3
+        return tp
+
+    def get_extra_config(self):
+        extra_config = {'TYPE': self.convert_type()}
+        if self.type == const.REPLAY_STORAGE_TYPE_SWIFT:
+            extra_config.update({'signer': 'S3SignerType'})
+        return extra_config
+
+    @property
+    def config(self):
+        config = self.meta
+        extra_config = self.get_extra_config()
+        config.update(extra_config)
+        return config
+
+    def in_defaults(self):
+        return self.type in self.TYPE_DEFAULTS
+
+    def is_valid(self):
+        if self.in_defaults():
+            return True
+        storage = jms_storage.get_object_storage(self.config)
+        target = 'tests.py'
+        src = os.path.join(settings.BASE_DIR, 'common', target)
+        return storage.is_valid(src, target)
+
+    def is_using(self):
+        return Terminal.objects.filter(replay_storage=self.name).exists()

apps/terminal/models/task.py

@@ -0,0 +1,25 @@
+from __future__ import unicode_literals
+
+import uuid
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from .terminal import Terminal
+
+
+class Task(models.Model):
+    NAME_CHOICES = (
+        ("kill_session", "Kill Session"),
+    )
+
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    name = models.CharField(max_length=128, choices=NAME_CHOICES, verbose_name=_("Name"))
+    args = models.CharField(max_length=1024, verbose_name=_("Args"))
+    terminal = models.ForeignKey(Terminal, null=True, on_delete=models.SET_NULL)
+    is_finished = models.BooleanField(default=False)
+    date_created = models.DateTimeField(auto_now_add=True)
+    date_finished = models.DateTimeField(null=True)
+
+    class Meta:
+        db_table = "terminal_task"
+

apps/terminal/models/terminal.py

@@ -0,0 +1,247 @@
+from __future__ import unicode_literals
+import uuid
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from django.conf import settings
+from django.core.cache import cache
+
+from users.models import User
+from .. import const
+
+
+class ComputeStatusMixin:
+
+    # system status
+    @staticmethod
+    def _common_compute_system_status(value, thresholds):
+        if thresholds[0] <= value <= thresholds[1]:
+            return const.ComponentStatusChoices.normal.value
+        elif thresholds[1] < value <= thresholds[2]:
+            return const.ComponentStatusChoices.high.value
+        else:
+            return const.ComponentStatusChoices.critical.value
+
+    def _compute_system_cpu_load_1_status(self, value):
+        thresholds = [0, 5, 20]
+        return self._common_compute_system_status(value, thresholds)
+
+    def _compute_system_memory_used_percent_status(self, value):
+        thresholds = [0, 85, 95]
+        return self._common_compute_system_status(value, thresholds)
+
+    def _compute_system_disk_used_percent_status(self, value):
+        thresholds = [0, 80, 99]
+        return self._common_compute_system_status(value, thresholds)
+
+    def _compute_system_status(self, state):
+        system_status_keys = [
+            'system_cpu_load_1', 'system_memory_used_percent', 'system_disk_used_percent'
+        ]
+        system_status = []
+        for system_status_key in system_status_keys:
+            state_value = state[system_status_key]
+            status = getattr(self, f'_compute_{system_status_key}_status')(state_value)
+            system_status.append(status)
+        return system_status
+
+    def _compute_component_status(self, state):
+        system_status = self._compute_system_status(state)
+        if const.ComponentStatusChoices.critical in system_status:
+            return const.ComponentStatusChoices.critical
+        elif const.ComponentStatusChoices.high in system_status:
+            return const.ComponentStatusChoices.high
+        else:
+            return const.ComponentStatusChoices.normal
+
+    @staticmethod
+    def _compute_component_status_display(status):
+        return getattr(const.ComponentStatusChoices, status).label
+
+
+class TerminalStateMixin(ComputeStatusMixin):
+    CACHE_KEY_COMPONENT_STATE = 'CACHE_KEY_COMPONENT_STATE_TERMINAL_{}'
+    CACHE_TIMEOUT = 120
+
+    @property
+    def cache_key(self):
+        return self.CACHE_KEY_COMPONENT_STATE.format(str(self.id))
+
+    # get
+    def _get_from_cache(self):
+        return cache.get(self.cache_key)
+
+    def _set_to_cache(self, state):
+        cache.set(self.cache_key, state, self.CACHE_TIMEOUT)
+
+    # set
+    def _add_status(self, state):
+        status = self._compute_component_status(state)
+        status_display = self._compute_component_status_display(status)
+        state.update({
+            'status': status,
+            'status_display': status_display
+        })
+
+    @property
+    def state(self):
+        state = self._get_from_cache()
+        return state or {}
+
+    @state.setter
+    def state(self, state):
+        self._add_status(state)
+        self._set_to_cache(state)
+
+
+class TerminalStatusMixin(TerminalStateMixin):
+
+    # alive
+    @property
+    def is_alive(self):
+        return bool(self.state)
+
+    # status
+    @property
+    def status(self):
+        if self.is_alive:
+            return self.state['status']
+        else:
+            return const.ComponentStatusChoices.critical.value
+
+    @property
+    def status_display(self):
+        return self._compute_component_status_display(self.status)
+
+    @property
+    def is_normal(self):
+        return self.status == const.ComponentStatusChoices.normal.value
+
+    @property
+    def is_high(self):
+        return self.status == const.ComponentStatusChoices.high.value
+
+    @property
+    def is_critical(self):
+        return self.status == const.ComponentStatusChoices.critical.value
+
+
+class Terminal(TerminalStatusMixin, models.Model):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    name = models.CharField(max_length=128, verbose_name=_('Name'))
+    type = models.CharField(choices=const.TerminalTypeChoices.choices, max_length=64, verbose_name=_('type'))
+    remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address'))
+    ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222)
+    http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000)
+    command_storage = models.CharField(max_length=128, verbose_name=_("Command storage"), default='default')
+    replay_storage = models.CharField(max_length=128, verbose_name=_("Replay storage"), default='default')
+    user = models.OneToOneField(User, related_name='terminal', verbose_name='Application User', null=True, on_delete=models.CASCADE)
+    is_accepted = models.BooleanField(default=False, verbose_name='Is Accepted')
+    is_deleted = models.BooleanField(default=False)
+    date_created = models.DateTimeField(auto_now_add=True)
+    comment = models.TextField(blank=True, verbose_name=_('Comment'))
+
+    @property
+    def is_active(self):
+        if self.user and self.user.is_active:
+            return True
+        return False
+
+    @is_active.setter
+    def is_active(self, active):
+        if self.user:
+            self.user.is_active = active
+            self.user.save()
+
+    def get_command_storage(self):
+        from .storage import CommandStorage
+        storage = CommandStorage.objects.filter(name=self.command_storage).first()
+        return storage
+
+    def get_command_storage_config(self):
+        s = self.get_command_storage()
+        if s:
+            config = s.config
+        else:
+            config = settings.DEFAULT_TERMINAL_COMMAND_STORAGE
+        return config
+
+    def get_command_storage_setting(self):
+        config = self.get_command_storage_config()
+        return {"TERMINAL_COMMAND_STORAGE": config}
+
+    def get_replay_storage(self):
+        from .storage import ReplayStorage
+        storage = ReplayStorage.objects.filter(name=self.replay_storage).first()
+        return storage
+
+    def get_replay_storage_config(self):
+        s = self.get_replay_storage()
+        if s:
+            config = s.config
+        else:
+            config = settings.DEFAULT_TERMINAL_REPLAY_STORAGE
+        return config
+
+    def get_replay_storage_setting(self):
+        config = self.get_replay_storage_config()
+        return {"TERMINAL_REPLAY_STORAGE": config}
+
+    @staticmethod
+    def get_login_title_setting():
+        login_title = None
+        if settings.XPACK_ENABLED:
+            from xpack.plugins.interface.models import Interface
+            login_title = Interface.get_login_title()
+        return {'TERMINAL_HEADER_TITLE': login_title}
+
+    @property
+    def config(self):
+        configs = {}
+        for k in dir(settings):
+            if not k.startswith('TERMINAL'):
+                continue
+            configs[k] = getattr(settings, k)
+        configs.update(self.get_command_storage_setting())
+        configs.update(self.get_replay_storage_setting())
+        configs.update(self.get_login_title_setting())
+        configs.update({
+            'SECURITY_MAX_IDLE_TIME': settings.SECURITY_MAX_IDLE_TIME
+        })
+        return configs
+
+    @property
+    def service_account(self):
+        return self.user
+
+    def create_app_user(self):
+        random = uuid.uuid4().hex[:6]
+        user, access_key = User.create_app_user(
+            name="{}-{}".format(self.name, random), comment=self.comment
+        )
+        self.user = user
+        self.save()
+        return user, access_key
+
+    def delete(self, using=None, keep_parents=False):
+        if self.user:
+            self.user.delete()
+        self.user = None
+        self.is_deleted = True
+        self.save()
+        return
+
+    def __str__(self):
+        status = "Active"
+        if not self.is_accepted:
+            status = "NotAccept"
+        elif self.is_deleted:
+            status = "Deleted"
+        elif not self.is_active:
+            status = "Disable"
+        return '%s: %s' % (self.name, status)
+
+    class Meta:
+        ordering = ('is_accepted',)
+        db_table = "terminal"
+

apps/terminal/serializers/__init__.py

@@ -4,3 +4,4 @@ from .terminal import *
 from .session import *
 from .storage import *
 from .command import *
+from .components import *

apps/terminal/serializers/components.py

@@ -0,0 +1,25 @@
+
+from rest_framework import serializers
+from django.utils.translation import ugettext_lazy as _
+
+
+class ComponentsStateSerializer(serializers.Serializer):
+    # system
+    system_cpu_load_1 = serializers.FloatField(
+        required=False, default=0, label=_("System cpu load (1 minutes)")
+    )
+    system_memory_used_percent = serializers.FloatField(
+        required=False, default=0, label=_('System memory used percent')
+    )
+    system_disk_used_percent = serializers.FloatField(
+        required=False, default=0, label=_('System disk used percent')
+    )
+    # sessions
+    session_active_count = serializers.IntegerField(
+        required=False, default=0, label=_("Session active count")
+    )
+
+    def save(self, **kwargs):
+        request = self.context['request']
+        terminal = request.user.terminal
+        terminal.state = self.validated_data

apps/terminal/serializers/terminal.py

@@ -6,19 +6,25 @@ from common.utils import is_uuid
 from ..models import (
     Terminal, Status, Session, Task, CommandStorage, ReplayStorage
 )
+from .components import ComponentsStateSerializer
 
 
 class TerminalSerializer(BulkModelSerializer):
     session_online = serializers.SerializerMethodField()
     is_alive = serializers.BooleanField(read_only=True)
+    status = serializers.CharField(read_only=True)
+    status_display = serializers.CharField(read_only=True)
+    state = ComponentsStateSerializer(read_only=True)
 
     class Meta:
         model = Terminal
         fields = [
-            'id', 'name', 'remote_addr', 'http_port', 'ssh_port',
+            'id', 'name', 'type', 'remote_addr', 'http_port', 'ssh_port',
             'comment', 'is_accepted', "is_active", 'session_online',
-            'is_alive', 'date_created', 'command_storage', 'replay_storage'
+            'is_alive', 'date_created', 'command_storage', 'replay_storage',
+            'status', 'status_display', 'state'
         ]
+        read_only_fields = ['type', 'date_created']
 
     @staticmethod
     def get_kwargs_may_be_uuid(value):

apps/terminal/urls/api_urls.py

@@ -33,7 +33,10 @@ urlpatterns = [
     path('commands/export/', api.CommandExportApi.as_view(), name="command-export"),
     path('commands/insecure-command/', api.InsecureCommandAlertAPI.as_view(), name="command-alert"),
     path('replay-storages/<uuid:pk>/test-connective/', api.ReplayStorageTestConnectiveApi.as_view(), name='replay-storage-test-connective'),
-    path('command-storages/<uuid:pk>/test-connective/', api.CommandStorageTestConnectiveApi.as_view(), name='command-storage-test-connective')
+    path('command-storages/<uuid:pk>/test-connective/', api.CommandStorageTestConnectiveApi.as_view(), name='command-storage-test-connective'),
+    # components
+    path('components/metrics/', api.ComponentsMetricsAPIView.as_view(), name='components-metrics'),
+    path('components/state/', api.ComponentsStateAPIView.as_view(), name='components-state'),
     # v2: get session's replay
     # path('v2/sessions/<uuid:pk>/replay/',
     #     api.SessionReplayV2ViewSet.as_view({'get': 'retrieve'}),

apps/terminal/utils.py

@@ -11,6 +11,7 @@ import jms_storage
 from common.tasks import send_mail_async
 from common.utils import get_logger, reverse
 from settings.models import Setting
+from . import const
 
 from .models import ReplayStorage, Session, Command
 
@@ -101,3 +102,104 @@ def send_command_alert_mail(command):
     logger.debug(message)
 
     send_mail_async.delay(subject, message, recipient_list, html_message=message)
+
+
+class ComponentsMetricsUtil(object):
+
+    def __init__(self, component_type=None):
+        self.type = component_type
+        self.components = []
+        self.initial_components()
+
+    def initial_components(self):
+        from .models import Terminal
+        terminals = Terminal.objects.all().order_by('type')
+        if self.type:
+            terminals = terminals.filter(type=self.type)
+        self.components = list(terminals)
+
+    def get_metrics(self):
+        total_count = normal_count = high_count = critical_count = session_active_total = 0
+        for component in self.components:
+            total_count += 1
+            if not component.is_alive:
+                critical_count += 1
+                continue
+            session_active_total += component.state.get('session_active_count', 0)
+            if component.is_normal:
+                normal_count += 1
+            elif component.is_high:
+                high_count += 1
+            else:
+                critical_count += 1
+        metrics = {
+            'total': total_count,
+            'normal': normal_count,
+            'high': high_count,
+            'critical': critical_count,
+            'session_active': session_active_total
+        }
+        return metrics
+
+
+class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil):
+
+    @staticmethod
+    def get_status_metrics(metrics):
+        return {
+            'any': metrics['total'],
+            'normal': metrics['normal'],
+            'high': metrics['high'],
+            'critical': metrics['critical']
+        }
+
+    def get_prometheus_metrics_text(self):
+        prometheus_metrics = []
+        prometheus_metrics.append('# JumpServer 各组件状态个数汇总')
+        base_status_metric_text = 'jumpserver_components_status_total{component_type="%s", status="%s"} %s'
+        for component in self.components:
+            component_type = component.type
+            base_metrics = self.get_metrics()
+
+            prometheus_metrics.append(f'## 组件: {component_type}')
+            status_metrics = self.get_status_metrics(base_metrics)
+            for status, value in status_metrics.items():
+                metric_text = base_status_metric_text % (component_type, status, value)
+                prometheus_metrics.append(metric_text)
+
+        prometheus_metrics.append('\n')
+        prometheus_metrics.append('# JumpServer 各组件在线会话数汇总')
+        base_session_active_metric_text = 'jumpserver_components_session_active_total{component_type="%s"} %s'
+        for component in self.components:
+            component_type = component.type
+            prometheus_metrics.append(f'## 组件: {component_type}')
+            base_metrics = self.get_metrics()
+            metric_text = base_session_active_metric_text % (
+                component_type,
+                base_metrics['session_active']
+            )
+            prometheus_metrics.append(metric_text)
+
+        prometheus_metrics.append('\n')
+        prometheus_metrics.append('# JumpServer 各组件节点一些指标')
+        base_system_state_metric_text = 'jumpserver_components_%s{component_type="%s", component="%s"} %s'
+        system_states_name = [
+            'system_cpu_load_1', 'system_memory_used_percent',
+            'system_disk_used_percent', 'session_active_count'
+        ]
+        for system_state_name in system_states_name:
+            prometheus_metrics.append(f'## 指标: {system_state_name}')
+            for component in self.components:
+                if not component.is_alive:
+                    continue
+                component_type = component.type
+                metric_text = base_system_state_metric_text % (
+                    system_state_name,
+                    component_type,
+                    component.name,
+                    component.state.get(system_state_name)
+                )
+                prometheus_metrics.append(metric_text)
+
+        prometheus_metrics_text = '\n'.join(prometheus_metrics)
+        return prometheus_metrics_text

apps/users/api/user.py

@@ -6,8 +6,8 @@ from rest_framework.decorators import action
 from rest_framework import generics
 from rest_framework.response import Response
 from rest_framework_bulk import BulkModelViewSet
+from django.db.models import Prefetch
 
-from common.db.aggregates import GroupConcat
 from common.permissions import (
     IsOrgAdmin, IsOrgAdminOrAppUser,
     CanUpdateDeleteUser, IsSuperUser
@@ -44,9 +44,18 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet):
     extra_filter_backends = [OrgRoleUserFilterBackend]
 
     def get_queryset(self):
-        return super().get_queryset().annotate(
-            gc_m2m_org_members__role=GroupConcat('m2m_org_members__role'),
-        ).prefetch_related('groups')
+        queryset = super().get_queryset().prefetch_related(
+            'groups'
+        )
+        if current_org.is_real():
+            # 为在列表中计算用户在真实组织里的角色
+            queryset = queryset.prefetch_related(
+                Prefetch(
+                    'm2m_org_members',
+                    queryset=OrganizationMember.objects.filter(org__id=current_org.id)
+                )
+            )
+        return queryset
 
     def send_created_signal(self, users):
         if not isinstance(users, list):

apps/users/forms/user.py

@@ -28,7 +28,7 @@ class UserCreateUpdateFormMixin(OrgModelForm):
     )
     source = forms.ChoiceField(
         choices=get_source_choices, required=True,
-        initial=User.SOURCE_LOCAL, label=_("Source")
+        initial=User.Source.local.value, label=_("Source")
     )
     public_key = forms.CharField(
         label=_('ssh public key'), max_length=5000, required=False,

apps/users/migrations/0031_auto_20201118_1801.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1 on 2020-11-18 10:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0030_auto_20200819_2041'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='first_name',
+            field=models.CharField(blank=True, max_length=150, verbose_name='first name'),
+        ),
+    ]

apps/users/models/user.py

@@ -7,10 +7,10 @@ import string
 import random
 
 from django.conf import settings
-from django.contrib.auth.hashers import make_password
 from django.contrib.auth.models import AbstractUser
 from django.core.cache import cache
 from django.db import models
+from django.db.models import TextChoices
 
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
@@ -170,22 +170,18 @@ class RoleMixin:
         from orgs.models import ROLE as ORG_ROLE
 
         if not current_org.is_real():
+            # 不是真实的组织，取 User 本身的角色
             if self.is_superuser:
                 return [ORG_ROLE.ADMIN]
             else:
                 return [ORG_ROLE.USER]
 
-        if hasattr(self, 'gc_m2m_org_members__role'):
-            names = self.gc_m2m_org_members__role
-            if isinstance(names, str):
-                roles = set(self.gc_m2m_org_members__role.split(','))
-            else:
-                roles = set()
-        else:
-            roles = set(self.m2m_org_members.filter(
-                org_id=current_org.id
-            ).values_list('role', flat=True))
-        roles = list(roles)
+        # 是真实组织，取 OrganizationMember 中的角色
+        roles = [
+            org_member.role
+            for org_member in self.m2m_org_members.all()
+            if org_member.org_id == current_org.id
+        ]
         roles.sort()
         return roles
 
@@ -485,18 +481,12 @@ class MFAMixin:
 
 
 class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser):
-    SOURCE_LOCAL = 'local'
-    SOURCE_LDAP = 'ldap'
-    SOURCE_OPENID = 'openid'
-    SOURCE_RADIUS = 'radius'
-    SOURCE_CAS = 'cas'
-    SOURCE_CHOICES = (
-        (SOURCE_LOCAL, _('Local')),
-        (SOURCE_LDAP, 'LDAP/AD'),
-        (SOURCE_OPENID, 'OpenID'),
-        (SOURCE_RADIUS, 'Radius'),
-        (SOURCE_CAS, 'CAS'),
-    )
+    class Source(TextChoices):
+        local = 'local', _('Local')
+        ldap = 'ldap', 'LDAP/AD'
+        openid = 'openid', 'OpenID'
+        radius = 'radius', 'Radius'
+        cas = 'cas', 'CAS'
 
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     username = models.CharField(
@@ -546,7 +536,7 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser):
         max_length=30, default='', blank=True, verbose_name=_('Created by')
     )
     source = models.CharField(
-        max_length=30, default=SOURCE_LOCAL, choices=SOURCE_CHOICES,
+        max_length=30, default=Source.local.value, choices=Source.choices,
         verbose_name=_('Source')
     )
     date_password_last_updated = models.DateTimeField(
@@ -597,7 +587,7 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser):
 
     @property
     def is_local(self):
-        return self.source == self.SOURCE_LOCAL
+        return self.source == self.Source.local.value
 
     def set_unprovide_attr_if_need(self):
         if not self.name:
@@ -667,6 +657,6 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser):
         user.groups.add(UserGroup.initial())
 
     def can_send_created_mail(self):
-        if self.email and self.source == self.SOURCE_LOCAL:
+        if self.email and self.source == self.Source.local.value:
             return True
         return False

apps/users/serializers/user.py

@@ -68,6 +68,10 @@ class UserSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer):
             'can_update', 'can_delete', 'login_blocked', 'org_roles'
         ]
 
+        read_only_fields = [
+            'date_joined', 'last_login', 'created_by', 'is_first_login', 'source'
+        ]
+
         extra_kwargs = {
             'password': {'write_only': True, 'required': False, 'allow_null': True, 'allow_blank': True},
             'public_key': {'write_only': True},

apps/users/signals_handler.py

@@ -2,14 +2,12 @@
 #
 
 from django.dispatch import receiver
-from django.db.models.signals import m2m_changed
 from django_auth_ldap.backend import populate_user
 from django.conf import settings
 from django_cas_ng.signals import cas_user_authenticated
 
 from jms_oidc_rp.signals import openid_create_or_update_user
 
-from perms.tasks import create_rebuild_user_tree_task
 from common.utils import get_logger
 from .signals import post_user_create
 from .models import User
@@ -27,19 +25,10 @@ def on_user_create(sender, user=None, **kwargs):
         send_user_created_mail(user)
 
 
-@receiver(m2m_changed, sender=User.groups.through)
-def on_user_groups_change(instance, action, reverse, pk_set, **kwargs):
-    if action.startswith('post'):
-        if reverse:
-            create_rebuild_user_tree_task(pk_set)
-        else:
-            create_rebuild_user_tree_task([instance.id])
-
-
 @receiver(cas_user_authenticated)
 def on_cas_user_authenticated(sender, user, created, **kwargs):
     if created:
-        user.source = user.SOURCE_CAS
+        user.source = user.Source.cas.value
         user.save()
 
 
@@ -48,7 +37,7 @@ def on_ldap_create_user(sender, user, ldap_user, **kwargs):
     if user and user.username not in ['admin']:
         exists = User.objects.filter(username=user.username).exists()
         if not exists:
-            user.source = user.SOURCE_LDAP
+            user.source = user.Source.ldap.value
             user.save()
 
 
@@ -57,9 +46,9 @@ def on_openid_create_or_update_user(sender, request, user, created, name, userna
     if created:
         logger.debug(
             "Receive OpenID user created signal: {}, "
-            "Set user source is: {}".format(user, User.SOURCE_OPENID)
+            "Set user source is: {}".format(user, User.Source.openid.value)
         )
-        user.source = User.SOURCE_OPENID
+        user.source = User.Source.openid.value
         user.save()
     elif not created and settings.AUTH_OPENID_ALWAYS_UPDATE_USER:
         logger.debug(

apps/users/tasks.py

@@ -22,7 +22,7 @@ logger = get_logger(__file__)
 
 @shared_task
 def check_password_expired():
-    users = User.objects.filter(source=User.SOURCE_LOCAL).exclude(role=User.ROLE.APP)
+    users = User.objects.filter(source=User.Source.local.value).exclude(role=User.ROLE.APP)
     for user in users:
         if not user.is_valid:
             continue