Merge pull request #3260 from jumpserver/dev

Dev

README.md

@@ -1,4 +1,4 @@
-## Jumpserver 多云环境下更好用的堡垒机
+# Jumpserver 多云环境下更好用的堡垒机
 
 ![Total visitor](https://visitor-count-badge.herokuapp.com/total.svg?repo_id=jumpserver)
 ![Visitors in today](https://visitor-count-badge.herokuapp.com/today.svg?repo_id=jumpserver)
@@ -7,18 +7,17 @@
 [![Ansible](https://img.shields.io/badge/ansible-2.4.2.0-blue.svg?style=plastic)](https://www.ansible.com/)
 [![Paramiko](https://img.shields.io/badge/paramiko-2.4.1-green.svg?style=plastic)](http://www.paramiko.org/)
 
-----
+Jumpserver 是全球首款完全开源的堡垒机，使用 GNU GPL v2.0 开源协议，是符合 4A 机制的运维安全审计系统。
 
-Jumpserver 是全球首款完全开源的堡垒机，使用 GNU GPL v2.0 开源协议，是符合 4A 的运维安全审计系统。
-
-Jumpserver 使用 Python / Django 进行开发，遵循 Web 2.0 规范，配备了业界领先的 Web Terminal 解决方案，交互界面美观、用户体验好。
+Jumpserver 使用 Python / Django 进行开发，遵循 Web 2.0 规范，配备了业界领先的 Web Terminal 方案，交互界面美观、用户体验好。
 
 Jumpserver 采纳分布式架构，支持多机房跨区域部署，支持横向扩展，无资产数量及并发限制。
 
 改变世界，从一点点开始。
 
-### 核心功能列表
-----
+注: [KubeOperator](https://github.com/KubeOperator/KubeOperator) 是 Jumpserver 团队在 Kubernetes 领域的的又一全新力作，欢迎关注和使用。
+
+## 核心功能列表
 
 <table class="subscription-level-table">
     <tr class="subscription-level-tr-border">
@@ -173,31 +172,27 @@ Jumpserver 采纳分布式架构，支持多机房跨区域部署，支持横向
     </tr>
 </table>
 
-### 安装及使用文档
-----
+## 安装及使用指南
 
 -  [Docker 快速安装文档](http://docs.jumpserver.org/zh/docs/dockerinstall.html)
 -  [Step by Step 安装文档](http://docs.jumpserver.org/zh/docs/step_by_step.html)
 -  [完整文档](http://docs.jumpserver.org)
 
-### 演示视频和系统截图
-----
+## 演示视频和截屏
 
-我们提供了演示视频和系统截图可以让你快速了解 Jumpserver。
+我们提供了演示视频和系统截图可以让你快速了解 Jumpserver：
 
 - [演示视频](https://jumpserver.oss-cn-hangzhou.aliyuncs.com/jms-media/%E3%80%90%E6%BC%94%E7%A4%BA%E8%A7%86%E9%A2%91%E3%80%91Jumpserver%20%E5%A0%A1%E5%9E%92%E6%9C%BA%20V1.5.0%20%E6%BC%94%E7%A4%BA%E8%A7%86%E9%A2%91%20-%20final.mp4)
 - [系统截图](http://docs.jumpserver.org/zh/docs/snapshot.html)
 
-### SDK
-----
+## SDK
 
-我们编写了一些SDK，供你的其它系统快速和 Jumpserver API 交互。
+我们编写了一些SDK，供您的其它系统快速和 Jumpserver API 交互：
 
 - [Python](https://github.com/jumpserver/jumpserver-python-sdk) Jumpserver 其它组件使用这个 SDK 完成交互
 - [Java](https://github.com/KaiJunYan/jumpserver-java-sdk.git) 恺珺同学提供的 Java 版本的 SDK
 
-### License & Copyright
-----
+## License & Copyright
 
 Copyright (c) 2014-2019 飞致云 FIT2CLOUD, All rights reserved.
 

apps/applications/api/remote_app.py

@@ -3,9 +3,8 @@
 
 
 from rest_framework import generics
-from rest_framework.pagination import LimitOffsetPagination
-from rest_framework_bulk import BulkModelViewSet
 
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..hands import IsOrgAdmin, IsAppUser
 from ..models import RemoteApp
 from ..serializers import RemoteAppSerializer, RemoteAppConnectionInfoSerializer
@@ -16,13 +15,12 @@ __all__ = [
 ]
 
 
-class RemoteAppViewSet(BulkModelViewSet):
+class RemoteAppViewSet(OrgBulkModelViewSet):
     filter_fields = ('name',)
     search_fields = filter_fields
     permission_classes = (IsOrgAdmin,)
     queryset = RemoteApp.objects.all()
     serializer_class = RemoteAppSerializer
-    pagination_class = LimitOffsetPagination
 
 
 class RemoteAppConnectionInfoApi(generics.RetrieveAPIView):

apps/applications/forms/remote_app.py

@@ -4,7 +4,7 @@
 from django.utils.translation import ugettext as _
 from django import forms
 
-from orgs.mixins import OrgModelForm
+from orgs.mixins.forms import OrgModelForm
 from assets.models import SystemUser
 
 from ..models import RemoteApp
@@ -89,23 +89,16 @@ class RemoteAppCreateUpdateForm(RemoteAppTypeForms, OrgModelForm):
         super().__init__(*args, **kwargs)
         field_asset = self.fields['asset']
         field_asset.queryset = field_asset.queryset.has_protocol('rdp')
-        field_system_user = self.fields['system_user']
-        field_system_user.queryset = field_system_user.queryset.filter(
-            protocol=SystemUser.PROTOCOL_RDP
-        )
 
     class Meta:
         model = RemoteApp
         fields = [
-            'name', 'asset', 'system_user', 'type', 'path', 'comment'
+            'name', 'asset', 'type', 'path', 'comment'
         ]
         widgets = {
             'asset': forms.Select(attrs={
                 'class': 'select2', 'data-placeholder': _('Asset')
             }),
-            'system_user': forms.Select(attrs={
-                'class': 'select2', 'data-placeholder': _('System user')
-            })
         }
 
     def _clean_params(self):

apps/applications/migrations/0002_remove_remoteapp_system_user.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.7 on 2019-09-09 09:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0001_initial'),
+        ('perms', '0009_remoteapppermission_system_users'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='remoteapp',
+            name='system_user',
+        ),
+    ]

apps/applications/models/remote_app.py

@@ -5,7 +5,7 @@ import uuid
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from orgs.mixins import OrgModelMixin
+from orgs.mixins.models import OrgModelMixin
 from common.fields.model import EncryptJsonDictTextField
 
 from .. import const
@@ -22,10 +22,6 @@ class RemoteApp(OrgModelMixin):
     asset = models.ForeignKey(
         'assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset')
     )
-    system_user = models.ForeignKey(
-        'assets.SystemUser', on_delete=models.CASCADE,
-        verbose_name=_('System user')
-    )
     type = models.CharField(
         default=const.REMOTE_APP_TYPE_CHROME,
         choices=const.REMOTE_APP_TYPE_CHOICES,
@@ -80,10 +76,3 @@ class RemoteApp(OrgModelMixin):
             'id': self.asset.id,
             'hostname': self.asset.hostname
         }
-
-    @property
-    def system_user_info(self):
-        return {
-            'id': self.system_user.id,
-            'name': self.system_user.name
-        }

apps/applications/serializers/remote_app.py

@@ -5,7 +5,7 @@
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from .. import const
 from ..models import RemoteApp
@@ -73,13 +73,13 @@ class RemoteAppSerializer(BulkOrgResourceModelSerializer):
         model = RemoteApp
         list_serializer_class = AdaptedBulkListSerializer
         fields = [
-            'id', 'name', 'asset', 'system_user', 'type', 'path', 'params',
+            'id', 'name', 'asset', 'type', 'path', 'params',
             'comment', 'created_by', 'date_created', 'asset_info',
-            'system_user_info', 'get_type_display',
+            'get_type_display',
         ]
         read_only_fields = [
             'created_by', 'date_created', 'asset_info',
-            'system_user_info', 'get_type_display'
+            'get_type_display'
         ]
 
 
@@ -89,7 +89,7 @@ class RemoteAppConnectionInfoSerializer(serializers.ModelSerializer):
     class Meta:
         model = RemoteApp
         fields = [
-            'id', 'name', 'asset', 'system_user', 'parameter_remote_app',
+            'id', 'name', 'asset', 'parameter_remote_app',
         ]
         read_only_fields = ['parameter_remote_app']
 

apps/applications/templates/applications/remote_app_create_update.html

@@ -13,7 +13,6 @@
         {% csrf_token %}
         {% bootstrap_field form.name layout="horizontal" %}
         {% bootstrap_field form.asset layout="horizontal" %}
-        {% bootstrap_field form.system_user layout="horizontal" %}
         {% bootstrap_field form.type layout="horizontal" %}
         {% bootstrap_field form.path layout="horizontal" %}
 

apps/applications/templates/applications/remote_app_detail.html

@@ -57,10 +57,6 @@
                                             <td>{% trans 'Asset' %}:</td>
                                             <td><b><a href="{% url 'assets:asset-detail' pk=remote_app.asset.id %}">{{ remote_app.asset.hostname }}</a></b></td>
                                         </tr>
-                                        <tr>
-                                            <td>{% trans 'System user' %}:</td>
-                                            <td><b><a href="{% url 'assets:system-user-detail' pk=remote_app.system_user.id %}">{{ remote_app.system_user.name }}</a></b></td>
-                                        </tr>
                                         <tr>
                                             <td>{% trans 'App type' %}:</td>
                                             <td><b>{{ remote_app.get_type_display }}</b></td>

apps/applications/templates/applications/remote_app_list.html

@@ -20,7 +20,6 @@
             <th class="text-center">{% trans 'Name' %}</th>
             <th class="text-center">{% trans 'App type' %}</th>
             <th class="text-center">{% trans 'Asset' %}</th>
-            <th class="text-center">{% trans 'System user' %}</th>
             <th class="text-center">{% trans 'Comment' %}</th>
             <th class="text-center">{% trans 'Action' %}</th>
         </tr>
@@ -47,12 +46,11 @@ function initTable() {
                     var detail_btn = '<a href="{% url 'assets:asset-detail' pk=DEFAULT_PK %}">' + hostname + '</a>';
                     $(td).html(detail_btn.replace('{{ DEFAULT_PK }}', cellData.id));
                 }},
-            {targets: 4, createdCell: function (td, cellData, rowData) {
-                    var name = htmlEscape(cellData.name);
-                    var detail_btn = '<a href="{% url 'assets:system-user-detail' pk=DEFAULT_PK %}">' + name + '</a>';
-                    $(td).html(detail_btn.replace('{{ DEFAULT_PK }}', cellData.id));
+            {targets: 3, createdCell: function (td, cellData, rowData) {
+                    var comment = htmlEscape(cellData);
+                    $(td).html(comment)
                 }},
-            {targets: 6, createdCell: function (td, cellData, rowData) {
+            {targets: 5, createdCell: function (td, cellData, rowData) {
                     var update_btn = '<a href="{% url "applications:remote-app-update" pk=DEFAULT_PK %}" class="btn btn-xs btn-info">{% trans "Update" %}</a>'.replace("{{ DEFAULT_PK }}", cellData);
                     var del_btn = '<a class="btn btn-xs btn-danger m-l-xs btn-delete" data-rid="{{ DEFAULT_PK }}">{% trans "Delete" %}</a>'.replace('{{ DEFAULT_PK }}', cellData);
                     $(td).html(update_btn + del_btn)
@@ -64,7 +62,6 @@ function initTable() {
             {data: "name" },
             {data: "get_type_display", orderable: false},
             {data: "asset_info", orderable: false},
-            {data: "system_user_info", orderable: false},
             {data: "comment"},
             {data: "id", orderable: false}
         ],

apps/applications/templates/applications/user_remote_app_list.html

@@ -16,7 +16,6 @@
             <th class="text-center">{% trans 'Name' %}</th>
             <th class="text-center">{% trans 'App type' %}</th>
             <th class="text-center">{% trans 'Asset' %}</th>
-            <th class="text-center">{% trans 'System user' %}</th>
             <th class="text-center">{% trans 'Comment' %}</th>
             <th class="text-center">{% trans 'Action' %}</th>
         </tr>
@@ -49,12 +48,8 @@ function initTable() {
                     var hostname = htmlEscape(cellData.hostname);
                     $(td).html(hostname);
                 }},
-            {targets: 4, createdCell: function (td, cellData, rowData) {
-                    var name = htmlEscape(cellData.name);
-                    $(td).html(name);
-                }},
-            {targets: 6, createdCell: function (td, cellData, rowData) {
-                    var conn_btn = '<a href="{% url "luna-view" %}?login_to=' +  cellData +'" class="btn btn-xs btn-primary">{% trans "Connect" %}</a>'.replace("{{ DEFAULT_PK }}", cellData);
+            {targets: 5, createdCell: function (td, cellData, rowData) {
+                    var conn_btn = '<a href="{% url "luna-view" %}?type=remote_app&login_to=' +  cellData +'" class="btn btn-xs btn-primary">{% trans "Connect" %}</a>'.replace("{{ DEFAULT_PK }}", cellData);
                     $(td).html(conn_btn)
                 }}
         ],
@@ -64,7 +59,6 @@ function initTable() {
             {data: "name"},
             {data: "get_type_display", orderable: false},
             {data: "asset_info", orderable: false},
-            {data: "system_user_info", orderable: false},
             {data: "comment", orderable: false},
             {data: "id", orderable: false}
         ]

apps/applications/urls/api_urls.py

@@ -1,20 +1,22 @@
 # coding:utf-8
 #
 
-from django.urls import path
+from django.urls import path, re_path
 from rest_framework_bulk.routes import BulkRouter
 
+from common import api as capi
 from .. import api
 
 app_name = 'applications'
 
 router = BulkRouter()
-router.register(r'remote-app', api.RemoteAppViewSet, 'remote-app')
+router.register(r'remote-apps', api.RemoteAppViewSet, 'remote-app')
 
 urlpatterns = [
-    path('remote-apps/<uuid:pk>/connection-info/',
-         api.RemoteAppConnectionInfoApi.as_view(),
-         name='remote-app-connection-info')
+    path('remote-apps/<uuid:pk>/connection-info/', api.RemoteAppConnectionInfoApi.as_view(), name='remote-app-connection-info'),
+]
+old_version_urlpatterns = [
+    re_path('(?P<resource>remote-app)/.*', capi.redirect_plural_name_api)
 ]
 
-urlpatterns += router.urls
+urlpatterns += router.urls + old_version_urlpatterns

apps/assets/api/__init__.py

@@ -6,3 +6,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/api/admin_user.py

@@ -17,10 +17,9 @@ from django.db import transaction
 from django.shortcuts import get_object_or_404
 from rest_framework import generics
 from rest_framework.response import Response
-from rest_framework_bulk import BulkModelViewSet
-from rest_framework.pagination import LimitOffsetPagination
+from orgs.mixins.api import OrgBulkModelViewSet
 
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from common.utils import get_logger
 from ..hands import IsOrgAdmin
 from ..models import AdminUser, Asset
@@ -36,7 +35,7 @@ __all__ = [
 ]
 
 
-class AdminUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class AdminUserViewSet(OrgBulkModelViewSet):
     """
     Admin user api set, for add,delete,update,list,retrieve resource
     """
@@ -46,11 +45,6 @@ class AdminUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
     queryset = AdminUser.objects.all()
     serializer_class = serializers.AdminUserSerializer
     permission_classes = (IsOrgAdmin,)
-    pagination_class = LimitOffsetPagination
-
-    def get_queryset(self):
-        queryset = super().get_queryset().all()
-        return queryset
 
 
 class AdminUserAuthApi(generics.UpdateAPIView):
@@ -98,7 +92,6 @@ class AdminUserTestConnectiveApi(generics.RetrieveAPIView):
 class AdminUserAssetsListView(generics.ListAPIView):
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.AssetSimpleSerializer
-    pagination_class = LimitOffsetPagination
     filter_fields = ("hostname", "ip")
     http_method_names = ['get']
     search_fields = filter_fields

apps/assets/api/asset.py

@@ -1,43 +1,31 @@
 # -*- coding: utf-8 -*-
 #
 
-import uuid
 import random
 
 from rest_framework import generics
-from rest_framework.views import APIView
 from rest_framework.response import Response
-from rest_framework_bulk import BulkModelViewSet
-from rest_framework_bulk import ListBulkCreateUpdateDestroyAPIView
-from rest_framework.pagination import LimitOffsetPagination
-from django.utils.translation import ugettext_lazy as _
 from django.shortcuts import get_object_or_404
-from django.urls import reverse_lazy
-from django.core.cache import cache
-from django.db.models import Q
-
-from common.mixins import IDInCacheFilterMixin, ApiMessageMixin
 
 from common.utils import get_logger, get_object_or_none
 from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
-from orgs.mixins import OrgBulkModelViewSet
-from ..const import CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..models import Asset, AdminUser, Node
 from .. import serializers
 from ..tasks import update_asset_hardware_info_manual, \
     test_asset_connectivity_manual
-from ..utils import LabelFilter
+from ..filters import AssetByNodeFilterBackend, LabelFilterBackend
 
 
 logger = get_logger(__file__)
 __all__ = [
-    'AssetViewSet', 'AssetListUpdateApi',
+    'AssetViewSet',
     'AssetRefreshHardwareApi', 'AssetAdminUserTestApi',
-    'AssetGatewayApi', 'AssetBulkUpdateSelectAPI'
+    'AssetGatewayApi',
 ]
 
 
-class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
+class AssetViewSet(OrgBulkModelViewSet):
     """
     API endpoint that allows Asset to be viewed or edited.
     """
@@ -46,9 +34,8 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
     ordering_fields = ("hostname", "ip", "port", "cpu_cores")
     queryset = Asset.objects.all()
     serializer_class = serializers.AssetSerializer
-    pagination_class = LimitOffsetPagination
     permission_classes = (IsOrgAdminOrAppUser,)
-    success_message = _("%(hostname)s was %(action)s successfully")
+    extra_filter_backends = [AssetByNodeFilterBackend, LabelFilterBackend]
 
     def set_assets_node(self, assets):
         if not isinstance(assets, list):
@@ -65,66 +52,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
         assets = serializer.save()
         self.set_assets_node(assets)
 
-    def filter_node(self, queryset):
-        node_id = self.request.query_params.get("node_id")
-        if not node_id:
-            return queryset
-
-        node = get_object_or_404(Node, id=node_id)
-        show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true')
-
-        if node.is_root() and show_current_asset:
-            queryset = queryset.filter(
-                Q(nodes=node_id) | Q(nodes__isnull=True)
-            )
-        elif node.is_root() and not show_current_asset:
-            pass
-        elif not node.is_root() and show_current_asset:
-            queryset = queryset.filter(nodes=node)
-        else:
-            queryset = queryset.filter(
-                nodes__key__regex='^{}(:[0-9]+)*$'.format(node.key),
-            )
-        return queryset.distinct()
-
-    def filter_admin_user_id(self, queryset):
-        admin_user_id = self.request.query_params.get('admin_user_id')
-        if not admin_user_id:
-            return queryset
-        admin_user = get_object_or_404(AdminUser, id=admin_user_id)
-        queryset = queryset.filter(admin_user=admin_user)
-        return queryset
-
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        queryset = self.filter_node(queryset)
-        queryset = self.filter_admin_user_id(queryset)
-        return queryset
-
-
-class AssetListUpdateApi(IDInCacheFilterMixin, ListBulkCreateUpdateDestroyAPIView):
-    """
-    Asset bulk update api
-    """
-    queryset = Asset.objects.all()
-    serializer_class = serializers.AssetSerializer
-    permission_classes = (IsOrgAdmin,)
-
-
-class AssetBulkUpdateSelectAPI(APIView):
-    permission_classes = (IsOrgAdmin,)
-
-    def post(self, request, *args, **kwargs):
-        assets_id = request.data.get('assets_id', '')
-        if assets_id:
-            spm = uuid.uuid4().hex
-            key = CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX.format(spm)
-            cache.set(key, assets_id, 300)
-            url = reverse_lazy('assets:asset-bulk-update') + '?spm=%s' % spm
-            return Response({'url': url})
-        error = _('Please select assets that need to be updated')
-        return Response({'error': error}, status=400)
-
 
 class AssetRefreshHardwareApi(generics.RetrieveAPIView):
     """

apps/assets/api/asset_user.py

@@ -2,15 +2,15 @@
 #
 
 from rest_framework.response import Response
-from rest_framework import viewsets, status, generics
-from rest_framework.pagination import LimitOffsetPagination
+from rest_framework import generics
 from rest_framework import filters
 from rest_framework_bulk import BulkModelViewSet
 from django.shortcuts import get_object_or_404
+from django.http import Http404
 
 from common.permissions import IsOrgAdminOrAppUser, NeedMFAVerify
 from common.utils import get_object_or_none, get_logger
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from ..backends import AssetUserManager
 from ..models import Asset, Node, SystemUser, AdminUser
 from .. import serializers
@@ -49,11 +49,10 @@ class AssetUserSearchBackend(filters.BaseFilterBackend):
             if field in ("node_id", "system_user_id", "admin_user_id"):
                 continue
             _queryset |= queryset.filter(**{field: value})
-        return _queryset
+        return _queryset.distinct()
 
 
-class AssetUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
-    pagination_class = LimitOffsetPagination
+class AssetUserViewSet(CommonApiMixin, BulkModelViewSet):
     serializer_class = serializers.AssetUserSerializer
     permission_classes = [IsOrgAdminOrAppUser]
     http_method_names = ['get', 'post']
@@ -67,6 +66,9 @@ class AssetUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
         AssetUserFilterBackend, AssetUserSearchBackend,
     )
 
+    def allow_bulk_destroy(self, qs, filtered):
+        return False
+
     def get_queryset(self):
         # 尽可能先返回更少的数据
         username = self.request.GET.get('username')
@@ -115,14 +117,6 @@ class AssetUserAuthInfoApi(generics.RetrieveAPIView):
     serializer_class = serializers.AssetUserAuthInfoSerializer
     permission_classes = [IsOrgAdminOrAppUser, NeedMFAVerify]
 
-    def retrieve(self, request, *args, **kwargs):
-        instance = self.get_object()
-        serializer = self.get_serializer(instance)
-        status_code = status.HTTP_200_OK
-        if not instance:
-            status_code = status.HTTP_400_BAD_REQUEST
-        return Response(serializer.data, status=status_code)
-
     def get_object(self):
         query_params = self.request.query_params
         username = query_params.get('username')
@@ -133,8 +127,7 @@ class AssetUserAuthInfoApi(generics.RetrieveAPIView):
             manger = AssetUserManager()
             instance = manger.get(username, asset, prefer=prefer)
         except Exception as e:
-            logger.error(e, exc_info=True)
-            return None
+            raise Http404("Not found")
         else:
             return instance
 

apps/assets/api/cmd_filter.py

@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 
-from rest_framework_bulk import BulkModelViewSet
-from rest_framework.pagination import LimitOffsetPagination
 from django.shortcuts import get_object_or_404
 
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..hands import IsOrgAdmin
 from ..models import CommandFilter, CommandFilterRule
 from .. import serializers
@@ -13,21 +12,19 @@ from .. import serializers
 __all__ = ['CommandFilterViewSet', 'CommandFilterRuleViewSet']
 
 
-class CommandFilterViewSet(BulkModelViewSet):
+class CommandFilterViewSet(OrgBulkModelViewSet):
     filter_fields = ("name",)
     search_fields = filter_fields
     permission_classes = (IsOrgAdmin,)
     queryset = CommandFilter.objects.all()
     serializer_class = serializers.CommandFilterSerializer
-    pagination_class = LimitOffsetPagination
 
 
-class CommandFilterRuleViewSet(BulkModelViewSet):
+class CommandFilterRuleViewSet(OrgBulkModelViewSet):
     filter_fields = ("content",)
     search_fields = filter_fields
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.CommandFilterRuleSerializer
-    pagination_class = LimitOffsetPagination
 
     def get_queryset(self):
         fpk = self.kwargs.get('filter_pk')

apps/assets/api/domain.py

@@ -1,13 +1,11 @@
 # ~*~ coding: utf-8 ~*~
 
-from rest_framework_bulk import BulkModelViewSet
 from rest_framework.views import APIView, Response
-from rest_framework.pagination import LimitOffsetPagination
-
 from django.views.generic.detail import SingleObjectMixin
 
 from common.utils import get_logger
-from common.permissions import IsOrgAdmin, IsAppUser, IsOrgAdminOrAppUser
+from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..models import Domain, Gateway
 from .. import serializers
 
@@ -16,34 +14,23 @@ logger = get_logger(__file__)
 __all__ = ['DomainViewSet', 'GatewayViewSet', "GatewayTestConnectionApi"]
 
 
-class DomainViewSet(BulkModelViewSet):
+class DomainViewSet(OrgBulkModelViewSet):
     queryset = Domain.objects.all()
-    permission_classes = (IsOrgAdmin,)
+    permission_classes = (IsOrgAdminOrAppUser,)
     serializer_class = serializers.DomainSerializer
-    pagination_class = LimitOffsetPagination
-
-    def get_queryset(self):
-        queryset = super().get_queryset().all()
-        return queryset
 
     def get_serializer_class(self):
         if self.request.query_params.get('gateway'):
             return serializers.DomainWithGatewaySerializer
         return super().get_serializer_class()
 
-    def get_permissions(self):
-        if self.request.query_params.get('gateway'):
-            self.permission_classes = (IsOrgAdminOrAppUser,)
-        return super().get_permissions()
 
-
-class GatewayViewSet(BulkModelViewSet):
+class GatewayViewSet(OrgBulkModelViewSet):
     filter_fields = ("domain__name", "name", "username", "ip", "domain")
     search_fields = filter_fields
     queryset = Gateway.objects.all()
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.GatewaySerializer
-    pagination_class = LimitOffsetPagination
 
 
 class GatewayTestConnectionApi(SingleObjectMixin, APIView):

apps/assets/api/gathered_user.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+
+from orgs.mixins.api import OrgModelViewSet
+from assets.models import GatheredUser
+from common.permissions import IsOrgAdmin
+
+from ..serializers import GatheredUserSerializer
+from ..filters import AssetRelatedByNodeFilterBackend
+
+
+__all__ = ['GatheredUserViewSet']
+
+
+class GatheredUserViewSet(OrgModelViewSet):
+    queryset = GatheredUser.objects.all()
+    serializer_class = GatheredUserSerializer
+    permission_classes = [IsOrgAdmin]
+    extra_filter_backends = [AssetRelatedByNodeFilterBackend]
+
+    filter_fields = ['asset', 'username', 'present']
+    search_fields = ['username', 'asset__ip', 'asset__hostname']
+
+

apps/assets/api/label.py

@@ -13,11 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from rest_framework.pagination import LimitOffsetPagination
 from django.db.models import Count
 
 from common.utils import get_logger
-from orgs.mixins import OrgBulkModelViewSet
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..hands import IsOrgAdmin
 from ..models import Label
 from .. import serializers
@@ -32,7 +31,6 @@ class LabelViewSet(OrgBulkModelViewSet):
     search_fields = filter_fields
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.LabelSerializer
-    pagination_class = LimitOffsetPagination
 
     def list(self, request, *args, **kwargs):
         if request.query_params.get("distinct"):

apps/assets/api/node.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from rest_framework import generics, mixins, viewsets
+from rest_framework import generics
 from rest_framework.serializers import ValidationError
 from rest_framework.views import APIView
 from rest_framework.response import Response
@@ -22,11 +22,11 @@ from django.shortcuts import get_object_or_404
 
 from common.utils import get_logger, get_object_or_none
 from common.tree import TreeNodeSerializer
+from orgs.mixins.api import OrgModelViewSet
 from ..hands import IsOrgAdmin
 from ..models import Node
 from ..tasks import update_assets_hardware_info_util, test_asset_connectivity_util
 from .. import serializers
-from ..utils import NodeUtil
 
 
 logger = get_logger(__file__)
@@ -35,33 +35,29 @@ __all__ = [
     'NodeAddAssetsApi', 'NodeRemoveAssetsApi', 'NodeReplaceAssetsApi',
     'NodeAddChildrenApi', 'RefreshNodeHardwareInfoApi',
     'TestNodeConnectiveApi', 'NodeListAsTreeApi',
-    'NodeChildrenAsTreeApi', 'RefreshAssetsAmount',
+    'NodeChildrenAsTreeApi', 'RefreshNodesCacheApi',
 ]
 
 
-class NodeViewSet(viewsets.ModelViewSet):
-    filter_fields = ('value', 'key', )
-    search_fields = filter_fields
+class NodeViewSet(OrgModelViewSet):
+    filter_fields = ('value', 'key', 'id')
+    search_fields = ('value', )
     queryset = Node.objects.all()
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.NodeSerializer
 
+    # 仅支持根节点指直接创建，子节点下的节点需要通过children接口创建
     def perform_create(self, serializer):
-        child_key = Node.root().get_next_child_key()
+        child_key = Node.org_root().get_next_child_key()
         serializer.validated_data["key"] = child_key
         serializer.save()
 
-    def update(self, request, *args, **kwargs):
+    def perform_update(self, serializer):
         node = self.get_object()
-        if node.is_root():
-            node_value = node.value
-            post_value = request.data.get('value')
-            if node_value != post_value:
-                return Response(
-                    {"msg": _("You can't update the root node name")},
-                    status=400
-                )
-        return super().update(request, *args, **kwargs)
+        if node.is_org_root() and node.value != serializer.validated_data['value']:
+            msg = _("You can't update the root node name")
+            raise ValidationError({"error": msg})
+        return super().perform_update(serializer)
 
 
 class NodeListAsTreeApi(generics.ListAPIView):
@@ -79,21 +75,74 @@ class NodeListAsTreeApi(generics.ListAPIView):
     permission_classes = (IsOrgAdmin,)
     serializer_class = TreeNodeSerializer
 
+    @staticmethod
+    def to_tree_queryset(queryset):
+        queryset = [node.as_tree_node() for node in queryset]
+        return queryset
+
     def get_queryset(self):
         queryset = Node.objects.all()
-        util = NodeUtil()
-        nodes = util.get_nodes_by_queryset(queryset)
-        queryset = [node.as_tree_node() for node in nodes]
         return queryset
 
-    @staticmethod
-    def refresh_nodes(queryset):
-        Node.expire_nodes_assets_amount()
-        Node.expire_nodes_full_value()
+    def filter_queryset(self, queryset):
+        queryset = super().filter_queryset(queryset)
+        queryset = self.to_tree_queryset(queryset)
         return queryset
 
 
-class NodeChildrenAsTreeApi(generics.ListAPIView):
+class NodeChildrenApi(generics.ListCreateAPIView):
+    queryset = Node.objects.all()
+    permission_classes = (IsOrgAdmin,)
+    serializer_class = serializers.NodeSerializer
+    instance = None
+    is_initial = False
+
+    def initial(self, request, *args, **kwargs):
+        self.instance = self.get_object()
+        return super().initial(request, *args, **kwargs)
+
+    def perform_create(self, serializer):
+        data = serializer.validated_data
+        _id = data.get("id")
+        value = data.get("value")
+        if not value:
+            value = self.instance.get_next_child_preset_name()
+        node = self.instance.create_child(value=value, _id=_id)
+        # 避免查询 full value
+        node._full_value = node.value
+        serializer.instance = node
+
+    def get_object(self):
+        pk = self.kwargs.get('pk') or self.request.query_params.get('id')
+        key = self.request.query_params.get("key")
+        if not pk and not key:
+            node = Node.org_root()
+            self.is_initial = True
+            return node
+        if pk:
+            node = get_object_or_404(Node, pk=pk)
+        else:
+            node = get_object_or_404(Node, key=key)
+        return node
+
+    def get_queryset(self):
+        query_all = self.request.query_params.get("all", "0") == "all"
+        if not self.instance:
+            return Node.objects.none()
+
+        if self.is_initial:
+            with_self = True
+        else:
+            with_self = False
+
+        if query_all:
+            queryset = self.instance.get_all_children(with_self=with_self)
+        else:
+            queryset = self.instance.get_children(with_self=with_self)
+        return queryset
+
+
+class NodeChildrenAsTreeApi(NodeChildrenApi):
     """
     节点子节点作为树返回，
     [
@@ -106,39 +155,26 @@ class NodeChildrenAsTreeApi(generics.ListAPIView):
     ]
 
     """
-    permission_classes = (IsOrgAdmin,)
     serializer_class = TreeNodeSerializer
-    node = None
-    is_root = False
+    http_method_names = ['get']
 
     def get_queryset(self):
-        self.check_need_refresh_nodes()
-        node_key = self.request.query_params.get('key')
-        util = NodeUtil()
-        # 是否包含自己
-        with_self = False
-        if not node_key:
-            node_key = Node.root().key
-            with_self = True
-        self.node = util.get_node_by_key(node_key)
-        queryset = self.node.get_children(with_self=with_self)
+        queryset = super().get_queryset()
         queryset = [node.as_tree_node() for node in queryset]
+        queryset = self.add_assets_if_need(queryset)
         queryset = sorted(queryset)
         return queryset
 
-    def filter_assets(self, queryset):
+    def add_assets_if_need(self, queryset):
         include_assets = self.request.query_params.get('assets', '0') == '1'
         if not include_assets:
             return queryset
-        assets = self.node.get_assets().only(
-            "id", "hostname", "ip", 'platform', "os", "org_id", "protocols",
+        assets = self.instance.get_assets().only(
+            "id", "hostname", "ip", 'platform', "os",
+            "org_id", "protocols",
         )
         for asset in assets:
-            queryset.append(asset.as_tree_node(self.node))
-        return queryset
-
-    def filter_queryset(self, queryset):
-        queryset = self.filter_assets(queryset)
+            queryset.append(asset.as_tree_node(self.instance))
         return queryset
 
     def check_need_refresh_nodes(self):
@@ -146,59 +182,6 @@ class NodeChildrenAsTreeApi(generics.ListAPIView):
             Node.refresh_nodes()
 
 
-class NodeChildrenApi(mixins.ListModelMixin, generics.CreateAPIView):
-    queryset = Node.objects.all()
-    permission_classes = (IsOrgAdmin,)
-    serializer_class = serializers.NodeSerializer
-    instance = None
-
-    def get(self, request, *args, **kwargs):
-        return self.list(request, *args, **kwargs)
-
-    def post(self, request, *args, **kwargs):
-        instance = self.get_object()
-        if not request.data.get("value"):
-            request.data["value"] = instance.get_next_child_preset_name()
-        return super().post(request, *args, **kwargs)
-
-    def create(self, request, *args, **kwargs):
-        instance = self.get_object()
-        value = request.data.get("value")
-        _id = request.data.get('id') or None
-        values = [child.value for child in instance.get_children()]
-        if value in values:
-            raise ValidationError(
-                'The same level node name cannot be the same'
-            )
-        node = instance.create_child(value=value, _id=_id)
-        return Response(self.serializer_class(instance=node).data, status=201)
-
-    def get_object(self):
-        pk = self.kwargs.get('pk') or self.request.query_params.get('id')
-        if not pk:
-            node = Node.root()
-        else:
-            node = get_object_or_404(Node, pk=pk)
-        return node
-
-    def get_queryset(self):
-        queryset = []
-        query_all = self.request.query_params.get("all")
-        node = self.get_object()
-
-        if node is None:
-            node = Node.root()
-            node.assets__count = node.get_all_assets().count()
-            queryset.append(node)
-
-        if query_all:
-            children = node.get_all_children()
-        else:
-            children = node.get_children()
-        queryset.extend(list(children))
-        return queryset
-
-
 class NodeAssetsApi(generics.ListAPIView):
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.AssetSerializer
@@ -251,7 +234,7 @@ class NodeRemoveAssetsApi(generics.UpdateAPIView):
     def perform_update(self, serializer):
         assets = serializer.validated_data.get('assets')
         instance = self.get_object()
-        if instance != Node.root():
+        if instance != Node.org_root():
             instance.assets.remove(*tuple(assets))
         else:
             assets = [asset for asset in assets if asset.nodes.count() > 1]
@@ -299,10 +282,13 @@ class TestNodeConnectiveApi(APIView):
         return Response({"task": task.id})
 
 
-class RefreshAssetsAmount(APIView):
+class RefreshNodesCacheApi(APIView):
     permission_classes = (IsOrgAdmin,)
-    model = Node
 
     def get(self, request, *args, **kwargs):
-        self.model.expire_nodes_assets_amount()
+        Node.refresh_nodes()
         return Response("Ok")
+
+    def delete(self, *args, **kwargs):
+        self.get(*args, **kwargs)
+        return Response(status=204)

apps/assets/api/system_user.py

@@ -16,18 +16,17 @@
 from django.shortcuts import get_object_or_404
 from rest_framework import generics
 from rest_framework.response import Response
-from rest_framework_bulk import BulkModelViewSet
-from rest_framework.pagination import LimitOffsetPagination
 
+from common.serializers import CeleryTaskSerializer
 from common.utils import get_logger
 from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
-from common.mixins import IDInCacheFilterMixin
-from orgs.mixins import OrgBulkModelViewSet
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..models import SystemUser, Asset
 from .. import serializers
-from ..tasks import push_system_user_to_assets_manual, \
-    test_system_user_connectivity_manual, push_system_user_a_asset_manual, \
-    test_system_user_connectivity_a_asset
+from ..tasks import (
+    push_system_user_to_assets_manual, test_system_user_connectivity_manual,
+    push_system_user_a_asset_manual, test_system_user_connectivity_a_asset,
+)
 
 
 logger = get_logger(__file__)
@@ -49,7 +48,6 @@ class SystemUserViewSet(OrgBulkModelViewSet):
     queryset = SystemUser.objects.all()
     serializer_class = serializers.SystemUserSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
-    pagination_class = LimitOffsetPagination
 
     def get_queryset(self):
         queryset = super().get_queryset().all()
@@ -92,6 +90,7 @@ class SystemUserPushApi(generics.RetrieveAPIView):
     """
     queryset = SystemUser.objects.all()
     permission_classes = (IsOrgAdmin,)
+    serializer_class = CeleryTaskSerializer
 
     def retrieve(self, request, *args, **kwargs):
         system_user = self.get_object()
@@ -108,6 +107,7 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView):
     """
     queryset = SystemUser.objects.all()
     permission_classes = (IsOrgAdmin,)
+    serializer_class = CeleryTaskSerializer
 
     def retrieve(self, request, *args, **kwargs):
         system_user = self.get_object()
@@ -118,7 +118,6 @@ class SystemUserTestConnectiveApi(generics.RetrieveAPIView):
 class SystemUserAssetsListView(generics.ListAPIView):
     permission_classes = (IsOrgAdmin,)
     serializer_class = serializers.AssetSimpleSerializer
-    pagination_class = LimitOffsetPagination
     filter_fields = ("hostname", "ip")
     http_method_names = ['get']
     search_fields = filter_fields

apps/assets/apps.py

@@ -1,11 +1,25 @@
 from __future__ import unicode_literals
 
 from django.apps import AppConfig
+from django.db.models.signals import post_migrate
+
+
+def initial_some_nodes():
+    from .models import Node
+    Node.initial_some_nodes()
+
+
+def initial_some_nodes_callback(sender, **kwargs):
+    initial_some_nodes()
 
 
 class AssetsConfig(AppConfig):
     name = 'assets'
 
     def ready(self):
-        from . import signals_handler
         super().ready()
+        from . import signals_handler
+        try:
+            initial_some_nodes()
+        except Exception:
+            post_migrate.connect(initial_some_nodes_callback, sender=self)

apps/assets/backends/asset_user.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 #
+from collections import defaultdict
 from .base import BaseBackend
 
 
@@ -23,6 +24,7 @@ class AssetUserBackend(BaseBackend):
             queryset = queryset.filter(username=username)
         if assets:
             queryset = queryset.filter(assets__in=assets).distinct()
+
         queryset = cls.filter_queryset_more(queryset)
         instances = cls.construct_authbook_objects(queryset, assets)
         return instances
@@ -30,10 +32,26 @@ class AssetUserBackend(BaseBackend):
     @classmethod
     def construct_authbook_objects(cls, asset_users, assets):
         instances = []
+        assets_user_assets_map = defaultdict(set)
+        if isinstance(asset_users, list):
+            assets_user_assets_map = {
+                asset_user.id: asset_user.assets.values_list('id', flat=True)
+                for asset_user in asset_users
+            }
+        else:
+            assets_user_assets = asset_users.values_list('id', 'assets')
+            for i, asset_id in assets_user_assets:
+                assets_user_assets_map[i].add(asset_id)
+
         for asset_user in asset_users:
             if not assets:
-                assets = asset_user.assets.all()
-            for asset in assets:
+                related_assets = asset_user.assets.all()
+            else:
+                assets_map = {a.id: a for a in assets}
+                related_assets = [
+                    assets_map.get(i) for i in assets_user_assets_map.get(asset_user.id) if i in assets_map
+                ]
+            for asset in related_assets:
                 instance = asset_user.construct_to_authbook(asset)
                 instance.backend = cls.backend
                 instances.append(instance)

apps/assets/backends/base.py

@@ -81,6 +81,11 @@ class AssetUserQuerySet(list):
         queryset = self.filter_in(kwargs).filter_equal(kwargs)
         return queryset
 
+    def distinct(self):
+        items = list(set(self))
+        self[:] = items
+        return self
+
     def __or__(self, other):
         self.extend(other)
         return self

apps/assets/filters.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+
+import coreapi
+from rest_framework import filters
+from django.db.models import Q
+
+from common.utils import dict_get_any, is_uuid, get_object_or_none
+from .models import Node, Label
+
+
+class AssetByNodeFilterBackend(filters.BaseFilterBackend):
+    fields = ['node', 'all']
+
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name=field, location='query', required=False,
+                type='string', example='', description='', schema=None,
+            )
+            for field in self.fields
+        ]
+
+    @staticmethod
+    def is_query_all(request):
+        query_all_arg = request.query_params.get('all')
+        show_current_asset_arg = request.query_params.get('show_current_asset')
+
+        query_all = query_all_arg == '1'
+        if show_current_asset_arg is not None:
+            query_all = show_current_asset_arg != '1'
+        return query_all
+
+    @staticmethod
+    def get_query_node(request):
+        node_id = dict_get_any(request.query_params, ['node', 'node_id'])
+        if not node_id:
+            return None, False
+
+        if is_uuid(node_id):
+            node = get_object_or_none(Node, id=node_id)
+        else:
+            node = get_object_or_none(Node, key=node_id)
+        return node, True
+
+    @staticmethod
+    def perform_query(pattern, queryset):
+        return queryset.filter(nodes__key__regex=pattern).distinct()
+
+    def filter_queryset(self, request, queryset, view):
+        node, has_query_arg = self.get_query_node(request)
+        if not has_query_arg:
+            return queryset
+
+        if node is None:
+            return queryset.none()
+        query_all = self.is_query_all(request)
+        if query_all:
+            pattern = node.get_all_children_pattern(with_self=True)
+        else:
+            pattern = node.get_children_key_pattern(with_self=True)
+        return self.perform_query(pattern, queryset)
+
+
+class LabelFilterBackend(filters.BaseFilterBackend):
+    sep = '#'
+    query_arg = 'label'
+
+    def get_schema_fields(self, view):
+        example = self.sep.join(['os', 'linux'])
+        return [
+            coreapi.Field(
+                name=self.query_arg, location='query', required=False,
+                type='string', example=example, description=''
+            )
+        ]
+
+    def get_query_labels(self, request):
+        labels_query = request.query_params.getlist(self.query_arg)
+        if not labels_query:
+            return None
+
+        q = None
+        for kv in labels_query:
+            if self.sep not in kv:
+                continue
+            key, value = kv.strip().split(self.sep)[:2]
+            if not all([key, value]):
+                continue
+            if q:
+                q |= Q(name=key, value=value)
+            else:
+                q = Q(name=key, value=value)
+        if not q:
+            return []
+        labels = Label.objects.filter(q, is_active=True)\
+            .values_list('id', flat=True)
+        return labels
+
+    def filter_queryset(self, request, queryset, view):
+        labels = self.get_query_labels(request)
+        if labels is None:
+            return queryset
+        if len(labels) == 0:
+            return queryset.none()
+        for label in labels:
+            queryset = queryset.filter(labels=label)
+        return queryset
+
+
+class AssetRelatedByNodeFilterBackend(AssetByNodeFilterBackend):
+    @staticmethod
+    def perform_query(pattern, queryset):
+        return queryset.filter(asset__nodes__key__regex=pattern).distinct()
+

apps/assets/forms/asset.py

@@ -4,7 +4,7 @@ from django import forms
 from django.utils.translation import gettext_lazy as _
 
 from common.utils import get_logger
-from orgs.mixins import OrgModelForm
+from orgs.mixins.forms import OrgModelForm
 
 from ..models import Asset, Node
 
@@ -29,9 +29,19 @@ class ProtocolForm(forms.Form):
 class AssetCreateForm(OrgModelForm):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        if self.data:
+            return
         nodes_field = self.fields['nodes']
-        nodes_field.choices = ((n.id, n.full_value) for n in
-                               Node.get_queryset())
+        if self.instance:
+            nodes_field.choices = [(n.id, n.full_value) for n in
+                                   self.instance.nodes.all()]
+        else:
+            nodes_field.choices = []
+
+    def add_nodes_initial(self, node):
+        nodes_field = self.fields['nodes']
+        nodes_field.choices.append((node.id, node.full_value))
+        nodes_field.initial = [node]
 
     class Meta:
         model = Asset
@@ -42,7 +52,7 @@ class AssetCreateForm(OrgModelForm):
         ]
         widgets = {
             'nodes': forms.SelectMultiple(attrs={
-                'class': 'select2', 'data-placeholder': _('Nodes')
+                'class': 'nodes-select2', 'data-placeholder': _('Nodes')
             }),
             'admin_user': forms.Select(attrs={
                 'class': 'select2', 'data-placeholder': _('Admin user')
@@ -58,6 +68,8 @@ class AssetCreateForm(OrgModelForm):
             'nodes': _("Node"),
         }
         help_texts = {
+            'hostname': _('Only Numbers, letters, and characters ( {} ) '
+                          'are allowed').format(" ".join(['.', '_', '@'])),
             'admin_user': _(
                 'root or other NOPASSWD sudo privilege user existed in asset,'
                 'If asset is windows or other set any one, more see admin user left menu'
@@ -68,6 +80,17 @@ class AssetCreateForm(OrgModelForm):
 
 
 class AssetUpdateForm(OrgModelForm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.data:
+            return
+        nodes_field = self.fields['nodes']
+        if self.instance:
+            nodes_field.choices = ((n.id, n.full_value) for n in
+                                   self.instance.nodes.all())
+        else:
+            nodes_field.choices = []
+
     class Meta:
         model = Asset
         fields = [
@@ -77,7 +100,7 @@ class AssetUpdateForm(OrgModelForm):
         ]
         widgets = {
             'nodes': forms.SelectMultiple(attrs={
-                'class': 'select2', 'data-placeholder': _('Node')
+                'class': 'nodes-select2', 'data-placeholder': _('Node')
             }),
             'admin_user': forms.Select(attrs={
                 'class': 'select2', 'data-placeholder': _('Admin user')
@@ -93,6 +116,8 @@ class AssetUpdateForm(OrgModelForm):
             'nodes': _("Node"),
         }
         help_texts = {
+            'hostname': _('Only Numbers, letters, and characters ( {} ) '
+                          'are allowed').format(" ".join(['.', '_', '@'])),
             'admin_user': _(
                 'root or other NOPASSWD sudo privilege user existed in asset,'
                 'If asset is windows or other set any one, more see admin user left menu'

apps/assets/forms/cmd_filter.py

@@ -5,7 +5,7 @@ from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
 import re
 
-from orgs.mixins import OrgModelForm
+from orgs.mixins.forms import OrgModelForm
 from ..models import CommandFilter, CommandFilterRule
 
 __all__ = ['CommandFilterForm', 'CommandFilterRuleForm']

apps/assets/forms/domain.py

@@ -3,7 +3,7 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
 
-from orgs.mixins import OrgModelForm
+from orgs.mixins.forms import OrgModelForm
 from ..models import Domain, Asset, Gateway
 from .user import PasswordAndKeyAuthForm
 

apps/assets/forms/user.py

@@ -4,7 +4,7 @@ from django import forms
 from django.utils.translation import gettext_lazy as _
 
 from common.utils import validate_ssh_private_key, ssh_pubkey_gen, get_logger
-from orgs.mixins import OrgModelForm
+from orgs.mixins.forms import OrgModelForm
 from ..models import AdminUser, SystemUser
 
 logger = get_logger(__file__)

apps/assets/migrations/0037_auto_20190724_2002.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.7 on 2019-07-24 12:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0036_auto_20190716_1535'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adminuser',
+            name='_become_pass',
+            field=models.CharField(blank=True, default='', max_length=128),
+        ),
+    ]

apps/assets/migrations/0038_auto_20190911_1634.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.1.7 on 2019-09-11 08:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0037_auto_20190724_2002'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='asset',
+            name='protocol',
+            field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('telnet', 'telnet'), ('vnc', 'vnc')], default='ssh', max_length=128, verbose_name='Protocol'),
+        ),
+        migrations.AlterField(
+            model_name='systemuser',
+            name='protocol',
+            field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('telnet', 'telnet'), ('vnc', 'vnc')], default='ssh', max_length=16, verbose_name='Protocol'),
+        ),
+    ]

apps/assets/migrations/0039_authbook_is_active.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.7 on 2019-09-17 12:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0038_auto_20190911_1634'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='authbook',
+            name='is_active',
+            field=models.BooleanField(default=True, verbose_name='Is active'),
+        ),
+    ]

apps/assets/migrations/0040_auto_20190917_2056.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.1.7 on 2019-09-17 12:56
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0039_authbook_is_active'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adminuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='authbook',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='gateway',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='systemuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+    ]

apps/assets/migrations/0041_gathereduser.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.1.7 on 2019-09-18 04:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0040_auto_20190917_2056'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GatheredUser',
+            fields=[
+                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('username', models.CharField(blank=True, db_index=True, max_length=32, verbose_name='Username')),
+                ('present', models.BooleanField(default=True, verbose_name='Present')),
+                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.Asset', verbose_name='Asset')),
+            ],
+            options={'ordering': ['asset'], 'verbose_name': 'GatherUser'},
+        ),
+    ]

apps/assets/models/__init__.py

@@ -9,3 +9,4 @@ from .cmd_filter import *
 from .authbook import *
 from .utils import *
 from .authbook import *
+from .gathered_user import *

apps/assets/models/asset.py

@@ -6,14 +6,13 @@ import uuid
 import logging
 import random
 from functools import reduce
-from collections import OrderedDict, defaultdict
-from django.core.cache import cache
+from collections import OrderedDict
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
 from .utils import Connectivity
-from orgs.mixins import OrgModelMixin, OrgManager
+from orgs.mixins.models import OrgModelMixin, OrgManager
 
 __all__ = ['Asset', 'ProtocolsMixin']
 logger = logging.getLogger(__name__)
@@ -32,7 +31,7 @@ def default_cluster():
 def default_node():
     try:
         from .node import Node
-        root = Node.root()
+        root = Node.org_root()
         return root
     except:
         return None
@@ -58,7 +57,7 @@ class ProtocolsMixin:
     PROTOCOL_CHOICES = (
         (PROTOCOL_SSH, 'ssh'),
         (PROTOCOL_RDP, 'rdp'),
-        (PROTOCOL_TELNET, 'telnet (beta)'),
+        (PROTOCOL_TELNET, 'telnet'),
         (PROTOCOL_VNC, 'vnc'),
     )
 
@@ -103,39 +102,17 @@ class NodesRelationMixin:
     id = ""
     _all_nodes_keys = None
 
-    @classmethod
-    def get_all_nodes_keys(cls):
-        """
-        :return: {asset.id: [node.key, ]}
-        """
-        from .node import Node
-        cache_key = cls.ALL_ASSET_NODES_CACHE_KEY
-        cached = cache.get(cache_key)
-        if cached:
-            return cached
-        assets = Asset.objects.all().only('id').prefetch_related(
-            models.Prefetch('nodes', queryset=Node.objects.all().only('key'))
-        )
-        assets_nodes_keys = {}
-        for asset in assets:
-            assets_nodes_keys[asset.id] = [n.key for n in asset.nodes.all()]
-        cache.set(cache_key, assets_nodes_keys, cls.CACHE_TIME)
-        return assets_nodes_keys
-
-    @classmethod
-    def expire_all_nodes_keys_cache(cls):
-        cache_key = cls.ALL_ASSET_NODES_CACHE_KEY
-        cache.delete(cache_key)
-
     def get_nodes(self):
         from .node import Node
-        nodes = self.nodes.all() or [Node.root()]
+        nodes = self.nodes.all()
+        if not nodes:
+            nodes = Node.objects.filter(id=Node.org_root().id)
         return nodes
 
     def get_all_nodes(self, flat=False):
         nodes = []
         for node in self.get_nodes():
-            _nodes = node.get_ancestor(with_self=True)
+            _nodes = node.get_ancestors(with_self=True)
             nodes.append(_nodes)
         if flat:
             nodes = list(reduce(lambda x, y: set(x) | set(y), nodes))
@@ -345,7 +322,6 @@ class Asset(ProtocolsMixin, NodesRelationMixin, OrgModelMixin):
                 else:
                     _nodes = [Node.default_node()]
                 asset.nodes.set(_nodes)
-                asset.system_users = [choice(SystemUser.objects.all()) for i in range(3)]
                 logger.debug('Generate fake asset : %s' % asset.ip)
             except IntegrityError:
                 print('Error continue')

apps/assets/models/authbook.py

@@ -4,7 +4,7 @@
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from orgs.mixins import OrgManager
+from orgs.mixins.models import OrgManager
 from .base import AssetUser
 
 __all__ = ['AuthBook']
@@ -13,7 +13,7 @@ __all__ = ['AuthBook']
 class AuthBookQuerySet(models.QuerySet):
 
     def latest_version(self):
-        return self.filter(is_latest=True)
+        return self.filter(is_latest=True).filter(is_active=True)
 
 
 class AuthBookManager(OrgManager):
@@ -24,6 +24,7 @@ class AuthBook(AssetUser):
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
     is_latest = models.BooleanField(default=False, verbose_name=_('Latest version'))
     version = models.IntegerField(default=1, verbose_name=_('Version'))
+    is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
 
     objects = AuthBookManager.from_queryset(AuthBookQuerySet)()
     backend = "db"
@@ -34,25 +35,25 @@ class AuthBook(AssetUser):
     class Meta:
         verbose_name = _('AuthBook')
 
-    def _set_latest(self):
-        self._remove_pre_obj_latest()
+    def set_to_latest(self):
+        self.remove_pre_latest()
         self.is_latest = True
         self.save()
 
-    def _get_pre_obj(self):
+    def get_pre_latest(self):
         pre_obj = self.__class__.objects.filter(
             username=self.username, asset=self.asset
         ).latest_version().first()
         return pre_obj
 
-    def _remove_pre_obj_latest(self):
-        pre_obj = self._get_pre_obj()
+    def remove_pre_latest(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             pre_obj.is_latest = False
             pre_obj.save()
 
-    def _set_version(self):
-        pre_obj = self._get_pre_obj()
+    def set_version(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             self.version = pre_obj.version + 1
         else:
@@ -60,8 +61,8 @@ class AuthBook(AssetUser):
         self.save()
 
     def set_version_and_latest(self):
-        self._set_version()
-        self._set_latest()
+        self.set_version()
+        self.set_to_latest()
 
     def get_related_assets(self):
         return [self.asset]

apps/assets/models/base.py

@@ -15,7 +15,7 @@ from common.utils import (
 )
 from common.validators import alphanumeric
 from common import fields
-from orgs.mixins import OrgModelMixin
+from orgs.mixins.models import OrgModelMixin
 from .utils import private_key_validator, Connectivity
 
 signer = get_signer()
@@ -26,7 +26,7 @@ logger = get_logger(__file__)
 class AssetUser(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
-    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric])
+    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric], db_index=True)
     password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
     private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
     public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))

apps/assets/models/cmd_filter.py

@@ -7,7 +7,7 @@ from django.db import models
 from django.core.validators import MinValueValidator, MaxValueValidator
 from django.utils.translation import ugettext_lazy as _
 
-from orgs.mixins import OrgModelMixin
+from orgs.mixins.models import OrgModelMixin
 
 
 __all__ = [

apps/assets/models/domain.py

@@ -9,7 +9,7 @@ import paramiko
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from orgs.mixins import OrgModelMixin
+from orgs.mixins.models import OrgModelMixin
 from .base import AssetUser
 
 __all__ = ['Domain', 'Gateway']

apps/assets/models/gathered_user.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#
+import uuid
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.models import OrgModelMixin
+
+__all__ = ['GatheredUser']
+
+
+class GatheredUser(OrgModelMixin):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_("Asset"))
+    username = models.CharField(max_length=32, blank=True, db_index=True,
+                                verbose_name=_('Username'))
+    present = models.BooleanField(default=True, verbose_name=_("Present"))
+    date_created = models.DateTimeField(auto_now_add=True,
+                                        verbose_name=_("Date created"))
+    date_updated = models.DateTimeField(auto_now=True,
+                                        verbose_name=_("Date updated"))
+
+    @property
+    def hostname(self):
+        return self.asset.hostname
+
+    @property
+    def ip(self):
+        return self.asset.ip
+
+    class Meta:
+        verbose_name = _('GatherUser')
+        ordering = ['asset']
+
+    def __str__(self):
+        return '{}: {}'.format(self.asset.hostname, self.username)
+
+
+

apps/assets/models/label.py

@@ -4,7 +4,7 @@
 import uuid
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
-from orgs.mixins import OrgModelMixin
+from orgs.mixins.models import OrgModelMixin
 
 
 class Label(OrgModelMixin):

apps/assets/models/node.py

@@ -2,6 +2,7 @@
 #
 import uuid
 import re
+import time
 
 from django.db import models, transaction
 from django.db.models import Q
@@ -9,11 +10,14 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext
 from django.core.cache import cache
 
-from orgs.mixins import OrgModelMixin, OrgManager
-from orgs.utils import set_current_org, get_current_org
+from common.utils import get_logger, timeit, lazyproperty
+from orgs.mixins.models import OrgModelMixin, OrgManager
+from orgs.utils import set_current_org, get_current_org, tmp_to_org
 from orgs.models import Organization
 
+
 __all__ = ['Node']
+logger = get_logger(__name__)
 
 
 class NodeQuerySet(models.QuerySet):
@@ -21,277 +25,133 @@ class NodeQuerySet(models.QuerySet):
         raise PermissionError("Bulk delete node deny")
 
 
+class TreeMixin:
+    tree_created_time = None
+    tree_updated_time_cache_key = 'NODE_TREE_UPDATED_AT'
+    tree_cache_time = 3600
+    tree_assets_cache_key = 'NODE_TREE_ASSETS_UPDATED_AT'
+    tree_assets_created_time = None
+    _tree_service = None
+
+    @classmethod
+    def tree(cls):
+        from ..utils import TreeService
+        tree_updated_time = cache.get(cls.tree_updated_time_cache_key, 0)
+        now = time.time()
+        # 什么时候重新初始化 _tree_service
+        if not cls.tree_created_time or \
+                tree_updated_time > cls.tree_created_time:
+            logger.debug("Create node tree")
+            tree = TreeService.new()
+            cls.tree_created_time = now
+            cls.tree_assets_created_time = now
+            cls._tree_service = tree
+            return tree
+        # 是否要重新初始化节点资产
+        node_assets_updated_time = cache.get(cls.tree_assets_cache_key, 0)
+        if not cls.tree_assets_created_time or \
+                node_assets_updated_time > cls.tree_assets_created_time:
+            cls._tree_service.init_assets()
+            cls.tree_assets_created_time = now
+            logger.debug("Refresh node tree assets")
+        return cls._tree_service
+
+    @classmethod
+    def refresh_tree(cls, t=None):
+        logger.debug("Refresh node tree")
+        key = cls.tree_updated_time_cache_key
+        ttl = cls.tree_cache_time
+        if not t:
+            t = time.time()
+        cache.set(key, t, ttl)
+
+    @classmethod
+    def refresh_node_assets(cls, t=None):
+        logger.debug("Refresh node tree assets")
+        key = cls.tree_assets_cache_key
+        ttl = cls.tree_cache_time
+        if not t:
+            t = time.time()
+        cache.set(key, t, ttl)
+
+    @staticmethod
+    def refresh_user_tree_cache():
+        """
+        当节点-节点关系，节点-资产关系发生变化时，应该刷新用户授权树缓存
+        :return:
+        """
+        from perms.utils.asset_permission import AssetPermissionUtilV2
+        AssetPermissionUtilV2.expire_all_user_tree_cache()
+
+
 class FamilyMixin:
-    _parents = None
-    _children = None
-    _all_children = None
+    __parents = None
+    __children = None
+    __all_children = None
     is_node = True
 
+    @staticmethod
+    def clean_children_keys(nodes_keys):
+        nodes_keys = sorted(list(nodes_keys), key=lambda x: (len(x), x))
+        nodes_keys_clean = []
+        for key in nodes_keys[::-1]:
+            found = False
+            for k in nodes_keys:
+                if key.startswith(k + ':'):
+                    found = True
+                    break
+            if not found:
+                nodes_keys_clean.append(key)
+        return nodes_keys_clean
+
+    @classmethod
+    def get_node_all_children_key_pattern(cls, key, with_self=True):
+        pattern = r'^{0}:'.format(key)
+        if with_self:
+            pattern += r'|^{0}$'.format(key)
+        return pattern
+
+    @classmethod
+    def get_node_children_key_pattern(cls, key, with_self=True):
+        pattern = r'^{0}:[0-9]+$'.format(key)
+        if with_self:
+            pattern += r'|^{0}$'.format(key)
+        return pattern
+
+    def get_children_key_pattern(self, with_self=False):
+        return self.get_node_children_key_pattern(self.key, with_self=with_self)
+
+    def get_all_children_pattern(self, with_self=False):
+        return self.get_node_all_children_key_pattern(self.key, with_self=with_self)
+
+    def is_children(self, other):
+        children_pattern = other.get_children_key_pattern(with_self=False)
+        return re.match(children_pattern, self.key)
+
+    def get_children(self, with_self=False):
+        pattern = self.get_children_key_pattern(with_self=with_self)
+        return Node.objects.filter(key__regex=pattern)
+
+    def get_all_children(self, with_self=False):
+        pattern = self.get_all_children_pattern(with_self=with_self)
+        children = Node.objects.filter(key__regex=pattern)
+        return children
+
     @property
     def children(self):
-        if self._children:
-            return self._children
-        pattern = r'^{0}:[0-9]+$'.format(self.key)
-        return Node.objects.filter(key__regex=pattern)
-
-    @children.setter
-    def children(self, value):
-        self._children = value
+        return self.get_children(with_self=False)
 
     @property
     def all_children(self):
-        if self._all_children:
-            return self._all_children
-        pattern = r'^{0}:'.format(self.key)
-        return Node.objects.filter(
-            key__regex=pattern
-        )
+        return self.get_all_children(with_self=False)
 
-    def get_children(self, with_self=False):
-        children = list(self.children)
-        if with_self:
-            children.append(self)
-        return children
-
-    def get_all_children(self, with_self=False):
-        children = self.all_children
-        if with_self:
-            children = list(children)
-            children.append(self)
-        return children
-
-    @property
-    def parents(self):
-        if self._parents:
-            return self._parents
-        ancestor_keys = self.get_ancestor_keys()
-        ancestor = Node.objects.filter(
-            key__in=ancestor_keys
-        ).order_by('key')
-        return ancestor
-
-    @parents.setter
-    def parents(self, value):
-        self._parents = value
-
-    def get_ancestor(self, with_self=False):
-        parents = self.parents
-        if with_self:
-            parents = list(parents)
-            parents.append(self)
-        return parents
-
-    @property
-    def parent(self):
-        if self._parents:
-            return self._parents[0]
-        if self.is_root():
-            return self
-        try:
-            parent = Node.objects.get(key=self.parent_key)
-            return parent
-        except Node.DoesNotExist:
-            return Node.root()
-
-    @parent.setter
-    def parent(self, parent):
-        if not self.is_node:
-            self.key = parent.key + ':fake'
-            return
-        children = self.get_all_children()
-        old_key = self.key
+    def create_child(self, value, _id=None):
         with transaction.atomic():
-            self.key = parent.get_next_child_key()
-            for child in children:
-                child.key = child.key.replace(old_key, self.key, 1)
-                child.save()
-            self.save()
-
-    def get_sibling(self, with_self=False):
-        key = ':'.join(self.key.split(':')[:-1])
-        pattern = r'^{}:[0-9]+$'.format(key)
-        sibling = Node.objects.filter(
-            key__regex=pattern.format(self.key)
-        )
-        if not with_self:
-            sibling = sibling.exclude(key=self.key)
-        return sibling
-
-    def get_family(self):
-        ancestor = self.get_ancestor()
-        children = self.get_all_children()
-        return [*tuple(ancestor), self, *tuple(children)]
-
-    def get_ancestor_keys(self, with_self=False):
-        parent_keys = []
-        key_list = self.key.split(":")
-        if not with_self:
-            key_list.pop()
-        for i in range(len(key_list)):
-            parent_keys.append(":".join(key_list))
-            key_list.pop()
-        return parent_keys
-
-    def is_children(self, other):
-        pattern = re.compile(r'^{0}:[0-9]+$'.format(self.key))
-        return pattern.match(other.key)
-
-    def is_parent(self, other):
-        pattern = re.compile(r'^{0}:[0-9]+$'.format(other.key))
-        return pattern.match(self.key)
-
-    @property
-    def parent_key(self):
-        parent_key = ":".join(self.key.split(":")[:-1])
-        return parent_key
-
-    @property
-    def parents_keys(self, with_self=False):
-        keys = []
-        key_list = self.key.split(":")
-        if not with_self:
-            key_list.pop()
-        for i in range(len(key_list)):
-            keys.append(':'.join(key_list))
-            key_list.pop()
-        return keys
-
-
-class FullValueMixin:
-    _full_value_cache_key = '_NODE_VALUE_{}'
-    _full_value = ''
-    key = ''
-
-    @property
-    def full_value(self):
-        if self._full_value:
-            return self._full_value
-        key = self._full_value_cache_key.format(self.key)
-        cached = cache.get(key)
-        if cached:
-            return cached
-        if self.is_root():
-            return self.value
-        parent_full_value = self.parent.full_value
-        value = parent_full_value + ' / ' + self.value
-        self.full_value = value
-        return value
-
-    @full_value.setter
-    def full_value(self, value):
-        self._full_value = value
-        key = self._full_value_cache_key.format(self.key)
-        cache.set(key, value, 3600*24)
-
-    def expire_full_value(self):
-        key = self._full_value_cache_key.format(self.key)
-        cache.delete_pattern(key+'*')
-
-    @classmethod
-    def expire_nodes_full_value(cls, nodes=None):
-        key = cls._full_value_cache_key.format('*')
-        cache.delete_pattern(key+'*')
-
-
-class AssetsAmountMixin:
-    _assets_amount_cache_key = '_NODE_ASSETS_AMOUNT_{}'
-    _assets_amount = None
-    key = ''
-    cache_time = 3600 * 24 * 7
-
-    @property
-    def assets_amount(self):
-        """
-        获取节点下所有资产数量速度太慢，所以需要重写，使用cache等方案
-        :return:
-        """
-        if self._assets_amount is not None:
-            return self._assets_amount
-        cache_key = self._assets_amount_cache_key.format(self.key)
-        cached = cache.get(cache_key)
-        if cached is not None:
-            return cached
-        assets_amount = self.get_all_assets().count()
-        self.assets_amount = assets_amount
-        return assets_amount
-
-    @assets_amount.setter
-    def assets_amount(self, value):
-        self._assets_amount = value
-        cache_key = self._assets_amount_cache_key.format(self.key)
-        cache.set(cache_key, value, self.cache_time)
-
-    def expire_assets_amount(self):
-        ancestor_keys = self.get_ancestor_keys(with_self=True)
-        cache_keys = [self._assets_amount_cache_key.format(k) for k in
-                      ancestor_keys]
-        cache.delete_many(cache_keys)
-
-    @classmethod
-    def expire_nodes_assets_amount(cls, nodes=None):
-        key = cls._assets_amount_cache_key.format('*')
-        cache.delete_pattern(key)
-
-    @classmethod
-    def refresh_nodes(cls):
-        from ..utils import NodeUtil
-        util = NodeUtil(with_assets_amount=True)
-        util.set_assets_amount()
-        util.set_full_value()
-
-
-class Node(OrgModelMixin, FamilyMixin, FullValueMixin, AssetsAmountMixin):
-    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    key = models.CharField(unique=True, max_length=64, verbose_name=_("Key"))  # '1:1:1:1'
-    value = models.CharField(max_length=128, verbose_name=_("Value"))
-    child_mark = models.IntegerField(default=0)
-    date_create = models.DateTimeField(auto_now_add=True)
-
-    objects = OrgManager.from_queryset(NodeQuerySet)()
-    is_node = True
-    _parents = None
-
-    class Meta:
-        verbose_name = _("Node")
-        ordering = ['key']
-
-    def __str__(self):
-        return self.full_value
-
-    def __eq__(self, other):
-        if not other:
-            return False
-        return self.id == other.id
-
-    def __gt__(self, other):
-        # if self.is_root() and not other.is_root():
-        #     return False
-        # elif not self.is_root() and other.is_root():
-        #     return True
-        self_key = [int(k) for k in self.key.split(':')]
-        other_key = [int(k) for k in other.key.split(':')]
-        self_parent_key = self_key[:-1]
-        other_parent_key = other_key[:-1]
-
-        if self_parent_key and other_parent_key and \
-                self_parent_key == other_parent_key:
-            return self.value > other.value
-        # if len(self_parent_key) < len(other_parent_key):
-        #     return True
-        # elif len(self_parent_key) > len(other_parent_key):
-        #     return False
-        return self_key > other_key
-
-    def __lt__(self, other):
-        return not self.__gt__(other)
-
-    @property
-    def name(self):
-        return self.value
-
-    @property
-    def level(self):
-        return len(self.key.split(':'))
+            child_key = self.get_next_child_key()
+            child = self.__class__.objects.create(
+                id=_id, key=child_key, value=value
+            )
+            return child
 
     def get_next_child_key(self):
         mark = self.child_mark
@@ -310,76 +170,285 @@ class Node(OrgModelMixin, FamilyMixin, FullValueMixin, AssetsAmountMixin):
         count = max(values) + 1 if values else 1
         return '{} {}'.format(name, count)
 
-    def create_child(self, value, _id=None):
+    # Parents
+    @classmethod
+    def get_node_ancestor_keys(cls, key, with_self=False):
+        parent_keys = []
+        key_list = key.split(":")
+        if not with_self:
+            key_list.pop()
+        for i in range(len(key_list)):
+            parent_keys.append(":".join(key_list))
+            key_list.pop()
+        return parent_keys
+
+    def get_ancestor_keys(self, with_self=False):
+        return self.get_node_ancestor_keys(
+            self.key, with_self=with_self
+        )
+
+    @property
+    def ancestors(self):
+        return self.get_ancestors(with_self=False)
+
+    def get_ancestors(self, with_self=False):
+        ancestor_keys = self.get_ancestor_keys(with_self=with_self)
+        return self.__class__.objects.filter(key__in=ancestor_keys)
+
+    @property
+    def parent_key(self):
+        parent_key = ":".join(self.key.split(":")[:-1])
+        return parent_key
+
+    def is_parent(self, other):
+        return other.is_children(self)
+
+    @property
+    def parent(self):
+        if self.is_org_root():
+            return self
+        parent_key = self.parent_key
+        return Node.objects.get(key=parent_key)
+
+    @parent.setter
+    def parent(self, parent):
+        if not self.is_node:
+            self.key = parent.key + ':fake'
+            return
+        children = self.get_all_children()
+        old_key = self.key
         with transaction.atomic():
-            child_key = self.get_next_child_key()
-            child = self.__class__.objects.create(id=_id, key=child_key, value=value)
-            return child
+            self.key = parent.get_next_child_key()
+            self.save()
+            for child in children:
+                child.key = child.key.replace(old_key, self.key, 1)
+                child.save()
+
+    def get_siblings(self, with_self=False):
+        key = ':'.join(self.key.split(':')[:-1])
+        pattern = r'^{}:[0-9]+$'.format(key)
+        sibling = Node.objects.filter(
+            key__regex=pattern.format(self.key)
+        )
+        if not with_self:
+            sibling = sibling.exclude(key=self.key)
+        return sibling
+
+    def get_family(self):
+        ancestors = self.get_ancestors()
+        children = self.get_all_children()
+        return [*tuple(ancestors), self, *tuple(children)]
+
+
+class FullValueMixin:
+    key = ''
+
+    @lazyproperty
+    def full_value(self):
+        if self.is_org_root():
+            return self.value
+        value = self.tree().get_node_full_tag(self.key)
+        return value
+
+
+class NodeAssetsMixin:
+    key = ''
+    id = None
+
+    @lazyproperty
+    def assets_amount(self):
+        """
+        获取节点下所有资产数量速度太慢，所以需要重写，使用cache等方案
+        :return:
+        """
+        amount = self.tree().assets_amount(self.key)
+        return amount
+
+    def get_all_assets(self):
+        from .asset import Asset
+        if self.is_org_root():
+            return Asset.objects.filter(org_id=self.org_id)
+        pattern = '^{0}$|^{0}:'.format(self.key)
+        return Asset.objects.filter(nodes__key__regex=pattern).distinct()
 
     def get_assets(self):
         from .asset import Asset
-        if self.is_default_node():
-            assets = Asset.objects.filter(Q(nodes__id=self.id) | Q(nodes__isnull=True))
+        if self.is_org_root():
+            assets = Asset.objects.filter(Q(nodes=self) | Q(nodes__isnull=True))
         else:
-            assets = Asset.objects.filter(nodes__id=self.id)
+            assets = Asset.objects.filter(nodes=self)
         return assets.distinct()
 
     def get_valid_assets(self):
         return self.get_assets().valid()
 
-    def get_all_assets(self):
-        from .asset import Asset
-        pattern = r'^{0}$|^{0}:'.format(self.key)
-        args = []
-        kwargs = {}
-        if self.is_root():
-            args.append(Q(nodes__key__regex=pattern) | Q(nodes=None))
-        else:
-            kwargs['nodes__key__regex'] = pattern
-        assets = Asset.objects.filter(*args, **kwargs).distinct()
-        return assets
-
     def get_all_valid_assets(self):
         return self.get_all_assets().valid()
 
-    def is_default_node(self):
-        return self.is_root() and self.key == '1'
+    @classmethod
+    def _get_nodes_all_assets(cls, nodes_keys):
+        """
+        当节点比较多的时候，这种正则方式性能差极了
+        :param nodes_keys:
+        :return:
+        """
+        from .asset import Asset
+        nodes_keys = cls.clean_children_keys(nodes_keys)
+        nodes_children_pattern = set()
+        for key in nodes_keys:
+            children_pattern = cls.get_node_all_children_key_pattern(key)
+            nodes_children_pattern.add(children_pattern)
+        pattern = '|'.join(nodes_children_pattern)
+        return Asset.objects.filter(nodes__key__regex=pattern).distinct()
 
-    def is_root(self):
+    @classmethod
+    def get_nodes_all_assets_ids(cls, nodes_keys):
+        nodes_keys = cls.clean_children_keys(nodes_keys)
+        assets_ids = set()
+        for key in nodes_keys:
+            node_assets_ids = cls.tree().all_assets(key)
+            assets_ids.update(set(node_assets_ids))
+        return assets_ids
+
+    @classmethod
+    def get_nodes_all_assets(cls, nodes_keys, extra_assets_ids=None):
+        from .asset import Asset
+        nodes_keys = cls.clean_children_keys(nodes_keys)
+        assets_ids = cls.get_nodes_all_assets_ids(nodes_keys)
+        if extra_assets_ids:
+            assets_ids.update(set(extra_assets_ids))
+        return Asset.objects.filter(id__in=assets_ids)
+
+
+class SomeNodesMixin:
+    key = ''
+    default_key = '1'
+    default_value = 'Default'
+    ungrouped_key = '-10'
+    ungrouped_value = _('ungrouped')
+    empty_key = '-11'
+    empty_value = _("empty")
+
+    def is_default_node(self):
+        return self.key == self.default_key
+
+    def is_org_root(self):
         if self.key.isdigit():
             return True
         else:
             return False
 
     @classmethod
-    def create_root_node(cls):
+    def create_org_root_node(cls):
         # 如果使用current_org 在set_current_org时会死循环
-        _current_org = get_current_org()
+        ori_org = get_current_org()
         with transaction.atomic():
-            if not _current_org.is_real():
+            if not ori_org.is_real():
                 return cls.default_node()
             set_current_org(Organization.root())
             org_nodes_roots = cls.objects.filter(key__regex=r'^[0-9]+$')
-            org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True) or ['1']
+            org_nodes_roots_keys = org_nodes_roots.values_list('key', flat=True)
+            if not org_nodes_roots_keys:
+                org_nodes_roots_keys = ['1']
             key = max([int(k) for k in org_nodes_roots_keys])
             key = str(key + 1) if key != 0 else '2'
-            set_current_org(_current_org)
-            root = cls.objects.create(key=key, value=_current_org.name)
+            set_current_org(ori_org)
+            root = cls.objects.create(key=key, value=ori_org.name)
             return root
 
     @classmethod
-    def root(cls):
+    def org_root(cls):
         root = cls.objects.filter(key__regex=r'^[0-9]+$')
         if root:
             return root[0]
         else:
-            return cls.create_root_node()
+            return cls.create_org_root_node()
+
+    @classmethod
+    def ungrouped_node(cls):
+        with tmp_to_org(Organization.system()):
+            defaults = {'value': cls.ungrouped_key}
+            obj, created = cls.objects.get_or_create(
+                defaults=defaults, key=cls.ungrouped_key
+            )
+            return obj
+
+    @classmethod
+    def empty_node(cls):
+        with tmp_to_org(Organization.system()):
+            defaults = {'value': cls.empty_value}
+            obj, created = cls.objects.get_or_create(
+                defaults=defaults, key=cls.empty_key
+            )
+            return obj
 
     @classmethod
     def default_node(cls):
-        defaults = {'value': 'Default'}
-        obj, created = cls.objects.get_or_create(defaults=defaults, key='1')
-        return obj
+        with tmp_to_org(Organization.default()):
+            defaults = {'value': cls.default_value}
+            obj, created = cls.objects.get_or_create(
+                defaults=defaults, key=cls.default_key,
+            )
+            return obj
+
+    @classmethod
+    def initial_some_nodes(cls):
+        cls.default_node()
+        cls.empty_node()
+        cls.ungrouped_node()
+
+
+class Node(OrgModelMixin, SomeNodesMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixin):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    key = models.CharField(unique=True, max_length=64, verbose_name=_("Key"))  # '1:1:1:1'
+    value = models.CharField(max_length=128, verbose_name=_("Value"))
+    child_mark = models.IntegerField(default=0)
+    date_create = models.DateTimeField(auto_now_add=True)
+
+    objects = OrgManager.from_queryset(NodeQuerySet)()
+    is_node = True
+    _parents = None
+
+    class Meta:
+        verbose_name = _("Node")
+        ordering = ['key']
+
+    def __str__(self):
+        return self.value
+
+    def __eq__(self, other):
+        if not other:
+            return False
+        return self.id == other.id
+
+    def __gt__(self, other):
+        self_key = [int(k) for k in self.key.split(':')]
+        other_key = [int(k) for k in other.key.split(':')]
+        self_parent_key = self_key[:-1]
+        other_parent_key = other_key[:-1]
+
+        if self_parent_key and self_parent_key == other_parent_key:
+            return self.value > other.value
+        return self_key > other_key
+
+    def __lt__(self, other):
+        return not self.__gt__(other)
+
+    @property
+    def name(self):
+        return self.value
+
+    @property
+    def level(self):
+        return len(self.key.split(':'))
+
+    @classmethod
+    def refresh_nodes(cls):
+        cls.refresh_tree()
+
+    @classmethod
+    def refresh_assets(cls):
+        cls.refresh_node_assets()
 
     def as_tree_node(self):
         from common.tree import TreeNode
@@ -390,7 +459,7 @@ class Node(OrgModelMixin, FamilyMixin, FullValueMixin, AssetsAmountMixin):
             'title': name,
             'pId': self.parent_key,
             'isParent': True,
-            'open': self.is_root(),
+            'open': self.is_org_root(),
             'meta': {
                 'node': {
                     "id": self.id,
@@ -410,19 +479,20 @@ class Node(OrgModelMixin, FamilyMixin, FullValueMixin, AssetsAmountMixin):
             return
         return super().delete(using=using, keep_parents=keep_parents)
 
-    @classmethod
-    def get_queryset(cls):
-        from ..utils import NodeUtil
-        util = NodeUtil()
-        return sorted(util.nodes)
-
     @classmethod
     def generate_fake(cls, count=100):
         import random
         org = get_current_org()
         if not org or not org.is_real():
             Organization.default().change_to()
+        i = 0
+        while i < count:
+            nodes = list(cls.objects.all())
+            if count > 100:
+                length = 100
+            else:
+                length = count
 
-        for i in range(count):
-            node = random.choice(cls.objects.all())
-            node.create_child('Node {}'.format(i))
+            for i in range(length):
+                node = random.choice(nodes)
+                node.create_child('Node {}'.format(i))

apps/assets/models/user.py

@@ -31,7 +31,7 @@ class AdminUser(AssetUser):
     become = models.BooleanField(default=True)
     become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4)
     become_user = models.CharField(default='root', max_length=64)
-    _become_pass = models.CharField(default='', max_length=128)
+    _become_pass = models.CharField(default='', blank=True, max_length=128)
     CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}'
     _prefer = "admin_user"
 
@@ -96,7 +96,7 @@ class SystemUser(AssetUser):
     PROTOCOL_CHOICES = (
         (PROTOCOL_SSH, 'ssh'),
         (PROTOCOL_RDP, 'rdp'),
-        (PROTOCOL_TELNET, 'telnet (beta)'),
+        (PROTOCOL_TELNET, 'telnet'),
         (PROTOCOL_VNC, 'vnc'),
     )
 
@@ -148,16 +148,12 @@ class SystemUser(AssetUser):
         return True, None
 
     def get_all_assets(self):
-        args = [Q(systemuser=self)]
-        pattern = set()
+        from assets.models import Node
         nodes_keys = self.nodes.all().values_list('key', flat=True)
-        for key in nodes_keys:
-            pattern.add(r'^{0}$|^{0}:'.format(key))
-        pattern = '|'.join(list(pattern))
-        if pattern:
-            args.append(Q(nodes__key__regex=pattern))
-        args = reduce(lambda x, y: x | y, args)
-        assets = Asset.objects.filter(args).distinct()
+        assets_ids = set(self.assets.all().values_list('id', flat=True))
+        nodes_assets_ids = Node.get_nodes_all_assets_ids(nodes_keys)
+        assets_ids.update(nodes_assets_ids)
+        assets = Asset.objects.filter(id__in=assets_ids)
         return assets
 
     class Meta:

apps/assets/serializers/__init__.py

@@ -9,3 +9,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/serializers/admin_user.py

@@ -6,7 +6,7 @@ from rest_framework import serializers
 from common.serializers import AdaptedBulkListSerializer
 
 from ..models import Node, AdminUser
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from .base import AuthSerializer, AuthSerializerMixin
 

apps/assets/serializers/asset.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 #
+import re
 from rest_framework import serializers
 from django.db.models import Prefetch
 from django.utils.translation import ugettext_lazy as _
 
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from common.serializers import AdaptedBulkListSerializer
 from ..models import Asset, Node, Label
 from .base import ConnectivitySerializer
@@ -91,6 +92,15 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
             'org_name': {'label': _('Org name')}
         }
 
+    @staticmethod
+    def validate_hostname(hostname):
+        pattern = r"^[\._@a-zA-Z0-9-]+$"
+        res = re.match(pattern, hostname)
+        if res is None:
+            msg = _("* The hostname contains characters that are not allowed")
+            raise serializers.ValidationError(msg)
+        return hostname
+
     @classmethod
     def setup_eager_loading(cls, queryset):
         """ Perform necessary eager loading of data. """

apps/assets/serializers/asset_user.py

@@ -5,7 +5,7 @@ from django.utils.translation import ugettext as _
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from ..models import AuthBook, Asset
 from ..backends import AssetUserManager
 from .base import ConnectivitySerializer, AuthSerializerMixin
@@ -53,6 +53,7 @@ class AssetUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         if not validated_data.get("name") and validated_data.get("username"):
             validated_data["name"] = validated_data["username"]
         instance = AssetUserManager.create(**validated_data)
+        instance.set_version_and_latest()
         return instance
 
 

apps/assets/serializers/cmd_filter.py

@@ -7,7 +7,7 @@ from django.utils.translation import ugettext_lazy as _
 from common.fields import ChoiceDisplayField
 from common.serializers import AdaptedBulkListSerializer
 from ..models import CommandFilter, CommandFilterRule, SystemUser
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 
 class CommandFilterSerializer(BulkOrgResourceModelSerializer):

apps/assets/serializers/domain.py

@@ -3,7 +3,7 @@
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from ..models import Domain, Gateway
 from .base import AuthSerializerMixin

apps/assets/serializers/gathered_user.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.serializers import OrgResourceModelSerializerMixin
+from ..models import GatheredUser
+
+
+class GatheredUserSerializer(OrgResourceModelSerializerMixin):
+    class Meta:
+        model = GatheredUser
+        fields = [
+            'id', 'asset', 'hostname', 'ip', 'username',
+            'present', 'date_created', 'date_updated'
+        ]
+        read_only_fields = fields
+        labels = {
+            'hostname': _("Hostname"),
+            'ip': "IP"
+        }

apps/assets/serializers/label.py

@@ -3,7 +3,7 @@
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from ..models import Label
 

apps/assets/serializers/node.py

@@ -2,7 +2,7 @@
 from rest_framework import serializers
 from django.utils.translation import ugettext as _
 
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from ..models import Asset, Node
 
 
@@ -13,22 +13,23 @@ __all__ = [
 
 
 class NodeSerializer(BulkOrgResourceModelSerializer):
-    assets_amount = serializers.IntegerField(read_only=True)
     name = serializers.ReadOnlyField(source='value')
+    value = serializers.CharField(
+        required=False, allow_blank=True, allow_null=True, label=_("value")
+    )
 
     class Meta:
         model = Node
         only_fields = ['id', 'key', 'value', 'org_id']
-        fields = only_fields + ['name', 'assets_amount']
-        read_only_fields = [
-            'key', 'name', 'assets_amount', 'org_id',
-        ]
+        fields = only_fields + ['name', 'full_value']
+        read_only_fields = ['key', 'org_id']
 
     def validate_value(self, data):
-        instance = self.instance if self.instance else Node.root()
-        children = instance.parent.get_children()
-        children_values = [node.value for node in children if node != instance]
-        if data in children_values:
+        if not self.instance and not data:
+            return data
+        instance = self.instance
+        siblings = instance.get_siblings()
+        if siblings.filter(value=data):
             raise serializers.ValidationError(
                 _('The same level node name cannot be the same')
             )

apps/assets/serializers/system_user.py

@@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from common.serializers import AdaptedBulkListSerializer
 from common.utils import ssh_pubkey_gen
-from orgs.mixins import BulkOrgResourceModelSerializer
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from ..models import SystemUser
 from .base import AuthSerializer, AuthSerializerMixin
 

apps/assets/signals_handler.py

@@ -1,16 +1,20 @@
 # -*- coding: utf-8 -*-
 #
 from collections import defaultdict
-from django.db.models.signals import post_save, m2m_changed, post_delete
+from django.db.models.signals import (
+    post_save, m2m_changed, post_delete
+)
+from django.db.models.aggregates import Count
 from django.dispatch import receiver
 
-from common.utils import get_logger
+from common.utils import get_logger, timeit
 from common.decorator import on_transaction_commit
-from .models import Asset, SystemUser, Node, AuthBook
+from .models import Asset, SystemUser, Node
 from .tasks import (
     update_assets_hardware_info_util,
     test_asset_connectivity_util,
-    push_system_user_to_assets
+    push_system_user_to_assets,
+    add_nodes_assets_to_system_users
 )
 
 
@@ -27,96 +31,161 @@ def test_asset_conn_on_created(asset):
     test_asset_connectivity_util.delay([asset])
 
 
-@receiver(post_save, sender=Asset, dispatch_uid="my_unique_identifier")
+@receiver(post_save, sender=Asset)
 @on_transaction_commit
 def on_asset_created_or_update(sender, instance=None, created=False, **kwargs):
+    """
+    当资产创建时，更新硬件信息，更新可连接性
+    确保资产必须属于一个节点
+    """
     if created:
-        logger.info("Asset `{}` create signal received".format(instance))
+        logger.info("Asset create signal recv: {}".format(instance))
 
         # 获取资产硬件信息
         update_asset_hardware_info_on_created(instance)
         test_asset_conn_on_created(instance)
 
-        # 过期节点资产数量
-        nodes = instance.nodes.all()
-        Node.expire_nodes_assets_amount(nodes)
+        # 确保资产存在一个节点
+        has_node = instance.nodes.all().exists()
+        if not has_node:
+            instance.nodes.add(Node.org_root())
 
 
-@receiver(post_delete, sender=Asset, dispatch_uid="my_unique_identifier")
+@receiver(post_delete, sender=Asset)
 def on_asset_delete(sender, instance=None, **kwargs):
-    # 过期节点资产数量
-    nodes = instance.nodes.all()
-    Node.expire_nodes_assets_amount(nodes)
+    """
+    当资产删除时，刷新节点，节点中存在节点和资产的关系
+    """
+    logger.debug("Asset delete signal recv: {}".format(instance))
+    Node.refresh_assets()
 
 
-@receiver(post_save, sender=SystemUser, dispatch_uid="my_unique_identifier")
+@receiver(post_save, sender=SystemUser, dispatch_uid="jms")
 def on_system_user_update(sender, instance=None, created=True, **kwargs):
+    """
+    当系统用户更新时，可能更新了秘钥，用户名等，这时要自动推送系统用户到资产上,
+    其实应该当 用户名，密码，秘钥 sudo等更新时再推送，这里偷个懒,
+    这里直接取了 instance.assets 因为nodes和系统用户发生变化时，会自动将nodes下的资产
+    关联到上面
+    """
     if instance and not created:
-        logger.info("System user `{}` update signal received".format(instance))
-        assets = instance.assets.all()
+        logger.info("System user update signal recv: {}".format(instance))
+        assets = instance.assets.all().valid()
         push_system_user_to_assets.delay(instance, assets)
 
 
-@receiver(m2m_changed, sender=SystemUser.nodes.through)
-def on_system_user_nodes_change(sender, instance=None, **kwargs):
-    if instance and kwargs["action"] == "post_add":
-        logger.info("System user `{}` nodes update signal received".format(instance))
-        assets = set()
-        nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
-        for node in nodes:
-            assets.update(set(node.get_all_assets()))
-        instance.assets.add(*tuple(assets))
-
-
 @receiver(m2m_changed, sender=SystemUser.assets.through)
-def on_system_user_assets_change(sender, instance=None, **kwargs):
-    if instance and kwargs["action"] == "post_add":
-        assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
-        push_system_user_to_assets.delay(instance, assets)
+def on_system_user_assets_change(sender, instance=None, action='', model=None, pk_set=None, **kwargs):
+    """
+    当系统用户和资产关系发生变化时，应该重新推送系统用户到新添加的资产中
+    """
+    if action != "post_add":
+        return
+    logger.debug("System user assets change signal recv: {}".format(instance))
+    queryset = model.objects.filter(pk__in=pk_set)
+    if model == Asset:
+        system_users = [instance]
+        assets = queryset
+    else:
+        system_users = queryset
+        assets = [instance]
+    for system_user in system_users:
+        push_system_user_to_assets.delay(system_user, assets)
+
+
+@receiver(m2m_changed, sender=SystemUser.nodes.through)
+def on_system_user_nodes_change(sender, instance=None, action=None, model=None, pk_set=None, **kwargs):
+    """
+    当系统用户和节点关系发生变化时，应该将节点下资产关联到新的系统用户上
+    """
+    if action != "post_add":
+        return
+    logger.info("System user nodes update signal recv: {}".format(instance))
+
+    queryset = model.objects.filter(pk__in=pk_set)
+    if model == Node:
+        nodes_keys = queryset.values_list('key', flat=True)
+        system_users = [instance]
+    else:
+        nodes_keys = [instance.key]
+        system_users = queryset
+    add_nodes_assets_to_system_users.delay(nodes_keys, system_users)
 
 
 @receiver(m2m_changed, sender=Asset.nodes.through)
-def on_asset_node_changed(sender, instance=None, **kwargs):
-    logger.debug("Asset nodes change signal received")
-    Asset.expire_all_nodes_keys_cache()
-    if isinstance(instance, Asset):
-        if kwargs['action'] == 'pre_remove':
-            nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
-            Node.expire_nodes_assets_amount(nodes)
-        if kwargs['action'] == 'post_add':
-            nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
-            Node.expire_nodes_assets_amount(nodes)
-            system_users_assets = defaultdict(set)
-            system_users = SystemUser.objects.filter(nodes__in=nodes)
-            # 清理节点缓存
-            for system_user in system_users:
-                system_users_assets[system_user].update({instance})
-            for system_user, assets in system_users_assets.items():
-                system_user.assets.add(*tuple(assets))
+def on_asset_nodes_change(sender, instance=None, action='', **kwargs):
+    """
+    资产节点发生变化时，刷新节点
+    """
+    if action.startswith('post'):
+        logger.debug("Asset nodes change signal recv: {}".format(instance))
+        Node.refresh_assets()
 
 
 @receiver(m2m_changed, sender=Asset.nodes.through)
-def on_node_assets_changed(sender, instance=None, **kwargs):
-    if isinstance(instance, Node):
-        logger.debug("Node assets change signal {} received".format(instance))
-        # 当节点和资产关系发生改变时，过期资产数量缓存
-        instance.expire_assets_amount()
-        assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
-        if kwargs['action'] == 'post_add':
-            # 重新关联系统用户和资产的关系
-            system_users = SystemUser.objects.filter(nodes=instance)
-            for system_user in system_users:
-                system_user.assets.add(*tuple(assets))
+def on_asset_nodes_add(sender, instance=None, action='', model=None, pk_set=None, **kwargs):
+    """
+    当资产的节点发生变化时，或者 当节点的资产关系发生变化时，
+    节点下新增的资产，添加到节点关联的系统用户中
+    """
+    if action != "post_add":
+        return
+    logger.debug("Assets node add signal recv: {}".format(action))
+    queryset = model.objects.filter(pk__in=pk_set).values_list('id', flat=True)
+    if model == Node:
+        nodes = queryset
+        assets = [instance]
+    else:
+        nodes = [instance]
+        assets = queryset
+    # 节点资产发生变化时，将资产关联到节点关联的系统用户, 只关注新增的
+    system_users_assets = defaultdict(set)
+    system_users = SystemUser.objects.filter(nodes__in=nodes)
+    for system_user in system_users:
+        system_users_assets[system_user].update(set(assets))
+    for system_user, _assets in system_users_assets.items():
+        system_user.assets.add(*tuple(_assets))
 
 
-@receiver(post_save, sender=Node)
-def on_node_update_or_created(sender, instance=None, created=False, **kwargs):
-    if instance and not created:
-        instance.expire_full_value()
+@receiver(m2m_changed, sender=Asset.nodes.through)
+def on_asset_nodes_remove(sender, instance=None, action='', model=None,
+                          pk_set=None, **kwargs):
+
+    """
+    监控资产删除节点关系, 或节点删除资产，避免产生游离资产
+    """
+    if action not in ["post_remove", "pre_clear", "post_clear"]:
+        return
+    if action == "pre_clear":
+        if model == Node:
+            instance._nodes = list(instance.nodes.all())
+        else:
+            instance._assets = list(instance.assets.all())
+        return
+    logger.debug("Assets node remove signal recv: {}".format(action))
+    if action == "post_remove":
+        queryset = model.objects.filter(pk__in=pk_set)
+    else:
+        if model == Node:
+            queryset = instance._nodes
+        else:
+            queryset = instance._assets
+    if model == Node:
+        assets = [instance]
+    else:
+        assets = queryset
+    if isinstance(assets, list):
+        assets_not_has_node = []
+        for asset in assets:
+            if asset.nodes.all().count() == 0:
+                assets_not_has_node.append(asset.id)
+    else:
+        assets_not_has_node = assets.annotate(nodes_count=Count('nodes'))\
+            .filter(nodes_count=0).values_list('id', flat=True)
+    Node.org_root().assets.add(*tuple(assets_not_has_node))
 
 
-@receiver(post_save, sender=AuthBook)
-def on_auth_book_created(sender, instance=None, created=False, **kwargs):
-    if created:
-        logger.debug('Receive create auth book object signal.')
-        instance.set_version_and_latest()
+@receiver([post_save, post_delete], sender=Node)
+def on_node_update_or_created(sender, **kwargs):
+    # 刷新节点
+    Node.refresh_nodes()

apps/assets/tasks.py

@@ -1,625 +0,0 @@
-# ~*~ coding: utf-8 ~*~
-import json
-import re
-import os
-
-from collections import defaultdict
-from celery import shared_task
-from django.utils.translation import ugettext as _
-from django.core.cache import cache
-
-from common.utils import (
-    capacity_convert, sum_capacity, encrypt_password, get_logger
-)
-from ops.celery.decorator import (
-    register_as_period_task, after_app_shutdown_clean_periodic
-)
-
-from .models import SystemUser, AdminUser
-from .models.utils import Connectivity
-from . import const
-
-
-FORKS = 10
-TIMEOUT = 60
-logger = get_logger(__file__)
-CACHE_MAX_TIME = 60*60*2
-disk_pattern = re.compile(r'^hd|sd|xvd|vd')
-PERIOD_TASK = os.environ.get("PERIOD_TASK", "on")
-
-
-def check_asset_can_run_ansible(asset):
-    if not asset.is_active:
-        msg = _("Asset has been disabled, skipped: {}").format(asset)
-        logger.info(msg)
-        return False
-    if not asset.is_support_ansible():
-        msg = _("Asset may not be support ansible, skipped: {}").format(asset)
-        logger.info(msg)
-        return False
-    return True
-
-
-def clean_hosts(assets):
-    clean_assets = []
-    for asset in assets:
-        if not check_asset_can_run_ansible(asset):
-            continue
-        clean_assets.append(asset)
-    if not clean_assets:
-        logger.info(_("No assets matched, stop task"))
-    return clean_assets
-
-
-def clean_hosts_by_protocol(system_user, assets):
-    hosts = [
-        asset for asset in assets
-        if asset.has_protocol(system_user.protocol)
-    ]
-    if not hosts:
-        msg = _("No assets matched related system user protocol, stop task")
-        logger.info(msg)
-    return hosts
-
-
-@shared_task
-def set_assets_hardware_info(assets, result, **kwargs):
-    """
-    Using ops task run result, to update asset info
-
-    @shared_task must be exit, because we using it as a task callback, is must
-    be a celery task also
-    :param assets:
-    :param result:
-    :param kwargs: {task_name: ""}
-    :return:
-    """
-    result_raw = result[0]
-    assets_updated = []
-    success_result = result_raw.get('ok', {})
-
-    for asset in assets:
-        hostname = asset.hostname
-        info = success_result.get(hostname, {})
-        info = info.get('setup', {}).get('ansible_facts', {})
-        if not info:
-            logger.error(_("Get asset info failed: {}").format(hostname))
-            continue
-        ___vendor = info.get('ansible_system_vendor', 'Unknown')
-        ___model = info.get('ansible_product_name', 'Unknown')
-        ___sn = info.get('ansible_product_serial', 'Unknown')
-
-        for ___cpu_model in info.get('ansible_processor', []):
-            if ___cpu_model.endswith('GHz') or ___cpu_model.startswith("Intel"):
-                break
-        else:
-            ___cpu_model = 'Unknown'
-        ___cpu_model = ___cpu_model[:48]
-        ___cpu_count = info.get('ansible_processor_count', 0)
-        ___cpu_cores = info.get('ansible_processor_cores', None) or \
-                       len(info.get('ansible_processor', []))
-        ___cpu_vcpus = info.get('ansible_processor_vcpus', 0)
-        ___memory = '%s %s' % capacity_convert(
-            '{} MB'.format(info.get('ansible_memtotal_mb'))
-        )
-        disk_info = {}
-        for dev, dev_info in info.get('ansible_devices', {}).items():
-            if disk_pattern.match(dev) and dev_info['removable'] == '0':
-                disk_info[dev] = dev_info['size']
-        ___disk_total = '%s %s' % sum_capacity(disk_info.values())
-        ___disk_info = json.dumps(disk_info)
-
-        # ___platform = info.get('ansible_system', 'Unknown')
-        ___os = info.get('ansible_distribution', 'Unknown')
-        ___os_version = info.get('ansible_distribution_version', 'Unknown')
-        ___os_arch = info.get('ansible_architecture', 'Unknown')
-        ___hostname_raw = info.get('ansible_hostname', 'Unknown')
-
-        for k, v in locals().items():
-            if k.startswith('___'):
-                setattr(asset, k.strip('_'), v)
-        asset.save()
-        assets_updated.append(asset)
-    return assets_updated
-
-
-@shared_task
-def update_assets_hardware_info_util(assets, task_name=None):
-    """
-    Using ansible api to update asset hardware info
-    :param assets:  asset seq
-    :param task_name: task_name running
-    :return: result summary ['contacted': {}, 'dark': {}]
-    """
-    from ops.utils import update_or_create_ansible_task
-    if task_name is None:
-        task_name = _("Update some assets hardware info")
-    tasks = const.UPDATE_ASSETS_HARDWARE_TASKS
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-    created_by = str(assets[0].org_id)
-    task, created = update_or_create_ansible_task(
-        task_name, hosts=hosts, tasks=tasks, created_by=created_by,
-        pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
-    )
-    result = task.run()
-    set_assets_hardware_info(assets, result)
-    return result
-
-
-@shared_task
-def update_asset_hardware_info_manual(asset):
-    task_name = _("Update asset hardware info: {}").format(asset.hostname)
-    update_assets_hardware_info_util(
-        [asset], task_name=task_name
-    )
-
-
-@shared_task
-def update_assets_hardware_info_period():
-    """
-    Update asset hardware period task
-    :return:
-    """
-    if PERIOD_TASK != "on":
-        logger.debug("Period task disabled, update assets hardware info pass")
-        return
-
-
-##  ADMIN USER CONNECTIVE  ##
-
-
-@shared_task
-def test_asset_connectivity_util(assets, task_name=None):
-    from ops.utils import update_or_create_ansible_task
-
-    if task_name is None:
-        task_name = _("Test assets connectivity")
-
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-
-    hosts_category = {
-        'linux': {
-            'hosts': [],
-            'tasks': const.TEST_ADMIN_USER_CONN_TASKS
-        },
-        'windows': {
-            'hosts': [],
-            'tasks': const.TEST_WINDOWS_ADMIN_USER_CONN_TASKS
-        }
-    }
-    for host in hosts:
-        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
-            else hosts_category['linux']['hosts']
-        hosts_list.append(host)
-
-    results_summary = dict(
-        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
-    )
-    created_by = assets[0].org_id
-    for k, value in hosts_category.items():
-        if not value['hosts']:
-            continue
-        task, created = update_or_create_ansible_task(
-            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
-            pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
-            created_by=created_by,
-        )
-        raw, summary = task.run()
-        success = summary.get('success', False)
-        contacted = summary.get('contacted', {})
-        dark = summary.get('dark', {})
-
-        results_summary['success'] &= success
-        results_summary['contacted'].update(contacted)
-        results_summary['dark'].update(dark)
-
-    for asset in assets:
-        if asset.hostname in results_summary.get('dark', {}).keys():
-            asset.connectivity = Connectivity.unreachable()
-        elif asset.hostname in results_summary.get('contacted', {}).keys():
-            asset.connectivity = Connectivity.reachable()
-        else:
-            asset.connectivity = Connectivity.unknown()
-    return results_summary
-
-
-@shared_task
-def test_asset_connectivity_manual(asset):
-    task_name = _("Test assets connectivity: {}").format(asset)
-    summary = test_asset_connectivity_util([asset], task_name=task_name)
-
-    if summary.get('dark'):
-        return False, summary['dark']
-    else:
-        return True, ""
-
-
-@shared_task
-def test_admin_user_connectivity_util(admin_user, task_name):
-    """
-    Test asset admin user can connect or not. Using ansible api do that
-    :param admin_user:
-    :param task_name:
-    :return:
-    """
-    assets = admin_user.get_related_assets()
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-    summary = test_asset_connectivity_util(hosts, task_name)
-    return summary
-
-
-@shared_task
-@register_as_period_task(interval=3600)
-def test_admin_user_connectivity_period():
-    """
-    A period task that update the ansible task period
-    """
-    if PERIOD_TASK != "on":
-        logger.debug('Period task off, skip')
-        return
-    key = '_JMS_TEST_ADMIN_USER_CONNECTIVITY_PERIOD'
-    prev_execute_time = cache.get(key)
-    if prev_execute_time:
-        logger.debug("Test admin user connectivity, less than 40 minutes, skip")
-        return
-    cache.set(key, 1, 60*40)
-    admin_users = AdminUser.objects.all()
-    for admin_user in admin_users:
-        task_name = _("Test admin user connectivity period: {}").format(admin_user.name)
-        test_admin_user_connectivity_util(admin_user, task_name)
-    cache.set(key, 1, 60*40)
-
-
-@shared_task
-def test_admin_user_connectivity_manual(admin_user):
-    task_name = _("Test admin user connectivity: {}").format(admin_user.name)
-    test_admin_user_connectivity_util(admin_user, task_name)
-    return True
-
-
-##  System user connective ##
-
-
-@shared_task
-def test_system_user_connectivity_util(system_user, assets, task_name):
-    """
-    Test system cant connect his assets or not.
-    :param system_user:
-    :param assets:
-    :param task_name:
-    :return:
-    """
-    from ops.utils import update_or_create_ansible_task
-
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-
-    hosts = clean_hosts_by_protocol(system_user, hosts)
-    if not hosts:
-        return {}
-
-    hosts_category = {
-        'linux': {
-            'hosts': [],
-            'tasks': const.TEST_SYSTEM_USER_CONN_TASKS
-        },
-        'windows': {
-            'hosts': [],
-            'tasks': const.TEST_WINDOWS_SYSTEM_USER_CONN_TASKS
-        }
-    }
-    for host in hosts:
-        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
-            else hosts_category['linux']['hosts']
-        hosts_list.append(host)
-
-    results_summary = dict(
-        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
-    )
-    for k, value in hosts_category.items():
-        if not value['hosts']:
-            continue
-        task, created = update_or_create_ansible_task(
-            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
-            pattern='all', options=const.TASK_OPTIONS,
-            run_as=system_user.username, created_by=system_user.org_id,
-        )
-        raw, summary = task.run()
-        success = summary.get('success', False)
-        contacted = summary.get('contacted', {})
-        dark = summary.get('dark', {})
-
-        results_summary['success'] &= success
-        results_summary['contacted'].update(contacted)
-        results_summary['dark'].update(dark)
-
-    system_user.set_connectivity(results_summary)
-    return results_summary
-
-
-@shared_task
-def test_system_user_connectivity_manual(system_user):
-    task_name = _("Test system user connectivity: {}").format(system_user)
-    assets = system_user.get_all_assets()
-    return test_system_user_connectivity_util(system_user, assets, task_name)
-
-
-@shared_task
-def test_system_user_connectivity_a_asset(system_user, asset):
-    task_name = _("Test system user connectivity: {} => {}").format(
-        system_user, asset
-    )
-    return test_system_user_connectivity_util(system_user, [asset], task_name)
-
-
-@shared_task
-def test_system_user_connectivity_period():
-    if PERIOD_TASK != "on":
-        logger.debug("Period task disabled, test system user connectivity pass")
-        return
-    system_users = SystemUser.objects.all()
-    for system_user in system_users:
-        task_name = _("Test system user connectivity period: {}").format(system_user)
-        assets = system_user.get_all_assets()
-        test_system_user_connectivity_util(system_user, assets, task_name)
-
-
-####  Push system user tasks ####
-
-def get_push_linux_system_user_tasks(system_user):
-    tasks = [
-        {
-            'name': 'Add user {}'.format(system_user.username),
-            'action': {
-                'module': 'user',
-                'args': 'name={} shell={} state=present'.format(
-                    system_user.username, system_user.shell,
-                ),
-            }
-        },
-        {
-            'name': 'Check home dir exists',
-            'action': {
-                'module': 'stat',
-                'args': 'path=/home/{}'.format(system_user.username)
-            },
-            'register': 'home_existed'
-        },
-        {
-            'name': "Set home dir permission",
-            'action': {
-                'module': 'file',
-                'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username)
-            },
-            'when': 'home_existed.stat.exists == true'
-        }
-    ]
-    if system_user.password:
-        tasks.append({
-            'name': 'Set {} password'.format(system_user.username),
-            'action': {
-                'module': 'user',
-                'args': 'name={} shell={} state=present password={}'.format(
-                    system_user.username, system_user.shell,
-                    encrypt_password(system_user.password, salt="K3mIlKK"),
-                ),
-            }
-        })
-    if system_user.public_key:
-        tasks.append({
-            'name': 'Set {} authorized key'.format(system_user.username),
-            'action': {
-                'module': 'authorized_key',
-                'args': "user={} state=present key='{}'".format(
-                    system_user.username, system_user.public_key
-                )
-            }
-        })
-    if system_user.sudo:
-        sudo = system_user.sudo.replace('\r\n', '\n').replace('\r', '\n')
-        sudo_list = sudo.split('\n')
-        sudo_tmp = []
-        for s in sudo_list:
-            sudo_tmp.append(s.strip(','))
-        sudo = ','.join(sudo_tmp)
-        tasks.append({
-            'name': 'Set {} sudo setting'.format(system_user.username),
-            'action': {
-                'module': 'lineinfile',
-                'args': "dest=/etc/sudoers state=present regexp='^{0} ALL=' "
-                        "line='{0} ALL=(ALL) NOPASSWD: {1}' "
-                        "validate='visudo -cf %s'".format(
-                    system_user.username, sudo,
-                )
-            }
-        })
-
-    return tasks
-
-
-def get_push_windows_system_user_tasks(system_user):
-    tasks = []
-    if system_user.password:
-        tasks.append({
-            'name': 'Add user {}'.format(system_user.username),
-            'action': {
-                'module': 'win_user',
-                'args': 'fullname={} '
-                        'name={} '
-                        'password={} '
-                        'state=present '
-                        'update_password=always '
-                        'password_expired=no '
-                        'password_never_expires=yes '
-                        'groups="Users,Remote Desktop Users" '
-                        'groups_action=add '
-                        ''.format(system_user.name,
-                                  system_user.username,
-                                  system_user.password),
-            }
-        })
-    return tasks
-
-
-def get_push_system_user_tasks(host, system_user):
-    if host.is_unixlike():
-        tasks = get_push_linux_system_user_tasks(system_user)
-    elif host.is_windows():
-        tasks = get_push_windows_system_user_tasks(system_user)
-    else:
-        msg = _(
-            "The asset {} system platform {} does not "
-            "support run Ansible tasks".format(host.hostname, host.platform)
-        )
-        logger.info(msg)
-        tasks = []
-    return tasks
-
-
-@shared_task
-def push_system_user_util(system_user, assets, task_name):
-    from ops.utils import update_or_create_ansible_task
-    if not system_user.is_need_push():
-        msg = _("Push system user task skip, auto push not enable or "
-                "protocol is not ssh or rdp: {}").format(system_user.name)
-        logger.info(msg)
-        return {}
-
-    # Set root as system user is dangerous
-    if system_user.username.lower() in ["root", "administrator"]:
-        msg = _("For security, do not push user {}".format(system_user.username))
-        logger.info(msg)
-        return {}
-
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-
-    hosts = clean_hosts_by_protocol(system_user, hosts)
-    if not hosts:
-        return {}
-
-    for host in hosts:
-        system_user.load_specific_asset_auth(host)
-        tasks = get_push_system_user_tasks(host, system_user)
-        if not tasks:
-            continue
-        task, created = update_or_create_ansible_task(
-            task_name=task_name, hosts=[host], tasks=tasks, pattern='all',
-            options=const.TASK_OPTIONS, run_as_admin=True,
-            created_by=system_user.org_id,
-        )
-        task.run()
-
-
-@shared_task
-def push_system_user_to_assets_manual(system_user):
-    assets = system_user.get_all_assets()
-    task_name = _("Push system users to assets: {}").format(system_user.name)
-    return push_system_user_util(system_user, assets, task_name=task_name)
-
-
-@shared_task
-def push_system_user_a_asset_manual(system_user, asset):
-    task_name = _("Push system users to asset: {} => {}").format(
-        system_user.name, asset
-    )
-    return push_system_user_util(system_user, [asset], task_name=task_name)
-
-
-@shared_task
-def push_system_user_to_assets(system_user, assets):
-    task_name = _("Push system users to assets: {}").format(system_user.name)
-    return push_system_user_util(system_user, assets, task_name)
-
-
-@shared_task
-@after_app_shutdown_clean_periodic
-def test_system_user_connectability_period():
-    pass
-
-
-@shared_task
-@after_app_shutdown_clean_periodic
-def test_admin_user_connectability_period():
-    pass
-
-
-#### Test Asset user connectivity task ####
-
-def get_test_asset_user_connectivity_tasks(asset):
-    if asset.is_unixlike():
-        tasks = const.TEST_ASSET_USER_CONN_TASKS
-    elif asset.is_windows():
-        tasks = const.TEST_WINDOWS_ASSET_USER_CONN_TASKS
-    else:
-        msg = _(
-            "The asset {} system platform {} does not "
-            "support run Ansible tasks".format(asset.hostname, asset.platform)
-        )
-        logger.info(msg)
-        tasks = []
-    return tasks
-
-
-@shared_task
-def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False):
-    """
-    :param asset_user: <AuthBook>对象
-    :param task_name:
-    :param run_as_admin:
-    :return:
-    """
-    from ops.utils import update_or_create_ansible_task
-
-    if not check_asset_can_run_ansible(asset_user.asset):
-        return
-
-    tasks = get_test_asset_user_connectivity_tasks(asset_user.asset)
-    if not tasks:
-        logger.debug("No tasks ")
-        return
-
-    args = (task_name,)
-    kwargs = {
-        'hosts': [asset_user.asset], 'tasks': tasks,
-        'pattern': 'all', 'options': const.TASK_OPTIONS,
-        'created_by': asset_user.org_id,
-    }
-    if run_as_admin:
-        kwargs["run_as_admin"] = True
-    else:
-        kwargs["run_as"] = asset_user.username
-    task, created = update_or_create_ansible_task(*args, **kwargs)
-    raw, summary = task.run()
-    asset_user.set_connectivity(summary)
-
-
-@shared_task
-def test_asset_users_connectivity_manual(asset_users, run_as_admin=False):
-    """
-    :param asset_users: <AuthBook>对象
-    """
-    for asset_user in asset_users:
-        task_name = _("Test asset user connectivity: {}").format(asset_user)
-        test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=run_as_admin)
-
-
-# @shared_task
-# @register_as_period_task(interval=3600)
-# @after_app_ready_start
-# @after_app_shutdown_clean_periodic
-# def push_system_user_period():
-#     for system_user in SystemUser.objects.all():
-#         push_system_user_related_nodes(system_user)
-
-
-
-

apps/assets/tasks/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+from .utils import *
+from .common import *
+from .admin_user_connectivity import *
+from .asset_connectivity import *
+from .asset_user_connectivity import *
+from .gather_asset_users import *
+from .gather_asset_hardware_info import *
+from .push_system_user import *
+from .system_user_connectivity import *

apps/assets/tasks/admin_user_connectivity.py

@@ -0,0 +1,65 @@
+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+from django.core.cache import cache
+
+from common.utils import get_logger
+from ops.celery.decorator import register_as_period_task
+
+from ..models import AdminUser
+from .utils import clean_hosts
+from .asset_connectivity import test_asset_connectivity_util
+from . import const
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'test_admin_user_connectivity_util', 'test_admin_user_connectivity_manual',
+    'test_admin_user_connectivity_period'
+]
+
+
+@shared_task(queue="ansible")
+def test_admin_user_connectivity_util(admin_user, task_name):
+    """
+    Test asset admin user can connect or not. Using ansible api do that
+    :param admin_user:
+    :param task_name:
+    :return:
+    """
+    assets = admin_user.get_related_assets()
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+    summary = test_asset_connectivity_util(hosts, task_name)
+    return summary
+
+
+@shared_task(queue="ansible")
+@register_as_period_task(interval=3600)
+def test_admin_user_connectivity_period():
+    """
+    A period task that update the ansible task period
+    """
+    if const.PERIOD_TASK_ENABLED:
+        logger.debug('Period task off, skip')
+        return
+    key = '_JMS_TEST_ADMIN_USER_CONNECTIVITY_PERIOD'
+    prev_execute_time = cache.get(key)
+    if prev_execute_time:
+        logger.debug("Test admin user connectivity, less than 40 minutes, skip")
+        return
+    cache.set(key, 1, 60*40)
+    admin_users = AdminUser.objects.all()
+    for admin_user in admin_users:
+        task_name = _("Test admin user connectivity period: {}").format(admin_user.name)
+        test_admin_user_connectivity_util(admin_user, task_name)
+    cache.set(key, 1, 60*40)
+
+
+@shared_task(queue="ansible")
+def test_admin_user_connectivity_manual(admin_user):
+    task_name = _("Test admin user connectivity: {}").format(admin_user.name)
+    test_admin_user_connectivity_util(admin_user, task_name)
+    return True

apps/assets/tasks/asset_connectivity.py

@@ -0,0 +1,81 @@
+# ~*~ coding: utf-8 ~*~
+from collections import defaultdict
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+from ..models.utils import Connectivity
+from . import const
+from .utils import clean_hosts
+
+
+logger = get_logger(__file__)
+__all__ = ['test_asset_connectivity_util', 'test_asset_connectivity_manual']
+
+
+@shared_task(queue="ansible")
+def test_asset_connectivity_util(assets, task_name=None):
+    from ops.utils import update_or_create_ansible_task
+
+    if task_name is None:
+        task_name = _("Test assets connectivity")
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.TEST_ADMIN_USER_CONN_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.TEST_WINDOWS_ADMIN_USER_CONN_TASKS
+        }
+    }
+    for host in hosts:
+        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(host)
+
+    results_summary = dict(
+        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
+    )
+    created_by = assets[0].org_id
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
+            created_by=created_by,
+        )
+        raw, summary = task.run()
+        success = summary.get('success', False)
+        contacted = summary.get('contacted', {})
+        dark = summary.get('dark', {})
+
+        results_summary['success'] &= success
+        results_summary['contacted'].update(contacted)
+        results_summary['dark'].update(dark)
+
+    for asset in assets:
+        if asset.hostname in results_summary.get('dark', {}).keys():
+            asset.connectivity = Connectivity.unreachable()
+        elif asset.hostname in results_summary.get('contacted', {}).keys():
+            asset.connectivity = Connectivity.reachable()
+        else:
+            asset.connectivity = Connectivity.unknown()
+    return results_summary
+
+
+@shared_task(queue="ansible")
+def test_asset_connectivity_manual(asset):
+    task_name = _("Test assets connectivity: {}").format(asset)
+    summary = test_asset_connectivity_util([asset], task_name=task_name)
+
+    if summary.get('dark'):
+        return False, summary['dark']
+    else:
+        return True, ""

apps/assets/tasks/asset_user_connectivity.py

@@ -0,0 +1,77 @@
+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+from . import const
+from .utils import check_asset_can_run_ansible
+
+
+logger = get_logger(__file__)
+
+
+__all__ = [
+    'test_asset_user_connectivity_util', 'test_asset_users_connectivity_manual',
+    'get_test_asset_user_connectivity_tasks',
+]
+
+
+def get_test_asset_user_connectivity_tasks(asset):
+    if asset.is_unixlike():
+        tasks = const.TEST_ASSET_USER_CONN_TASKS
+    elif asset.is_windows():
+        tasks = const.TEST_WINDOWS_ASSET_USER_CONN_TASKS
+    else:
+        msg = _(
+            "The asset {} system platform {} does not "
+            "support run Ansible tasks".format(asset.hostname, asset.platform)
+        )
+        logger.info(msg)
+        tasks = []
+    return tasks
+
+
+@shared_task(queue="ansible")
+def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False):
+    """
+    :param asset_user: <AuthBook>对象
+    :param task_name:
+    :param run_as_admin:
+    :return:
+    """
+    from ops.utils import update_or_create_ansible_task
+
+    if not check_asset_can_run_ansible(asset_user.asset):
+        return
+
+    tasks = get_test_asset_user_connectivity_tasks(asset_user.asset)
+    if not tasks:
+        logger.debug("No tasks ")
+        return
+
+    args = (task_name,)
+    kwargs = {
+        'hosts': [asset_user.asset], 'tasks': tasks,
+        'pattern': 'all', 'options': const.TASK_OPTIONS,
+        'created_by': asset_user.org_id,
+    }
+    if run_as_admin:
+        kwargs["run_as_admin"] = True
+    else:
+        kwargs["run_as"] = asset_user.username
+    task, created = update_or_create_ansible_task(*args, **kwargs)
+    raw, summary = task.run()
+    asset_user.set_connectivity(summary)
+
+
+@shared_task(queue="ansible")
+def test_asset_users_connectivity_manual(asset_users, run_as_admin=False):
+    """
+    :param asset_users: <AuthBook>对象
+    """
+    for asset_user in asset_users:
+        task_name = _("Test asset user connectivity: {}").format(asset_user)
+        test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=run_as_admin)
+
+

apps/assets/tasks/common.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+#
+
+from celery import shared_task
+
+__all__ = ['add_nodes_assets_to_system_users']
+
+
+@shared_task
+def add_nodes_assets_to_system_users(nodes_keys, system_users):
+    from ..models import Node
+    assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True)
+    for system_user in system_users:
+        system_user.assets.add(*tuple(assets))

apps/assets/tasks/const.py

@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 #
+import os
 from django.utils.translation import ugettext_lazy as _
 
+
+PERIOD_TASK_ENABLED = os.environ.get("PERIOD_TASK", "on") == 'on'
+
 UPDATE_ASSETS_HARDWARE_TASKS = [
    {
        'name': "setup",
@@ -79,3 +83,22 @@ CONNECTIVITY_CHOICES = (
     (CONN_UNKNOWN, _("Unknown")),
 )
 
+GATHER_ASSET_USERS_TASKS = [
+    {
+        "name": "gather host users",
+        "action": {
+            "module": 'getent',
+            "args": "database=passwd"
+        },
+    },
+]
+
+GATHER_ASSET_USERS_TASKS_WINDOWS = [
+    {
+        "name": "gather windows host users",
+        "action": {
+            "module": 'win_shell',
+            "args": "net user"
+        }
+    }
+]

apps/assets/tasks/gather_asset_hardware_info.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+#
+import json
+import re
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import (
+    capacity_convert, sum_capacity, get_logger
+)
+from . import const
+from .utils import clean_hosts
+
+
+logger = get_logger(__file__)
+disk_pattern = re.compile(r'^hd|sd|xvd|vd|nv')
+__all__ = [
+    'update_assets_hardware_info_util', 'update_asset_hardware_info_manual',
+    'update_assets_hardware_info_period',
+]
+
+
+def set_assets_hardware_info(assets, result, **kwargs):
+    """
+    Using ops task run result, to update asset info
+
+    @shared_task must be exit, because we using it as a task callback, is must
+    be a celery task also
+    :param assets:
+    :param result:
+    :param kwargs: {task_name: ""}
+    :return:
+    """
+    result_raw = result[0]
+    assets_updated = []
+    success_result = result_raw.get('ok', {})
+
+    for asset in assets:
+        hostname = asset.hostname
+        info = success_result.get(hostname, {})
+        info = info.get('setup', {}).get('ansible_facts', {})
+        if not info:
+            logger.error(_("Get asset info failed: {}").format(hostname))
+            continue
+        ___vendor = info.get('ansible_system_vendor', 'Unknown')
+        ___model = info.get('ansible_product_name', 'Unknown')
+        ___sn = info.get('ansible_product_serial', 'Unknown')
+
+        for ___cpu_model in info.get('ansible_processor', []):
+            if ___cpu_model.endswith('GHz') or ___cpu_model.startswith("Intel"):
+                break
+        else:
+            ___cpu_model = 'Unknown'
+        ___cpu_model = ___cpu_model[:48]
+        ___cpu_count = info.get('ansible_processor_count', 0)
+        ___cpu_cores = info.get('ansible_processor_cores', None) or \
+                       len(info.get('ansible_processor', []))
+        ___cpu_vcpus = info.get('ansible_processor_vcpus', 0)
+        ___memory = '%s %s' % capacity_convert(
+            '{} MB'.format(info.get('ansible_memtotal_mb'))
+        )
+        disk_info = {}
+        for dev, dev_info in info.get('ansible_devices', {}).items():
+            if disk_pattern.match(dev) and dev_info['removable'] == '0':
+                disk_info[dev] = dev_info['size']
+        ___disk_total = '%.1f %s' % sum_capacity(disk_info.values())
+        ___disk_info = json.dumps(disk_info)
+
+        # ___platform = info.get('ansible_system', 'Unknown')
+        ___os = info.get('ansible_distribution', 'Unknown')
+        ___os_version = info.get('ansible_distribution_version', 'Unknown')
+        ___os_arch = info.get('ansible_architecture', 'Unknown')
+        ___hostname_raw = info.get('ansible_hostname', 'Unknown')
+
+        for k, v in locals().items():
+            if k.startswith('___'):
+                setattr(asset, k.strip('_'), v)
+        asset.save()
+        assets_updated.append(asset)
+    return assets_updated
+
+
+@shared_task
+def update_assets_hardware_info_util(assets, task_name=None):
+    """
+    Using ansible api to update asset hardware info
+    :param assets:  asset seq
+    :param task_name: task_name running
+    :return: result summary ['contacted': {}, 'dark': {}]
+    """
+    from ops.utils import update_or_create_ansible_task
+    if task_name is None:
+        task_name = _("Update some assets hardware info")
+    tasks = const.UPDATE_ASSETS_HARDWARE_TASKS
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+    created_by = str(assets[0].org_id)
+    task, created = update_or_create_ansible_task(
+        task_name, hosts=hosts, tasks=tasks, created_by=created_by,
+        pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
+    )
+    result = task.run()
+    set_assets_hardware_info(assets, result)
+    return True
+
+
+@shared_task(queue="ansible")
+def update_asset_hardware_info_manual(asset):
+    task_name = _("Update asset hardware info: {}").format(asset.hostname)
+    update_assets_hardware_info_util(
+        [asset], task_name=task_name
+    )
+
+
+@shared_task(queue="ansible")
+def update_assets_hardware_info_period():
+    """
+    Update asset hardware period task
+    :return:
+    """
+    if not const.PERIOD_TASK_ENABLED:
+        logger.debug("Period task disabled, update assets hardware info pass")
+        return

apps/assets/tasks/gather_asset_users.py

@@ -0,0 +1,135 @@
+# ~*~ coding: utf-8 ~*~
+
+import re
+from collections import defaultdict
+from celery import shared_task
+
+from django.utils.translation import ugettext as _
+
+from orgs.utils import tmp_to_org
+from common.utils import get_logger
+from ..models import GatheredUser, Node
+from .utils import clean_hosts
+from . import const
+
+__all__ = ['gather_asset_users', 'gather_nodes_asset_users']
+logger = get_logger(__name__)
+space = re.compile('\s+')
+ignore_login_shell = re.compile(r'nologin$|sync$|shutdown$|halt$')
+
+
+def parse_linux_result_to_users(result):
+    task_result = {}
+    for task_name, raw in result.items():
+        res = raw.get('ansible_facts', {}).get('getent_passwd')
+        if res:
+            task_result = res
+            break
+    if not task_result or not isinstance(task_result, dict):
+        return []
+    users = []
+    for username, attr in task_result.items():
+        if ignore_login_shell.search(attr[-1]):
+            continue
+        users.append(username)
+    return users
+
+
+def parse_windows_result_to_users(result):
+    task_result = []
+    for task_name, raw in result.items():
+        res = raw.get('stdout_lines', {})
+        if res:
+            task_result = res
+            break
+    if not task_result:
+        return []
+
+    users = []
+
+    for i in range(4):
+        task_result.pop(0)
+    for i in range(2):
+        task_result.pop()
+
+    for line in task_result:
+        user = space.split(line)
+        if user[0]:
+            users.append(user[0])
+    return users
+
+
+def add_asset_users(assets, results):
+    assets_map = {a.hostname: a for a in assets}
+    parser_map = {
+        'linux': parse_linux_result_to_users,
+        'windows': parse_windows_result_to_users
+    }
+
+    assets_users_map = {}
+
+    for platform, platform_results in results.items():
+        for hostname, res in platform_results.items():
+            parse = parser_map.get(platform)
+            users = parse(res)
+            logger.debug('Gathered host users: {} {}'.format(hostname, users))
+            asset = assets_map.get(hostname)
+            if not asset:
+                continue
+            assets_users_map[asset] = users
+
+    for asset, users in assets_users_map.items():
+        with tmp_to_org(asset.org_id):
+            GatheredUser.objects.filter(asset=asset, present=True)\
+                .update(present=False)
+            for username in users:
+                defaults = {'asset': asset, 'username': username, 'present': True}
+                GatheredUser.objects.update_or_create(
+                    defaults=defaults, asset=asset, username=username,
+                )
+
+
+@shared_task(queue="ansible")
+def gather_asset_users(assets, task_name=None):
+    from ops.utils import update_or_create_ansible_task
+    if task_name is None:
+        task_name = _("Gather assets users")
+    assets = clean_hosts(assets)
+    if not assets:
+        return
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.GATHER_ASSET_USERS_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.GATHER_ASSET_USERS_TASKS_WINDOWS
+        }
+    }
+    for asset in assets:
+        hosts_list = hosts_category['windows']['hosts'] if asset.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(asset)
+
+    results = {'linux': defaultdict(dict), 'windows': defaultdict(dict)}
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        _task_name = '{}: {}'.format(task_name, k)
+        task, created = update_or_create_ansible_task(
+            task_name=_task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS,
+            run_as_admin=True, created_by=value['hosts'][0].org_id,
+        )
+        raw, summary = task.run()
+        results[k].update(raw['ok'])
+    add_asset_users(assets, results)
+
+
+@shared_task(queue="ansible")
+def gather_nodes_asset_users(nodes_key):
+    assets = Node.get_nodes_all_assets(nodes_key)
+    assets_groups_by_100 = [assets[i:i+100] for i in range(0, len(assets), 100)]
+    for _assets in assets_groups_by_100:
+        gather_asset_users(_assets)

apps/assets/tasks/push_system_user.py

@@ -0,0 +1,202 @@
+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import encrypt_password, get_logger
+from . import const
+from .utils import clean_hosts_by_protocol, clean_hosts
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'push_system_user_util', 'push_system_user_to_assets',
+    'push_system_user_to_assets_manual', 'push_system_user_a_asset_manual',
+]
+
+
+def get_push_linux_system_user_tasks(system_user):
+    tasks = [
+        {
+            'name': 'Add user {}'.format(system_user.username),
+            'action': {
+                'module': 'user',
+                'args': 'name={} shell={} state=present'.format(
+                    system_user.username, system_user.shell,
+                ),
+            }
+        },
+        {
+            'name': 'Add group {}'.format(system_user.username),
+            'action': {
+                'module': 'group',
+                'args': 'name={} state=present'.format(
+                    system_user.username,
+                ),
+            }
+        },
+        {
+            'name': 'Check home dir exists',
+            'action': {
+                'module': 'stat',
+                'args': 'path=/home/{}'.format(system_user.username)
+            },
+            'register': 'home_existed'
+        },
+        {
+            'name': "Set home dir permission",
+            'action': {
+                'module': 'file',
+                'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username)
+            },
+            'when': 'home_existed.stat.exists == true'
+        }
+    ]
+    if system_user.password:
+        tasks.append({
+            'name': 'Set {} password'.format(system_user.username),
+            'action': {
+                'module': 'user',
+                'args': 'name={} shell={} state=present password={}'.format(
+                    system_user.username, system_user.shell,
+                    encrypt_password(system_user.password, salt="K3mIlKK"),
+                ),
+            }
+        })
+    if system_user.public_key:
+        tasks.append({
+            'name': 'Set {} authorized key'.format(system_user.username),
+            'action': {
+                'module': 'authorized_key',
+                'args': "user={} state=present key='{}'".format(
+                    system_user.username, system_user.public_key
+                )
+            }
+        })
+    if system_user.sudo:
+        sudo = system_user.sudo.replace('\r\n', '\n').replace('\r', '\n')
+        sudo_list = sudo.split('\n')
+        sudo_tmp = []
+        for s in sudo_list:
+            sudo_tmp.append(s.strip(','))
+        sudo = ','.join(sudo_tmp)
+        tasks.append({
+            'name': 'Set {} sudo setting'.format(system_user.username),
+            'action': {
+                'module': 'lineinfile',
+                'args': "dest=/etc/sudoers state=present regexp='^{0} ALL=' "
+                        "line='{0} ALL=(ALL) NOPASSWD: {1}' "
+                        "validate='visudo -cf %s'".format(
+                    system_user.username, sudo,
+                )
+            }
+        })
+
+    return tasks
+
+
+def get_push_windows_system_user_tasks(system_user):
+    tasks = []
+    if not system_user.password:
+        return tasks
+    tasks.append({
+        'name': 'Add user {}'.format(system_user.username),
+        'action': {
+            'module': 'win_user',
+            'args': 'fullname={} '
+                    'name={} '
+                    'password={} '
+                    'state=present '
+                    'update_password=always '
+                    'password_expired=no '
+                    'password_never_expires=yes '
+                    'groups="Users,Remote Desktop Users" '
+                    'groups_action=add '
+                    ''.format(system_user.name,
+                              system_user.username,
+                              system_user.password),
+        }
+    })
+    return tasks
+
+
+def get_push_system_user_tasks(host, system_user):
+    if host.is_unixlike():
+        tasks = get_push_linux_system_user_tasks(system_user)
+    elif host.is_windows():
+        tasks = get_push_windows_system_user_tasks(system_user)
+    else:
+        msg = _(
+            "The asset {} system platform {} does not "
+            "support run Ansible tasks".format(host.hostname, host.platform)
+        )
+        logger.info(msg)
+        tasks = []
+    return tasks
+
+
+@shared_task(queue="ansible")
+def push_system_user_util(system_user, assets, task_name):
+    from ops.utils import update_or_create_ansible_task
+    if not system_user.is_need_push():
+        msg = _("Push system user task skip, auto push not enable or "
+                "protocol is not ssh or rdp: {}").format(system_user.name)
+        logger.info(msg)
+        return {}
+
+    # Set root as system user is dangerous
+    if system_user.username.lower() in ["root", "administrator"]:
+        msg = _("For security, do not push user {}".format(system_user.username))
+        logger.info(msg)
+        return {}
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts = clean_hosts_by_protocol(system_user, hosts)
+    if not hosts:
+        return {}
+
+    for host in hosts:
+        system_user.load_specific_asset_auth(host)
+        tasks = get_push_system_user_tasks(host, system_user)
+        if not tasks:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=[host], tasks=tasks, pattern='all',
+            options=const.TASK_OPTIONS, run_as_admin=True,
+            created_by=system_user.org_id,
+        )
+        task.run()
+
+
+@shared_task(queue="ansible")
+def push_system_user_to_assets_manual(system_user):
+    assets = system_user.get_all_assets()
+    task_name = _("Push system users to assets: {}").format(system_user.name)
+    return push_system_user_util(system_user, assets, task_name=task_name)
+
+
+@shared_task(queue="ansible")
+def push_system_user_a_asset_manual(system_user, asset):
+    task_name = _("Push system users to asset: {} => {}").format(
+        system_user.name, asset
+    )
+    return push_system_user_util(system_user, [asset], task_name=task_name)
+
+
+@shared_task(queue="ansible")
+def push_system_user_to_assets(system_user, assets):
+    task_name = _("Push system users to assets: {}").format(system_user.name)
+    return push_system_user_util(system_user, assets, task_name)
+
+
+
+# @shared_task
+# @register_as_period_task(interval=3600)
+# @after_app_ready_start
+# @after_app_shutdown_clean_periodic
+# def push_system_user_period():
+#     for system_user in SystemUser.objects.all():
+#         push_system_user_related_nodes(system_user)

apps/assets/tasks/system_user_connectivity.py

@@ -0,0 +1,101 @@
+
+from collections import defaultdict
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+
+from ..models import SystemUser
+from . import const
+from .utils import clean_hosts, clean_hosts_by_protocol
+
+logger = get_logger(__name__)
+__all__ = [
+    'test_system_user_connectivity_util', 'test_system_user_connectivity_manual',
+    'test_system_user_connectivity_period', 'test_system_user_connectivity_a_asset',
+]
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_util(system_user, assets, task_name):
+    """
+    Test system cant connect his assets or not.
+    :param system_user:
+    :param assets:
+    :param task_name:
+    :return:
+    """
+    from ops.utils import update_or_create_ansible_task
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts = clean_hosts_by_protocol(system_user, hosts)
+    if not hosts:
+        return {}
+
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.TEST_SYSTEM_USER_CONN_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.TEST_WINDOWS_SYSTEM_USER_CONN_TASKS
+        }
+    }
+    for host in hosts:
+        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(host)
+
+    results_summary = dict(
+        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
+    )
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS,
+            run_as=system_user.username, created_by=system_user.org_id,
+        )
+        raw, summary = task.run()
+        success = summary.get('success', False)
+        contacted = summary.get('contacted', {})
+        dark = summary.get('dark', {})
+
+        results_summary['success'] &= success
+        results_summary['contacted'].update(contacted)
+        results_summary['dark'].update(dark)
+
+    system_user.set_connectivity(results_summary)
+    return results_summary
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_manual(system_user):
+    task_name = _("Test system user connectivity: {}").format(system_user)
+    assets = system_user.get_all_assets()
+    return test_system_user_connectivity_util(system_user, assets, task_name)
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_a_asset(system_user, asset):
+    task_name = _("Test system user connectivity: {} => {}").format(
+        system_user, asset
+    )
+    return test_system_user_connectivity_util(system_user, [asset], task_name)
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_period():
+    if not const.PERIOD_TASK_ENABLED:
+        logger.debug("Period task disabled, test system user connectivity pass")
+        return
+    system_users = SystemUser.objects.all()
+    for system_user in system_users:
+        task_name = _("Test system user connectivity period: {}").format(system_user)
+        assets = system_user.get_all_assets()
+        test_system_user_connectivity_util(system_user, assets, task_name)

apps/assets/tasks/utils.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'check_asset_can_run_ansible', 'clean_hosts', 'clean_hosts_by_protocol'
+]
+
+
+def check_asset_can_run_ansible(asset):
+    if not asset.is_active:
+        msg = _("Asset has been disabled, skipped: {}").format(asset)
+        logger.info(msg)
+        return False
+    if not asset.is_support_ansible():
+        msg = _("Asset may not be support ansible, skipped: {}").format(asset)
+        logger.info(msg)
+        return False
+    return True
+
+
+def clean_hosts(assets):
+    clean_assets = []
+    for asset in assets:
+        if not check_asset_can_run_ansible(asset):
+            continue
+        clean_assets.append(asset)
+    if not clean_assets:
+        logger.info(_("No assets matched, stop task"))
+    return clean_assets
+
+
+def clean_hosts_by_protocol(system_user, assets):
+    hosts = [
+        asset for asset in assets
+        if asset.has_protocol(system_user.protocol)
+    ]
+    if not hosts:
+        msg = _("No assets matched related system user protocol, stop task")
+        logger.info(msg)
+    return hosts

apps/assets/templates/assets/_asset_list_modal.html

@@ -15,11 +15,11 @@
     text-align: center;
 }
 
-#assetTree2.ztree * {
-    background-color: #f8fafb;
+#asset_modal_tree.ztree * {
+    background-color: white;
 }
-#assetTree2.ztree {
-    background-color: #f8fafb;
+#asset_modal_tree.ztree {
+    background-color: white;
 }
 </style>
 
@@ -29,7 +29,8 @@
            <div class="ibox float-e-margins">
                <div class="ibox-content mailbox-content" style="padding-top: 0;padding-left: 1px">
                    <div class="file-manager ">
-                       <div id="assetTree2" class="ztree">
+                       <div id="asset_modal_tree" class="ztree">
+                           {% trans 'Loading' %} ...
                        </div>
                        <div class="clearfix"></div>
                    </div>
@@ -55,12 +56,77 @@
 </div>
 
 <script>
-var zTree2, asset_table2 = 0;
-function initTable2() {
-    if(asset_table2){
-        return
+function syncTableSelectedAssetToSelect2(table) {
+    var assets = table.selected;
+    var options = [];
+    var select2Id = assetModalOption.select2Id;
+    $(select2Id + ' option').each(function (i, v) {
+        options.push(v.value)
+    });
+    table.selected_rows.forEach(function (i) {
+        var name = i.hostname + '(' + i.ip + ')';
+        var option = new Option(name, i.id, false, true);
+
+        if (options.indexOf(i.id) === -1) {
+            $(select2Id).append(option).trigger('change');
+        }
+    });
+    $(select2Id).val(assets).trigger('change');
+}
+
+
+
+// 解决input框中的资产和弹出表格中资产的显示不一致
+function syncSelectedAssetsToModalTable(assetModalTable) {
+    var select2Id = assetModalOption.select2Id;
+    var inputAssets = $(select2Id).val();
+    var selectedAssets = assetModalTable.selected.concat();
+
+    // input assets无，table assets选中，则取消勾选(再次click)
+    if (selectedAssets.length !== 0) {
+        $.each(selectedAssets, function (index, assetId) {
+            if ($.inArray(assetId, inputAssets) === -1) {
+                $('#' + assetId).trigger('click');  // 取消勾选
+            }
+        });
     }
 
+    // input assets有，table assets没选，则选中(click)
+    if (inputAssets) {
+        assetModalTable.selected = inputAssets;
+        $.each(inputAssets, function (index, assetId) {
+            var dom = document.getElementById(assetId);
+            if (dom !== null) {
+                var selected = dom.parentElement.parentElement.className.indexOf('selected')
+            }
+            if (selected === -1) {
+                $('#' + assetId).trigger('click');
+            }
+        });
+    }
+}
+
+defaultOnAssetModalConfirm = syncTableSelectedAssetToSelect2;
+defaultOnModalTableDone = syncSelectedAssetsToModalTable;
+
+
+var assetModalOption = {
+    selectStyle: 'multi',
+    select2Id: '#id_assets',
+    onModalTableDone: defaultOnModalTableDone,
+    onModalTreeDone: null,
+    onModalConfirm: defaultOnAssetModalConfirm,
+};
+
+var assetModalTable, assetModalTree = null;
+
+function initAssetModalTable() {
+    if(assetModalTable){
+        return
+    }
+    if (assetModalOption.selectStyle === 'single') {
+        $('.ipt_check_all').addClass('hidden')
+    }
     var options = {
         ele: $('#asset_list_modal_table'),
         ajax_url: '{% url "api-assets:asset-list" %}?show_current_asset=1',
@@ -68,22 +134,26 @@ function initTable2() {
             {data: "id"}, {data: "hostname" }, {data: "ip" }
         ],
         lengthMenu: [[10, 25, 50], [10, 25, 50]],
-        pageLength: 10
+        pageLength: 10,
+        select_style: assetModalOption.selectStyle
     };
-    asset_table2 = jumpserver.initServerSideDataTable(options);
-    return asset_table2
+    assetModalTable = jumpserver.initServerSideDataTable(options);
+    if (assetModalOption.onModalTableDone) {
+        assetModalOption.onModalTableDone(assetModalTable);
+    }
+    return assetModalTable
 }
 
-function onNodeSelected2(event, treeNode) {
-    var url = asset_table2.ajax.url();
+function onModalTreeNodeSelected(event, treeNode) {
+    var url = assetModalTable.ajax.url();
     url = setUrlParam(url, "node_id", treeNode.meta.node.id);
     url = setUrlParam(url, "show_current_asset", "");
-    asset_table2.ajax.url(url);
-    asset_table2.ajax.reload();
+    assetModalTable.ajax.url(url);
+    assetModalTable.ajax.reload();
 }
 
 
-function initTree2() {
+function initModalTree() {
     var url = '{% url 'api-assets:node-children-tree' %}?assets=0';
     var setting = {
         view: {
@@ -102,17 +172,34 @@ function initTree2() {
             type: 'get'
 		},
         callback: {
-            onSelected: onNodeSelected2
+            onSelected: onModalTreeNodeSelected
         }
     };
-    zTree2 = $.fn.zTree.init($("#assetTree2"), setting);
+    $.get(url, function(data, status){
+        $.fn.zTree.init($("#asset_modal_tree"), setting);
+        assetModalTree = $.fn.zTree.getZTreeObj("assetTree2");
+        if (assetModalOption.onModalTreeDone) {
+            assetModalOption.onModalTreeDone(assetModalTree);
+        }
+        return assetModalTree;
+    });
+}
+
+function setAssetModalOptions(options) {
+    assetModalOption = options;
 }
 
 
 $(document).ready(function(){
+
 }).on('show.bs.modal', function () {
-    initTable2();
-    initTree2();
+    initAssetModalTable();
+    initModalTree();
+}).on('click', '#btn_asset_modal_confirm', function () {
+    if (assetModalOption.onModalConfirm) {
+        assetModalOption.onModalConfirm(assetModalTable, assetModalTree);
+    }
+    $("#asset_list_modal").modal('hide');
 })
 </script>
 {% endblock %}

apps/assets/templates/assets/_node_tree.html

@@ -37,6 +37,7 @@
    <div class="ibox-content mailbox-content" style="padding-top: 0;padding-left: 1px">
        <div class="file-manager" id="tree-node-id">
            <div id="{% block treeID %}nodeTree{% endblock %}" class="ztree">
+           {% trans 'Loading' %} ...
            </div>
            <div class="clearfix"></div>
        </div>
@@ -73,7 +74,6 @@ function initNodeTree(options) {
     if (options.showAssets) {
         treeUrl = setUrlParam(treeUrl, 'assets', '1')
     }
-    var asyncTreeUrl = setUrlParam(treeUrl, 'refresh', '0');
     var setting = {
         view: {
             dblClickExpand: false,
@@ -86,7 +86,7 @@ function initNodeTree(options) {
         },
         async: {
 			enable: true,
-			url: asyncTreeUrl,
+			url: treeUrl,
 			autoParam: ["id=key", "name=n", "level=lv"],
             type: 'get'
 		},
@@ -114,9 +114,15 @@ function initNodeTree(options) {
     $.get(treeUrl, function (data, status) {
         zTree = $.fn.zTree.init($("#nodeTree"), setting, data);
         rootNodeAddDom(zTree, function () {
-            treeUrl = setUrlParam(treeUrl, 'refresh', '1');
-            initNodeTree(options);
-            treeUrl = setUrlParam(treeUrl, 'refresh', '0');
+            const url = '{% url 'api-assets:refresh-nodes-cache' %}';
+            requestApi({
+                url: url,
+                method: 'GET',
+                flash_message: false,
+                success: function () {
+                    initNodeTree(options);
+                }
+            });
         });
         inited = true;
     });
@@ -236,7 +242,8 @@ function onBodyMouseDown(event){
 }
 
 function onRename(event, treeId, treeNode, isCancel){
-    var url = "{% url 'api-assets:node-detail' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", current_node_id);
+    var url = "{% url 'api-assets:node-detail' pk=DEFAULT_PK %}"
+        .replace("{{ DEFAULT_PK }}", current_node_id);
     var data = {"value": treeNode.name};
     if (isCancel){
         return
@@ -247,10 +254,13 @@ function onRename(event, treeId, treeNode, isCancel){
         method: "PATCH",
         success_message: "{% trans 'Rename success' %}",
         success: function () {
-            treeNode.name = treeNode.name + ' (' + treeNode.meta.node.assets_amount + ')';
+            var assets_amount = treeNode.meta.node.assets_amount;
+            if (!assets_amount) {
+                assets_amount = 0;
+            }
+            treeNode.name = treeNode.name + ' (' + assets_amount + ')';
             zTree.updateNode(treeNode);
-            console.log("Success: " + treeNode.name)
-        }
+        },
     })
 }
 

apps/assets/templates/assets/_system_user.html

@@ -119,7 +119,7 @@ function autoLoginModeProtocol() {
         $(sudo_id).closest('.form-group').addClass('hidden');
         $(shell_id).closest('.form-group').addClass('hidden');
     }
-    else if (protocol === 'telnet (beta)') {
+    else if (protocol === 'telnet') {
         $('.auth-fields').removeClass('hidden');
         $(auto_generate_key).closest('.form-group').addClass('hidden');
         $(private_key_id).closest('.form-group').addClass('hidden');
@@ -165,7 +165,7 @@ function manualLoginModeProtocol() {
         $(sudo_id).closest('.form-group').addClass('hidden');
         $(shell_id).closest('.form-group').addClass('hidden');
     }
-    else if (protocol === 'telnet (beta)') {
+    else if (protocol === 'telnet') {
         $('.auth-fields').addClass('hidden');
         $(auto_generate_key).closest('.form-group').addClass('hidden');
         $(password_id).closest('.form-group').addClass('hidden');

apps/assets/templates/assets/admin_user_detail.html

@@ -88,9 +88,9 @@
                                         <form>
                                             <tr>
                                                 <td colspan="2" class="no-borders">
-                                                    <select data-placeholder="{% trans 'Select nodes' %}"  id="nodes_selected"  class="select2" style="width: 100%" multiple="" tabindex="4">
+                                                    <select data-placeholder="{% trans 'Select nodes' %}"  id="nodes_selected"  class="nodes-select2" style="width: 100%" multiple="" tabindex="4">
                                                         {% for node in nodes %}
-                                                        <option value="{{ node.id }}" id="opt_{{ node.id }}" >{{ node }}</option>
+                                                        <option value="{{ node.id }}" id="opt_{{ node.id }}" >{{ node.full_value }}</option>
                                                         {% endfor %}
                                                     </select>
                                                 </td>
@@ -140,7 +140,8 @@ function replaceNodeAssetsAdminUser(nodes) {
 
 jumpserver.nodes_selected = {};
 $(document).ready(function () {
-    $('.select2').select2()
+    var url = "{% url 'api-assets:node-list' %}";
+    nodesSelect2Init(".nodes-select2", url)
     .on('select2:select', function(evt) {
         var data = evt.params.data;
         jumpserver.nodes_selected[data.id] = data.text;

apps/assets/templates/assets/asset_create.html

@@ -28,7 +28,7 @@
             <div class="col-md-2 col-md-offset-2" style="text-align: right">{{ fm.name }}</div>
             <div class="col-md-6">{{ fm.port }}</div>
             <div class="col-md-1" style="padding: 6px 0">
-                <a class="btn btn-danger btn-xs btn-protocol btn-del"><span class="fa fa-minus"></span> </a>
+                <a class="btn btn-danger btn-xs btn-protocol btn-delete"><span class="fa fa-minus"></span> </a>
                 <a class="btn btn-primary btn-xs btn-protocol btn-add" style="display: none"><span class="fa fa-plus"></span></a>
             </div>
         </div>
@@ -97,7 +97,7 @@ function format(item) {
 function protocolBtnShow() {
     $(".btn-protocol.btn-add").hide();
     $(".btn-protocol.btn-add:last").show();
-    var btnDel = $(".btn-protocol.btn-del");
+    var btnDel = $(".btn-protocol.btn-delete");
     if (btnDel.length === 1) {
         btnDel.addClass("disabled")
     } else {
@@ -110,6 +110,8 @@ $(document).ready(function () {
     $('.select2').select2({
         allowClear: true
     });
+    var url = "{% url 'api-assets:node-list' %}";
+    nodesSelect2Init(".nodes-select2", url);
     $(".labels").select2({
         allowClear: true,
         templateSelection: format
@@ -138,7 +140,7 @@ $(document).ready(function () {
         protocolRef.trigger("change")
     }
 })
-.on("click", ".btn-protocol.btn-del", function () {
+.on("click", ".btn-protocol.btn-delete", function () {
     $(this).parent().parent().remove();
     protocolBtnShow()
 })

apps/assets/templates/assets/asset_detail.html

@@ -195,10 +195,7 @@
                                         <form>
                                             <tr>
                                                 <td colspan="2" class="no-borders">
-                                                    <select data-placeholder="{% trans 'Nodes' %}" id="groups_selected" class="select2 groups" style="width: 100%" multiple="" tabindex="4">
-                                                        {% for node in nodes_remain %}
-                                                        <option value="{{ node.id }}" id="opt_{{ node.id }}" >{{ node }}</option>
-                                                        {% endfor %}
+                                                    <select data-placeholder="{% trans 'Nodes' %}" id="groups_selected" class="nodes-select2 groups" style="width: 100%" multiple="" tabindex="4">
                                                     </select>
                                                 </td>
                                             </tr>
@@ -211,7 +208,7 @@
 
                                         {% for node in asset.nodes.all %}
                                         <tr>
-                                          <td ><b class="bdg_node" data-gid={{ node.id }}>{{ node }}</b></td>
+                                          <td ><b class="bdg_node" data-gid={{ node.id }}>{{ node.full_value }}</b></td>
                                           <td>
                                               <button class="btn btn-danger pull-right btn-xs btn-leave-node" type="button"><i class="fa fa-minus"></i></button>
                                           </td>
@@ -291,7 +288,9 @@ function refreshAssetHardware() {
 
 
 $(document).ready(function () {
-    $('.select2.groups').select2().on('select2:select', function(evt) {
+    var url = "{% url 'api-assets:node-list' %}";
+    nodesSelect2Init(".nodes-select2", url)
+    .on('select2:select', function(evt) {
         var data = evt.params.data;
         jumpserver.nodes_selected[data.id] = data.text;
     }).on('select2:unselect', function(evt) {

apps/assets/templates/assets/asset_list.html

@@ -85,7 +85,7 @@
                   <button data-toggle="dropdown" class="btn btn-default btn-sm dropdown-toggle">{% trans 'Label' %} <span class="caret"></span></button>
                   <ul class="dropdown-menu labels">
                       {% for label in labels %}
-                          <li><a style="font-weight: bolder">{{ label.name }}:{{ label.value }}</a></li>
+                          <li><a style="font-weight: bolder">{{ label.name }}#{{ label.value }}</a></li>
                       {% endfor %}
                   </ul>
                </div>
@@ -124,7 +124,6 @@
    </div>
 </div>
 
-{% include 'assets/_node_tree.html' %}
 {% include 'assets/_asset_update_modal.html' %}
 {% include 'assets/_asset_import_modal.html' %}
 {% include 'assets/_asset_list_modal.html' %}
@@ -172,9 +171,13 @@ function initTable() {
         ],
         ajax_url: '{% url "api-assets:asset-list" %}',
         columns: [
-            {data: "id"}, {data: "hostname" }, {data: "ip" },
+            {data: "id"}, {data: "hostname"}, {data: "ip"},
             {data: "cpu_cores", orderable: false},
-            {data: "connectivity", orderable: false}, {data: "id", orderable: false }
+            {
+                data: "connectivity",
+                orderable: false,
+                width: '60px'
+            }, {data: "id", orderable: false}
         ],
         op_html: $('#actions').html()
     };
@@ -227,6 +230,33 @@ function onNodeSelected(event, treeNode) {
     asset_table.ajax.reload();
 }
 
+function onAssetModalConfirmAddAssetToNode(table) {
+    var assets_selected = table.selected;
+    if (!current_node_id) {
+        return
+    }
+
+    var data = {'assets': assets_selected};
+    var success = function () {
+        table.selected = [];
+        table.ajax.reload()
+    };
+
+    var url = '';
+    if (update_node_action === "move") {
+        url = "{% url 'api-assets:node-replace-assets' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", current_node_id);
+    } else {
+        url = "{% url 'api-assets:node-add-assets' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", current_node_id);
+    }
+
+    requestApi({
+        'url': url,
+        'method': 'PUT',
+        'body': JSON.stringify(data),
+        'success': success
+    })
+}
+
 $(document).ready(function(){
     initTable();
     initTree();
@@ -237,9 +267,15 @@ $(document).ready(function(){
     else{
         $('#show_current_asset').css('display', 'inline-block');
     }
+
+    var modalOption = {
+        onModalConfirm: onAssetModalConfirmAddAssetToNode,
+        onModalTableDone: null
+    };
+    setAssetModalOptions(modalOption);
 })
 .on('click', '.labels li', function () {
-    var val = $(this).text();
+    var val = 'label:' + $(this).text();
     $("#asset_list_table_filter input").val(val);
     asset_table.search(val).draw();
  })
@@ -430,7 +466,6 @@ $(document).ready(function(){
      }
 
     function doRemove() {
-       var nodes = zTree.getSelectedNodes();
        if (!current_node_id) {
            return
        }
@@ -442,9 +477,10 @@ $(document).ready(function(){
        var success = function () {
            asset_table.ajax.reload()
        };
+       var url = "{% url 'api-assets:node-remove-assets' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", current_node_id);
 
        requestApi({
-           'url': '/api/assets/v1/nodes/' + current_node_id + '/assets/remove/',
+           'url': url,
            'method': 'PUT',
            'body': JSON.stringify(data),
            'success': success
@@ -472,32 +508,7 @@ $(document).ready(function(){
     }
     $(".ipt_check_all").prop("checked", false)
 })
-.on('click', '#btn_asset_modal_confirm', function () {
-    var assets_selected = asset_table2.selected;
-    if (!current_node_id) {
-        return
-    }
-
-    var data = {'assets': assets_selected};
-    var success = function () {
-        asset_table2.selected = [];
-        asset_table2.ajax.reload()
-    };
-
-    var url = '';
-    if (update_node_action === "move") {
-        url = "{% url 'api-assets:node-replace-assets' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", current_node_id);
-    } else {
-        url = "{% url 'api-assets:node-add-assets' pk=DEFAULT_PK %}".replace("{{ DEFAULT_PK }}", current_node_id);
-    }
-
-    requestApi({
-        'url': url,
-        'method': 'PUT',
-        'body': JSON.stringify(data),
-        'success': success
-    })
-}).on('hidden.bs.modal', '#asset_list_modal', function () {
+.on('hidden.bs.modal', '#asset_list_modal', function () {
     window.location.reload();
 }).on('click', '#menu_asset_add', function () {
     update_node_action = "add"

apps/assets/templates/assets/cmd_filter_rule_list.html

@@ -103,7 +103,7 @@ $(document).ready(function(){
 .on('click', '.btn-delete', function () {
     var $this = $(this);
     var $data_table = $('#cmd_filter_rule_list_table').DataTable();
-    var name = $(this).closest("tr").find(":nth-child(2)").children('a').html();
+    var name = $(this).closest("tr").find(":nth-child(2)").html();
     var uid = $this.data('uid');
     var the_url = '{% url "api-assets:cmd-filter-rule-detail" filter_pk=object.id pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', uid);
     objectDelete($this, name, the_url);

apps/assets/templates/assets/domain_create_update.html

@@ -28,25 +28,6 @@ $(document).ready(function () {
 }).on('click', '.select2-selection__rendered', function (e) {
     e.preventDefault();
     $("#asset_list_modal").modal();
-    initSelectedAssets2Table();
-})
-.on('click', '#btn_asset_modal_confirm', function () {
-    var assets = asset_table2.selected;
-    var options = [];
-    $('#id_assets option').each(function (i, v) {
-        options.push(v.value)
-    });
-    asset_table2.selected_rows.forEach(function (i) {
-        var name = i.hostname + '(' + i.ip + ')';
-        var option = new Option(name, i.id, false, true);
-
-        if (options.indexOf(i.id) === -1) {
-            $('#id_assets').append(option).trigger('change');
-        }
-    });
-    $('.select2').val(assets).trigger('change');
-    $("#asset_list_modal").modal('hide');
-
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();

apps/assets/templates/assets/domain_gateway_list.html

@@ -118,7 +118,7 @@ $(document).ready(function(){
 .on('click', '.btn-delete', function () {
     var $this = $(this);
     var $data_table = $('#domain_list_table').DataTable();
-    var name = $(this).closest("tr").find(":nth-child(2)").children('a').html();
+    var name = $(this).closest("tr").find(":nth-child(2)").html();
     var uid = $this.data('uid');
     var the_url = '{% url "api-assets:gateway-detail" pk=DEFAULT_PK %}'.replace('{{ DEFAULT_PK }}', uid);
     objectDelete($this, name, the_url);

apps/assets/templates/assets/label_create_update.html

@@ -32,24 +32,6 @@ $(document).ready(function () {
 }).on('click', '.select2-selection__rendered', function (e) {
     e.preventDefault();
     $("#asset_list_modal").modal();
-    initSelectedAssets2Table();
-})
-.on('click', '#btn_asset_modal_confirm', function () {
-    var assets = asset_table2.selected;
-    var options = [];
-    $('#id_assets option').each(function (i, v) {
-        options.push(v.value)
-    });
-    asset_table2.selected_rows.forEach(function (i) {
-        var name = i.hostname + '(' + i.ip + ')';
-        var option = new Option(name, i.id, false, true);
-
-        if (options.indexOf(i.id) === -1) {
-            $('#id_assets').append(option).trigger('change');
-        }
-    });
-    $('#id_assets').val(assets).trigger('change');
-    $("#asset_list_modal").modal('hide');
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();

apps/assets/templates/assets/system_user_assets.html

@@ -88,10 +88,7 @@
                                         <form>
                                             <tr>
                                                 <td colspan="2" class="no-borders">
-                                                    <select data-placeholder="{% trans 'Add to node' %}" id="node_selected" class="select2" style="width: 100%" multiple="" tabindex="4">
-                                                        {% for node in nodes_remain %}
-                                                        <option value="{{ node.id }}" id="opt_{{ node.id }}" >{{ node }}</option>
-                                                        {% endfor %}
+                                                    <select data-placeholder="{% trans 'Add to node' %}" id="node_selected" class="nodes-select2" style="width: 100%" multiple="" tabindex="4">
                                                     </select>
                                                 </td>
                                             </tr>
@@ -104,7 +101,7 @@
 
                                         {% for node in system_user.nodes.all|sort %}
                                         <tr>
-                                          <td ><b class="bdg_node" data-gid={{ node.id }}>{{ node }}</b></td>
+                                          <td ><b class="bdg_node" data-gid={{ node.id }}>{{ node.full_value }}</b></td>
                                           <td>
                                               <button class="btn btn-danger pull-right btn-xs btn-remove-from-node" type="button"><i class="fa fa-minus"></i></button>
                                           </td>
@@ -156,6 +153,8 @@ jumpserver.nodes_selected = {};
 
 $(document).ready(function () {
 	$('.select2').select2()
+    var url = "{% url 'api-assets:node-list' %}";
+    nodesSelect2Init(".nodes-select2", url)
 	.on('select2:select', function(evt) {
             var data = evt.params.data;
             jumpserver.nodes_selected[data.id] = data.text;

apps/assets/templates/assets/user_asset_list.html

@@ -21,9 +21,10 @@
 
 {% block custom_foot_js %}
 <script>
-var treeUrl = "{% url 'api-perms:my-nodes-as-tree' %}?&cache_policy=1";
+var treeUrl = "{% url 'api-perms:my-nodes-children-as-tree' %}?&cache_policy=1";
 var assetTableUrl = "{% url 'api-perms:my-assets' %}?cache_policy=1";
 var selectUrl = '{% url "api-perms:my-node-assets" node_id=DEFAULT_PK %}?cache_policy=1&all=1';
+var systemUsersUrl = "{% url 'api-perms:my-asset-system-users' asset_id=DEFAULT_PK %}?cache_policy=1";
 var showAssetHref = false; // Need input default true
 var actions = {
     targets: 4, createdCell: function (td, cellData) {
@@ -33,7 +34,6 @@ var actions = {
 }};
 $(document).ready(function () {
     initTree();
-    initTable();
 }).on('click', '.labels li', function () {
     var val = $(this).text();
     $("#user_assets_table_filter input").val(val);

apps/assets/urls/api_urls.py

@@ -1,33 +1,33 @@
 # coding:utf-8
-from django.urls import path
+from django.urls import path, re_path
 from rest_framework_nested import routers
 # from rest_framework.routers import DefaultRouter
 from rest_framework_bulk.routes import BulkRouter
 
+from common import api as capi
+
 from .. import api
 
 app_name = 'assets'
 
 router = BulkRouter()
 router.register(r'assets', api.AssetViewSet, 'asset')
-router.register(r'admin-user', api.AdminUserViewSet, 'admin-user')
-router.register(r'system-user', api.SystemUserViewSet, 'system-user')
+router.register(r'admin-users', api.AdminUserViewSet, 'admin-user')
+router.register(r'system-users', api.SystemUserViewSet, 'system-user')
 router.register(r'labels', api.LabelViewSet, 'label')
 router.register(r'nodes', api.NodeViewSet, 'node')
-router.register(r'domain', api.DomainViewSet, 'domain')
-router.register(r'gateway', api.GatewayViewSet, 'gateway')
-router.register(r'cmd-filter', api.CommandFilterViewSet, 'cmd-filter')
-router.register(r'asset-user', api.AssetUserViewSet, 'asset-user')
-router.register(r'asset-user-info', api.AssetUserExportViewSet, 'asset-user-info')
+router.register(r'domains', api.DomainViewSet, 'domain')
+router.register(r'gateways', api.GatewayViewSet, 'gateway')
+router.register(r'cmd-filters', api.CommandFilterViewSet, 'cmd-filter')
+router.register(r'asset-users', api.AssetUserViewSet, 'asset-user')
+router.register(r'asset-users-info', api.AssetUserExportViewSet, 'asset-user-info')
+router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 
-cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filter', lookup='filter')
+cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
 cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')
 
 
 urlpatterns = [
-    path('assets-bulk/', api.AssetListUpdateApi.as_view(), name='asset-bulk-update'),
-    path('asset/update/select/',
-         api.AssetBulkUpdateSelectAPI.as_view(), name='asset-bulk-update-select'),
     path('assets/<uuid:pk>/refresh/',
          api.AssetRefreshHardwareApi.as_view(), name='asset-refresh'),
     path('assets/<uuid:pk>/alive/',
@@ -35,36 +35,36 @@ urlpatterns = [
     path('assets/<uuid:pk>/gateway/',
          api.AssetGatewayApi.as_view(), name='asset-gateway'),
 
-    path('asset-user/auth-info/',
+    path('asset-users/auth-info/',
          api.AssetUserAuthInfoApi.as_view(), name='asset-user-auth-info'),
-    path('asset-user/test-connective/',
+    path('asset-users/test-connective/',
          api.AssetUserTestConnectiveApi.as_view(), name='asset-user-connective'),
 
 
-    path('admin-user/<uuid:pk>/nodes/',
+    path('admin-users/<uuid:pk>/nodes/',
          api.ReplaceNodesAdminUserApi.as_view(), name='replace-nodes-admin-user'),
-    path('admin-user/<uuid:pk>/auth/',
+    path('admin-users/<uuid:pk>/auth/',
          api.AdminUserAuthApi.as_view(), name='admin-user-auth'),
-    path('admin-user/<uuid:pk>/connective/',
+    path('admin-users/<uuid:pk>/connective/',
          api.AdminUserTestConnectiveApi.as_view(), name='admin-user-connective'),
-    path('admin-user/<uuid:pk>/assets/',
+    path('admin-users/<uuid:pk>/assets/',
          api.AdminUserAssetsListView.as_view(), name='admin-user-assets'),
 
-    path('system-user/<uuid:pk>/auth-info/',
+    path('system-users/<uuid:pk>/auth-info/',
          api.SystemUserAuthInfoApi.as_view(), name='system-user-auth-info'),
-    path('system-user/<uuid:pk>/asset/<uuid:aid>/auth-info/',
+    path('system-users/<uuid:pk>/assets/<uuid:aid>/auth-info/',
          api.SystemUserAssetAuthInfoApi.as_view(), name='system-user-asset-auth-info'),
-    path('system-user/<uuid:pk>/assets/',
+    path('system-users/<uuid:pk>/assets/',
          api.SystemUserAssetsListView.as_view(), name='system-user-assets'),
-    path('system-user/<uuid:pk>/push/',
+    path('system-users/<uuid:pk>/push/',
          api.SystemUserPushApi.as_view(), name='system-user-push'),
-    path('system-user/<uuid:pk>/asset/<uuid:aid>/push/',
+    path('system-users/<uuid:pk>/assets/<uuid:aid>/push/',
          api.SystemUserPushToAssetApi.as_view(), name='system-user-push-to-asset'),
-    path('system-user/<uuid:pk>/asset/<uuid:aid>/test/',
+    path('system-users/<uuid:pk>/assets/<uuid:aid>/test/',
          api.SystemUserTestAssetConnectivityApi.as_view(), name='system-user-test-to-asset'),
-    path('system-user/<uuid:pk>/connective/',
+    path('system-users/<uuid:pk>/connective/',
          api.SystemUserTestConnectiveApi.as_view(), name='system-user-connective'),
-    path('system-user/<uuid:pk>/cmd-filter-rules/',
+    path('system-users/<uuid:pk>/cmd-filter-rules/',
          api.SystemUserCommandFilterRuleListApi.as_view(), name='system-user-cmd-filter-rule-list'),
 
     path('nodes/tree/', api.NodeListAsTreeApi.as_view(), name='node-tree'),
@@ -86,13 +86,17 @@ urlpatterns = [
          api.RefreshNodeHardwareInfoApi.as_view(), name='node-refresh-hardware-info'),
     path('nodes/<uuid:pk>/test-connective/',
          api.TestNodeConnectiveApi.as_view(), name='node-test-connective'),
-    path('nodes/refresh-assets-amount/',
-         api.RefreshAssetsAmount.as_view(), name='refresh-assets-amount'),
 
-    path('gateway/<uuid:pk>/test-connective/',
+    path('nodes/cache/', api.RefreshNodesCacheApi.as_view(), name='refresh-nodes-cache'),
+
+    path('gateways/<uuid:pk>/test-connective/',
          api.GatewayTestConnectionApi.as_view(), name='test-gateway-connective'),
 
 ]
 
-urlpatterns += router.urls + cmd_filter_router.urls
+old_version_urlpatterns = [
+    re_path('(?P<resource>admin-user|system-user|domain|gateway|cmd-filter|asset-user)/.*', capi.redirect_plural_name_api)
+]
+
+urlpatterns += router.urls + cmd_filter_router.urls + old_version_urlpatterns
 

apps/assets/utils.py

@@ -1,12 +1,14 @@
 # ~*~ coding: utf-8 ~*~
 #
-import time
 from functools import reduce
-from django.db.models import Prefetch, Q
+from treelib import Tree
+from collections import defaultdict
+from copy import deepcopy
+import threading
+from django.db.models import Q
 
-from common.utils import get_object_or_none, get_logger
-from common.struct import Stack
-from .models import SystemUser, Label, Node, Asset
+from common.utils import get_object_or_none, get_logger, timeit
+from .models import SystemUser, Label, Asset
 
 
 logger = get_logger(__file__)
@@ -22,235 +24,144 @@ def get_system_user_by_id(id):
     return system_user
 
 
-class LabelFilterMixin:
-    def get_filter_labels_ids(self):
-        query_params = self.request.query_params
-        query_keys = query_params.keys()
-        all_label_keys = Label.objects.values_list('name', flat=True)
-        valid_keys = set(all_label_keys) & set(query_keys)
+class TreeService(Tree):
+    tag_sep = ' / '
+    cache_key = '_NODE_FULL_TREE'
+    cache_time = 3600
+    has_empty_node = False
+    has_ungrouped_node = False
 
-        if not valid_keys:
-            return []
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.nodes_assets_map = defaultdict(set)
+        self.all_nodes_assets_map = {}
 
-        labels_query = [
-            {"name": key, "value": query_params[key]}
-            for key in valid_keys
-        ]
-        args = [Q(**kwargs) for kwargs in labels_query]
-        args = reduce(lambda x, y: x | y, args)
-        labels_id = Label.objects.filter(args).values_list('id', flat=True)
-        return labels_id
+    @classmethod
+    @timeit
+    def new(cls):
+        from .models import Node
+        from orgs.utils import tmp_to_root_org
 
+        with tmp_to_root_org():
+            all_nodes = Node.objects.all()
+            tree = cls()
+            tree.create_node(tag='', identifier='')
+            for node in all_nodes:
+                tree.create_node(
+                    tag=node.value, identifier=node.key,
+                    parent=node.parent_key,
+                )
+            tree.init_assets()
+        return tree
 
-class LabelFilter(LabelFilterMixin):
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        labels_ids = self.get_filter_labels_ids()
-        if not labels_ids:
-            return queryset
-        for labels_id in labels_ids:
-            queryset = queryset.filter(labels=labels_id)
-        return queryset
+    def init_assets_async(self):
+        t = threading.Thread(target=self.init_assets)
+        t.start()
 
+    def init_assets(self):
+        from orgs.utils import tmp_to_root_org
+        self.all_nodes_assets_map = {}
+        self.nodes_assets_map = defaultdict(set)
+        logger.debug('Init tree assets')
+        with tmp_to_root_org():
+            queryset = Asset.objects.all().values_list('id', 'nodes__key')
+            for asset_id, key in queryset:
+                if not key:
+                    continue
+                self.nodes_assets_map[key].add(asset_id)
 
-class NodeUtil:
-    def __init__(self, with_assets_amount=False, debug=False):
-        self.stack = Stack()
-        self._nodes = {}
-        self.with_assets_amount = with_assets_amount
-        self._debug = debug
-        self.init()
+    def all_children_ids(self, nid, with_self=True):
+        children_ids = self.expand_tree(nid)
+        if not with_self:
+            next(children_ids)
+        return list(children_ids)
+
+    def all_children(self, nid, with_self=True, deep=False):
+        children_ids = self.all_children_ids(nid, with_self=with_self)
+        return [self.get_node(i, deep=deep) for i in children_ids]
+
+    def ancestors(self, nid, with_self=False, deep=False):
+        ancestor_ids = list(self.rsearch(nid))
+        ancestor_ids.pop()
+        if not with_self:
+            ancestor_ids.pop(0)
+        return [self.get_node(i, deep=deep) for i in ancestor_ids]
+
+    def get_node_full_tag(self, nid):
+        ancestors = self.ancestors(nid, with_self=True)
+        ancestors.reverse()
+        return self.tag_sep.join([n.tag for n in ancestors])
+
+    def get_family(self, nid, deep=False):
+        ancestors = self.ancestors(nid, with_self=False, deep=deep)
+        children = self.all_children(nid, with_self=False)
+        return ancestors + [self[nid]] + children
+
+    def root_node(self):
+        return self.get_node(self.root)
+
+    def get_node(self, nid, deep=False):
+        node = super().get_node(nid)
+        if deep:
+            node = self.copy_node(node)
+        return node
+
+    def parent(self, nid, deep=False):
+        parent = super().parent(nid)
+        if deep:
+            parent = self.copy_node(parent)
+        return parent
+
+    def assets(self, nid):
+        assets = self.nodes_assets_map[nid]
+        return assets
+
+    def set_assets(self, nid, assets):
+        self.nodes_assets_map[nid] = assets
+
+    def all_assets(self, nid):
+        assets = self.all_nodes_assets_map.get(nid)
+        if assets:
+            return assets
+        assets = set(self.assets(nid))
+        children = self.children(nid)
+        for child in children:
+            assets.update(self.all_assets(child.identifier))
+        return assets
+
+    def assets_amount(self, nid):
+        return len(self.all_assets(nid))
 
     @staticmethod
-    def sorted_by(node):
-        return [int(i) for i in node.key.split(':')]
+    def copy_node(node):
+        new_node = deepcopy(node)
+        new_node.fpointer = None
+        return new_node
 
-    def get_queryset(self):
-        all_nodes = Node.objects.all()
-        if self.with_assets_amount:
-            all_nodes = all_nodes.prefetch_related(
-                Prefetch('assets', queryset=Asset.objects.all().only('id'))
-            )
-            all_nodes = list(all_nodes)
-            for node in all_nodes:
-                node._assets = set(node.assets.all())
-        return all_nodes
-
-    def get_all_nodes(self):
-        all_nodes = sorted(self.get_queryset(), key=self.sorted_by)
-
-        guarder = Node(key='', value='Guarder')
-        guarder._assets = []
-        all_nodes.append(guarder)
-        return all_nodes
-
-    def push_to_stack(self, node):
-        # 入栈之前检查
-        # 如果栈是空的，证明是一颗树的根部
-        if self.stack.is_empty():
-            node._full_value = node.value
-            node._parents = []
+    def safe_add_ancestors(self, ancestors):
+        # 如果祖先节点为1个，那么添加该节点, 父节点是root node
+        if len(ancestors) == 1:
+            node = ancestors[0]
+            parent = self.root_node()
         else:
-            # 如果不是根节点,
-            # 该节点的祖先应该是父节点的祖先加上父节点
-            # 该节点的名字是父节点的名字+自己的名字
-            node._parents = [self.stack.top] + self.stack.top._parents
-            node._full_value = ' / '.join(
-                [self.stack.top._full_value, node.value]
-            )
-        node._children = []
-        node._all_children = []
-        self.debug("入栈: {}".format(node.key))
-        self.stack.push(node)
-
-    # 出栈
-    def pop_from_stack(self):
-        _node = self.stack.pop()
-        self.debug("出栈: {} 栈顶: {}".format(_node.key, self.stack.top.key if self.stack.top else None))
-        self._nodes[_node.key] = _node
-        if not self.stack.top:
-            return
-        if self.with_assets_amount:
-            self.stack.top._assets.update(_node._assets)
-            _node._assets_amount = len(_node._assets)
-            delattr(_node, '_assets')
-        self.stack.top._children.append(_node)
-        self.stack.top._all_children.extend([_node] + _node._all_children)
-
-    def init(self):
-        all_nodes = self.get_all_nodes()
-        for node in all_nodes:
-            self.debug("准备: {} 栈顶: {}".format(node.key, self.stack.top.key if self.stack.top else None))
-            # 入栈之前检查，该节点是不是栈顶节点的子节点
-            # 如果不是，则栈顶出栈
-            while self.stack.top and not self.stack.top.is_children(node):
-                self.pop_from_stack()
-            self.push_to_stack(node)
-        # 出栈最后一个
-        self.debug("剩余: {}".format(', '.join([n.key for n in self.stack])))
-
-    def get_nodes_by_queryset(self, queryset):
-        nodes = []
-        for n in queryset:
-            node = self.get_node_by_key(n.key)
-            if not node:
-                continue
-            nodes.append(node)
-        return nodes
-
-    def get_node_by_key(self, key):
-        return self._nodes.get(key)
-
-    def debug(self, msg):
-        self._debug and logger.debug(msg)
-
-    def set_assets_amount(self):
-        for node in self._nodes.values():
-            node.assets_amount = node._assets_amount
-
-    def set_full_value(self):
-        for node in self._nodes.values():
-            node.full_value = node._full_value
-
-    @property
-    def nodes(self):
-        return list(self._nodes.values())
-
-    def get_family_by_key(self, key):
-        tree_nodes = set()
-        node = self.get_node_by_key(key)
-        if not node:
-            return []
-        tree_nodes.update(node._parents)
-        tree_nodes.add(node)
-        tree_nodes.update(node._all_children)
-        return list(tree_nodes)
-
-    # 使用给定节点生成一颗树
-    # 找到他们的祖先节点
-    # 可选找到他们的子孙节点
-    def get_family(self, node):
-        return self.get_family_by_key(node.key)
-
-    def get_family_keys_by_key(self, key):
-        nodes = self.get_family_by_key(key)
-        return [n.key for n in nodes]
-
-    def get_some_nodes_family_by_keys(self, keys):
-        family = set()
-        for key in keys:
-            family.update(self.get_family_by_key(key))
-        return family
-
-    def get_some_nodes_family_keys_by_keys(self, keys):
-        family = self.get_some_nodes_family_by_keys(keys)
-        return [n.key for n in family]
-
-    def get_nodes_parents_by_key(self, key, with_self=True):
-        parents = set()
-        node = self.get_node_by_key(key)
-        if not node:
-            return []
-        parents.update(set(node._parents))
-        if with_self:
-            parents.add(node)
-        return list(parents)
-
-    def get_node_parents(self, node, with_self=True):
-        return self.get_nodes_parents_by_key(node.key, with_self=with_self)
-
-    def get_nodes_parents_keys_by_key(self, key, with_self=True):
-        nodes = self.get_nodes_parents_by_key(key, with_self=with_self)
-        return [n.key for n in nodes]
-
-    def get_all_children_by_key(self, key, with_self=True):
-        children = set()
-        node = self.get_node_by_key(key)
-        if not node:
-            return []
-        children.update(set(node._all_children))
-        if with_self:
-            children.add(node)
-        return list(children)
-
-    def get_all_children(self, node, with_self=True):
-        return self.get_all_children_by_key(node.key, with_self=with_self)
-
-    def get_all_children_keys_by_key(self, key, with_self=True):
-        nodes = self.get_all_children_by_key(key, with_self=with_self)
-        return [n.key for n in nodes]
-
-
-def test_node_tree():
-    tree = NodeUtil()
-    for node in tree._nodes.values():
-        print("Check {}".format(node.key))
-        children_wanted = node.get_all_children().count()
-        children = len(node._children)
-        if children != children_wanted:
-            print("{} children not equal: {} != {}".format(node.key, children, children_wanted))
-
-        assets_amount_wanted = node.get_all_assets().count()
-        if node._assets_amount != assets_amount_wanted:
-            print("{} assets amount not equal: {} != {}".format(
-                node.key, node._assets_amount, assets_amount_wanted)
-            )
-
-        full_value_wanted = node.full_value
-        if node._full_value != full_value_wanted:
-            print("{} full value not equal: {} != {}".format(
-                node.key, node._full_value, full_value_wanted)
-            )
-
-        parents_wanted = node.get_ancestor().count()
-        parents = len(node._parents)
-        if parents != parents_wanted:
-            print("{} parents count not equal: {} != {}".format(
-                node.key, parents, parents_wanted)
-            )
-
-
-
-
-
+            node, ancestors = ancestors[0], ancestors[1:]
+            parent_id = ancestors[0].identifier
+            # 如果父节点不存在, 则先添加父节点
+            if not self.contains(parent_id):
+                self.safe_add_ancestors(ancestors)
+            parent = self.get_node(parent_id)
 
+        # 如果当前节点已再树中，则移动当前节点到父节点中
+        # 这个是由于 当前节点放到了二级节点中
+        if self.contains(node.identifier):
+            self.move_node(node.identifier, parent.identifier)
+        else:
+            self.add_node(node, parent)
+    #
+    # def __getstate__(self):
+    #     self.mutex = None
+    #     return self.__dict__
+    #
+    # def __setstate__(self, state):
+    #     self.__dict__ = state
+    #     self.mutex = threading.Lock()

apps/assets/views/admin_user.py

@@ -83,7 +83,6 @@ class AdminUserDetailView(PermissionsMixin, DetailView):
         context = {
             'app': _('Assets'),
             'action': _('Admin user detail'),
-            'nodes': Node.get_queryset(),
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/assets/views/asset.py

@@ -16,8 +16,7 @@ from common.utils import get_object_or_none, get_logger
 from common.permissions import PermissionsMixin, IsOrgAdmin, IsValidUser
 from common.const import KEY_CACHE_RESOURCES_ID
 from .. import forms
-from ..utils import NodeUtil
-from ..models import Asset, SystemUser, Label, Node
+from ..models import Asset, Label, Node
 
 
 __all__ = [
@@ -33,7 +32,7 @@ class AssetListView(PermissionsMixin, TemplateView):
     permission_classes = [IsOrgAdmin]
 
     def get_context_data(self, **kwargs):
-        Node.root()
+        Node.org_root()
         context = {
             'app': _('Assets'),
             'action': _('Asset list'),
@@ -86,8 +85,8 @@ class AssetCreateView(PermissionsMixin, FormMixin, TemplateView):
         if node_id:
             node = get_object_or_none(Node, id=node_id)
         else:
-            node = Node.root()
-        form["nodes"].initial = node
+            node = Node.org_root()
+        form.add_nodes_initial(node)
         return form
 
     def get_protocol_formset(self):
@@ -196,13 +195,9 @@ class AssetDetailView(PermissionsMixin, DetailView):
         ).select_related('admin_user', 'domain')
 
     def get_context_data(self, **kwargs):
-        nodes_remain = Node.objects.exclude(assets=self.object).only('key')
-        util = NodeUtil()
-        nodes_remain = util.get_nodes_by_queryset(nodes_remain)
         context = {
             'app': _('Assets'),
             'action': _('Asset detail'),
-            'nodes_remain': nodes_remain,
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/assets/views/system_user.py

@@ -98,14 +98,9 @@ class SystemUserAssetView(PermissionsMixin, DetailView):
     permission_classes = [IsOrgAdmin]
 
     def get_context_data(self, **kwargs):
-        from ..utils import NodeUtil
-        nodes_remain = Node.objects.exclude(systemuser=self.object)
-        util = NodeUtil()
-        nodes_remain = util.get_nodes_by_queryset(nodes_remain)
         context = {
             'app': _('assets'),
             'action': _('System user asset'),
-            'nodes_remain': nodes_remain
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/audits/api.py

@@ -3,7 +3,7 @@
 
 from rest_framework import viewsets
 
-from common.permissions import IsOrgAdminOrAppUser, IsAuditor
+from common.permissions import IsOrgAdminOrAppUser, IsOrgAuditor
 from .models import FTPLog
 from .serializers import FTPLogSerializer
 
@@ -11,4 +11,4 @@ from .serializers import FTPLogSerializer
 class FTPLogViewSet(viewsets.ModelViewSet):
     queryset = FTPLog.objects.all()
     serializer_class = FTPLogSerializer
-    permission_classes = (IsOrgAdminOrAppUser | IsAuditor,)
+    permission_classes = (IsOrgAdminOrAppUser | IsOrgAuditor,)

apps/audits/apps.py

@@ -1,4 +1,6 @@
 from django.apps import AppConfig
+from django.conf import settings
+from django.db.models.signals import post_save
 
 
 class AuditsConfig(AppConfig):
@@ -6,3 +8,5 @@ class AuditsConfig(AppConfig):
 
     def ready(self):
         from . import signals_handler
+        if settings.SYSLOG_ENABLE:
+            post_save.connect(signals_handler.on_audits_log_create)

apps/audits/migrations/0006_auto_20190726_1753.py

@@ -0,0 +1,35 @@
+# Generated by Django 2.1.7 on 2019-07-26 09:53
+
+from django.db import migrations, models
+
+
+def migrate_loginlog_reason_to_str(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+
+    reason_map = {
+        "0": "",
+        "1": 'Username/password check failed',
+        "2": 'MFA authentication failed',
+        "3": "Username does not exist",
+        "4": "Password expired",
+    }
+
+    model = apps.get_model("audits", "UserLoginLog")
+    for k, v in reason_map.items():
+        model.objects.using(db_alias).filter(reason=k).update(reason=v)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0005_auto_20190228_1715'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userloginlog',
+            name='reason',
+            field=models.CharField(blank=True, default='', max_length=128, verbose_name='Reason'),
+        ),
+        migrations.RunPython(migrate_loginlog_reason_to_str),
+    ]

apps/audits/models.py

@@ -5,7 +5,7 @@ from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 from django.utils import timezone
 
-from orgs.mixins import OrgModelMixin
+from orgs.mixins.models import OrgModelMixin
 
 __all__ = [
     'FTPLog', 'OperateLog', 'PasswordChangeLog', 'UserLoginLog',
@@ -72,20 +72,6 @@ class UserLoginLog(models.Model):
         (MFA_UNKNOWN, _('-')),
     )
 
-    REASON_NOTHING = 0
-    REASON_PASSWORD = 1
-    REASON_MFA = 2
-    REASON_NOT_EXIST = 3
-    REASON_PASSWORD_EXPIRED = 4
-
-    REASON_CHOICE = (
-        (REASON_NOTHING, _('-')),
-        (REASON_PASSWORD, _('Username/password check failed')),
-        (REASON_MFA, _('MFA authentication failed')),
-        (REASON_NOT_EXIST, _("Username does not exist")),
-        (REASON_PASSWORD_EXPIRED, _("Password expired")),
-    )
-
     STATUS_CHOICE = (
         (True, _('Success')),
         (False, _('Failed'))
@@ -97,7 +83,7 @@ class UserLoginLog(models.Model):
     city = models.CharField(max_length=254, blank=True, null=True, verbose_name=_('Login city'))
     user_agent = models.CharField(max_length=254, blank=True, null=True, verbose_name=_('User agent'))
     mfa = models.SmallIntegerField(default=MFA_UNKNOWN, choices=MFA_CHOICE, verbose_name=_('MFA'))
-    reason = models.SmallIntegerField(default=0, choices=REASON_CHOICE, verbose_name=_('Reason'))
+    reason = models.CharField(default='', max_length=128, blank=True, verbose_name=_('Reason'))
     status = models.BooleanField(max_length=2, default=True, choices=STATUS_CHOICE, verbose_name=_('Status'))
     datetime = models.DateTimeField(default=timezone.now, verbose_name=_('Date login'))
 

apps/audits/serializers.py

@@ -3,11 +3,36 @@
 
 from rest_framework import serializers
 
-from .models import FTPLog
+from terminal.models import Session
+from . import models
 
 
 class FTPLogSerializer(serializers.ModelSerializer):
 
     class Meta:
-        model = FTPLog
+        model = models.FTPLog
+        fields = '__all__'
+
+
+class LoginLogSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.UserLoginLog
+        fields = '__all__'
+
+
+class OperateLogSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.OperateLog
+        fields = '__all__'
+
+
+class PasswordChangeLogSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.PasswordChangeLog
+        fields = '__all__'
+
+
+class SessionAuditSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Session
         fields = '__all__'

apps/audits/signals_handler.py

@@ -4,13 +4,18 @@
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
 from django.db import transaction
+from rest_framework.renderers import JSONRenderer
 
 from jumpserver.utils import current_request
-from common.utils import get_request_ip, get_logger
+from common.utils import get_request_ip, get_logger, get_syslogger
 from users.models import User
-from .models import OperateLog, PasswordChangeLog
+from terminal.models import Session
+from . import models
+from . import serializers
 
 logger = get_logger(__name__)
+sys_logger = get_syslogger("audits")
+json_render = JSONRenderer()
 
 
 MODELS_NEED_RECORD = (
@@ -36,7 +41,7 @@ def create_operate_log(action, sender, resource):
     }
     with transaction.atomic():
         try:
-            OperateLog.objects.create(**data)
+            models.OperateLog.objects.create(**data)
         except Exception as e:
             logger.error("Create operate log error: {}".format(e))
 
@@ -44,15 +49,15 @@ def create_operate_log(action, sender, resource):
 @receiver(post_save, dispatch_uid="my_unique_identifier")
 def on_object_created_or_update(sender, instance=None, created=False, **kwargs):
     if created:
-        action = OperateLog.ACTION_CREATE
+        action = models.OperateLog.ACTION_CREATE
     else:
-        action = OperateLog.ACTION_UPDATE
+        action = models.OperateLog.ACTION_UPDATE
     create_operate_log(action, sender, instance)
 
 
 @receiver(post_delete, dispatch_uid="my_unique_identifier")
 def on_object_delete(sender, instance=None, **kwargs):
-    create_operate_log(OperateLog.ACTION_DELETE, sender, instance)
+    create_operate_log(models.OperateLog.ACTION_DELETE, sender, instance)
 
 
 @receiver(post_save, sender=User, dispatch_uid="my_unique_identifier")
@@ -61,7 +66,32 @@ def on_user_change_password(sender, instance=None, **kwargs):
         if not current_request or not current_request.user.is_authenticated:
             return
         with transaction.atomic():
-            PasswordChangeLog.objects.create(
+            models.PasswordChangeLog.objects.create(
                 user=instance, change_by=current_request.user,
                 remote_addr=get_request_ip(current_request),
             )
+
+
+def on_audits_log_create(sender, instance=None, **kwargs):
+    if sender == models.UserLoginLog:
+        category = "login_log"
+        serializer = serializers.LoginLogSerializer
+    elif sender == models.FTPLog:
+        serializer = serializers.FTPLogSerializer
+        category = "ftp_log"
+    elif sender == models.OperateLog:
+        category = "operation_log"
+        serializer = serializers.OperateLogSerializer
+    elif sender == models.PasswordChangeLog:
+        category = "password_change_log"
+        serializer = serializers.PasswordChangeLogSerializer
+    elif sender == Session:
+        category = "host_session_log"
+        serializer = serializers.SessionAuditSerializer
+    else:
+        return
+
+    s = serializer(instance=instance)
+    data = json_render.render(s.data).decode(errors='ignore')
+    msg = "{} - {}".format(category, data)
+    sys_logger.info(msg)

apps/audits/templates/audits/ftp_log_list.html

@@ -11,6 +11,9 @@
         #search_btn {
             margin-bottom: 0;
         }
+        .form-control {
+            height: 30px;
+        }
     </style>
 {% endblock %}
 
@@ -109,7 +112,7 @@
     <script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
     <script>
         $(document).ready(function() {
-            $('table').DataTable({
+            $('#editable').DataTable({
                 "searching": false,
                 "paging": false,
                 "bInfo" : false,

apps/audits/templates/audits/login_log_list.html

@@ -8,6 +8,12 @@
         #search_btn {
             margin-bottom: 0;
         }
+        .form-control {
+            height: 30px;
+        }
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
     </style>
 {% endblock %}
 
@@ -72,7 +78,7 @@
 	            <td class="text-center">{{ login_log.ip }}</td>
 	            <td class="text-center">{{ login_log.city }}</td>
 	            <td class="text-center">{{ login_log.get_mfa_display }}</td>
-	            <td class="text-center">{{ login_log.get_reason_display }}</td>
+	            <td class="text-center">{% trans login_log.reason %}</td>
 	            <td class="text-center">{{ login_log.get_status_display }}</td>
 	            <td class="text-center">{{ login_log.datetime }}</td>
 	        </tr>
@@ -99,7 +105,7 @@
     <script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
     <script>
         $(document).ready(function() {
-            jumpserver.initStaticTable('table');
+            jumpserver.initStaticTable('#login_log_table');
             $('#date .input-daterange').datepicker({
                 format: "yyyy-mm-dd",
                 todayBtn: "linked",

apps/audits/templates/audits/operate_log_list.html

@@ -11,6 +11,12 @@
         #search_btn {
             margin-bottom: 0;
         }
+        .form-control {
+            height: 30px;
+        }
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
     </style>
 {% endblock %}
 
@@ -94,7 +100,7 @@
     <script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
     <script>
         $(document).ready(function() {
-            $('table').DataTable({
+            $('#editable').DataTable({
                 "searching": false,
                 "paging": false,
                 "bInfo" : false,

apps/audits/templates/audits/password_change_log_list.html

@@ -11,6 +11,12 @@
         #search_btn {
             margin-bottom: 0;
         }
+        .form-control {
+            height: 30px;
+        }
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
     </style>
 {% endblock %}
 
@@ -71,7 +77,7 @@
     <script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
     <script>
         $(document).ready(function() {
-            $('table').DataTable({
+            $('#editable').DataTable({
                 "searching": false,
                 "paging": false,
                 "bInfo" : false,

apps/audits/urls/api_urls.py

@@ -1,17 +1,23 @@
 # ~*~ coding: utf-8 ~*~
 from __future__ import unicode_literals
 
-from django.conf.urls import url
+from django.urls.conf import re_path
 from rest_framework.routers import DefaultRouter
+
+from common import api as capi
 from .. import api
 
 
 app_name = "audits"
 
 router = DefaultRouter()
-router.register(r'ftp-log', api.FTPLogViewSet, 'ftp-log')
+router.register(r'ftp-logs', api.FTPLogViewSet, 'ftp-log')
 
 urlpatterns = [
 ]
 
+old_version_urlpatterns = [
+    re_path('(?P<resource>ftp-log)/.*', capi.redirect_plural_name_api)
+]
+
 urlpatterns += router.urls

apps/audits/views.py

@@ -18,8 +18,9 @@ from django.db.models import Q
 
 from audits.utils import get_excel_response, write_content_to_excel
 from common.mixins import DatetimeSearchMixin
-from common.permissions import PermissionsMixin, IsOrgAdmin, IsAuditor, IsValidUser
-
+from common.permissions import (
+    PermissionsMixin, IsOrgAdmin, IsValidUser, IsOrgAuditor
+)
 from orgs.utils import current_org
 from ops.views import CommandExecutionListView as UserCommandExecutionListView
 from .models import FTPLog, OperateLog, PasswordChangeLog, UserLoginLog
@@ -47,7 +48,7 @@ class FTPLogListView(PermissionsMixin, DatetimeSearchMixin, ListView):
     paginate_by = settings.DISPLAY_PER_PAGE
     user = asset = system_user = filename = ''
     date_from = date_to = None
-    permission_classes = [IsOrgAdmin | IsAuditor]
+    permission_classes = [IsOrgAdmin | IsOrgAuditor]
 
     def get_queryset(self):
         self.queryset = super().get_queryset()
@@ -96,7 +97,7 @@ class OperateLogListView(PermissionsMixin, DatetimeSearchMixin, ListView):
     user = action = resource_type = ''
     date_from = date_to = None
     actions_dict = dict(OperateLog.ACTION_CHOICES)
-    permission_classes = [IsOrgAdmin | IsAuditor]
+    permission_classes = [IsOrgAdmin | IsOrgAuditor]
 
     def get_queryset(self):
         self.queryset = super().get_queryset()
@@ -119,7 +120,7 @@ class OperateLogListView(PermissionsMixin, DatetimeSearchMixin, ListView):
 
     def get_context_data(self, **kwargs):
         context = {
-            'user_list': current_org.get_org_users(),
+            'user_list': current_org.get_org_members(),
             'actions': self.actions_dict,
             'resource_type_list': get_resource_type_list(),
             'date_from': self.date_from,
@@ -139,10 +140,10 @@ class PasswordChangeLogList(PermissionsMixin, DatetimeSearchMixin, ListView):
     paginate_by = settings.DISPLAY_PER_PAGE
     user = ''
     date_from = date_to = None
-    permission_classes = [IsOrgAdmin | IsAuditor]
+    permission_classes = [IsOrgAdmin | IsOrgAuditor]
 
     def get_queryset(self):
-        users = current_org.get_org_users()
+        users = current_org.get_org_members()
         self.queryset = super().get_queryset().filter(
             user__in=[user.__str__() for user in users]
         )
@@ -159,7 +160,7 @@ class PasswordChangeLogList(PermissionsMixin, DatetimeSearchMixin, ListView):
 
     def get_context_data(self, **kwargs):
         context = {
-            'user_list': current_org.get_org_users(),
+            'user_list': current_org.get_org_members(),
             'date_from': self.date_from,
             'date_to': self.date_to,
             'user': self.user,
@@ -176,18 +177,18 @@ class LoginLogListView(PermissionsMixin, DatetimeSearchMixin, ListView):
     paginate_by = settings.DISPLAY_PER_PAGE
     user = keyword = ""
     date_to = date_from = None
-    permission_classes = [IsOrgAdmin | IsAuditor]
+    permission_classes = [IsOrgAdmin | IsOrgAuditor]
 
     @staticmethod
-    def get_org_users():
-        users = current_org.get_org_users().values_list('username', flat=True)
+    def get_org_members():
+        users = current_org.get_org_members().values_list('username', flat=True)
         return users
 
     def get_queryset(self):
         if current_org.is_default():
             queryset = super().get_queryset()
         else:
-            users = self.get_org_users()
+            users = self.get_org_members()
             queryset = super().get_queryset().filter(username__in=users)
 
         self.user = self.request.GET.get('user', '')
@@ -214,7 +215,7 @@ class LoginLogListView(PermissionsMixin, DatetimeSearchMixin, ListView):
             'date_to': self.date_to,
             'user': self.user,
             'keyword': self.keyword,
-            'user_list': self.get_org_users(),
+            'user_list': self.get_org_members(),
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)
@@ -223,6 +224,10 @@ class LoginLogListView(PermissionsMixin, DatetimeSearchMixin, ListView):
 class CommandExecutionListView(UserCommandExecutionListView):
     user_id = None
 
+    def get_user_list(self):
+        users = current_org.get_org_members(exclude=('Auditor',))
+        return users
+
     def get_queryset(self):
         queryset = self._get_queryset()
         self.user_id = self.request.GET.get('user')
@@ -233,10 +238,6 @@ class CommandExecutionListView(UserCommandExecutionListView):
             queryset = queryset.filter(user__in=org_users)
         return queryset
 
-    def get_user_list(self):
-        users = current_org.get_org_users()
-        return users
-
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context.update({

apps/authentication/api/__init__.py

@@ -2,3 +2,6 @@
 #
 
 from .auth import *
+from .token import *
+from .mfa import *
+from .access_key import *

apps/authentication/api/access_key.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+#
+
+from rest_framework.viewsets import ModelViewSet
+
+from common.permissions import IsValidUser
+from .. import serializers
+
+
+class AccessKeyViewSet(ModelViewSet):
+    permission_classes = (IsValidUser,)
+    serializer_class = serializers.AccessKeySerializer
+    search_fields = ['^id', '^secret']
+
+    def get_queryset(self):
+        return self.request.user.access_keys.all()
+
+    def perform_create(self, serializer):
+        user = self.request.user
+        user.create_access_key()

apps/authentication/api/auth.py

@@ -16,15 +16,17 @@ from rest_framework.views import APIView
 
 from common.utils import get_logger, get_request_ip
 from common.permissions import IsOrgAdminOrAppUser, IsValidUser
-from orgs.mixins import RootOrgViewMixin
+from orgs.mixins.api import RootOrgViewMixin
 from users.serializers import UserSerializer
 from users.models import User
 from assets.models import Asset, SystemUser
 from audits.models import UserLoginLog as LoginLog
 from users.utils import (
-    check_user_valid, check_otp_code, increase_login_failed_count,
+    check_otp_code, increase_login_failed_count,
     is_block_login, clean_failed_count
 )
+from .. import const
+from ..utils import check_user_valid
 from ..serializers import OtpVerifySerializer
 from ..signals import post_auth_success, post_auth_failed
 
@@ -39,6 +41,16 @@ class UserAuthApi(RootOrgViewMixin, APIView):
     permission_classes = (AllowAny,)
     serializer_class = UserSerializer
 
+    def get_serializer_context(self):
+        return {
+            'request': self.request,
+            'view': self
+        }
+
+    def get_serializer(self, *args, **kwargs):
+        kwargs['context'] = self.get_serializer_context()
+        return self.serializer_class(*args, **kwargs)
+
     def post(self, request):
         # limit login
         username = request.data.get('username')
@@ -53,29 +65,17 @@ class UserAuthApi(RootOrgViewMixin, APIView):
         user, msg = self.check_user_valid(request)
         if not user:
             username = request.data.get('username', '')
-            exist = User.objects.filter(username=username).first()
-            reason = LoginLog.REASON_PASSWORD if exist else LoginLog.REASON_NOT_EXIST
-            self.send_auth_signal(success=False, username=username, reason=reason)
+            self.send_auth_signal(success=False, username=username, reason=msg)
             increase_login_failed_count(username, ip)
             return Response({'msg': msg}, status=401)
 
-        if user.password_has_expired:
-            self.send_auth_signal(
-                success=False, username=username,
-                reason=LoginLog.REASON_PASSWORD_EXPIRED
-            )
-            msg = _("The user {} password has expired, please update.".format(
-                user.username))
-            logger.info(msg)
-            return Response({'msg': msg}, status=401)
-
         if not user.otp_enabled:
             self.send_auth_signal(success=True, user=user)
             # 登陆成功，清除原来的缓存计数
             clean_failed_count(username, ip)
-            token = user.create_bearer_token(request)
+            token, expired_at = user.create_bearer_token(request)
             return Response(
-                {'token': token, 'user': self.serializer_class(user).data}
+                {'token': token, 'user': self.get_serializer(user).data}
             )
 
         seed = uuid.uuid4().hex
@@ -87,7 +87,7 @@ class UserAuthApi(RootOrgViewMixin, APIView):
                          'conduct MFA secondary certification'),
                 'otp_url': reverse('api-auth:user-otp-auth'),
                 'seed': seed,
-                'user': self.serializer_class(user).data
+                'user': self.get_serializer(user).data
             }, status=300
         )
 
@@ -157,6 +157,16 @@ class UserOtpAuthApi(RootOrgViewMixin, APIView):
     permission_classes = (AllowAny,)
     serializer_class = UserSerializer
 
+    def get_serializer_context(self):
+        return {
+            'request': self.request,
+            'view': self
+        }
+
+    def get_serializer(self, *args, **kwargs):
+        kwargs['context'] = self.get_serializer_context()
+        return self.serializer_class(*args, **kwargs)
+
     def post(self, request):
         otp_code = request.data.get('otp_code', '')
         seed = request.data.get('seed', '')
@@ -167,11 +177,11 @@ class UserOtpAuthApi(RootOrgViewMixin, APIView):
                 status=401
             )
         if not check_otp_code(user.otp_secret_key, otp_code):
-            self.send_auth_signal(success=False, username=user.username, reason=LoginLog.REASON_MFA)
+            self.send_auth_signal(success=False, username=user.username, reason=const.mfa_failed)
             return Response({'msg': _('MFA certification failed')}, status=401)
         self.send_auth_signal(success=True, user=user)
-        token = user.create_bearer_token(request)
-        data = {'token': token, 'user': self.serializer_class(user).data}
+        token, expired_at = user.create_bearer_token(request)
+        data = {'token': token, 'user': self.get_serializer(user).data}
         return Response(data)
 
     def send_auth_signal(self, success=True, user=None, username='', reason=''):