Merge pull request #3243 from jumpserver/gather_asset_user

[Feature] 收集资产用户

apps/assets/api/__init__.py

@@ -6,3 +6,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/api/admin_user.py

@@ -19,7 +19,7 @@ from rest_framework import generics
 from rest_framework.response import Response
 from orgs.mixins.api import OrgBulkModelViewSet
 
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from common.utils import get_logger
 from ..hands import IsOrgAdmin
 from ..models import AdminUser, Asset

apps/assets/api/asset.py

@@ -5,9 +5,7 @@ import random
 
 from rest_framework import generics
 from rest_framework.response import Response
-from django.utils.translation import ugettext_lazy as _
 from django.shortcuts import get_object_or_404
-from django.db.models import Q
 
 from common.utils import get_logger, get_object_or_none
 from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser
@@ -16,7 +14,7 @@ from ..models import Asset, AdminUser, Node
 from .. import serializers
 from ..tasks import update_asset_hardware_info_manual, \
     test_asset_connectivity_manual
-from ..utils import LabelFilter
+from ..filters import AssetByNodeFilterBackend, LabelFilterBackend
 
 
 logger = get_logger(__file__)
@@ -27,7 +25,7 @@ __all__ = [
 ]
 
 
-class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
+class AssetViewSet(OrgBulkModelViewSet):
     """
     API endpoint that allows Asset to be viewed or edited.
     """
@@ -37,7 +35,8 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
     queryset = Asset.objects.all()
     serializer_class = serializers.AssetSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
-    success_message = _("%(hostname)s was %(action)s successfully")
+    extra_filter_backends = [AssetByNodeFilterBackend, LabelFilterBackend]
+    custom_filter_fields = ['admin_user_id']
 
     def set_assets_node(self, assets):
         if not isinstance(assets, list):
@@ -54,30 +53,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
         assets = serializer.save()
         self.set_assets_node(assets)
 
-    def filter_node(self, queryset):
-        node_id = self.request.query_params.get("node_id")
-        if not node_id:
-            return queryset
-
-        node = get_object_or_404(Node, id=node_id)
-        show_current_asset = self.request.query_params.get("show_current_asset") in ('1', 'true')
-
-        # 当前节点是顶层节点, 并且仅显示直接资产
-        if node.is_org_root() and show_current_asset:
-            queryset = queryset.filter(
-                Q(nodes=node_id) | Q(nodes__isnull=True)
-            ).distinct()
-        # 当前节点是顶层节点，显示所有资产
-        elif node.is_org_root() and not show_current_asset:
-            return queryset
-        # 当前节点不是鼎城节点，只显示直接资产
-        elif not node.is_org_root() and show_current_asset:
-            queryset = queryset.filter(nodes=node)
-        else:
-            children = node.get_all_children(with_self=True)
-            queryset = queryset.filter(nodes__in=children).distinct()
-        return queryset
-
     def filter_admin_user_id(self, queryset):
         admin_user_id = self.request.query_params.get('admin_user_id')
         if not admin_user_id:
@@ -88,7 +63,6 @@ class AssetViewSet(LabelFilter, OrgBulkModelViewSet):
 
     def filter_queryset(self, queryset):
         queryset = super().filter_queryset(queryset)
-        queryset = self.filter_node(queryset)
         queryset = self.filter_admin_user_id(queryset)
         return queryset
 

apps/assets/api/asset_user.py

@@ -10,7 +10,7 @@ from django.http import Http404
 
 from common.permissions import IsOrgAdminOrAppUser, NeedMFAVerify
 from common.utils import get_object_or_none, get_logger
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from ..backends import AssetUserManager
 from ..models import Asset, Node, SystemUser, AdminUser
 from .. import serializers
@@ -52,7 +52,7 @@ class AssetUserSearchBackend(filters.BaseFilterBackend):
         return _queryset
 
 
-class AssetUserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class AssetUserViewSet(CommonApiMixin, BulkModelViewSet):
     serializer_class = serializers.AssetUserSerializer
     permission_classes = [IsOrgAdminOrAppUser]
     http_method_names = ['get', 'post']

apps/assets/api/gathered_user.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+#
+
+from orgs.mixins.api import OrgModelViewSet
+from assets.models import GatheredUser
+from common.permissions import IsOrgAdmin
+
+from ..serializers import GatheredUserSerializer
+from ..filters import AssetRelatedByNodeFilterBackend
+
+
+__all__ = ['GatheredUserViewSet']
+
+
+class GatheredUserViewSet(OrgModelViewSet):
+    queryset = GatheredUser.objects.all()
+    serializer_class = GatheredUserSerializer
+    permission_classes = [IsOrgAdmin]
+    extra_filter_backends = [AssetRelatedByNodeFilterBackend]
+
+    filter_fields = ['asset', 'username', 'present']
+    search_fields = ['username', 'asset__ip', 'asset__hostname']
+
+

apps/assets/filters.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+#
+
+import coreapi
+from rest_framework import filters
+from django.db.models import Q
+
+from common.utils import dict_get_any, is_uuid, get_object_or_none
+from .models import Node, Label
+
+
+class AssetByNodeFilterBackend(filters.BaseFilterBackend):
+    fields = ['node', 'all']
+
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name=field, location='query', required=False,
+                type='string', example='', description='', schema=None,
+            )
+            for field in self.fields
+        ]
+
+    @staticmethod
+    def is_query_all(request):
+        query_all_arg = request.query_params.get('all')
+        show_current_asset_arg = request.query_params.get('show_current_asset')
+
+        query_all = query_all_arg == '1'
+        if show_current_asset_arg is not None:
+            query_all = show_current_asset_arg != '1'
+        return query_all
+
+    @staticmethod
+    def get_query_node(request):
+        node_id = dict_get_any(request.query_params, ['node', 'node_id'])
+        if not node_id:
+            return None, False
+
+        if is_uuid(node_id):
+            node = get_object_or_none(Node, id=node_id)
+        else:
+            node = get_object_or_none(Node, key=node_id)
+        return node, True
+
+    @staticmethod
+    def perform_query(pattern, queryset):
+        return queryset.filter(nodes__key__regex=pattern)
+
+    def filter_queryset(self, request, queryset, view):
+        node, has_query_arg = self.get_query_node(request)
+        if not has_query_arg:
+            return queryset
+
+        if node is None:
+            return queryset.none()
+        query_all = self.is_query_all(request)
+        if query_all:
+            pattern = node.get_all_children_pattern(with_self=True)
+        else:
+            pattern = node.get_children_key_pattern(with_self=True)
+        return self.perform_query(pattern, queryset)
+
+
+class LabelFilterBackend(filters.BaseFilterBackend):
+    sep = '#'
+    query_arg = 'label'
+
+    def get_schema_fields(self, view):
+        example = self.sep.join(['os', 'linux'])
+        return [
+            coreapi.Field(
+                name=self.query_arg, location='query', required=False,
+                type='string', example=example, description=''
+            )
+        ]
+
+    def get_query_labels(self, request):
+        labels_query = request.query_params.getlist(self.query_arg)
+        if not labels_query:
+            return None
+
+        q = None
+        for kv in labels_query:
+            if self.sep not in kv:
+                continue
+            key, value = kv.strip().split(self.sep)[:2]
+            if not all([key, value]):
+                continue
+            if q:
+                q |= Q(name=key, value=value)
+            else:
+                q = Q(name=key, value=value)
+        if not q:
+            return []
+        labels = Label.objects.filter(q, is_active=True)\
+            .values_list('id', flat=True)
+        return labels
+
+    def filter_queryset(self, request, queryset, view):
+        labels = self.get_query_labels(request)
+        if labels is None:
+            return queryset
+        if len(labels) == 0:
+            return queryset.none()
+        for label in labels:
+            queryset = queryset.filter(labels=label)
+        return queryset
+
+
+class AssetRelatedByNodeFilterBackend(AssetByNodeFilterBackend):
+    @staticmethod
+    def perform_query(pattern, queryset):
+        return queryset.filter(asset__nodes__key__regex=pattern).distinct()
+

apps/assets/migrations/0039_authbook_is_active.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.1.7 on 2019-09-17 12:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0038_auto_20190911_1634'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='authbook',
+            name='is_active',
+            field=models.BooleanField(default=True, verbose_name='Is active'),
+        ),
+    ]

apps/assets/migrations/0040_auto_20190917_2056.py

@@ -0,0 +1,36 @@
+# Generated by Django 2.1.7 on 2019-09-17 12:56
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0039_authbook_is_active'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adminuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='authbook',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='gateway',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+        migrations.AlterField(
+            model_name='systemuser',
+            name='username',
+            field=models.CharField(blank=True, db_index=True, max_length=32, validators=[django.core.validators.RegexValidator('^[0-9a-zA-Z_@\\-\\.]*$', 'Special char not allowed')], verbose_name='Username'),
+        ),
+    ]

apps/assets/migrations/0041_gathereduser.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.1.7 on 2019-09-18 04:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0040_auto_20190917_2056'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GatheredUser',
+            fields=[
+                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('username', models.CharField(blank=True, db_index=True, max_length=32, verbose_name='Username')),
+                ('present', models.BooleanField(default=True, verbose_name='Present')),
+                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.Asset', verbose_name='Asset')),
+            ],
+            options={'ordering': ['asset'], 'verbose_name': 'GatherUser'},
+        ),
+    ]

apps/assets/models/__init__.py

@@ -9,3 +9,4 @@ from .cmd_filter import *
 from .authbook import *
 from .utils import *
 from .authbook import *
+from .gathered_user import *

apps/assets/models/authbook.py

@@ -13,7 +13,7 @@ __all__ = ['AuthBook']
 class AuthBookQuerySet(models.QuerySet):
 
     def latest_version(self):
-        return self.filter(is_latest=True)
+        return self.filter(is_latest=True).filter(is_active=True)
 
 
 class AuthBookManager(OrgManager):
@@ -24,6 +24,7 @@ class AuthBook(AssetUser):
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
     is_latest = models.BooleanField(default=False, verbose_name=_('Latest version'))
     version = models.IntegerField(default=1, verbose_name=_('Version'))
+    is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
 
     objects = AuthBookManager.from_queryset(AuthBookQuerySet)()
     backend = "db"
@@ -34,25 +35,25 @@ class AuthBook(AssetUser):
     class Meta:
         verbose_name = _('AuthBook')
 
-    def _set_latest(self):
-        self._remove_pre_obj_latest()
+    def set_to_latest(self):
+        self.remove_pre_latest()
         self.is_latest = True
         self.save()
 
-    def _get_pre_obj(self):
+    def get_pre_latest(self):
         pre_obj = self.__class__.objects.filter(
             username=self.username, asset=self.asset
         ).latest_version().first()
         return pre_obj
 
-    def _remove_pre_obj_latest(self):
-        pre_obj = self._get_pre_obj()
+    def remove_pre_latest(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             pre_obj.is_latest = False
             pre_obj.save()
 
-    def _set_version(self):
-        pre_obj = self._get_pre_obj()
+    def set_version(self):
+        pre_obj = self.get_pre_latest()
         if pre_obj:
             self.version = pre_obj.version + 1
         else:
@@ -60,8 +61,8 @@ class AuthBook(AssetUser):
         self.save()
 
     def set_version_and_latest(self):
-        self._set_version()
-        self._set_latest()
+        self.set_version()
+        self.set_to_latest()
 
     def get_related_assets(self):
         return [self.asset]

apps/assets/models/base.py

@@ -26,7 +26,7 @@ logger = get_logger(__file__)
 class AssetUser(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
-    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric])
+    username = models.CharField(max_length=32, blank=True, verbose_name=_('Username'), validators=[alphanumeric], db_index=True)
     password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
     private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
     public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))

apps/assets/models/gathered_user.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#
+import uuid
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.models import OrgModelMixin
+
+__all__ = ['GatheredUser']
+
+
+class GatheredUser(OrgModelMixin):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_("Asset"))
+    username = models.CharField(max_length=32, blank=True, db_index=True,
+                                verbose_name=_('Username'))
+    present = models.BooleanField(default=True, verbose_name=_("Present"))
+    date_created = models.DateTimeField(auto_now_add=True,
+                                        verbose_name=_("Date created"))
+    date_updated = models.DateTimeField(auto_now=True,
+                                        verbose_name=_("Date updated"))
+
+    @property
+    def hostname(self):
+        return self.asset.hostname
+
+    @property
+    def ip(self):
+        return self.asset.ip
+
+    class Meta:
+        verbose_name = _('GatherUser')
+        ordering = ['asset']
+
+    def __str__(self):
+        return '{}: {}'.format(self.asset.hostname, self.username)
+
+
+

apps/assets/models/node.py

@@ -10,7 +10,7 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ugettext
 from django.core.cache import cache
 
-from common.utils import get_logger
+from common.utils import get_logger, timeit
 from orgs.mixins.models import OrgModelMixin, OrgManager
 from orgs.utils import set_current_org, get_current_org, tmp_to_org
 from orgs.models import Organization
@@ -116,16 +116,24 @@ class FamilyMixin:
     def all_children(self):
         return self.get_all_children(with_self=False)
 
-    def get_children(self, with_self=False):
+    def get_children_key_pattern(self, with_self=False):
         pattern = r'^{0}:[0-9]+$'.format(self.key)
         if with_self:
             pattern += r'|^{0}$'.format(self.key)
+        return pattern
+
+    def get_children(self, with_self=False):
+        pattern = self.get_children_key_pattern(with_self=with_self)
         return Node.objects.filter(key__regex=pattern)
 
-    def get_all_children(self, with_self=False):
+    def get_all_children_pattern(self, with_self=False):
         pattern = r'^{0}:'.format(self.key)
         if with_self:
             pattern += r'|^{0}$'.format(self.key)
+        return pattern
+
+    def get_all_children(self, with_self=False):
+        pattern = self.get_all_children_pattern(with_self=with_self)
         children = Node.objects.filter(key__regex=pattern)
         return children
 
@@ -290,14 +298,16 @@ class NodeAssetsMixin:
         return self.get_all_assets().valid()
 
     @classmethod
-    def get_nodes_all_assets(cls, nodes_keys):
+    def get_nodes_all_assets(cls, nodes_keys, extra_assets_ids=None):
         from .asset import Asset
         nodes_keys = cls.clean_children_keys(nodes_keys)
-        pattern = set()
+        assets_ids = set()
         for key in nodes_keys:
-            pattern.add(r'^{0}$|^{0}:'.format(key))
-        pattern = '|'.join(list(pattern))
-        return Asset.objects.filter(nodes__key__regex=pattern)
+            node_assets_ids = cls.tree().all_assets(key)
+            assets_ids.update(set(node_assets_ids))
+        if extra_assets_ids:
+            assets_ids.update(set(extra_assets_ids))
+        return Asset.objects.filter(id__in=assets_ids)
 
 
 class SomeNodesMixin:

apps/assets/serializers/__init__.py

@@ -9,3 +9,4 @@ from .node import *
 from .domain import *
 from .cmd_filter import *
 from .asset_user import *
+from .gathered_user import *

apps/assets/serializers/asset_user.py

@@ -53,6 +53,7 @@ class AssetUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         if not validated_data.get("name") and validated_data.get("username"):
             validated_data["name"] = validated_data["username"]
         instance = AssetUserManager.create(**validated_data)
+        instance.set_version_and_latest()
         return instance
 
 

apps/assets/serializers/gathered_user.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.serializers import OrgResourceModelSerializerMixin
+from ..models import GatheredUser
+
+
+class GatheredUserSerializer(OrgResourceModelSerializerMixin):
+    class Meta:
+        model = GatheredUser
+        fields = [
+            'id', 'asset', 'hostname', 'ip', 'username',
+            'present', 'date_created', 'date_updated'
+        ]
+        read_only_fields = fields
+        labels = {
+            'hostname': _("Hostname"),
+            'ip': "IP"
+        }

apps/assets/signals_handler.py

@@ -7,13 +7,14 @@ from django.db.models.signals import (
 from django.db.models.aggregates import Count
 from django.dispatch import receiver
 
-from common.utils import get_logger
+from common.utils import get_logger, timeit
 from common.decorator import on_transaction_commit
-from .models import Asset, SystemUser, Node, AuthBook
+from .models import Asset, SystemUser, Node
 from .tasks import (
     update_assets_hardware_info_util,
     test_asset_connectivity_util,
-    push_system_user_to_assets
+    push_system_user_to_assets,
+    add_nodes_assets_to_system_users
 )
 
 
@@ -99,7 +100,7 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None,
     """
     if action != "post_add":
         return
-    logger.info("System user `{}` nodes update signal recv".format(instance))
+    logger.info("System user nodes update signal recv: {}".format(instance))
 
     queryset = model.objects.filter(pk__in=pk_set)
     if model == Node:
@@ -108,9 +109,7 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None,
     else:
         nodes_keys = [instance.key]
         system_users = queryset
-    assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True)
-    for system_user in system_users:
-        system_user.assets.add(*tuple(assets))
+    add_nodes_assets_to_system_users.delay(nodes_keys, system_users)
 
 
 @receiver(m2m_changed, sender=Asset.nodes.through)
@@ -190,10 +189,3 @@ def on_asset_nodes_remove(sender, instance=None, action='', model=None,
 def on_node_update_or_created(sender, **kwargs):
     # 刷新节点
     Node.refresh_nodes()
-
-
-@receiver(post_save, sender=AuthBook)
-def on_auth_book_created(sender, instance=None, created=False, **kwargs):
-    if created:
-        logger.debug('Receive create auth book object signal.')
-        instance.set_version_and_latest()

apps/assets/tasks.py

@@ -1,634 +0,0 @@
-# ~*~ coding: utf-8 ~*~
-import json
-import re
-import os
-
-from collections import defaultdict
-from celery import shared_task
-from django.utils.translation import ugettext as _
-from django.core.cache import cache
-
-from common.utils import (
-    capacity_convert, sum_capacity, encrypt_password, get_logger
-)
-from ops.celery.decorator import (
-    register_as_period_task, after_app_shutdown_clean_periodic
-)
-
-from .models import SystemUser, AdminUser
-from .models.utils import Connectivity
-from . import const
-
-
-FORKS = 10
-TIMEOUT = 60
-logger = get_logger(__file__)
-CACHE_MAX_TIME = 60*60*2
-disk_pattern = re.compile(r'^hd|sd|xvd|vd|nv')
-PERIOD_TASK = os.environ.get("PERIOD_TASK", "on")
-
-
-def check_asset_can_run_ansible(asset):
-    if not asset.is_active:
-        msg = _("Asset has been disabled, skipped: {}").format(asset)
-        logger.info(msg)
-        return False
-    if not asset.is_support_ansible():
-        msg = _("Asset may not be support ansible, skipped: {}").format(asset)
-        logger.info(msg)
-        return False
-    return True
-
-
-def clean_hosts(assets):
-    clean_assets = []
-    for asset in assets:
-        if not check_asset_can_run_ansible(asset):
-            continue
-        clean_assets.append(asset)
-    if not clean_assets:
-        logger.info(_("No assets matched, stop task"))
-    return clean_assets
-
-
-def clean_hosts_by_protocol(system_user, assets):
-    hosts = [
-        asset for asset in assets
-        if asset.has_protocol(system_user.protocol)
-    ]
-    if not hosts:
-        msg = _("No assets matched related system user protocol, stop task")
-        logger.info(msg)
-    return hosts
-
-
-@shared_task(queue="ansible")
-def set_assets_hardware_info(assets, result, **kwargs):
-    """
-    Using ops task run result, to update asset info
-
-    @shared_task must be exit, because we using it as a task callback, is must
-    be a celery task also
-    :param assets:
-    :param result:
-    :param kwargs: {task_name: ""}
-    :return:
-    """
-    result_raw = result[0]
-    assets_updated = []
-    success_result = result_raw.get('ok', {})
-
-    for asset in assets:
-        hostname = asset.hostname
-        info = success_result.get(hostname, {})
-        info = info.get('setup', {}).get('ansible_facts', {})
-        if not info:
-            logger.error(_("Get asset info failed: {}").format(hostname))
-            continue
-        ___vendor = info.get('ansible_system_vendor', 'Unknown')
-        ___model = info.get('ansible_product_name', 'Unknown')
-        ___sn = info.get('ansible_product_serial', 'Unknown')
-
-        for ___cpu_model in info.get('ansible_processor', []):
-            if ___cpu_model.endswith('GHz') or ___cpu_model.startswith("Intel"):
-                break
-        else:
-            ___cpu_model = 'Unknown'
-        ___cpu_model = ___cpu_model[:48]
-        ___cpu_count = info.get('ansible_processor_count', 0)
-        ___cpu_cores = info.get('ansible_processor_cores', None) or \
-                       len(info.get('ansible_processor', []))
-        ___cpu_vcpus = info.get('ansible_processor_vcpus', 0)
-        ___memory = '%s %s' % capacity_convert(
-            '{} MB'.format(info.get('ansible_memtotal_mb'))
-        )
-        disk_info = {}
-        for dev, dev_info in info.get('ansible_devices', {}).items():
-            if disk_pattern.match(dev) and dev_info['removable'] == '0':
-                disk_info[dev] = dev_info['size']
-        ___disk_total = '%.1f %s' % sum_capacity(disk_info.values())
-        ___disk_info = json.dumps(disk_info)
-
-        # ___platform = info.get('ansible_system', 'Unknown')
-        ___os = info.get('ansible_distribution', 'Unknown')
-        ___os_version = info.get('ansible_distribution_version', 'Unknown')
-        ___os_arch = info.get('ansible_architecture', 'Unknown')
-        ___hostname_raw = info.get('ansible_hostname', 'Unknown')
-
-        for k, v in locals().items():
-            if k.startswith('___'):
-                setattr(asset, k.strip('_'), v)
-        asset.save()
-        assets_updated.append(asset)
-    return assets_updated
-
-
-@shared_task
-def update_assets_hardware_info_util(assets, task_name=None):
-    """
-    Using ansible api to update asset hardware info
-    :param assets:  asset seq
-    :param task_name: task_name running
-    :return: result summary ['contacted': {}, 'dark': {}]
-    """
-    from ops.utils import update_or_create_ansible_task
-    if task_name is None:
-        task_name = _("Update some assets hardware info")
-    tasks = const.UPDATE_ASSETS_HARDWARE_TASKS
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-    created_by = str(assets[0].org_id)
-    task, created = update_or_create_ansible_task(
-        task_name, hosts=hosts, tasks=tasks, created_by=created_by,
-        pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
-    )
-    result = task.run()
-    set_assets_hardware_info(assets, result)
-    return result
-
-
-@shared_task(queue="ansible")
-def update_asset_hardware_info_manual(asset):
-    task_name = _("Update asset hardware info: {}").format(asset.hostname)
-    update_assets_hardware_info_util(
-        [asset], task_name=task_name
-    )
-
-
-@shared_task(queue="ansible")
-def update_assets_hardware_info_period():
-    """
-    Update asset hardware period task
-    :return:
-    """
-    if PERIOD_TASK != "on":
-        logger.debug("Period task disabled, update assets hardware info pass")
-        return
-
-
-##  ADMIN USER CONNECTIVE  ##
-
-
-@shared_task(queue="ansible")
-def test_asset_connectivity_util(assets, task_name=None):
-    from ops.utils import update_or_create_ansible_task
-
-    if task_name is None:
-        task_name = _("Test assets connectivity")
-
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-
-    hosts_category = {
-        'linux': {
-            'hosts': [],
-            'tasks': const.TEST_ADMIN_USER_CONN_TASKS
-        },
-        'windows': {
-            'hosts': [],
-            'tasks': const.TEST_WINDOWS_ADMIN_USER_CONN_TASKS
-        }
-    }
-    for host in hosts:
-        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
-            else hosts_category['linux']['hosts']
-        hosts_list.append(host)
-
-    results_summary = dict(
-        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
-    )
-    created_by = assets[0].org_id
-    for k, value in hosts_category.items():
-        if not value['hosts']:
-            continue
-        task, created = update_or_create_ansible_task(
-            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
-            pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
-            created_by=created_by,
-        )
-        raw, summary = task.run()
-        success = summary.get('success', False)
-        contacted = summary.get('contacted', {})
-        dark = summary.get('dark', {})
-
-        results_summary['success'] &= success
-        results_summary['contacted'].update(contacted)
-        results_summary['dark'].update(dark)
-
-    for asset in assets:
-        if asset.hostname in results_summary.get('dark', {}).keys():
-            asset.connectivity = Connectivity.unreachable()
-        elif asset.hostname in results_summary.get('contacted', {}).keys():
-            asset.connectivity = Connectivity.reachable()
-        else:
-            asset.connectivity = Connectivity.unknown()
-    return results_summary
-
-
-@shared_task(queue="ansible")
-def test_asset_connectivity_manual(asset):
-    task_name = _("Test assets connectivity: {}").format(asset)
-    summary = test_asset_connectivity_util([asset], task_name=task_name)
-
-    if summary.get('dark'):
-        return False, summary['dark']
-    else:
-        return True, ""
-
-
-@shared_task(queue="ansible")
-def test_admin_user_connectivity_util(admin_user, task_name):
-    """
-    Test asset admin user can connect or not. Using ansible api do that
-    :param admin_user:
-    :param task_name:
-    :return:
-    """
-    assets = admin_user.get_related_assets()
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-    summary = test_asset_connectivity_util(hosts, task_name)
-    return summary
-
-
-@shared_task(queue="ansible")
-@register_as_period_task(interval=3600)
-def test_admin_user_connectivity_period():
-    """
-    A period task that update the ansible task period
-    """
-    if PERIOD_TASK != "on":
-        logger.debug('Period task off, skip')
-        return
-    key = '_JMS_TEST_ADMIN_USER_CONNECTIVITY_PERIOD'
-    prev_execute_time = cache.get(key)
-    if prev_execute_time:
-        logger.debug("Test admin user connectivity, less than 40 minutes, skip")
-        return
-    cache.set(key, 1, 60*40)
-    admin_users = AdminUser.objects.all()
-    for admin_user in admin_users:
-        task_name = _("Test admin user connectivity period: {}").format(admin_user.name)
-        test_admin_user_connectivity_util(admin_user, task_name)
-    cache.set(key, 1, 60*40)
-
-
-@shared_task(queue="ansible")
-def test_admin_user_connectivity_manual(admin_user):
-    task_name = _("Test admin user connectivity: {}").format(admin_user.name)
-    test_admin_user_connectivity_util(admin_user, task_name)
-    return True
-
-
-##  System user connective ##
-
-
-@shared_task(queue="ansible")
-def test_system_user_connectivity_util(system_user, assets, task_name):
-    """
-    Test system cant connect his assets or not.
-    :param system_user:
-    :param assets:
-    :param task_name:
-    :return:
-    """
-    from ops.utils import update_or_create_ansible_task
-
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-
-    hosts = clean_hosts_by_protocol(system_user, hosts)
-    if not hosts:
-        return {}
-
-    hosts_category = {
-        'linux': {
-            'hosts': [],
-            'tasks': const.TEST_SYSTEM_USER_CONN_TASKS
-        },
-        'windows': {
-            'hosts': [],
-            'tasks': const.TEST_WINDOWS_SYSTEM_USER_CONN_TASKS
-        }
-    }
-    for host in hosts:
-        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
-            else hosts_category['linux']['hosts']
-        hosts_list.append(host)
-
-    results_summary = dict(
-        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
-    )
-    for k, value in hosts_category.items():
-        if not value['hosts']:
-            continue
-        task, created = update_or_create_ansible_task(
-            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
-            pattern='all', options=const.TASK_OPTIONS,
-            run_as=system_user.username, created_by=system_user.org_id,
-        )
-        raw, summary = task.run()
-        success = summary.get('success', False)
-        contacted = summary.get('contacted', {})
-        dark = summary.get('dark', {})
-
-        results_summary['success'] &= success
-        results_summary['contacted'].update(contacted)
-        results_summary['dark'].update(dark)
-
-    system_user.set_connectivity(results_summary)
-    return results_summary
-
-
-@shared_task(queue="ansible")
-def test_system_user_connectivity_manual(system_user):
-    task_name = _("Test system user connectivity: {}").format(system_user)
-    assets = system_user.get_all_assets()
-    return test_system_user_connectivity_util(system_user, assets, task_name)
-
-
-@shared_task(queue="ansible")
-def test_system_user_connectivity_a_asset(system_user, asset):
-    task_name = _("Test system user connectivity: {} => {}").format(
-        system_user, asset
-    )
-    return test_system_user_connectivity_util(system_user, [asset], task_name)
-
-
-@shared_task(queue="ansible")
-def test_system_user_connectivity_period():
-    if PERIOD_TASK != "on":
-        logger.debug("Period task disabled, test system user connectivity pass")
-        return
-    system_users = SystemUser.objects.all()
-    for system_user in system_users:
-        task_name = _("Test system user connectivity period: {}").format(system_user)
-        assets = system_user.get_all_assets()
-        test_system_user_connectivity_util(system_user, assets, task_name)
-
-
-####  Push system user tasks ####
-
-def get_push_linux_system_user_tasks(system_user):
-    tasks = [
-        {
-            'name': 'Add user {}'.format(system_user.username),
-            'action': {
-                'module': 'user',
-                'args': 'name={} shell={} state=present'.format(
-                    system_user.username, system_user.shell,
-                ),
-            }
-        },
-        {
-            'name': 'Add group {}'.format(system_user.username),
-            'action': {
-                'module': 'group',
-                'args': 'name={} state=present'.format(
-                    system_user.username,
-                ),
-            }
-        },
-        {
-            'name': 'Check home dir exists',
-            'action': {
-                'module': 'stat',
-                'args': 'path=/home/{}'.format(system_user.username)
-            },
-            'register': 'home_existed'
-        },
-        {
-            'name': "Set home dir permission",
-            'action': {
-                'module': 'file',
-                'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username)
-            },
-            'when': 'home_existed.stat.exists == true'
-        }
-    ]
-    if system_user.password:
-        tasks.append({
-            'name': 'Set {} password'.format(system_user.username),
-            'action': {
-                'module': 'user',
-                'args': 'name={} shell={} state=present password={}'.format(
-                    system_user.username, system_user.shell,
-                    encrypt_password(system_user.password, salt="K3mIlKK"),
-                ),
-            }
-        })
-    if system_user.public_key:
-        tasks.append({
-            'name': 'Set {} authorized key'.format(system_user.username),
-            'action': {
-                'module': 'authorized_key',
-                'args': "user={} state=present key='{}'".format(
-                    system_user.username, system_user.public_key
-                )
-            }
-        })
-    if system_user.sudo:
-        sudo = system_user.sudo.replace('\r\n', '\n').replace('\r', '\n')
-        sudo_list = sudo.split('\n')
-        sudo_tmp = []
-        for s in sudo_list:
-            sudo_tmp.append(s.strip(','))
-        sudo = ','.join(sudo_tmp)
-        tasks.append({
-            'name': 'Set {} sudo setting'.format(system_user.username),
-            'action': {
-                'module': 'lineinfile',
-                'args': "dest=/etc/sudoers state=present regexp='^{0} ALL=' "
-                        "line='{0} ALL=(ALL) NOPASSWD: {1}' "
-                        "validate='visudo -cf %s'".format(
-                    system_user.username, sudo,
-                )
-            }
-        })
-
-    return tasks
-
-
-def get_push_windows_system_user_tasks(system_user):
-    tasks = []
-    if system_user.password:
-        tasks.append({
-            'name': 'Add user {}'.format(system_user.username),
-            'action': {
-                'module': 'win_user',
-                'args': 'fullname={} '
-                        'name={} '
-                        'password={} '
-                        'state=present '
-                        'update_password=always '
-                        'password_expired=no '
-                        'password_never_expires=yes '
-                        'groups="Users,Remote Desktop Users" '
-                        'groups_action=add '
-                        ''.format(system_user.name,
-                                  system_user.username,
-                                  system_user.password),
-            }
-        })
-    return tasks
-
-
-def get_push_system_user_tasks(host, system_user):
-    if host.is_unixlike():
-        tasks = get_push_linux_system_user_tasks(system_user)
-    elif host.is_windows():
-        tasks = get_push_windows_system_user_tasks(system_user)
-    else:
-        msg = _(
-            "The asset {} system platform {} does not "
-            "support run Ansible tasks".format(host.hostname, host.platform)
-        )
-        logger.info(msg)
-        tasks = []
-    return tasks
-
-
-@shared_task(queue="ansible")
-def push_system_user_util(system_user, assets, task_name):
-    from ops.utils import update_or_create_ansible_task
-    if not system_user.is_need_push():
-        msg = _("Push system user task skip, auto push not enable or "
-                "protocol is not ssh or rdp: {}").format(system_user.name)
-        logger.info(msg)
-        return {}
-
-    # Set root as system user is dangerous
-    if system_user.username.lower() in ["root", "administrator"]:
-        msg = _("For security, do not push user {}".format(system_user.username))
-        logger.info(msg)
-        return {}
-
-    hosts = clean_hosts(assets)
-    if not hosts:
-        return {}
-
-    hosts = clean_hosts_by_protocol(system_user, hosts)
-    if not hosts:
-        return {}
-
-    for host in hosts:
-        system_user.load_specific_asset_auth(host)
-        tasks = get_push_system_user_tasks(host, system_user)
-        if not tasks:
-            continue
-        task, created = update_or_create_ansible_task(
-            task_name=task_name, hosts=[host], tasks=tasks, pattern='all',
-            options=const.TASK_OPTIONS, run_as_admin=True,
-            created_by=system_user.org_id,
-        )
-        task.run()
-
-
-@shared_task(queue="ansible")
-def push_system_user_to_assets_manual(system_user):
-    assets = system_user.get_all_assets()
-    task_name = _("Push system users to assets: {}").format(system_user.name)
-    return push_system_user_util(system_user, assets, task_name=task_name)
-
-
-@shared_task(queue="ansible")
-def push_system_user_a_asset_manual(system_user, asset):
-    task_name = _("Push system users to asset: {} => {}").format(
-        system_user.name, asset
-    )
-    return push_system_user_util(system_user, [asset], task_name=task_name)
-
-
-@shared_task(queue="ansible")
-def push_system_user_to_assets(system_user, assets):
-    task_name = _("Push system users to assets: {}").format(system_user.name)
-    return push_system_user_util(system_user, assets, task_name)
-
-
-@shared_task
-@after_app_shutdown_clean_periodic
-def test_system_user_connectability_period():
-    pass
-
-
-@shared_task
-@after_app_shutdown_clean_periodic
-def test_admin_user_connectability_period():
-    pass
-
-
-#### Test Asset user connectivity task ####
-
-def get_test_asset_user_connectivity_tasks(asset):
-    if asset.is_unixlike():
-        tasks = const.TEST_ASSET_USER_CONN_TASKS
-    elif asset.is_windows():
-        tasks = const.TEST_WINDOWS_ASSET_USER_CONN_TASKS
-    else:
-        msg = _(
-            "The asset {} system platform {} does not "
-            "support run Ansible tasks".format(asset.hostname, asset.platform)
-        )
-        logger.info(msg)
-        tasks = []
-    return tasks
-
-
-@shared_task(queue="ansible")
-def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False):
-    """
-    :param asset_user: <AuthBook>对象
-    :param task_name:
-    :param run_as_admin:
-    :return:
-    """
-    from ops.utils import update_or_create_ansible_task
-
-    if not check_asset_can_run_ansible(asset_user.asset):
-        return
-
-    tasks = get_test_asset_user_connectivity_tasks(asset_user.asset)
-    if not tasks:
-        logger.debug("No tasks ")
-        return
-
-    args = (task_name,)
-    kwargs = {
-        'hosts': [asset_user.asset], 'tasks': tasks,
-        'pattern': 'all', 'options': const.TASK_OPTIONS,
-        'created_by': asset_user.org_id,
-    }
-    if run_as_admin:
-        kwargs["run_as_admin"] = True
-    else:
-        kwargs["run_as"] = asset_user.username
-    task, created = update_or_create_ansible_task(*args, **kwargs)
-    raw, summary = task.run()
-    asset_user.set_connectivity(summary)
-
-
-@shared_task(queue="ansible")
-def test_asset_users_connectivity_manual(asset_users, run_as_admin=False):
-    """
-    :param asset_users: <AuthBook>对象
-    """
-    for asset_user in asset_users:
-        task_name = _("Test asset user connectivity: {}").format(asset_user)
-        test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=run_as_admin)
-
-
-# @shared_task
-# @register_as_period_task(interval=3600)
-# @after_app_ready_start
-# @after_app_shutdown_clean_periodic
-# def push_system_user_period():
-#     for system_user in SystemUser.objects.all():
-#         push_system_user_related_nodes(system_user)
-
-
-
-

apps/assets/tasks/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+from .utils import *
+from .common import *
+from .admin_user_connectivity import *
+from .asset_connectivity import *
+from .asset_user_connectivity import *
+from .gather_asset_users import *
+from .gather_asset_hardware_info import *
+from .push_system_user import *
+from .system_user_connectivity import *

apps/assets/tasks/admin_user_connectivity.py

@@ -0,0 +1,65 @@
+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+from django.core.cache import cache
+
+from common.utils import get_logger
+from ops.celery.decorator import register_as_period_task
+
+from ..models import AdminUser
+from .utils import clean_hosts
+from .asset_connectivity import test_asset_connectivity_util
+from . import const
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'test_admin_user_connectivity_util', 'test_admin_user_connectivity_manual',
+    'test_admin_user_connectivity_period'
+]
+
+
+@shared_task(queue="ansible")
+def test_admin_user_connectivity_util(admin_user, task_name):
+    """
+    Test asset admin user can connect or not. Using ansible api do that
+    :param admin_user:
+    :param task_name:
+    :return:
+    """
+    assets = admin_user.get_related_assets()
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+    summary = test_asset_connectivity_util(hosts, task_name)
+    return summary
+
+
+@shared_task(queue="ansible")
+@register_as_period_task(interval=3600)
+def test_admin_user_connectivity_period():
+    """
+    A period task that update the ansible task period
+    """
+    if const.PERIOD_TASK_ENABLED:
+        logger.debug('Period task off, skip')
+        return
+    key = '_JMS_TEST_ADMIN_USER_CONNECTIVITY_PERIOD'
+    prev_execute_time = cache.get(key)
+    if prev_execute_time:
+        logger.debug("Test admin user connectivity, less than 40 minutes, skip")
+        return
+    cache.set(key, 1, 60*40)
+    admin_users = AdminUser.objects.all()
+    for admin_user in admin_users:
+        task_name = _("Test admin user connectivity period: {}").format(admin_user.name)
+        test_admin_user_connectivity_util(admin_user, task_name)
+    cache.set(key, 1, 60*40)
+
+
+@shared_task(queue="ansible")
+def test_admin_user_connectivity_manual(admin_user):
+    task_name = _("Test admin user connectivity: {}").format(admin_user.name)
+    test_admin_user_connectivity_util(admin_user, task_name)
+    return True

apps/assets/tasks/asset_connectivity.py

@@ -0,0 +1,81 @@
+# ~*~ coding: utf-8 ~*~
+from collections import defaultdict
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+from ..models.utils import Connectivity
+from . import const
+from .utils import clean_hosts
+
+
+logger = get_logger(__file__)
+__all__ = ['test_asset_connectivity_util', 'test_asset_connectivity_manual']
+
+
+@shared_task(queue="ansible")
+def test_asset_connectivity_util(assets, task_name=None):
+    from ops.utils import update_or_create_ansible_task
+
+    if task_name is None:
+        task_name = _("Test assets connectivity")
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.TEST_ADMIN_USER_CONN_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.TEST_WINDOWS_ADMIN_USER_CONN_TASKS
+        }
+    }
+    for host in hosts:
+        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(host)
+
+    results_summary = dict(
+        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
+    )
+    created_by = assets[0].org_id
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
+            created_by=created_by,
+        )
+        raw, summary = task.run()
+        success = summary.get('success', False)
+        contacted = summary.get('contacted', {})
+        dark = summary.get('dark', {})
+
+        results_summary['success'] &= success
+        results_summary['contacted'].update(contacted)
+        results_summary['dark'].update(dark)
+
+    for asset in assets:
+        if asset.hostname in results_summary.get('dark', {}).keys():
+            asset.connectivity = Connectivity.unreachable()
+        elif asset.hostname in results_summary.get('contacted', {}).keys():
+            asset.connectivity = Connectivity.reachable()
+        else:
+            asset.connectivity = Connectivity.unknown()
+    return results_summary
+
+
+@shared_task(queue="ansible")
+def test_asset_connectivity_manual(asset):
+    task_name = _("Test assets connectivity: {}").format(asset)
+    summary = test_asset_connectivity_util([asset], task_name=task_name)
+
+    if summary.get('dark'):
+        return False, summary['dark']
+    else:
+        return True, ""

apps/assets/tasks/asset_user_connectivity.py

@@ -0,0 +1,77 @@
+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+from . import const
+from .utils import check_asset_can_run_ansible
+
+
+logger = get_logger(__file__)
+
+
+__all__ = [
+    'test_asset_user_connectivity_util', 'test_asset_users_connectivity_manual',
+    'get_test_asset_user_connectivity_tasks',
+]
+
+
+def get_test_asset_user_connectivity_tasks(asset):
+    if asset.is_unixlike():
+        tasks = const.TEST_ASSET_USER_CONN_TASKS
+    elif asset.is_windows():
+        tasks = const.TEST_WINDOWS_ASSET_USER_CONN_TASKS
+    else:
+        msg = _(
+            "The asset {} system platform {} does not "
+            "support run Ansible tasks".format(asset.hostname, asset.platform)
+        )
+        logger.info(msg)
+        tasks = []
+    return tasks
+
+
+@shared_task(queue="ansible")
+def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False):
+    """
+    :param asset_user: <AuthBook>对象
+    :param task_name:
+    :param run_as_admin:
+    :return:
+    """
+    from ops.utils import update_or_create_ansible_task
+
+    if not check_asset_can_run_ansible(asset_user.asset):
+        return
+
+    tasks = get_test_asset_user_connectivity_tasks(asset_user.asset)
+    if not tasks:
+        logger.debug("No tasks ")
+        return
+
+    args = (task_name,)
+    kwargs = {
+        'hosts': [asset_user.asset], 'tasks': tasks,
+        'pattern': 'all', 'options': const.TASK_OPTIONS,
+        'created_by': asset_user.org_id,
+    }
+    if run_as_admin:
+        kwargs["run_as_admin"] = True
+    else:
+        kwargs["run_as"] = asset_user.username
+    task, created = update_or_create_ansible_task(*args, **kwargs)
+    raw, summary = task.run()
+    asset_user.set_connectivity(summary)
+
+
+@shared_task(queue="ansible")
+def test_asset_users_connectivity_manual(asset_users, run_as_admin=False):
+    """
+    :param asset_users: <AuthBook>对象
+    """
+    for asset_user in asset_users:
+        task_name = _("Test asset user connectivity: {}").format(asset_user)
+        test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=run_as_admin)
+
+

apps/assets/tasks/common.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+#
+
+from celery import shared_task
+
+__all__ = ['add_nodes_assets_to_system_users']
+
+
+@shared_task
+def add_nodes_assets_to_system_users(nodes_keys, system_users):
+    from ..models import Node
+    assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True)
+    for system_user in system_users:
+        system_user.assets.add(*tuple(assets))

apps/assets/tasks/const.py

@@ -1,7 +1,11 @@
 # -*- coding: utf-8 -*-
 #
+import os
 from django.utils.translation import ugettext_lazy as _
 
+
+PERIOD_TASK_ENABLED = os.environ.get("PERIOD_TASK", "on") == 'on'
+
 UPDATE_ASSETS_HARDWARE_TASKS = [
    {
        'name': "setup",
@@ -79,3 +83,22 @@ CONNECTIVITY_CHOICES = (
     (CONN_UNKNOWN, _("Unknown")),
 )
 
+GATHER_ASSET_USERS_TASKS = [
+    {
+        "name": "gather host users",
+        "action": {
+            "module": 'getent',
+            "args": "database=passwd"
+        },
+    },
+]
+
+GATHER_ASSET_USERS_TASKS_WINDOWS = [
+    {
+        "name": "gather windows host users",
+        "action": {
+            "module": 'win_shell',
+            "args": "net user"
+        }
+    }
+]

apps/assets/tasks/gather_asset_hardware_info.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+#
+import json
+import re
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import (
+    capacity_convert, sum_capacity, get_logger
+)
+from . import const
+from .utils import clean_hosts
+
+
+logger = get_logger(__file__)
+disk_pattern = re.compile(r'^hd|sd|xvd|vd|nv')
+__all__ = [
+    'update_assets_hardware_info_util', 'update_asset_hardware_info_manual',
+    'update_assets_hardware_info_period',
+]
+
+
+def set_assets_hardware_info(assets, result, **kwargs):
+    """
+    Using ops task run result, to update asset info
+
+    @shared_task must be exit, because we using it as a task callback, is must
+    be a celery task also
+    :param assets:
+    :param result:
+    :param kwargs: {task_name: ""}
+    :return:
+    """
+    result_raw = result[0]
+    assets_updated = []
+    success_result = result_raw.get('ok', {})
+
+    for asset in assets:
+        hostname = asset.hostname
+        info = success_result.get(hostname, {})
+        info = info.get('setup', {}).get('ansible_facts', {})
+        if not info:
+            logger.error(_("Get asset info failed: {}").format(hostname))
+            continue
+        ___vendor = info.get('ansible_system_vendor', 'Unknown')
+        ___model = info.get('ansible_product_name', 'Unknown')
+        ___sn = info.get('ansible_product_serial', 'Unknown')
+
+        for ___cpu_model in info.get('ansible_processor', []):
+            if ___cpu_model.endswith('GHz') or ___cpu_model.startswith("Intel"):
+                break
+        else:
+            ___cpu_model = 'Unknown'
+        ___cpu_model = ___cpu_model[:48]
+        ___cpu_count = info.get('ansible_processor_count', 0)
+        ___cpu_cores = info.get('ansible_processor_cores', None) or \
+                       len(info.get('ansible_processor', []))
+        ___cpu_vcpus = info.get('ansible_processor_vcpus', 0)
+        ___memory = '%s %s' % capacity_convert(
+            '{} MB'.format(info.get('ansible_memtotal_mb'))
+        )
+        disk_info = {}
+        for dev, dev_info in info.get('ansible_devices', {}).items():
+            if disk_pattern.match(dev) and dev_info['removable'] == '0':
+                disk_info[dev] = dev_info['size']
+        ___disk_total = '%.1f %s' % sum_capacity(disk_info.values())
+        ___disk_info = json.dumps(disk_info)
+
+        # ___platform = info.get('ansible_system', 'Unknown')
+        ___os = info.get('ansible_distribution', 'Unknown')
+        ___os_version = info.get('ansible_distribution_version', 'Unknown')
+        ___os_arch = info.get('ansible_architecture', 'Unknown')
+        ___hostname_raw = info.get('ansible_hostname', 'Unknown')
+
+        for k, v in locals().items():
+            if k.startswith('___'):
+                setattr(asset, k.strip('_'), v)
+        asset.save()
+        assets_updated.append(asset)
+    return assets_updated
+
+
+@shared_task
+def update_assets_hardware_info_util(assets, task_name=None):
+    """
+    Using ansible api to update asset hardware info
+    :param assets:  asset seq
+    :param task_name: task_name running
+    :return: result summary ['contacted': {}, 'dark': {}]
+    """
+    from ops.utils import update_or_create_ansible_task
+    if task_name is None:
+        task_name = _("Update some assets hardware info")
+    tasks = const.UPDATE_ASSETS_HARDWARE_TASKS
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+    created_by = str(assets[0].org_id)
+    task, created = update_or_create_ansible_task(
+        task_name, hosts=hosts, tasks=tasks, created_by=created_by,
+        pattern='all', options=const.TASK_OPTIONS, run_as_admin=True,
+    )
+    result = task.run()
+    set_assets_hardware_info(assets, result)
+    return True
+
+
+@shared_task(queue="ansible")
+def update_asset_hardware_info_manual(asset):
+    task_name = _("Update asset hardware info: {}").format(asset.hostname)
+    update_assets_hardware_info_util(
+        [asset], task_name=task_name
+    )
+
+
+@shared_task(queue="ansible")
+def update_assets_hardware_info_period():
+    """
+    Update asset hardware period task
+    :return:
+    """
+    if not const.PERIOD_TASK_ENABLED:
+        logger.debug("Period task disabled, update assets hardware info pass")
+        return

apps/assets/tasks/gather_asset_users.py

@@ -0,0 +1,135 @@
+# ~*~ coding: utf-8 ~*~
+
+import re
+from collections import defaultdict
+from celery import shared_task
+
+from django.utils.translation import ugettext as _
+
+from orgs.utils import tmp_to_org
+from common.utils import get_logger
+from ..models import GatheredUser, Node
+from .utils import clean_hosts
+from . import const
+
+__all__ = ['gather_asset_users', 'gather_nodes_asset_users']
+logger = get_logger(__name__)
+space = re.compile('\s+')
+ignore_login_shell = re.compile(r'nologin$|sync$|shutdown$|halt$')
+
+
+def parse_linux_result_to_users(result):
+    task_result = {}
+    for task_name, raw in result.items():
+        res = raw.get('ansible_facts', {}).get('getent_passwd')
+        if res:
+            task_result = res
+            break
+    if not task_result or not isinstance(task_result, dict):
+        return []
+    users = []
+    for username, attr in task_result.items():
+        if ignore_login_shell.search(attr[-1]):
+            continue
+        users.append(username)
+    return users
+
+
+def parse_windows_result_to_users(result):
+    task_result = []
+    for task_name, raw in result.items():
+        res = raw.get('stdout_lines', {})
+        if res:
+            task_result = res
+            break
+    if not task_result:
+        return []
+
+    users = []
+
+    for i in range(4):
+        task_result.pop(0)
+    for i in range(2):
+        task_result.pop()
+
+    for line in task_result:
+        user = space.split(line)
+        if user[0]:
+            users.append(user[0])
+    return users
+
+
+def add_asset_users(assets, results):
+    assets_map = {a.hostname: a for a in assets}
+    parser_map = {
+        'linux': parse_linux_result_to_users,
+        'windows': parse_windows_result_to_users
+    }
+
+    assets_users_map = {}
+
+    for platform, platform_results in results.items():
+        for hostname, res in platform_results.items():
+            parse = parser_map.get(platform)
+            users = parse(res)
+            logger.debug('Gathered host users: {} {}'.format(hostname, users))
+            asset = assets_map.get(hostname)
+            if not asset:
+                continue
+            assets_users_map[asset] = users
+
+    for asset, users in assets_users_map.items():
+        with tmp_to_org(asset.org_id):
+            GatheredUser.objects.filter(asset=asset, present=True)\
+                .update(present=False)
+            for username in users:
+                defaults = {'asset': asset, 'username': username, 'present': True}
+                GatheredUser.objects.update_or_create(
+                    defaults=defaults, asset=asset, username=username,
+                )
+
+
+@shared_task(queue="ansible")
+def gather_asset_users(assets, task_name=None):
+    from ops.utils import update_or_create_ansible_task
+    if task_name is None:
+        task_name = _("Gather assets users")
+    assets = clean_hosts(assets)
+    if not assets:
+        return
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.GATHER_ASSET_USERS_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.GATHER_ASSET_USERS_TASKS_WINDOWS
+        }
+    }
+    for asset in assets:
+        hosts_list = hosts_category['windows']['hosts'] if asset.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(asset)
+
+    results = {'linux': defaultdict(dict), 'windows': defaultdict(dict)}
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        _task_name = '{}: {}'.format(task_name, k)
+        task, created = update_or_create_ansible_task(
+            task_name=_task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS,
+            run_as_admin=True, created_by=value['hosts'][0].org_id,
+        )
+        raw, summary = task.run()
+        results[k].update(raw['ok'])
+    add_asset_users(assets, results)
+
+
+@shared_task(queue="ansible")
+def gather_nodes_asset_users(nodes_key):
+    assets = Node.get_nodes_all_assets(nodes_key)
+    assets_groups_by_100 = [assets[i:i+100] for i in range(0, len(assets), 100)]
+    for _assets in assets_groups_by_100:
+        gather_asset_users(_assets)

apps/assets/tasks/push_system_user.py

@@ -0,0 +1,202 @@
+# ~*~ coding: utf-8 ~*~
+
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import encrypt_password, get_logger
+from . import const
+from .utils import clean_hosts_by_protocol, clean_hosts
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'push_system_user_util', 'push_system_user_to_assets',
+    'push_system_user_to_assets_manual', 'push_system_user_a_asset_manual',
+]
+
+
+def get_push_linux_system_user_tasks(system_user):
+    tasks = [
+        {
+            'name': 'Add user {}'.format(system_user.username),
+            'action': {
+                'module': 'user',
+                'args': 'name={} shell={} state=present'.format(
+                    system_user.username, system_user.shell,
+                ),
+            }
+        },
+        {
+            'name': 'Add group {}'.format(system_user.username),
+            'action': {
+                'module': 'group',
+                'args': 'name={} state=present'.format(
+                    system_user.username,
+                ),
+            }
+        },
+        {
+            'name': 'Check home dir exists',
+            'action': {
+                'module': 'stat',
+                'args': 'path=/home/{}'.format(system_user.username)
+            },
+            'register': 'home_existed'
+        },
+        {
+            'name': "Set home dir permission",
+            'action': {
+                'module': 'file',
+                'args': "path=/home/{0} owner={0} group={0} mode=700".format(system_user.username)
+            },
+            'when': 'home_existed.stat.exists == true'
+        }
+    ]
+    if system_user.password:
+        tasks.append({
+            'name': 'Set {} password'.format(system_user.username),
+            'action': {
+                'module': 'user',
+                'args': 'name={} shell={} state=present password={}'.format(
+                    system_user.username, system_user.shell,
+                    encrypt_password(system_user.password, salt="K3mIlKK"),
+                ),
+            }
+        })
+    if system_user.public_key:
+        tasks.append({
+            'name': 'Set {} authorized key'.format(system_user.username),
+            'action': {
+                'module': 'authorized_key',
+                'args': "user={} state=present key='{}'".format(
+                    system_user.username, system_user.public_key
+                )
+            }
+        })
+    if system_user.sudo:
+        sudo = system_user.sudo.replace('\r\n', '\n').replace('\r', '\n')
+        sudo_list = sudo.split('\n')
+        sudo_tmp = []
+        for s in sudo_list:
+            sudo_tmp.append(s.strip(','))
+        sudo = ','.join(sudo_tmp)
+        tasks.append({
+            'name': 'Set {} sudo setting'.format(system_user.username),
+            'action': {
+                'module': 'lineinfile',
+                'args': "dest=/etc/sudoers state=present regexp='^{0} ALL=' "
+                        "line='{0} ALL=(ALL) NOPASSWD: {1}' "
+                        "validate='visudo -cf %s'".format(
+                    system_user.username, sudo,
+                )
+            }
+        })
+
+    return tasks
+
+
+def get_push_windows_system_user_tasks(system_user):
+    tasks = []
+    if not system_user.password:
+        return tasks
+    tasks.append({
+        'name': 'Add user {}'.format(system_user.username),
+        'action': {
+            'module': 'win_user',
+            'args': 'fullname={} '
+                    'name={} '
+                    'password={} '
+                    'state=present '
+                    'update_password=always '
+                    'password_expired=no '
+                    'password_never_expires=yes '
+                    'groups="Users,Remote Desktop Users" '
+                    'groups_action=add '
+                    ''.format(system_user.name,
+                              system_user.username,
+                              system_user.password),
+        }
+    })
+    return tasks
+
+
+def get_push_system_user_tasks(host, system_user):
+    if host.is_unixlike():
+        tasks = get_push_linux_system_user_tasks(system_user)
+    elif host.is_windows():
+        tasks = get_push_windows_system_user_tasks(system_user)
+    else:
+        msg = _(
+            "The asset {} system platform {} does not "
+            "support run Ansible tasks".format(host.hostname, host.platform)
+        )
+        logger.info(msg)
+        tasks = []
+    return tasks
+
+
+@shared_task(queue="ansible")
+def push_system_user_util(system_user, assets, task_name):
+    from ops.utils import update_or_create_ansible_task
+    if not system_user.is_need_push():
+        msg = _("Push system user task skip, auto push not enable or "
+                "protocol is not ssh or rdp: {}").format(system_user.name)
+        logger.info(msg)
+        return {}
+
+    # Set root as system user is dangerous
+    if system_user.username.lower() in ["root", "administrator"]:
+        msg = _("For security, do not push user {}".format(system_user.username))
+        logger.info(msg)
+        return {}
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts = clean_hosts_by_protocol(system_user, hosts)
+    if not hosts:
+        return {}
+
+    for host in hosts:
+        system_user.load_specific_asset_auth(host)
+        tasks = get_push_system_user_tasks(host, system_user)
+        if not tasks:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=[host], tasks=tasks, pattern='all',
+            options=const.TASK_OPTIONS, run_as_admin=True,
+            created_by=system_user.org_id,
+        )
+        task.run()
+
+
+@shared_task(queue="ansible")
+def push_system_user_to_assets_manual(system_user):
+    assets = system_user.get_all_assets()
+    task_name = _("Push system users to assets: {}").format(system_user.name)
+    return push_system_user_util(system_user, assets, task_name=task_name)
+
+
+@shared_task(queue="ansible")
+def push_system_user_a_asset_manual(system_user, asset):
+    task_name = _("Push system users to asset: {} => {}").format(
+        system_user.name, asset
+    )
+    return push_system_user_util(system_user, [asset], task_name=task_name)
+
+
+@shared_task(queue="ansible")
+def push_system_user_to_assets(system_user, assets):
+    task_name = _("Push system users to assets: {}").format(system_user.name)
+    return push_system_user_util(system_user, assets, task_name)
+
+
+
+# @shared_task
+# @register_as_period_task(interval=3600)
+# @after_app_ready_start
+# @after_app_shutdown_clean_periodic
+# def push_system_user_period():
+#     for system_user in SystemUser.objects.all():
+#         push_system_user_related_nodes(system_user)

apps/assets/tasks/system_user_connectivity.py

@@ -0,0 +1,101 @@
+
+from collections import defaultdict
+from celery import shared_task
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+
+from ..models import SystemUser
+from . import const
+from .utils import clean_hosts, clean_hosts_by_protocol
+
+logger = get_logger(__name__)
+__all__ = [
+    'test_system_user_connectivity_util', 'test_system_user_connectivity_manual',
+    'test_system_user_connectivity_period', 'test_system_user_connectivity_a_asset',
+]
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_util(system_user, assets, task_name):
+    """
+    Test system cant connect his assets or not.
+    :param system_user:
+    :param assets:
+    :param task_name:
+    :return:
+    """
+    from ops.utils import update_or_create_ansible_task
+
+    hosts = clean_hosts(assets)
+    if not hosts:
+        return {}
+
+    hosts = clean_hosts_by_protocol(system_user, hosts)
+    if not hosts:
+        return {}
+
+    hosts_category = {
+        'linux': {
+            'hosts': [],
+            'tasks': const.TEST_SYSTEM_USER_CONN_TASKS
+        },
+        'windows': {
+            'hosts': [],
+            'tasks': const.TEST_WINDOWS_SYSTEM_USER_CONN_TASKS
+        }
+    }
+    for host in hosts:
+        hosts_list = hosts_category['windows']['hosts'] if host.is_windows() \
+            else hosts_category['linux']['hosts']
+        hosts_list.append(host)
+
+    results_summary = dict(
+        contacted=defaultdict(dict), dark=defaultdict(dict), success=True
+    )
+    for k, value in hosts_category.items():
+        if not value['hosts']:
+            continue
+        task, created = update_or_create_ansible_task(
+            task_name=task_name, hosts=value['hosts'], tasks=value['tasks'],
+            pattern='all', options=const.TASK_OPTIONS,
+            run_as=system_user.username, created_by=system_user.org_id,
+        )
+        raw, summary = task.run()
+        success = summary.get('success', False)
+        contacted = summary.get('contacted', {})
+        dark = summary.get('dark', {})
+
+        results_summary['success'] &= success
+        results_summary['contacted'].update(contacted)
+        results_summary['dark'].update(dark)
+
+    system_user.set_connectivity(results_summary)
+    return results_summary
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_manual(system_user):
+    task_name = _("Test system user connectivity: {}").format(system_user)
+    assets = system_user.get_all_assets()
+    return test_system_user_connectivity_util(system_user, assets, task_name)
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_a_asset(system_user, asset):
+    task_name = _("Test system user connectivity: {} => {}").format(
+        system_user, asset
+    )
+    return test_system_user_connectivity_util(system_user, [asset], task_name)
+
+
+@shared_task(queue="ansible")
+def test_system_user_connectivity_period():
+    if not const.PERIOD_TASK_ENABLED:
+        logger.debug("Period task disabled, test system user connectivity pass")
+        return
+    system_users = SystemUser.objects.all()
+    for system_user in system_users:
+        task_name = _("Test system user connectivity period: {}").format(system_user)
+        assets = system_user.get_all_assets()
+        test_system_user_connectivity_util(system_user, assets, task_name)

apps/assets/tasks/utils.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+#
+from django.utils.translation import ugettext as _
+
+from common.utils import get_logger
+
+
+logger = get_logger(__file__)
+__all__ = [
+    'check_asset_can_run_ansible', 'clean_hosts', 'clean_hosts_by_protocol'
+]
+
+
+def check_asset_can_run_ansible(asset):
+    if not asset.is_active:
+        msg = _("Asset has been disabled, skipped: {}").format(asset)
+        logger.info(msg)
+        return False
+    if not asset.is_support_ansible():
+        msg = _("Asset may not be support ansible, skipped: {}").format(asset)
+        logger.info(msg)
+        return False
+    return True
+
+
+def clean_hosts(assets):
+    clean_assets = []
+    for asset in assets:
+        if not check_asset_can_run_ansible(asset):
+            continue
+        clean_assets.append(asset)
+    if not clean_assets:
+        logger.info(_("No assets matched, stop task"))
+    return clean_assets
+
+
+def clean_hosts_by_protocol(system_user, assets):
+    hosts = [
+        asset for asset in assets
+        if asset.has_protocol(system_user.protocol)
+    ]
+    if not hosts:
+        msg = _("No assets matched related system user protocol, stop task")
+        logger.info(msg)
+    return hosts

apps/assets/templates/assets/asset_list.html

@@ -85,7 +85,7 @@
                   <button data-toggle="dropdown" class="btn btn-default btn-sm dropdown-toggle">{% trans 'Label' %} <span class="caret"></span></button>
                   <ul class="dropdown-menu labels">
                       {% for label in labels %}
-                          <li><a style="font-weight: bolder">{{ label.name }}:{{ label.value }}</a></li>
+                          <li><a style="font-weight: bolder">{{ label.name }}#{{ label.value }}</a></li>
                       {% endfor %}
                   </ul>
                </div>
@@ -171,9 +171,13 @@ function initTable() {
         ],
         ajax_url: '{% url "api-assets:asset-list" %}',
         columns: [
-            {data: "id"}, {data: "hostname" }, {data: "ip" },
+            {data: "id"}, {data: "hostname"}, {data: "ip"},
             {data: "cpu_cores", orderable: false},
-            {data: "connectivity", orderable: false}, {data: "id", orderable: false }
+            {
+                data: "connectivity",
+                orderable: false,
+                width: '60px'
+            }, {data: "id", orderable: false}
         ],
         op_html: $('#actions').html()
     };
@@ -271,7 +275,7 @@ $(document).ready(function(){
     setAssetModalOptions(modalOption);
 })
 .on('click', '.labels li', function () {
-    var val = $(this).text();
+    var val = 'label:' + $(this).text();
     $("#asset_list_table_filter input").val(val);
     asset_table.search(val).draw();
  })

apps/assets/templates/assets/user_asset_list.html

@@ -24,7 +24,7 @@
 var treeUrl = "{% url 'api-perms:my-nodes-children-as-tree' %}?&cache_policy=1";
 var assetTableUrl = "{% url 'api-perms:my-assets' %}?cache_policy=1";
 var selectUrl = '{% url "api-perms:my-node-assets" node_id=DEFAULT_PK %}?cache_policy=1&all=1';
-var systemUsersUrl = "{% url 'api-perms:my-asset-system-users' asset_id=DEFAULT_PK %}";
+var systemUsersUrl = "{% url 'api-perms:my-asset-system-users' asset_id=DEFAULT_PK %}?cache_policy=1";
 var showAssetHref = false; // Need input default true
 var actions = {
     targets: 4, createdCell: function (td, cellData) {

apps/assets/urls/api_urls.py

@@ -21,6 +21,7 @@ router.register(r'gateways', api.GatewayViewSet, 'gateway')
 router.register(r'cmd-filters', api.CommandFilterViewSet, 'cmd-filter')
 router.register(r'asset-users', api.AssetUserViewSet, 'asset-user')
 router.register(r'asset-users-info', api.AssetUserExportViewSet, 'asset-user-info')
+router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 
 cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
 cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')

apps/assets/utils.py

@@ -24,37 +24,6 @@ def get_system_user_by_id(id):
     return system_user
 
 
-class LabelFilterMixin:
-    def get_filter_labels_ids(self):
-        query_params = self.request.query_params
-        query_keys = query_params.keys()
-        all_label_keys = Label.objects.values_list('name', flat=True)
-        valid_keys = set(all_label_keys) & set(query_keys)
-
-        if not valid_keys:
-            return []
-
-        labels_query = [
-            {"name": key, "value": query_params[key]}
-            for key in valid_keys
-        ]
-        args = [Q(**kwargs) for kwargs in labels_query]
-        args = reduce(lambda x, y: x | y, args)
-        labels_id = Label.objects.filter(args).values_list('id', flat=True)
-        return labels_id
-
-
-class LabelFilter(LabelFilterMixin):
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        labels_ids = self.get_filter_labels_ids()
-        if not labels_ids:
-            return queryset
-        for labels_id in labels_ids:
-            queryset = queryset.filter(labels=labels_id)
-        return queryset
-
-
 class TreeService(Tree):
     tag_sep = ' / '
     cache_key = '_NODE_FULL_TREE'

apps/common/filters.py

@@ -1,11 +1,15 @@
 # -*- coding: utf-8 -*-
 #
+import coreapi
 from rest_framework import filters
 from rest_framework.fields import DateTimeField
 from rest_framework.serializers import ValidationError
+from django.core.cache import cache
 import logging
 
-__all__ = ["DatetimeRangeFilter"]
+from . import const
+
+__all__ = ["DatetimeRangeFilter", "IDSpmFilter", "CustomFilter"]
 
 
 class DatetimeRangeFilter(filters.BaseFilterBackend):
@@ -40,3 +44,52 @@ class DatetimeRangeFilter(filters.BaseFilterBackend):
         if kwargs:
             queryset = queryset.filter(**kwargs)
         return queryset
+
+
+class IDSpmFilter(filters.BaseFilterBackend):
+    def get_schema_fields(self, view):
+        return [
+            coreapi.Field(
+                name='spm', location='query', required=False,
+                type='string', example='',
+                description='Pre post objects id get spm id, then using filter'
+            )
+        ]
+
+    def filter_queryset(self, request, queryset, view):
+        spm = request.query_params.get('spm')
+        if not spm:
+            return queryset
+        cache_key = const.KEY_CACHE_RESOURCES_ID.format(spm)
+        resources_id = cache.get(cache_key)
+        if not resources_id or not isinstance(resources_id, list):
+            queryset = queryset.none()
+            return queryset
+        queryset = queryset.filter(id__in=resources_id)
+        return queryset
+
+
+class CustomFilter(filters.BaseFilterBackend):
+
+    def get_schema_fields(self, view):
+        fields = []
+        defaults = dict(
+            location='query', required=False,
+            type='string', example='',
+            description=''
+        )
+        if not hasattr(view, 'custom_filter_fields'):
+            return []
+
+        for field in view.custom_filter_fields:
+            if isinstance(field, str):
+                defaults['name'] = field
+            elif isinstance(field, dict):
+                defaults.update(field)
+            else:
+                continue
+            fields.append(coreapi.Field(**defaults))
+        return fields
+
+    def filter_queryset(self, request, queryset, view):
+        return queryset

apps/common/mixins/api.py

@@ -1,15 +1,13 @@
 # -*- coding: utf-8 -*-
 #
 from django.http import JsonResponse
-from django.core.cache import cache
-from django.utils.translation import ugettext_lazy as _
-from django.contrib import messages
+from rest_framework.settings import api_settings
 
-from ..const import KEY_CACHE_RESOURCES_ID
+from ..filters import IDSpmFilter, CustomFilter
 
 __all__ = [
-    "JSONResponseMixin", "IDInCacheFilterMixin", "IDExportFilterMixin",
-    "IDInFilterMixin", "ApiMessageMixin"
+    "JSONResponseMixin", "CommonApiMixin",
+    "IDSpmFilterMixin", "CommonApiMixin",
 ]
 
 
@@ -20,69 +18,31 @@ class JSONResponseMixin(object):
         return JsonResponse(context)
 
 
-class IDInFilterMixin(object):
+class IDSpmFilterMixin:
+    def get_filter_backends(self):
+        backends = super().get_filter_backends()
+        backends.append(IDSpmFilter)
+        return backends
+
+
+class ExtraFilterFieldsMixin:
+    default_added_filters = [CustomFilter, IDSpmFilter]
+    filter_backends = api_settings.DEFAULT_FILTER_BACKENDS
+    extra_filter_fields = []
+    extra_filter_backends = []
+
+    def get_filter_backends(self):
+        if self.filter_backends != self.__class__.filter_backends:
+            return self.filter_backends
+        return list(self.filter_backends) + \
+               self.default_added_filters + \
+               list(self.extra_filter_backends)
+
     def filter_queryset(self, queryset):
-        queryset = super(IDInFilterMixin, self).filter_queryset(queryset)
-        id_list = self.request.query_params.get('id__in')
-        if id_list:
-            import json
-            try:
-                ids = json.loads(id_list)
-            except Exception as e:
-                return queryset
-            if isinstance(ids, list):
-                queryset = queryset.filter(id__in=ids)
+        for backend in self.get_filter_backends():
+            queryset = backend().filter_queryset(self.request, queryset, self)
         return queryset
 
 
-class IDInCacheFilterMixin(object):
-
-    def filter_queryset(self, queryset):
-        queryset = super().filter_queryset(queryset)
-        spm = self.request.query_params.get('spm')
-        if not spm:
-            return queryset
-        cache_key = KEY_CACHE_RESOURCES_ID.format(spm)
-        resources_id = cache.get(cache_key)
-        if not resources_id or not isinstance(resources_id, list):
-            queryset = queryset.none()
-            return queryset
-        queryset = queryset.filter(id__in=resources_id)
-        return queryset
-
-
-class IDExportFilterMixin(object):
-    def filter_queryset(self, queryset):
-        # 下载导入模版
-        if self.request.query_params.get('template') == 'import':
-            return []
-        else:
-            return super(IDExportFilterMixin, self).filter_queryset(queryset)
-
-
-class ApiMessageMixin:
-    success_message = _("%(name)s was %(action)s successfully")
-    _action_map = {"create": _("create"), "update": _("update")}
-
-    def get_success_message(self, cleaned_data):
-        if not isinstance(cleaned_data, dict):
-            return ''
-        data = {k: v for k, v in cleaned_data.items()}
-        action = getattr(self, "action", "create")
-        data["action"] = self._action_map.get(action)
-        try:
-            message = self.success_message % data
-        except:
-            message = ''
-        return message
-
-    def dispatch(self, request, *args, **kwargs):
-        resp = super().dispatch(request, *args, **kwargs)
-        if request.method.lower() in ("get", "delete", "patch"):
-            return resp
-        if resp.status_code >= 400:
-            return resp
-        message = self.get_success_message(resp.data)
-        if message:
-            messages.success(request, message)
-        return resp
+class CommonApiMixin(ExtraFilterFieldsMixin):
+    pass

apps/common/utils/common.py

@@ -8,7 +8,6 @@ import datetime
 import uuid
 from functools import wraps
 import time
-import copy
 import ipaddress
 
 
@@ -199,3 +198,31 @@ def timeit(func):
         logger.debug(msg)
         return result
     return wrapper
+
+
+def group_obj_by_count(objs, count=50):
+    objs_grouped = [
+        objs[i:i + count] for i in range(0, len(objs), count)
+    ]
+    return objs_grouped
+
+
+def dict_get_any(d, keys):
+    for key in keys:
+        value = d.get(key)
+        if value:
+            return value
+    return None
+
+
+class lazyproperty:
+    def __init__(self, func):
+        self.func = func
+
+    def __get__(self, instance, cls):
+        if instance is None:
+            return self
+        else:
+            value = self.func(instance)
+            setattr(instance, self.func.__name__, value)
+            return value

apps/jumpserver/celery_flower.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#
+from django.http import HttpResponse
+from django.conf import settings
+from django.utils.translation import ugettext as _
+from django.views.decorators.csrf import csrf_exempt
+
+from proxy.views import proxy_view
+
+flower_url = settings.FLOWER_URL
+
+
+@csrf_exempt
+def celery_flower_view(request, path):
+    if not request.user.is_superuser:
+        return HttpResponse("Forbidden")
+    remote_url = 'http://{}/{}'.format(flower_url, path)
+    try:
+        response = proxy_view(request, remote_url)
+    except Exception as e:
+        msg = _("<h1>Flow service unavailable, check it</h1>") + \
+              '<br><br> <div>{}</div>'.format(e)
+        response = HttpResponse(msg)
+    return response
+

apps/jumpserver/conf.py

@@ -382,6 +382,8 @@ defaults = {
     'SYSLOG_ADDR': '',  # '192.168.0.1:514'
     'SYSLOG_FACILITY': 'user',
     'PERM_SINGLE_ASSET_TO_UNGROUP_NODE': False,
+    'WINDOWS_SSH_DEFAULT_SHELL': 'cmd',
+    'FLOWER_URL': "127.0.0.1:5555"
 }
 
 

apps/jumpserver/settings.py

@@ -622,3 +622,5 @@ ASSETS_PERM_CACHE_TIME = CONFIG.ASSETS_PERM_CACHE_TIME
 BACKEND_ASSET_USER_AUTH_VAULT = False
 
 PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
+WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
+FLOWER_URL = CONFIG.FLOWER_URL

apps/jumpserver/swagger.py

@@ -33,6 +33,21 @@ class CustomSwaggerAutoSchema(SwaggerAutoSchema):
         operation.summary = operation.operation_id
         return operation
 
+    def get_filter_parameters(self):
+        if not self.should_filter():
+            return []
+
+        fields = []
+        if hasattr(self.view, 'get_filter_backends'):
+            backends = self.view.get_filter_backends()
+        elif hasattr(self.view, 'filter_backends'):
+            backends = self.view.filter_backends
+        else:
+            backends = []
+        for filter_backend in backends:
+            fields += self.probe_inspectors(self.filter_inspectors, 'get_filter_parameters', filter_backend()) or []
+        return fields
+
 
 def get_swagger_view(version='v1'):
     from .urls import api_v1, api_v2

apps/jumpserver/urls.py

@@ -7,7 +7,9 @@ from django.conf.urls.static import static
 from django.conf.urls.i18n import i18n_patterns
 from django.views.i18n import JavaScriptCatalog
 
-from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
+# from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
+from . import views
+from .celery_flower import celery_flower_view
 from .swagger import get_swagger_view
 
 api_v1 = [
@@ -40,6 +42,7 @@ app_view_patterns = [
     path('orgs/', include('orgs.urls.views_urls', namespace='orgs')),
     path('auth/', include('authentication.urls.view_urls'), name='auth'),
     path('applications/', include('applications.urls.views_urls', namespace='applications')),
+    re_path(r'flower/(?P<path>.*)', celery_flower_view, name='flower-view'),
 ]
 
 
@@ -57,13 +60,13 @@ js_i18n_patterns = i18n_patterns(
 
 
 urlpatterns = [
-    path('', IndexView.as_view(), name='index'),
+    path('', views.IndexView.as_view(), name='index'),
     path('api/v1/', include(api_v1)),
     path('api/v2/', include(api_v2)),
-    re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', redirect_format_api),
-    path('api/health/', HealthCheckView.as_view(), name="health"),
-    path('luna/', LunaView.as_view(), name='luna-view'),
-    path('i18n/<str:lang>/', I18NView.as_view(), name='i18n-switch'),
+    re_path('api/(?P<app>\w+)/(?P<version>v\d)/.*', views.redirect_format_api),
+    path('api/health/', views.HealthCheckView.as_view(), name="health"),
+    path('luna/', views.LunaView.as_view(), name='luna-view'),
+    path('i18n/<str:lang>/', views.I18NView.as_view(), name='i18n-switch'),
     path('settings/', include('settings.urls.view_urls', namespace='settings')),
 
     # External apps url

apps/jumpserver/views.py

@@ -224,3 +224,6 @@ class HealthCheckView(APIView):
 
     def get(self, request):
         return JsonResponse({"status": 1, "time": int(time.time())})
+
+
+

apps/ops/celery/utils.py

@@ -6,7 +6,9 @@ import os
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
 from django.db.utils import ProgrammingError, OperationalError
-from django_celery_beat.models import PeriodicTask, IntervalSchedule, CrontabSchedule
+from django_celery_beat.models import (
+    PeriodicTask, IntervalSchedule, CrontabSchedule, PeriodicTasks
+)
 
 
 def create_or_update_celery_periodic_tasks(tasks):
@@ -75,17 +77,20 @@ def create_or_update_celery_periodic_tasks(tasks):
         task = PeriodicTask.objects.update_or_create(
             defaults=defaults, name=name,
         )
+        PeriodicTasks.update_changed()
         return task
 
 
 def disable_celery_periodic_task(task_name):
     from django_celery_beat.models import PeriodicTask
     PeriodicTask.objects.filter(name=task_name).update(enabled=False)
+    PeriodicTasks.update_changed()
 
 
 def delete_celery_periodic_task(task_name):
     from django_celery_beat.models import PeriodicTask
     PeriodicTask.objects.filter(name=task_name).delete()
+    PeriodicTasks.update_changed()
 
 
 def get_celery_task_log_path(task_id):

apps/ops/inventory.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 
+from django.conf import settings
 from .ansible.inventory import BaseInventory
 
 from common.utils import get_logger
@@ -14,6 +15,7 @@ logger = get_logger(__file__)
 
 
 class JMSBaseInventory(BaseInventory):
+    windows_ssh_default_ssh = settings.WINDOWS_SSH_DEFAULT_SHELL
 
     def convert_to_ansible(self, asset, run_as_admin=False):
         info = {
@@ -33,7 +35,7 @@ class JMSBaseInventory(BaseInventory):
         if asset.is_windows():
             info["vars"].update({
                 "ansible_connection": "ssh",
-                "ansible_shell_type": "cmd",
+                "ansible_shell_type": self.windows_ssh_default_ssh,
             })
         for label in asset.labels.all():
             info["vars"].update({
@@ -49,7 +51,7 @@ class JMSBaseInventory(BaseInventory):
     def make_proxy_command(asset):
         gateway = asset.domain.random_gateway()
         proxy_command_list = [
-            "ssh", "-p", str(gateway.port),
+            "ssh", "-o", "Port={}".format(gateway.port),
             "-o", "StrictHostKeyChecking=no",
             "{}@{}".format(gateway.username, gateway.ip),
             "-W", "%h:%p", "-q",

apps/ops/migrations/0008_auto_20190919_2100.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.1.7 on 2019-09-19 13:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0007_auto_20190724_2002'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='date_updated',
+            field=models.DateTimeField(auto_now=True, verbose_name='Date updated'),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='date_created',
+            field=models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Date created'),
+        ),
+        migrations.AlterModelOptions(
+            name='task',
+            options={'get_latest_by': 'date_created',
+                     'ordering': ('-date_updated',)},
+        ),
+    ]

apps/ops/models/adhoc.py

@@ -13,7 +13,7 @@ from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django_celery_beat.models import PeriodicTask
 
-from common.utils import get_signer, get_logger
+from common.utils import get_signer, get_logger, lazyproperty
 from orgs.utils import set_to_root_org
 from ..celery.utils import delete_celery_periodic_task, \
     create_or_update_celery_periodic_tasks, \
@@ -42,7 +42,8 @@ class Task(models.Model):
     is_deleted = models.BooleanField(default=False)
     comment = models.TextField(blank=True, verbose_name=_("Comment"))
     created_by = models.CharField(max_length=128, blank=True, default='')
-    date_created = models.DateTimeField(auto_now_add=True, db_index=True)
+    date_created = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name=_("Date created"))
+    date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
     __latest_adhoc = None
     _ignore_auto_created_by = True
 
@@ -51,16 +52,39 @@ class Task(models.Model):
         return str(self.id).split('-')[-1]
 
     @property
-    def latest_adhoc(self):
-        if not self.__latest_adhoc:
-            self.__latest_adhoc = self.get_latest_adhoc()
-        return self.__latest_adhoc
-
-    @latest_adhoc.setter
-    def latest_adhoc(self, item):
-        self.__latest_adhoc = item
+    def versions(self):
+        return self.adhoc.all().count()
 
     @property
+    def is_success(self):
+        if self.latest_history:
+            return self.latest_history.is_success
+        else:
+            return False
+
+    @property
+    def timedelta(self):
+        if self.latest_history:
+            return self.latest_history.timedelta
+        else:
+            return 0
+
+    @property
+    def date_start(self):
+        if self.latest_history:
+            return self.latest_history.date_start
+        else:
+            return None
+
+    @property
+    def assets_amount(self):
+        return self.latest_adhoc.hosts.count()
+
+    @lazyproperty
+    def latest_adhoc(self):
+        return self.get_latest_adhoc()
+
+    @lazyproperty
     def latest_history(self):
         try:
             return self.history.all().latest()
@@ -139,6 +163,7 @@ class Task(models.Model):
     class Meta:
         db_table = 'ops_task'
         unique_together = ('name', 'created_by')
+        ordering = ('-date_updated',)
         get_latest_by = 'date_created'
 
 
@@ -218,30 +243,35 @@ class AdHoc(models.Model):
             hid = str(uuid.uuid4())
         history = AdHocRunHistory(id=hid, adhoc=self, task=self.task)
         time_start = time.time()
+        date_start = timezone.now()
+        is_success = False
+
         try:
-            date_start = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-            history.date_start = timezone.now()
-            print(_("{} Start task: {}").format(date_start, self.task.name))
+            date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            print(_("{} Start task: {}").format(date_start_s, self.task.name))
             raw, summary = self._run_only()
-            date_end = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-            print(_("{} Task finish").format(date_end))
-            history.is_finished = True
-            if summary.get('dark'):
-                history.is_success = False
-            else:
-                history.is_success = True
-            history.result = raw
-            history.summary = summary
+            is_success = summary.get('success', False)
             return raw, summary
         except Exception as e:
             logger.error(e, exc_info=True)
-            return {}, {"dark": {"all": str(e)}, "contacted": []}
+            summary = {}
+            raw = {"dark": {"all": str(e)}, "contacted": []}
+            return raw, summary
         finally:
-            history.date_finished = timezone.now()
-            history.timedelta = time.time() - time_start
-            history.save()
+            date_end = timezone.now()
+            date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S')
+            print(_("{} Task finish").format(date_end_s))
+            print('.\n\n.')
+            AdHocRunHistory.objects.filter(id=history.id).update(
+                date_start=date_start,
+                is_finished=True,
+                is_success=is_success,
+                date_finished=timezone.now(),
+                timedelta=time.time() - time_start
+            )
 
     def _run_only(self):
+        Task.objects.filter(id=self.task.id).update(date_updated=timezone.now())
         runner = AdHocRunner(self.inventory, options=self.options)
         try:
             result = runner.run(

apps/ops/serializers.py

@@ -19,7 +19,12 @@ class CeleryTaskSerializer(serializers.Serializer):
 class TaskSerializer(serializers.ModelSerializer):
     class Meta:
         model = Task
-        fields = '__all__'
+        fields = [
+            'id', 'name', 'interval', 'crontab', 'is_periodic',
+            'is_deleted', 'comment', 'created_by', 'date_created',
+            'versions', 'is_success', 'timedelta', 'assets_amount',
+            'date_updated', 'history_summary',
+        ]
 
 
 class AdHocSerializer(serializers.ModelSerializer):

apps/ops/tasks.py

@@ -2,6 +2,7 @@
 import os
 import subprocess
 import datetime
+import time
 
 from django.conf import settings
 from celery import shared_task, subtask
@@ -122,3 +123,22 @@ def hello123():
 def hello_callback(result):
     print(result)
     print("Hello callback")
+
+
+@shared_task
+def add(a, b):
+    time.sleep(5)
+    return a + b
+
+
+@shared_task
+def add_m(x):
+    from celery import chain
+    a = range(x)
+    b = [a[i:i + 10] for i in range(0, len(a), 10)]
+    s = list()
+    s.append(add.s(b[0], b[1]))
+    for i in b[1:]:
+        s.append(add.s(i))
+    res = chain(*tuple(s))()
+    return res

apps/ops/templates/ops/task_list.html

@@ -1,101 +1,87 @@
 {% extends '_base_list.html' %}
-{% load i18n %}
-{% load static %}
-
-{% block content_left_head %}
-{#    <div class="uc pull-left m-r-5"><a class="btn btn-sm btn-primary btn-create-asset"> {% trans "Create task" %} </a></div>#}
-{% endblock %}
-
-
-{% block table_search %}
-    <form id="search_form" method="get" action="" class="pull-right form-inline">
-        <div class="input-group">
-            <input type="text" class="form-control input-sm" name="keyword" placeholder="{% trans 'Search'  %}" value="{{ keyword }}">
-        </div>
-        <div class="input-group">
-            <div class="input-group-btn">
-                <button id='search_btn' type="submit" class="btn btn-sm btn-primary">
-                    {% trans "Search"  %}
-                </button>
-            </div>
-        </div>
-    </form>
-{% endblock %}
-
-{% block table_head %}
-    <th class="text-center"></th>
-    <th class="text-center">{% trans 'Name' %}</th>
-    <th class="text-center">{% trans 'Run times' %}</th>
-    <th class="text-center">{% trans 'Versions' %}</th>
-    <th class="text-center">{% trans 'Hosts' %}</th>
-    <th class="text-center">{% trans 'Success' %}</th>
-    <th class="text-center">{% trans 'Date' %}</th>
-    <th class="text-center">{% trans 'Time' %}</th>
-    <th class="text-center">{% trans 'Action' %}</th>
-{% endblock %}
-
-{% block table_body %}
-    {% for object in task_list %}
-        <tr class="gradeX">
-            <td class="text-center"><input type="checkbox" class="cbx-term"> </td>
-            <td class="text-left"><a href="{% url 'ops:task-detail' pk=object.id %}">{{ object.name }}</a></td>
-            <td class="text-center">
-                <span class="text-danger">{{ object.history_summary.failed }}</span>/<span class="text-navy">{{ object.history_summary.success}}</span>/{{ object.history_summary.total}}
-            </td>
-            <td class="text-center">{{ object.adhoc.all | length}}</td>
-            <td class="text-center">{{ object.latest_adhoc.hosts | length}}</td>
-            <td class="text-center">
-            {% if object.latest_history %}
-                {% if object.latest_history.is_success %}
-                    <i class="fa fa-check text-navy"></i>
-                {% else %}
-                    <i class="fa fa-times text-danger"></i>
-                {% endif %}
-            {% endif %}
-            </td>
-            <td class="text-center">{{ object.latest_history.date_start }}</td>
-            <td class="text-center">
-                {% if object.latest_history %}
-                {{ object.latest_history.timedelta|floatformat }} s
-                {% endif %}
-            </td>
-            <td class="text-center">
-                <a data-uid="{{ object.id }}" class="btn btn-xs btn-primary btn-run">{% trans "Run" %}</a>
-                <a data-uid="{{ object.id }}" class="btn btn-xs btn-danger btn-del">{% trans "Delete" %}</a>
-            </td>
+{% load i18n static %}
+{% block table_search %}{% endblock %}
+{% block table_container %}
+    <table class="table table-striped table-bordered table-hover " id="task_list_table">
+        <thead>
+        <tr>
+            <th class="text-center">
+                <input id="" type="checkbox" class="ipt_check_all">
+            </th>
+            <th class="text-left">{% trans 'Name' %}</th>
+            <th class="text-center">{% trans 'Run times' %}</th>
+            <th class="text-center">{% trans 'Versions' %}</th>
+            <th class="text-center">{% trans 'Hosts' %}</th>
+            <th class="text-center">{% trans 'Success' %}</th>
+            <th class="text-center">{% trans 'Date' %}</th>
+            <th class="text-center">{% trans 'Time' %}</th>
+            <th class="text-center">{% trans 'Action' %}</th>
         </tr>
-    {% endfor %}
+        </thead>
+    </table>
 {% endblock %}
 
+{% block content_bottom_left %}{% endblock %}
 {% block custom_foot_js %}
-<script src="{% static 'js/plugins/datepicker/bootstrap-datepicker.js' %}"></script>
 <script>
-$(document).ready(function() {
-    $('table').DataTable({
-        "searching": false,
-        "paging": false,
-        "bInfo" : false,
-        "order": [],
-        "columnDefs": [
-            { "targets": 0, "orderable": false },
-            { "targets": 4, "orderable": false },
-            { "targets": 5, "orderable": false },
-            { "targets": 8, "orderable": false }
-        ]
-    });
-    $('.select2').select2({
-        dropdownAutoWidth : true,
-        width: 'auto'
-    });
-    $('#date .input-daterange').datepicker({
-        format: "yyyy-mm-dd",
-        todayBtn: "linked",
-        keyboardNavigation: false,
-        forceParse: false,
-        calendarWeeks: true,
-        autoclose: true
-    });
-
+$(document).ready(function () {
+    var options = {
+        ele: $('#task_list_table'),
+        buttons: [],
+        columnDefs: [
+            {targets: 1, createdCell: function (td, cellData, rowData) {
+                cellData = htmlEscape(cellData);
+                var innerHtml = '<a href="{% url "ops:task-detail" pk=DEFAULT_PK %}">' + cellData + '</a>'
+                innerHtml = innerHtml.replace('{{ DEFAULT_PK }}', rowData.id);
+                $(td).html(innerHtml);
+            }},
+            {targets: 2, createdCell: function (td, cellData) {
+                var innerHtml = '<span class="text-danger">failed</span>/<span class="text-navy">success</span>/total';
+                if (cellData) {
+                    innerHtml = innerHtml.replace('failed', cellData.failed)
+                        .replace('success', cellData.success)
+                        .replace('total', cellData.total);
+                    $(td).html(innerHtml);
+                } else {
+                    $(td).html('')
+                }
+            }},
+            {targets: 5, createdCell: function (td, cellData) {
+                var successBtn = '<i class="fa fa-check text-navy"></i>';
+                var failedBtn = '<i class="fa fa-times text-danger"></i>';
+                if (cellData) {
+                    $(td).html(successBtn)
+                } else {
+                    $(td).html(failedBtn)
+                }
+            }},
+            {targets: 6, createdCell: function (td, cellData) {
+                $(td).html(toSafeLocalDateStr(cellData));
+            }},
+            {targets: 7, createdCell: function (td, cellData) {
+                var delta = readableSecond(cellData);
+                $(td).html(delta);
+            }},
+            {
+                targets: 8,
+                createdCell: function (td, cellData, rowData) {
+                    var runBtn = '<a data-uid="ID" class="btn btn-xs btn-primary btn-run">{% trans "Run" %}</a> '.replace('ID', cellData);
+                    var delBtn = '<a data-uid="ID" class="btn btn-xs btn-danger btn-del">{% trans "Delete" %}</a>'.replace('ID', cellData);
+                    $(td).html(runBtn + delBtn)
+                }
+            }
+        ],
+        ajax_url: '{% url "api-ops:task-list" %}',
+        columns: [
+            {data: "id"}, {data: "name", className: "text-left"}, {data: "history_summary", orderable: false},
+            {data: "versions", orderable: false}, {data: "assets_amount", orderable: false},
+            {data: "is_success", orderable: false}, {data: "date_updated"},
+            {data: "timedelta", orderable:false}, {data: "id", orderable: false},
+        ],
+        order: [],
+        op_html: $('#actions').html()
+    };
+    jumpserver.initServerSideDataTable(options);
 }).on('click', '.btn-del', function () {
     var $this = $(this);
     var name = $this.closest("tr").find(":nth-child(2)").children('a').html();
@@ -122,7 +108,6 @@ $(document).ready(function() {
         success: success,
         success_message: "{% trans 'Task start: ' %}" + " " + name
     });
-
 })
 </script>
 {% endblock %}

apps/ops/views/adhoc.py

@@ -2,7 +2,7 @@
 
 from django.utils.translation import ugettext as _
 from django.conf import settings
-from django.views.generic import ListView, DetailView
+from django.views.generic import ListView, DetailView, TemplateView
 
 from common.mixins import DatetimeSearchMixin
 from common.permissions import PermissionsMixin, IsOrgAdmin
@@ -17,7 +17,7 @@ __all__ = [
 ]
 
 
-class TaskListView(PermissionsMixin, DatetimeSearchMixin, ListView):
+class TaskListView(PermissionsMixin, TemplateView):
     paginate_by = settings.DISPLAY_PER_PAGE
     model = Task
     ordering = ('-date_created',)
@@ -26,27 +26,10 @@ class TaskListView(PermissionsMixin, DatetimeSearchMixin, ListView):
     keyword = ''
     permission_classes = [IsOrgAdmin]
 
-    def get_queryset(self):
-        queryset = super().get_queryset()
-        if current_org.is_real():
-            queryset = queryset.filter(created_by=current_org.id)
-        else:
-            queryset = queryset.filter(created_by='')
-
-        self.keyword = self.request.GET.get('keyword', '')
-        if self.keyword:
-            queryset = queryset.filter(
-                name__icontains=self.keyword,
-            )
-        return queryset
-
     def get_context_data(self, **kwargs):
         context = {
             'app': _('Ops'),
             'action': _('Task list'),
-            'date_from': self.date_from,
-            'date_to': self.date_to,
-            'keyword': self.keyword,
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/orgs/mixins/api.py

@@ -3,7 +3,7 @@
 from django.shortcuts import get_object_or_404
 from rest_framework.viewsets import ModelViewSet
 from rest_framework_bulk import BulkModelViewSet
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 
 from ..utils import set_to_root_org
 from ..models import Organization
@@ -20,20 +20,23 @@ class RootOrgViewMixin:
         return super().dispatch(request, *args, **kwargs)
 
 
-class OrgModelViewSet(IDInCacheFilterMixin, ModelViewSet):
-    def get_queryset(self):
-        return super().get_queryset().all()
-
-
-class OrgBulkModelViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class OrgQuerySetMixin:
     def get_queryset(self):
         queryset = super().get_queryset().all()
+        if hasattr(self, 'swagger_fake_view'):
+            return queryset[:1]
         if hasattr(self, 'action') and self.action == 'list' and \
             hasattr(self, 'serializer_class') and \
                 hasattr(self.serializer_class, 'setup_eager_loading'):
             queryset = self.serializer_class.setup_eager_loading(queryset)
         return queryset
 
+
+class OrgModelViewSet(CommonApiMixin, OrgQuerySetMixin, ModelViewSet):
+    pass
+
+
+class OrgBulkModelViewSet(CommonApiMixin, OrgQuerySetMixin, BulkModelViewSet):
     def allow_bulk_destroy(self, qs, filtered):
         if qs.count() <= filtered.count():
             return False

apps/orgs/mixins/serializers.py

@@ -11,7 +11,8 @@ from ..utils import get_current_org_id_for_serializer
 
 __all__ = [
     "OrgResourceSerializerMixin", "BulkOrgResourceSerializerMixin",
-    "BulkOrgResourceModelSerializer", "OrgMembershipSerializerMixin"
+    "BulkOrgResourceModelSerializer", "OrgMembershipSerializerMixin",
+    "OrgResourceModelSerializerMixin",
 ]
 
 
@@ -42,6 +43,10 @@ class OrgResourceSerializerMixin(serializers.Serializer):
         return fields
 
 
+class OrgResourceModelSerializerMixin(OrgResourceSerializerMixin, serializers.ModelSerializer):
+    pass
+
+
 class BulkOrgResourceSerializerMixin(BulkSerializerMixin, OrgResourceSerializerMixin):
     pass
 

apps/orgs/utils.py

@@ -18,6 +18,8 @@ def get_org_from_request(request):
 
 
 def set_current_org(org):
+    if isinstance(org, str):
+        org = Organization.get_instance(org)
     setattr(thread_local, 'current_org_id', org.id)
 
 

apps/perms/api/asset_permission.py

@@ -180,6 +180,7 @@ class AssetPermissionRemoveUserApi(RetrieveUpdateAPIView):
             users = serializer.validated_data.get('users')
             if users:
                 perm.users.remove(*tuple(users))
+                perm.save()
             return Response({"msg": "ok"})
         else:
             return Response({"error": serializer.errors})
@@ -197,6 +198,7 @@ class AssetPermissionAddUserApi(RetrieveUpdateAPIView):
             users = serializer.validated_data.get('users')
             if users:
                 perm.users.add(*tuple(users))
+                perm.save()
             return Response({"msg": "ok"})
         else:
             return Response({"error": serializer.errors})
@@ -217,6 +219,7 @@ class AssetPermissionRemoveAssetApi(RetrieveUpdateAPIView):
             assets = serializer.validated_data.get('assets')
             if assets:
                 perm.assets.remove(*tuple(assets))
+                perm.save()
             return Response({"msg": "ok"})
         else:
             return Response({"error": serializer.errors})
@@ -234,6 +237,7 @@ class AssetPermissionAddAssetApi(RetrieveUpdateAPIView):
             assets = serializer.validated_data.get('assets')
             if assets:
                 perm.assets.add(*tuple(assets))
+                perm.save()
             return Response({"msg": "ok"})
         else:
             return Response({"error": serializer.errors})

apps/perms/api/mixin.py

@@ -1,21 +1,17 @@
 # -*- coding: utf-8 -*-
 #
-import uuid
-from django.db.models import Q
 
 from rest_framework.generics import get_object_or_404
-from assets.utils import LabelFilterMixin
 from common.permissions import IsValidUser, IsOrgAdminOrAppUser
 from common.utils import get_logger
 from orgs.utils import set_to_root_org
-from ..hands import User, Asset, SystemUser
-from .. import serializers
+from ..hands import User, UserGroup
 
 
 logger = get_logger(__name__)
 
 __all__ = [
-    'UserPermissionMixin',
+    'UserPermissionMixin', 'UserGroupPermissionMixin',
 ]
 
 
@@ -45,101 +41,12 @@ class UserPermissionMixin:
         return super().get_permissions()
 
 
-class GrantAssetsMixin(LabelFilterMixin):
-    serializer_class = serializers.AssetGrantedSerializer
+class UserGroupPermissionMixin:
+    obj = None
 
-    def get_serializer_queryset(self, queryset):
-        assets_ids = []
-        system_users_ids = set()
-        for asset in queryset:
-            assets_ids.append(asset["id"])
-            system_users_ids.update(set(asset["system_users"]))
-        assets = Asset.objects.filter(id__in=assets_ids).only(
-            *self.serializer_class.Meta.only_fields
-        )
-        assets_map = {asset.id: asset for asset in assets}
-        system_users = SystemUser.objects.filter(id__in=system_users_ids).only(
-            *self.serializer_class.system_users_only_fields
-        )
-        system_users_map = {s.id: s for s in system_users}
-        data = []
-        for item in queryset:
-            i = item["id"]
-            asset = assets_map.get(i)
-            if not asset:
-                continue
+    def get_obj(self):
+        user_group_id = self.kwargs.get('pk', '')
+        user_group = get_object_or_404(UserGroup, id=user_group_id)
+        return user_group
 
-            _system_users = item["system_users"]
-            system_users_granted = []
-            for sid, action in _system_users.items():
-                system_user = system_users_map.get(sid)
-                if not system_user:
-                    continue
-                if not asset.has_protocol(system_user.protocol):
-                    continue
-                system_user.actions = action
-                system_users_granted.append(system_user)
-            asset.system_users_granted = system_users_granted
-            data.append(asset)
-        return data
 
-    def get_serializer(self, assets_items=None, many=True):
-        if assets_items is None:
-            assets_items = []
-        assets_items = self.get_serializer_queryset(assets_items)
-        return super().get_serializer(assets_items, many=many)
-
-    def filter_queryset_by_id(self, assets_items):
-        i = self.request.query_params.get("id")
-        if not i:
-            return assets_items
-        try:
-            pk = uuid.UUID(i)
-        except ValueError:
-            return assets_items
-        assets_map = {asset['id']: asset for asset in assets_items}
-        if pk in assets_map:
-            return [assets_map.get(pk)]
-        else:
-            return []
-
-    def search_queryset(self, assets_items):
-        search = self.request.query_params.get("search")
-        if not search:
-            return assets_items
-        assets_map = {asset['id']: asset for asset in assets_items}
-        assets_ids = set(assets_map.keys())
-        assets_ids_search = Asset.objects.filter(id__in=assets_ids).filter(
-            Q(hostname__icontains=search) | Q(ip__icontains=search)
-        ).values_list('id', flat=True)
-        return [assets_map.get(asset_id) for asset_id in assets_ids_search]
-
-    def filter_queryset_by_label(self, assets_items):
-        labels_id = self.get_filter_labels_ids()
-        if not labels_id:
-            return assets_items
-
-        assets_map = {asset['id']: asset for asset in assets_items}
-        assets_matched = Asset.objects.filter(id__in=assets_map.keys())
-        for label_id in labels_id:
-            assets_matched = assets_matched.filter(labels=label_id)
-        assets_ids_matched = assets_matched.values_list('id', flat=True)
-        return [assets_map.get(asset_id) for asset_id in assets_ids_matched]
-
-    def sort_queryset(self, assets_items):
-        order_by = self.request.query_params.get('order', 'hostname')
-
-        if order_by not in ['hostname', '-hostname', 'ip', '-ip']:
-            order_by = 'hostname'
-        assets_map = {asset['id']: asset for asset in assets_items}
-        assets_ids_search = Asset.objects.filter(id__in=assets_map.keys())\
-            .order_by(order_by)\
-            .values_list('id', flat=True)
-        return [assets_map.get(asset_id) for asset_id in assets_ids_search]
-
-    def filter_queryset(self, assets_items):
-        assets_items = self.filter_queryset_by_id(assets_items)
-        assets_items = self.search_queryset(assets_items)
-        assets_items = self.filter_queryset_by_label(assets_items)
-        assets_items = self.sort_queryset(assets_items)
-        return assets_items

apps/perms/api/user_group_permission.py

@@ -1,11 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 
-from django.shortcuts import get_object_or_404
-
-from ..hands import UserGroup
 
 from . import user_permission as uapi
+from .mixin import UserGroupPermissionMixin
 
 __all__ = [
     'UserGroupGrantedAssetsApi', 'UserGroupGrantedNodesApi',
@@ -17,15 +15,6 @@ __all__ = [
 ]
 
 
-class UserGroupPermissionMixin:
-    obj = None
-
-    def get_object(self):
-        user_group_id = self.kwargs.get('pk', '')
-        user_group = get_object_or_404(UserGroup, id=user_group_id)
-        return user_group
-
-
 class UserGroupGrantedAssetsApi(UserGroupPermissionMixin, uapi.UserGrantedAssetsApi):
     pass
 

apps/perms/api/user_permission/common.py

@@ -51,8 +51,7 @@ class GetUserAssetPermissionActionsApi(UserAssetPermissionMixin,
         asset = get_object_or_404(Asset, id=asset_id)
         system_user = get_object_or_404(SystemUser, id=system_id)
 
-        system_users_actions = self.util.get_asset_system_users_with_actions(
-            asset)
+        system_users_actions = self.util.get_asset_system_users_with_actions(asset)
         actions = system_users_actions.get(system_user)
         return {"actions": actions}
 
@@ -103,8 +102,7 @@ class UserGrantedAssetSystemUsersApi(UserAssetPermissionMixin, ListAPIView):
     def get_queryset(self):
         asset_id = self.kwargs.get('asset_id')
         asset = get_object_or_404(Asset, id=asset_id)
-        system_users_with_actions = self.util.get_asset_system_users_with_actions(
-            asset)
+        system_users_with_actions = self.util.get_asset_system_users_with_actions(asset)
         system_users = []
         for system_user, actions in system_users_with_actions.items():
             system_user.actions = actions

apps/perms/api/user_permission/mixin.py

@@ -1,23 +1,27 @@
 # -*- coding: utf-8 -*-
 #
+from common.utils import lazyproperty
+from common.tree import TreeNodeSerializer
 from ..mixin import UserPermissionMixin
 from ...utils import AssetPermissionUtilV2, ParserNode
-from ...hands import Node
-from common.tree import TreeNodeSerializer
+from ...hands import Node, Asset
 
 
 class UserAssetPermissionMixin(UserPermissionMixin):
     util = None
-    tree = None
 
-    def initial(self, *args, **kwargs):
-        super().initial(*args, *kwargs)
+    @lazyproperty
+    def util(self):
         cache_policy = self.request.query_params.get('cache_policy', '0')
         system_user_id = self.request.query_params.get("system_user")
-        self.util = AssetPermissionUtilV2(self.obj, cache_policy=cache_policy)
+        util = AssetPermissionUtilV2(self.obj, cache_policy=cache_policy)
         if system_user_id:
-            self.util.filter_permissions(system_users=system_user_id)
-        self.tree = self.util.get_user_tree()
+            util.filter_permissions(system_users=system_user_id)
+        return util
+
+    @lazyproperty
+    def tree(self):
+        return self.util.get_user_tree()
 
 
 class UserNodeTreeMixin:
@@ -41,7 +45,9 @@ class UserNodeTreeMixin:
         queryset = self.parse_nodes_to_queryset(queryset)
         return queryset
 
-    def get_serializer(self, queryset, many=True, **kwargs):
+    def get_serializer(self, queryset=None, many=True, **kwargs):
+        if queryset is None:
+            queryset = Node.objects.none()
         queryset = self.get_serializer_queryset(queryset)
         queryset.sort()
         return super().get_serializer(queryset, many=many, **kwargs)
@@ -64,7 +70,9 @@ class UserAssetTreeMixin:
         _queryset = self.parse_assets_to_queryset(queryset, None)
         return _queryset
 
-    def get_serializer(self, queryset, many=True, **kwargs):
+    def get_serializer(self, queryset=None, many=True, **kwargs):
+        if queryset is None:
+            queryset = Asset.objects.none()
         queryset = self.get_serializer_queryset(queryset)
         queryset.sort()
         return super().get_serializer(queryset, many=many, **kwargs)

apps/perms/api/user_permission/user_permission_assets.py

@@ -7,7 +7,7 @@ from rest_framework.generics import (
 )
 
 from common.permissions import IsOrgAdminOrAppUser
-from common.utils import get_logger
+from common.utils import get_logger, timeit
 from ...hands import Node
 from ... import serializers
 from .mixin import UserAssetPermissionMixin, UserAssetTreeMixin

apps/perms/signals_handler.py

@@ -19,18 +19,6 @@ permission_m2m_senders = (
 )
 
 
-@on_transaction_commit
-def on_permission_m2m_change(sender, action='', **kwargs):
-    if not action.startswith('post'):
-        return
-    logger.debug('Asset permission m2m changed, refresh user tree cache')
-    AssetPermissionUtilV2.expire_all_user_tree_cache()
-
-
-for sender in permission_m2m_senders:
-    m2m_changed.connect(on_permission_m2m_change, sender=sender)
-
-
 @receiver([post_save, post_delete], sender=AssetPermission)
 @on_transaction_commit
 def on_permission_change(sender, action='', **kwargs):

apps/perms/utils/asset_permission.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
+import time
 import pickle
-import threading
 from collections import defaultdict
 from functools import reduce
 from hashlib import md5
@@ -12,7 +12,7 @@ from django.db.models import Q
 from django.conf import settings
 
 from orgs.utils import set_to_root_org
-from common.utils import get_logger, timeit
+from common.utils import get_logger, timeit, lazyproperty
 from common.tree import TreeNode
 from assets.utils import TreeService
 from ..models import AssetPermission
@@ -126,27 +126,30 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
         'comment', 'is_active', 'os', 'org_id'
     )
 
-    def __init__(self, obj, cache_policy='0'):
+    def __init__(self, obj=None, cache_policy='0'):
         self.object = obj
         self.cache_policy = cache_policy
-        self.obj_id = str(obj.id)
+        self.obj_id = str(obj.id) if obj else None
         self._permissions = None
-        self._permissions_id = None  # 标记_permission的唯一值
         self._filter_id = 'None'  # 当通过filter更改 permission是标记
         self.change_org_if_need()
         self._user_tree = None
         self._user_tree_filter_id = 'None'
-        self.full_tree = Node.tree()
-        self.mutex = threading.Lock()
 
     @staticmethod
     def change_org_if_need():
         set_to_root_org()
 
+    @lazyproperty
+    def full_tree(self):
+        return Node.tree()
+
     @property
     def permissions(self):
         if self._permissions:
             return self._permissions
+        if self.object is None:
+            return AssetPermission.objects.none()
         object_cls = self.object.__class__.__name__
         func = self.get_permissions_map[object_cls]
         permissions = func(self.object)
@@ -159,7 +162,7 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
         self._permissions = self.permissions.filter(**filters)
         self._filter_id = md5(filters_json.encode()).hexdigest()
 
-    @property
+    @lazyproperty
     def user_tree(self):
         return self.get_user_tree()
 
@@ -303,27 +306,26 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
     @timeit
     def get_user_tree(self):
         # 使用锁，保证多次获取tree的时候顺序执行，可以使用缓存
-        with self.mutex:
-            user_tree = self.get_user_tree_from_local()
-            if user_tree:
-                return user_tree
-            user_tree = self.get_user_tree_from_cache_if_need()
-            if user_tree:
-                self.set_user_tree_to_local(user_tree)
-                return user_tree
-            user_tree = TreeService()
-            full_tree_root = self.full_tree.root_node()
-            user_tree.create_node(
-                tag=full_tree_root.tag,
-                identifier=full_tree_root.identifier
-            )
-            self.add_direct_nodes_to_user_tree(user_tree)
-            self.add_single_assets_node_to_user_tree(user_tree)
-            self.parse_user_tree_to_full_tree(user_tree)
-            self.add_empty_node_if_need(user_tree)
-            self.set_user_tree_to_cache_if_need(user_tree)
+        user_tree = self.get_user_tree_from_local()
+        if user_tree:
+            return user_tree
+        user_tree = self.get_user_tree_from_cache_if_need()
+        if user_tree:
             self.set_user_tree_to_local(user_tree)
             return user_tree
+        user_tree = TreeService()
+        full_tree_root = self.full_tree.root_node()
+        user_tree.create_node(
+            tag=full_tree_root.tag,
+            identifier=full_tree_root.identifier
+        )
+        self.add_direct_nodes_to_user_tree(user_tree)
+        self.add_single_assets_node_to_user_tree(user_tree)
+        self.parse_user_tree_to_full_tree(user_tree)
+        self.add_empty_node_if_need(user_tree)
+        self.set_user_tree_to_cache_if_need(user_tree)
+        self.set_user_tree_to_local(user_tree)
+        return user_tree
 
     # Todo: 是否可以获取多个资产的系统用户
     def get_asset_system_users_with_actions(self, asset):
@@ -332,17 +334,13 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
         for node in nodes:
             ancestor_keys = node.get_ancestor_keys(with_self=True)
             nodes_keys_related.update(set(ancestor_keys))
-        pattern = []
-        for key in nodes_keys_related:
-            pattern.append(r'^{0}$|^{0}:'.format(key))
-        pattern = '|'.join(list(pattern))
         kwargs = {"assets": asset}
 
-        if pattern:
-            kwargs["nodes__key__regex"] = pattern
+        if nodes_keys_related:
+            kwargs["nodes__key__in"] = nodes_keys_related
 
         queryset = self.permissions
-        if len(kwargs) == 1:
+        if kwargs == 1:
             queryset = queryset.filter(**kwargs)
         elif len(kwargs) > 1:
             kwargs = [{k: v} for k, v in kwargs.items()]
@@ -376,33 +374,11 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
         nodes_keys = Node.clean_children_keys(nodes_keys)
         return nodes_keys, assets_ids
 
-    @staticmethod
-    def filter_assets_by_or_kwargs(kwargs):
-        if len(kwargs) == 1:
-            queryset = Asset.objects.filter(**kwargs)
-        elif len(kwargs) > 1:
-            kwargs = [{k: v} for k, v in kwargs.items()]
-            args = [Q(**kw) for kw in kwargs]
-            args = reduce(lambda x, y: x | y, args)
-            queryset = Asset.objects.filter(args)
-        else:
-            queryset = Asset.objects.none()
-        return queryset
-
     @timeit
     def get_assets(self):
         nodes_keys, assets_ids = self.get_permissions_nodes_and_assets()
-        pattern = set()
-        for key in nodes_keys:
-            pattern.add(r'^{0}$|^{0}:'.format(key))
-        pattern = '|'.join(list(pattern))
-        kwargs = {}
-        if assets_ids:
-            kwargs["id__in"] = assets_ids
-        if pattern:
-            kwargs["nodes__key__regex"] = pattern
-        queryset = self.filter_assets_by_or_kwargs(kwargs)
-        return queryset.valid().distinct()
+        queryset = Node.get_nodes_all_assets(nodes_keys, extra_assets_ids=assets_ids)
+        return queryset.valid()
 
     def get_nodes_assets(self, node, deep=False):
         if deep:
@@ -410,7 +386,7 @@ class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin):
         else:
             assets_ids = self.user_tree.assets(node.key)
         queryset = Asset.objects.filter(id__in=assets_ids)
-        return queryset.valid().distinct()
+        return queryset.valid()
 
     def get_nodes(self):
         return [n.identifier for n in self.user_tree.all_nodes_itr()]

apps/settings/api.py

@@ -97,6 +97,8 @@ class LDAPUserListApi(generics.ListAPIView):
     serializer_class = LDAPUserSerializer
 
     def get_queryset(self):
+        if hasattr(self, 'swagger_fake_view'):
+            return []
         util = LDAPUtil()
         try:
             users = util.search_user_items()

apps/static/js/jumpserver.js

@@ -153,6 +153,7 @@ function activeNav() {
     } else {
         $("#" + app).addClass('active');
         $('#' + app + ' #' + resource).addClass('active');
+        $('#' + app + ' #' + resource.replace('-', '_')).addClass('active');
     }
 }
 
@@ -1138,7 +1139,10 @@ function timeOffset(a, b) {
     var start = safeDate(a);
     var end = safeDate(b);
     var offset = (end - start) / 1000;
+    return readableSecond(offset)
+}
 
+function readableSecond(offset) {
     var days = offset / 3600 / 24;
     var hours = offset / 3600;
     var minutes = offset / 60;

apps/templates/_nav.html

@@ -114,6 +114,9 @@
         <ul class="nav nav-second-level">
             <li id="task"><a href="{% url 'ops:task-list' %}">{% trans 'Task list' %}</a></li>
             <li id="command-execution"><a href="{% url 'ops:command-execution-start' %}">{% trans 'Batch command' %}</a></li>
+            {% if request.user.is_superuser %}
+            <li><a href="{% url 'flower-view' path='' %}" target="_blank" >{% trans 'Task monitor' %}</a></li>
+            {% endif %}
         </ul>
     </li>
 {% endif %}
@@ -171,7 +174,5 @@
 
 <script>
 $(document).ready(function () {
-    var current_org = '{{ CURRENT_ORG.name }}';
-    console.log(current_org);
 })
 </script>

apps/terminal/serializers_v2/terminal.py

@@ -58,5 +58,5 @@ class TerminalSerializer(serializers.ModelSerializer):
 
 class TerminalRegistrationSerializer(serializers.Serializer):
     name = serializers.CharField(max_length=128)
-    comment = serializers.CharField(max_length=128)
+    comment = serializers.CharField(max_length=128, )
     service_account = ServiceAccountSerializer(read_only=True)

apps/users/api/user.py

@@ -14,7 +14,7 @@ from common.permissions import (
     IsOrgAdmin, IsCurrentUserOrReadOnly, IsOrgAdminOrAppUser,
     CanUpdateDeleteUser,
 )
-from common.mixins import IDInCacheFilterMixin
+from common.mixins import CommonApiMixin
 from common.utils import get_logger
 from orgs.utils import current_org
 from .. import serializers
@@ -30,7 +30,7 @@ __all__ = [
 ]
 
 
-class UserViewSet(IDInCacheFilterMixin, BulkModelViewSet):
+class UserViewSet(CommonApiMixin, BulkModelViewSet):
     filter_fields = ('username', 'email', 'name', 'id')
     search_fields = filter_fields
     queryset = User.objects.exclude(role=User.ROLE_APP)

apps/users/templates/users/_granted_assets.html

@@ -72,16 +72,10 @@ function initTable() {
                 $(td).html(cellData);
             }},
             {targets: 3, createdCell: function (td, cellData) {
-                function success(systemUsers) {
-                    var users = [];
-                    $.each(systemUsers, function (id, data) {
-                        var name = htmlEscape(data.name);
-                        users.push(name);
-                    });
-                    $(td).html(users.join(','))
-                }
-                $(td).html("{% trans 'Loading' %}");
-                getGrantedAssetSystemUsers(cellData, success)
+                var innerHtml = '<a class="btn-show-system-users" data-aid="99999999"> {% trans "Show" %} </a>'
+                    .replace('99999999', cellData);
+                $(td).html(innerHtml);
+
             }},
         ],
         ajax_url: assetTableUrl,
@@ -185,5 +179,19 @@ $(document).ready(function () {
     var val = $(this).text();
     $("#user_assets_table_filter input").val(val);
     assetTable.search(val).draw();
- })
+}).on('click', '.btn-show-system-users', function () {
+    var $this = $(this);
+    var assetId = $this.data('aid');
+
+    function success(systemUsers) {
+        var users = [];
+        $.each(systemUsers, function (id, data) {
+            var name = htmlEscape(data.name);
+            users.push(name);
+        });
+        $this.parent().html(users.join(','))
+    }
+
+    getGrantedAssetSystemUsers(assetId, success)
+})
 </script>

apps/users/templates/users/user_granted_asset.html

@@ -33,9 +33,9 @@
 {% block custom_foot_js %}
 <script>
 var assetTableUrl = "{% url 'api-perms:user-assets' pk=object.id %}?cache_policy=1";
-var selectUrl = '{% url "api-perms:user-node-assets" pk=object.id node_id=DEFAULT_PK %}?all=1';
+var selectUrl = '{% url "api-perms:user-node-assets" pk=object.id node_id=DEFAULT_PK %}?cache_policy=1&all=1';
 var treeUrl = "{% url 'api-perms:user-nodes-children-as-tree' pk=object.id %}?cache_policy=1";
-var systemUsersUrl = "{% url 'api-perms:user-asset-system-users' pk=object.id asset_id=DEFAULT_PK %}";
+var systemUsersUrl = "{% url 'api-perms:user-asset-system-users' pk=object.id asset_id=DEFAULT_PK %}?cache_policy=1";
 
 $(document).ready(function () {
     initTree();

apps/users/templates/users/user_group_granted_asset.html

@@ -34,9 +34,9 @@
 {% block custom_foot_js %}
 <script>
 var assetTableUrl = "{% url 'api-perms:user-group-assets' pk=object.id %}?cache_policy=1";
-var selectUrl = '{% url "api-perms:user-group-node-assets" pk=object.id node_id=DEFAULT_PK %}?all=1';
+var selectUrl = '{% url "api-perms:user-group-node-assets" pk=object.id node_id=DEFAULT_PK %}?&cache_policy=1&all=1';
 var treeUrl = "{% url 'api-perms:user-group-nodes-children-as-tree' pk=object.id %}?cache_policy=1";
-var systemUsersUrl = "{% url 'api-perms:user-group-asset-system-users' pk=object.id asset_id=DEFAULT_PK %}";
+var systemUsersUrl = "{% url 'api-perms:user-group-asset-system-users' pk=object.id asset_id=DEFAULT_PK %}?cache_policy=1";
 var showAssetHref = true; // Need input default true
 
 

jms

@@ -200,9 +200,13 @@ def is_running(s, unlink=True):
 
 
 def parse_service(s):
-    all_services = ['gunicorn', 'celery_ansible', 'celery_default', 'beat']
+    all_services = [
+        'gunicorn', 'celery_ansible', 'celery_default', 'beat', 'flower'
+    ]
     if s == 'all':
         return all_services
+    elif s == 'gunicorn':
+        return ['gunicorn', 'flower']
     elif s == "celery":
         return ["celery_ansible", "celery_default"]
     elif "," in s:
@@ -240,17 +244,19 @@ def get_start_gunicorn_kwargs():
 
 
 def get_start_celery_ansible_kwargs():
-    print("\n- Start Celery as Distributed Task Queue")
+    print("\n- Start Celery as Distributed Task Queue: Ansible")
     return get_start_worker_kwargs('ansible', 4)
 
 
 def get_start_celery_default_kwargs():
+    print("\n- Start Celery as Distributed Task Queue: Celery")
     return get_start_worker_kwargs('celery', 2)
 
 
 def get_start_worker_kwargs(queue, num):
     # Todo: Must set this environment, otherwise not no ansible result return
     os.environ.setdefault('PYTHONOPTIMIZE', '1')
+    os.environ.setdefault('ANSIBLE_FORCE_COLOR', 'True')
 
     if os.getuid() == 0:
         os.environ.setdefault('C_FORCE_ROOT', '1')
@@ -261,6 +267,24 @@ def get_start_worker_kwargs(queue, num):
         '-l', 'INFO',
         '-c', str(num),
         '-Q', queue,
+        '-n', '{}@%h'.format(queue)
+    ]
+    return {"cmd": cmd, "cwd": APPS_DIR}
+
+
+def get_start_flower_kwargs():
+    print("\n- Start Flower as Task Monitor")
+    if os.getuid() == 0:
+        os.environ.setdefault('C_FORCE_ROOT', '1')
+
+    cmd = [
+        'celery', 'flower',
+        '-A', 'ops',
+        '-l', 'INFO',
+        '--url_prefix=flower',
+        '--auto_refresh=False',
+        '--max_tasks=1000',
+        '--tasks_columns=uuid,name,args,state,received,started,runtime,worker'
     ]
     return {"cmd": cmd, "cwd": APPS_DIR}
 
@@ -333,6 +357,7 @@ def start_service(s):
         "celery_ansible": get_start_celery_ansible_kwargs,
         "celery_default": get_start_celery_default_kwargs,
         "beat": get_start_beat_kwargs,
+        "flower": get_start_flower_kwargs,
     }
 
     kwargs = services_kwargs.get(s)()
@@ -449,7 +474,7 @@ if __name__ == '__main__':
     )
     parser.add_argument(
         "service", type=str, default="all", nargs="?",
-        choices=("all", "gunicorn", "celery", "beat", "celery,beat"),
+        choices=("all", "gunicorn", "celery", "beat", "celery,beat", "flower"),
         help="The service to start",
     )
     parser.add_argument('-d', '--daemon', nargs="?", const=1)