Merge branch 'dev' of github.com:jumpserver/jumpserver into github_dev

apps/assets/api/asset.py

@@ -40,7 +40,9 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet):
     permission_classes = (IsSuperUserOrAppUser,)
 
     def get_queryset(self):
-        queryset = super().get_queryset()
+        queryset = super().get_queryset()\
+            .prefetch_related('labels', 'nodes')\
+            .select_related('admin_user')
         admin_user_id = self.request.query_params.get('admin_user_id')
         node_id = self.request.query_params.get("node_id")
         show_current_asset = self.request.query_params.get("show_current_asset")
@@ -48,15 +50,24 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet):
         if admin_user_id:
             admin_user = get_object_or_404(AdminUser, id=admin_user_id)
             queryset = queryset.filter(admin_user=admin_user)
-        if node_id:
+
+        if node_id and show_current_asset:
             node = get_object_or_404(Node, id=node_id)
-            if not node.is_root():
+            if node.is_root():
+                queryset = queryset.filter(
+                    Q(nodes=node_id) | Q(nodes__isnull=True)
+                ).distinct()
+            else:
+                queryset = queryset.filter(nodes=node).distinct()
+
+        if node_id and not show_current_asset:
+            node = get_object_or_404(Node, id=node_id)
+            if node.is_root():
+                queryset = Asset.objects.all()
+            else:
                 queryset = queryset.filter(
                     nodes__key__regex='^{}(:[0-9]+)*$'.format(node.key),
                 ).distinct()
-        if show_current_asset and node_id:
-            queryset = queryset.filter(nodes=node_id).distinct()
-
         return queryset
 
 

apps/assets/models/asset.py

@@ -4,6 +4,7 @@
 
 import uuid
 import logging
+import random
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -58,42 +59,70 @@ class Asset(models.Model):
         ('Other', 'Other'),
     )
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True)
+    ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'),
-    hostname = models.CharField(max_length=128, unique=True, verbose_name=_('Hostname'))
+                                      db_index=True)
+    hostname = models.CharField(max_length=128, unique=True,
+                                verbose_name=_('Hostname'))
     port = models.IntegerField(default=22, verbose_name=_('Port'))
-    platform = models.CharField(max_length=128, choices=PLATFORM_CHOICES, default='Linux', verbose_name=_('Platform'))
+    platform = models.CharField(max_length=128, choices=PLATFORM_CHOICES,
-    domain = models.ForeignKey("assets.Domain", null=True, blank=True, related_name='assets', verbose_name=_("Domain"), on_delete=models.SET_NULL)
+                                default='Linux', verbose_name=_('Platform'))
-    nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets', verbose_name=_("Nodes"))
+    domain = models.ForeignKey("assets.Domain", null=True, blank=True,
+                               related_name='assets', verbose_name=_("Domain"),
+                               on_delete=models.SET_NULL)
+    nodes = models.ManyToManyField('assets.Node', default=default_node,
+                                   related_name='assets',
+                                   verbose_name=_("Nodes"))
     is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
 
     # Auth
-    admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT, null=True, verbose_name=_("Admin user"))
+    admin_user = models.ForeignKey('assets.AdminUser', on_delete=models.PROTECT,
+                                   null=True, verbose_name=_("Admin user"))
 
     # Some information
-    public_ip = models.GenericIPAddressField(max_length=32, blank=True, null=True, verbose_name=_('Public IP'))
+    public_ip = models.GenericIPAddressField(max_length=32, blank=True,
-    number = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Asset number'))
+                                             null=True,
+                                             verbose_name=_('Public IP'))
+    number = models.CharField(max_length=32, null=True, blank=True,
+                              verbose_name=_('Asset number'))
 
     # Collect
-    vendor = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Vendor'))
+    vendor = models.CharField(max_length=64, null=True, blank=True,
-    model = models.CharField(max_length=54, null=True, blank=True, verbose_name=_('Model'))
+                              verbose_name=_('Vendor'))
-    sn = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Serial number'))
+    model = models.CharField(max_length=54, null=True, blank=True,
+                             verbose_name=_('Model'))
+    sn = models.CharField(max_length=128, null=True, blank=True,
+                          verbose_name=_('Serial number'))
 
-    cpu_model = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('CPU model'))
+    cpu_model = models.CharField(max_length=64, null=True, blank=True,
+                                 verbose_name=_('CPU model'))
     cpu_count = models.IntegerField(null=True, verbose_name=_('CPU count'))
     cpu_cores = models.IntegerField(null=True, verbose_name=_('CPU cores'))
-    memory = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Memory'))
+    memory = models.CharField(max_length=64, null=True, blank=True,
-    disk_total = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk total'))
+                              verbose_name=_('Memory'))
-    disk_info = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk info'))
+    disk_total = models.CharField(max_length=1024, null=True, blank=True,
+                                  verbose_name=_('Disk total'))
+    disk_info = models.CharField(max_length=1024, null=True, blank=True,
+                                 verbose_name=_('Disk info'))
 
-    os = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('OS'))
+    os = models.CharField(max_length=128, null=True, blank=True,
-    os_version = models.CharField(max_length=16, null=True, blank=True, verbose_name=_('OS version'))
+                          verbose_name=_('OS'))
-    os_arch = models.CharField(max_length=16, blank=True, null=True, verbose_name=_('OS arch'))
+    os_version = models.CharField(max_length=16, null=True, blank=True,
-    hostname_raw = models.CharField(max_length=128, blank=True, null=True, verbose_name=_('Hostname raw'))
+                                  verbose_name=_('OS version'))
+    os_arch = models.CharField(max_length=16, blank=True, null=True,
+                               verbose_name=_('OS arch'))
+    hostname_raw = models.CharField(max_length=128, blank=True, null=True,
+                                    verbose_name=_('Hostname raw'))
 
-    labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels"))
+    labels = models.ManyToManyField('assets.Label', blank=True,
-    created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by'))
+                                    related_name='assets',
-    date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created'))
+                                    verbose_name=_("Labels"))
-    comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment'))
+    created_by = models.CharField(max_length=32, null=True, blank=True,
+                                  verbose_name=_('Created by'))
+    date_created = models.DateTimeField(auto_now_add=True, null=True,
+                                        blank=True,
+                                        verbose_name=_('Date created'))
+    comment = models.TextField(max_length=128, default='', blank=True,
+                               verbose_name=_('Comment'))
 
     objects = AssetManager()
 
@@ -117,7 +146,24 @@ class Asset(models.Model):
 
     def get_nodes(self):
         from .node import Node
-        return self.nodes.all() or [Node.root()]
+        nodes = self.nodes.all() or [Node.root()]
+        return nodes
+
+    @property
+    def nodes_cache_key(self):
+        key = "NODES_OF_{}".format(str(self.id))
+        return key
+
+    def get_nodes_or_cache(self):
+        cached = cache.get(self.nodes_cache_key)
+        if cached is not None:
+            return cached
+        nodes = list(self.get_nodes())
+        cache.set(self.nodes_cache_key, nodes, 3600)
+        return nodes
+
+    def expire_nodes_cache(self):
+        cache.delete(self.nodes_cache_key)
 
     @property
     def hardware_info(self):
@@ -190,7 +236,8 @@ class Asset(models.Model):
 
         seed()
         for i in range(count):
-            asset = cls(ip='%s.%s.%s.%s' % (i, i, i, i),
+            ip = [str(i) for i in random.sample(range(255), 4)]
+            asset = cls(ip='.'.join(ip),
                         hostname=forgery_py.internet.user_name(True),
                         admin_user=choice(AdminUser.objects.all()),
                         port=22,

apps/assets/models/node.py

@@ -3,6 +3,7 @@
 import uuid
 
 from django.db import models, transaction
+from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 
 
@@ -12,9 +13,6 @@ __all__ = ['Node']
 class Node(models.Model):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     key = models.CharField(unique=True, max_length=64, verbose_name=_("Key"))  # '1:1:1:1'
-    # value = models.CharField(
-    #     max_length=128, unique=True, verbose_name=_("Value")
-    # )
     value = models.CharField(max_length=128, verbose_name=_("Value"))
     child_mark = models.IntegerField(default=0)
     date_create = models.DateTimeField(auto_now_add=True)
@@ -30,10 +28,11 @@ class Node(models.Model):
 
     @property
     def full_value(self):
-        if self == self.__class__.root():
+        ancestor = [a.value for a in self.ancestor]
+        if self.is_root():
             return self.value
-        else:
+        ancestor.append(self.value)
-            return '{} / {}'.format(self.parent.full_value, self.value)
+        return ' / '.join(ancestor)
 
     @property
     def level(self):
@@ -87,6 +86,11 @@ class Node(models.Model):
 
     def get_assets(self):
         from .asset import Asset
+        if self.is_root():
+            assets = Asset.objects.filter(
+                Q(nodes__id=self.id) | Q(nodes__isnull=True)
+            )
+        else:
             assets = Asset.objects.filter(nodes__id=self.id)
         return assets
 
@@ -102,14 +106,6 @@ class Node(models.Model):
             assets = Asset.objects.filter(nodes__in=nodes).distinct()
         return assets
 
-    def get_current_assets(self):
-        from .asset import Asset
-        assets = Asset.objects.filter(nodes=self).distinct()
-        return assets
-
-    def has_assets(self):
-        return self.get_all_assets()
-
     def get_all_valid_assets(self):
         return self.get_all_assets().valid()
 
@@ -120,7 +116,6 @@ class Node(models.Model):
     def parent(self):
         if self.key == "0" or not self.key.startswith("0"):
             return self.__class__.root()
-
         parent_key = ":".join(self.key.split(":")[:-1])
         try:
             parent = self.__class__.objects.get(key=parent_key)
@@ -134,16 +129,17 @@ class Node(models.Model):
 
     @property
     def ancestor(self):
+        if self.is_root():
+            ancestor = self.__class__.objects.filter(key='0')
+        else:
             _key = self.key.split(':')
             ancestor_keys = []
-
-        if self.is_root():
-            return [self.__class__.root()]
-
             for i in range(len(_key)-1):
                 _key.pop()
                 ancestor_keys.append(':'.join(_key))
-        return self.__class__.objects.filter(key__in=ancestor_keys)
+            ancestor = self.__class__.objects.filter(key__in=ancestor_keys)
+        ancestor = list(ancestor)
+        return ancestor
 
     @property
     def ancestor_with_self(self):

apps/assets/serializers/asset.py

@@ -12,34 +12,11 @@ __all__ = [
 ]
 
 
-class NodeTMPSerializer(serializers.ModelSerializer):
-    parent = serializers.SerializerMethodField()
-    assets_amount = serializers.SerializerMethodField()
-
-    class Meta:
-        model = Node
-        fields = ['id', 'key', 'value', 'parent', 'assets_amount', 'is_node']
-        list_serializer_class = BulkListSerializer
-
-    @staticmethod
-    def get_parent(obj):
-        return obj.parent.id
-
-    @staticmethod
-    def get_assets_amount(obj):
-        return obj.get_all_assets().count()
-
-    def get_fields(self):
-        fields = super().get_fields()
-        field = fields["key"]
-        field.required = False
-        return fields
-
-
 class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer):
     """
     资产的数据结构
     """
+    nodes = serializers.SerializerMethodField()
 
     class Meta:
         model = Asset
@@ -54,6 +31,10 @@ class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer):
         ])
         return fields
 
+    @staticmethod
+    def get_nodes(obj):
+        return [n.id for n in obj.get_nodes_or_cache()]
+
 
 class AssetGrantedSerializer(serializers.ModelSerializer):
     """
@@ -61,13 +42,13 @@ class AssetGrantedSerializer(serializers.ModelSerializer):
     """
     system_users_granted = AssetSystemUserSerializer(many=True, read_only=True)
     system_users_join = serializers.SerializerMethodField()
-    nodes = NodeTMPSerializer(many=True, read_only=True)
+    # nodes = NodeTMPSerializer(many=True, read_only=True)
 
     class Meta:
         model = Asset
         fields = (
             "id", "hostname", "ip", "port", "system_users_granted",
-            "is_active", "system_users_join", "os", 'domain', "nodes",
+            "is_active", "system_users_join", "os", 'domain',
             "platform", "comment"
         )
 

apps/assets/serializers/node.py

@@ -80,7 +80,7 @@ class NodeSerializer(serializers.ModelSerializer):
 class NodeCurrentSerializer(NodeSerializer):
     @staticmethod
     def get_assets_amount(obj):
-        return obj.get_current_assets().count()
+        return obj.get_assets().count()
 
 
 class NodeAssetsSerializer(serializers.ModelSerializer):

apps/assets/signals_handler.py

@@ -63,11 +63,14 @@ def on_system_user_assets_change(sender, instance=None, **kwargs):
 
 @receiver(m2m_changed, sender=Asset.nodes.through)
 def on_asset_node_changed(sender, instance=None, **kwargs):
-    if isinstance(instance, Asset) and kwargs['action'] == 'post_add':
+    if isinstance(instance, Asset):
+        instance.expire_nodes_cache()
+        if kwargs['action'] == 'post_add':
             logger.debug("Asset node change signal received")
             nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
             system_users_assets = defaultdict(set)
             system_users = SystemUser.objects.filter(nodes__in=nodes)
+            # 清理节点缓存
             for system_user in system_users:
                 system_users_assets[system_user].update({instance})
             for system_user, assets in system_users_assets.items():
@@ -76,9 +79,15 @@ def on_asset_node_changed(sender, instance=None, **kwargs):
 
 @receiver(m2m_changed, sender=Asset.nodes.through)
 def on_node_assets_changed(sender, instance=None, **kwargs):
-    if isinstance(instance, Node) and kwargs['action'] == 'post_add':
+    if isinstance(instance, Node):
-        logger.debug("Node assets change signal received")
         assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set'])
+        # 清理资产节点缓存
+        for asset in assets:
+            asset.expire_nodes_cache()
+
+        if kwargs['action'] == 'post_add':
+            logger.debug("Node assets change signal received")
+            # 重新关联系统用户和资产的关系
             system_users = SystemUser.objects.filter(nodes=instance)
             for system_user in system_users:
                 system_user.assets.add(*tuple(assets))

apps/assets/templates/assets/_asset_list_modal.html

@@ -59,7 +59,7 @@ var zTree2, asset_table2 = 0;
 function initTable2() {
     var options = {
         ele: $('#asset_list_modal_table'),
-        ajax_url: '{% url "api-assets:asset-list" %}',
+        ajax_url: '{% url "api-assets:asset-list" %}?show_current_asset=1',
         columns: [
             {data: "id"}, {data: "hostname" }, {data: "ip" }
         ],
@@ -95,7 +95,7 @@ function initTree2() {
     };
 
     var zNodes = [];
-    $.get("{% url 'api-assets:node-list' %}", function(data, status){
+    $.get("{% url 'api-assets:node-list' %}?show_current_asset=1", function(data, status){
         $.each(data, function (index, value) {
             value["pId"] = value["parent"];
             {#value["open"] = true;#}

apps/assets/utils.py

@@ -1,7 +1,8 @@
 # ~*~ coding: utf-8 ~*~
 #
-
+import os
 import paramiko
+from paramiko.ssh_exception import SSHException
 
 from common.utils import get_object_or_none
 from .models import Asset, SystemUser, Label
@@ -49,22 +50,23 @@ def test_gateway_connectability(gateway):
     """
     client = paramiko.SSHClient()
     client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
+    proxy = paramiko.SSHClient()
-    proxy_command = [
+    proxy.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
-        "ssh", "{}@{}".format(gateway.username, gateway.ip),
+    proxy.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        "-p", str(gateway.port), "-W", "127.0.0.1:{}".format(gateway.port),
-    ]
-
-    if gateway.password:
-        proxy_command.insert(0, "sshpass -p '{}'".format(gateway.password))
-    if gateway.private_key:
-        proxy_command.append("-i {}".format(gateway.private_key_file))
 
     try:
-        sock = paramiko.ProxyCommand(" ".join(proxy_command))
+        proxy.connect(gateway.ip, username=gateway.username,
-    except paramiko.ProxyCommandFailure as e:
+                      password=gateway.password,
+                      pkey=gateway.private_key_obj)
+    except(paramiko.AuthenticationException,
+           paramiko.BadAuthenticationType,
+           SSHException) as e:
         return False, str(e)
 
+    sock = proxy.get_transport().open_channel(
+        'direct-tcpip', ('127.0.0.1', gateway.port), ('127.0.0.1', 0)
+    )
+
     try:
         client.connect("127.0.0.1", port=gateway.port,
                        username=gateway.username,

apps/perms/api.py

@@ -147,13 +147,8 @@ class UserGrantedNodeAssetsApi(ListAPIView):
             user = get_object_or_404(User, id=user_id)
         else:
             user = self.request.user
+        node = get_object_or_404(Node, id=node_id)
         nodes = AssetPermissionUtil.get_user_nodes_with_assets(user)
-        node = get_object_or_none(Node, id=node_id)
-
-        if not node:
-            unnode = [node for node in nodes if node.name == 'Unnode']
-            node = unnode[0] if unnode else None
-
         assets = nodes.get(node, [])
         for asset, system_users in assets.items():
             asset.system_users_granted = system_users

apps/perms/models.py

@@ -33,8 +33,8 @@ class AssetPermission(models.Model):
     nodes = models.ManyToManyField('assets.Node', related_name='granted_by_permissions', blank=True, verbose_name=_("Nodes"))
     system_users = models.ManyToManyField('assets.SystemUser', related_name='granted_by_permissions', verbose_name=_("System user"))
     is_active = models.BooleanField(default=True, verbose_name=_('Active'))
-    date_start = models.DateTimeField(default=timezone.now, verbose_name=_("Date start"))
+    date_start = models.DateTimeField(default=timezone.now, db_index=True, verbose_name=_("Date start"))
-    date_expired = models.DateTimeField(default=date_expired_default, verbose_name=_('Date expired'))
+    date_expired = models.DateTimeField(default=date_expired_default, db_index=True, verbose_name=_('Date expired'))
     created_by = models.CharField(max_length=128, blank=True, verbose_name=_('Created by'))
     date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created'))
     comment = models.TextField(verbose_name=_('Comment'), blank=True)

apps/perms/utils.py

@@ -15,9 +15,23 @@ logger = get_logger(__file__)
 
 class Tree:
     def __init__(self):
-        self.__all_nodes = list(Node.objects.all())
+        self.__all_nodes = list(Node.objects.all().prefetch_related('assets'))
+        self.__node_asset_map = defaultdict(set)
         self.nodes = defaultdict(dict)
         self.root = Node.root()
+        self.init_node_asset_map()
+
+    def init_node_asset_map(self):
+        for node in self.__all_nodes:
+            assets = node.get_assets().values_list('id', flat=True)
+            for asset in assets:
+                self.__node_asset_map[str(asset)].add(node)
+
+    def add_asset(self, asset, system_users):
+        nodes = self.__node_asset_map.get(str(asset.id), [])
+        self.add_nodes(nodes)
+        for node in nodes:
+            self.nodes[node][asset].update(system_users)
 
     def add_node(self, node):
         if node in self.nodes:
@@ -44,11 +58,15 @@ class AssetPermissionUtil:
 
     @staticmethod
     def get_user_group_permissions(user_group):
-        return AssetPermission.objects.all().valid().filter(user_groups=user_group)
+        return AssetPermission.objects.all().valid().filter(
+            user_groups=user_group
+        )
 
     @staticmethod
     def get_asset_permissions(asset):
-        return AssetPermission.objects.all().valid().filter(assets=asset)
+        return AssetPermission.objects.all().valid().filter(
+            assets=asset
+        )
 
     @staticmethod
     def get_node_permissions(node):
@@ -56,7 +74,9 @@ class AssetPermissionUtil:
 
     @staticmethod
     def get_system_user_permissions(system_user):
-        return AssetPermission.objects.valid().all().filter(system_users=system_user)
+        return AssetPermission.objects.valid().all().filter(
+            system_users=system_user
+        )
 
     @classmethod
     def get_user_group_nodes(cls, group):
@@ -114,7 +134,7 @@ class AssetPermissionUtil:
         _assets = cls.get_user_group_assets(group)
         tree = Tree()
         for asset, _system_users in _assets.items():
-            _nodes = asset.get_nodes()
+            _nodes = asset.get_nodes_or_cache()
             tree.add_nodes(_nodes)
             for node in _nodes:
                 tree.nodes[node][asset].update(_system_users)
@@ -207,11 +227,11 @@ class AssetPermissionUtil:
         tree = Tree()
         _assets = cls.get_user_assets(user)
         for asset, _system_users in _assets.items():
-            _nodes = asset.get_nodes()
+            tree.add_asset(asset, _system_users)
-            tree.add_nodes(_nodes)
+            # _nodes = asset.get_nodes()
-
+            # tree.add_nodes(_nodes)
-            for node in _nodes:
+            # for node in _nodes:
-                tree.nodes[node][asset].update(_system_users)
+            #     tree.nodes[node][asset].update(_system_users)
         return tree.nodes
 
     @classmethod

apps/users/models/user.py

@@ -77,7 +77,7 @@ class User(AbstractUser):
     is_first_login = models.BooleanField(default=True)
     date_expired = models.DateTimeField(
         default=date_expired_default, blank=True, null=True,
-        verbose_name=_('Date expired')
+        db_index=True, verbose_name=_('Date expired')
     )
     created_by = models.CharField(
         max_length=30, default='', verbose_name=_('Created by')

apps/users/views/login.py

@@ -50,7 +50,9 @@ class UserLoginView(FormView):
 
     def get(self, request, *args, **kwargs):
         if request.user.is_staff:
-            return redirect(self.get_success_url())
+            return redirect(redirect_user_first_login_or_index(
+                request, self.redirect_field_name)
+            )
         request.session.set_test_cookie()
         return super().get(request, *args, **kwargs)
 

jms

@@ -123,6 +123,7 @@ def start_gunicorn():
         'gunicorn', 'jumpserver.wsgi',
         '-b', bind,
         '-w', str(WORKERS),
+        '-k', 'eventlet',
         '--access-logformat', log_format,
         '-p', pid_file,
     ]