[update] 修改一些性能问题

apps/assets/api/asset_user.py

@@ -149,9 +149,10 @@ class AssetUserTestConnectiveApi(generics.RetrieveAPIView):
     def get_asset_users(self):
         username = self.request.GET.get('username')
         asset_id = self.request.GET.get('asset_id')
+        prefer = self.request.GET.get("prefer")
         asset = get_object_or_none(Asset, pk=asset_id)
         manager = AssetUserManager()
-        asset_users = manager.filter(username=username, assets=[asset])
+        asset_users = manager.filter(username=username, assets=[asset], prefer=prefer)
         return asset_users
 
     def retrieve(self, request, *args, **kwargs):

apps/assets/models/asset.py

@@ -126,6 +126,7 @@ class Asset(OrgModelMixin):
     comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment'))
 
     objects = OrgManager.from_queryset(AssetQuerySet)()
+    _connectivity = None
 
     def __str__(self):
         return '{0.hostname}({0.ip})'.format(self)
@@ -221,17 +222,18 @@ class Asset(OrgModelMixin):
 
     @property
     def connectivity(self):
+        if self._connectivity:
+            return self._connectivity
         if not self.admin_user:
             return Connectivity.unknown()
-        instance = self.admin_user.get_asset_user(self)
+        connectivity = self.admin_user.get_asset_connectivity(self)
-        return instance.connectivity
+        return connectivity
 
     @connectivity.setter
     def connectivity(self, value):
         if not self.admin_user:
             return
-        instance = self.admin_user.get_asset_user(self)
+        self.admin_user.set_asset_connectivity(self, value)
-        instance.set_asset_connectivity(self, value)
 
     def get_auth_info(self):
         if not self.admin_user:

apps/assets/models/base.py

@@ -35,8 +35,8 @@ class AssetUser(OrgModelMixin):
     date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
     created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by'))
 
-    CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_{}_ASSET_CONNECTIVITY"
+    CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_{}_{}_ASSET_CONNECTIVITY"
-    CONNECTIVITY_AMOUNT_CACHE_KEY = "ASSET_USER_{}_CONNECTIVITY_AMOUNT"
+    CONNECTIVITY_AMOUNT_CACHE_KEY = "ASSET_USER_{}_{}_CONNECTIVITY_AMOUNT"
     ASSETS_AMOUNT_CACHE_KEY = "ASSET_USER_{}_ASSETS_AMOUNT"
     ASSET_USER_CACHE_TIME = 3600 * 24
 
@@ -99,21 +99,24 @@ class AssetUser(OrgModelMixin):
         unreachable = summary.get('dark', {}).keys()
         reachable = summary.get('contacted', {}).keys()
 
-        for asset in self.get_related_assets():
+        assets = self.get_related_assets()
+        if not isinstance(assets, list):
+            assets = assets.only('id', 'hostname', 'admin_user__id')
+        for asset in assets:
             if asset.hostname in unreachable:
                 self.set_asset_connectivity(asset, Connectivity.unreachable())
             elif asset.hostname in reachable:
                 self.set_asset_connectivity(asset, Connectivity.reachable())
             else:
                 self.set_asset_connectivity(asset, Connectivity.unknown())
-        cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id)
+        cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.username, self.part_id)
         cache.delete(cache_key)
 
     @property
     def connectivity(self):
-        assets = self.get_related_assets()\
+        assets = self.get_related_assets()
-            .select_related('admin_user')\
+        if not isinstance(assets, list):
-            .only('id', 'hostname', 'admin_user')
+            assets = assets.only('id', 'hostname', 'admin_user__id')
         data = {
             'unreachable': [],
             'reachable': [],
@@ -131,11 +134,11 @@ class AssetUser(OrgModelMixin):
 
     @property
     def connectivity_amount(self):
-        cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id)
+        cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.username, self.part_id)
         amount = cache.get(cache_key)
         if not amount:
-            connectivity = {k: len(v) for k, v in self.connectivity.items()}
+            amount = {k: len(v) for k, v in self.connectivity.items()}
-            cache.set(cache_key, connectivity, self.ASSET_USER_CACHE_TIME)
+            cache.set(cache_key, amount, self.ASSET_USER_CACHE_TIME)
         return amount
 
     @property
@@ -152,17 +155,18 @@ class AssetUser(OrgModelMixin):
         cache.delete(cache_key)
 
     def get_asset_connectivity(self, asset):
-        i = self.generate_id_with_asset(asset)
+        key = self.get_asset_connectivity_key(asset)
-        key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
         return Connectivity.get(key)
 
+    def get_asset_connectivity_key(self, asset):
+        return self.CONNECTIVITY_ASSET_CACHE_KEY.format(self.username, asset.id)
+
     def set_asset_connectivity(self, asset, c):
-        i = self.generate_id_with_asset(asset)
+        key = self.get_asset_connectivity_key(asset)
-        key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i)
         Connectivity.set(key, c)
         # 当为某个系统用户或管理用户设置的的时候，失效掉他们的连接数量
-        amount_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id)
+        amount_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.username, '*')
-        cache.delete(amount_key)
+        cache.delete_pattern(amount_key)
 
     def get_asset_user(self, asset):
         from ..backends import AssetUserManager

apps/assets/models/utils.py

@@ -98,5 +98,11 @@ class Connectivity:
     def __eq__(self, other):
         return self.status == other.status
 
+    def __gt__(self, other):
+        return self.status > other.status
+
+    def __lt__(self, other):
+        return not self.__gt__(other)
+
     def __str__(self):
         return self.display()

apps/assets/serializers/asset.py

@@ -2,12 +2,12 @@
 #
 from rest_framework import serializers
 from rest_framework.validators import ValidationError
-
+from django.db.models import Prefetch
 from django.utils.translation import ugettext_lazy as _
 
 from orgs.mixins import BulkOrgResourceModelSerializer
 from common.serializers import AdaptedBulkListSerializer
-from ..models import Asset, Protocol
+from ..models import Asset, Protocol, Node, Label
 from .base import ConnectivitySerializer
 
 __all__ = [
@@ -58,7 +58,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
             'cpu_model', 'cpu_count', 'cpu_cores', 'cpu_vcpus', 'memory',
             'disk_total', 'disk_info', 'os', 'os_version', 'os_arch',
             'hostname_raw', 'comment', 'created_by', 'date_created',
-            'hardware_info', 'connectivity'
+            'hardware_info', 'connectivity',
         ]
         read_only_fields = (
             'vendor', 'model', 'sn', 'cpu_model', 'cpu_count',
@@ -76,8 +76,11 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
     @classmethod
     def setup_eager_loading(cls, queryset):
         """ Perform necessary eager loading of data. """
-        queryset = queryset.prefetch_related('labels', 'nodes', 'protocols')\
+        queryset = queryset.prefetch_related(
-            .select_related('admin_user', 'domain')
+            Prefetch('nodes', queryset=Node.objects.all().only('id')),
+            Prefetch('labels', queryset=Label.objects.all().only('id')),
+            'protocols'
+        ).select_related('admin_user', 'domain')
         return queryset
 
     @staticmethod

apps/assets/tasks.py

@@ -576,6 +576,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False)
 
     tasks = get_test_asset_user_connectivity_tasks(asset_user.asset)
     if not tasks:
+        logger.debug("No tasks ")
         return
 
     args = (task_name,)

apps/assets/templates/assets/_asset_user_list.html

@@ -92,8 +92,8 @@ function initAssetUserTable() {
         ajax_url: assetUserListUrl,
         columns: [
             {data: "id"}, {data: "hostname"}, {data: "ip"},
-            {data: "username", orderable: false}, {data: "version", orderable: false},
+            {data: "username"}, {data: "version", orderable: false},
-            {data: "connectivity", orderable: false},
+            {data: "connectivity"},
             {data: "date_created", orderable: false},
             {data: "asset", orderable: false}
         ],

apps/assets/templates/assets/admin_user_list.html

@@ -75,7 +75,7 @@ function initTable() {
             }},
             {targets: 4, createdCell: function (td, cellData) {
                 var innerHtml = "";
-                var data = cellData['reachable'];
+                var data = cellData.reachable;
                 if (data !== 0) {
                     innerHtml = "<span class='text-navy'>" + data + "</span>";
                 } else {

apps/common/renders/csv.py

@@ -57,10 +57,15 @@ class JMSCSVRender(BaseRenderer):
         request = renderer_context['request']
         template = request.query_params.get('template', 'export')
         view = renderer_context['view']
-        data = json.loads(json.dumps(data, cls=encoders.JSONEncoder))
+
+        if isinstance(data, dict) and data.get("count"):
+            data = data["results"]
+
         if template == 'import':
             data = [data[0]] if data else data
 
+        data = json.loads(json.dumps(data, cls=encoders.JSONEncoder))
+
         try:
             serializer = view.get_serializer()
             self.set_response_disposition(serializer, renderer_context)

apps/ops/tasks.py

@@ -47,7 +47,7 @@ def run_command_execution(cid, **kwargs):
         try:
             execution.run()
         except SoftTimeLimitExceeded:
-            print("HLLL")
+            logger.error("Run time out")
     else:
         logger.error("Not found the execution id: {}".format(cid))
 

apps/orgs/mixins/api.py

@@ -28,7 +28,8 @@ class OrgModelViewSet(IDInCacheFilterMixin, ModelViewSet):
 class OrgBulkModelViewSet(IDInCacheFilterMixin, BulkModelViewSet):
     def get_queryset(self):
         queryset = super().get_queryset().all()
-        if hasattr(self, 'serializer_class') and \
+        if hasattr(self, 'action') and self.action == 'list' and \
+            hasattr(self, 'serializer_class') and \
                 hasattr(self.serializer_class, 'setup_eager_loading'):
             queryset = self.serializer_class.setup_eager_loading(queryset)
         return queryset

apps/orgs/mixins/models.py

@@ -32,10 +32,10 @@ class OrgManager(models.Manager):
             kwargs['org_id'] = _current_org.id
         elif _current_org.is_default():
             queryset = queryset.filter(org_id="")
-
+        #
         # lines = traceback.format_stack()
         # print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-        # for line in lines[-10:-5]:
+        # for line in lines[-10:-1]:
         #     print(line)
         # print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
 

apps/perms/api/user_permission.py

@@ -262,7 +262,6 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView)
                 system_users=self.system_user_id
             )
         nodes = util.get_nodes_with_assets()
-        print(list(nodes.keys()))
         for node, assets in nodes.items():
             data = parse_node_to_tree_node(node)
             queryset.append(data)

apps/templates/_import_modal.html

@@ -10,7 +10,7 @@
         {% csrf_token %}
         <div class="form-group">
             <label class="control-label">{% trans "Download the imported template or use the exported CSV file format" %}</label>
-            <a href="{% block import_modal_download_template_url %}{% endblock %}?format=csv&template=import" style="display: block">{% trans 'Download the import template' %}</a>
+            <a href="{% block import_modal_download_template_url %}{% endblock %}?format=csv&template=import&limit=1" style="display: block">{% trans 'Download the import template' %}</a>
         </div>
 
         <div class="form-group">