From 1472f0437f17b39356b605804ce4ba065f3f15d1 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 4 Jul 2019 21:17:39 +0800 Subject: [PATCH] =?UTF-8?q?[update]=20=E4=BF=AE=E6=94=B9=E4=B8=80=E4=BA=9B?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/asset_user.py | 3 +- apps/assets/models/asset.py | 10 +++--- apps/assets/models/base.py | 36 ++++++++++--------- apps/assets/models/utils.py | 6 ++++ apps/assets/serializers/asset.py | 13 ++++--- apps/assets/tasks.py | 1 + .../templates/assets/_asset_user_list.html | 4 +-- .../templates/assets/admin_user_list.html | 2 +- apps/common/mixins/api.py | 2 +- apps/common/renders/csv.py | 7 +++- apps/ops/tasks.py | 2 +- apps/orgs/mixins/api.py | 3 +- apps/orgs/mixins/models.py | 4 +-- apps/perms/api/user_permission.py | 1 - apps/templates/_import_modal.html | 2 +- 15 files changed, 59 insertions(+), 37 deletions(-) diff --git a/apps/assets/api/asset_user.py b/apps/assets/api/asset_user.py index 8e8a217ee..38d820349 100644 --- a/apps/assets/api/asset_user.py +++ b/apps/assets/api/asset_user.py @@ -149,9 +149,10 @@ class AssetUserTestConnectiveApi(generics.RetrieveAPIView): def get_asset_users(self): username = self.request.GET.get('username') asset_id = self.request.GET.get('asset_id') + prefer = self.request.GET.get("prefer") asset = get_object_or_none(Asset, pk=asset_id) manager = AssetUserManager() - asset_users = manager.filter(username=username, assets=[asset]) + asset_users = manager.filter(username=username, assets=[asset], prefer=prefer) return asset_users def retrieve(self, request, *args, **kwargs): diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 704abf404..0e5fc2090 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -126,6 +126,7 @@ class Asset(OrgModelMixin): comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment')) objects = OrgManager.from_queryset(AssetQuerySet)() + _connectivity = None def __str__(self): return '{0.hostname}({0.ip})'.format(self) @@ -221,17 +222,18 @@ class Asset(OrgModelMixin): @property def connectivity(self): + if self._connectivity: + return self._connectivity if not self.admin_user: return Connectivity.unknown() - instance = self.admin_user.get_asset_user(self) - return instance.connectivity + connectivity = self.admin_user.get_asset_connectivity(self) + return connectivity @connectivity.setter def connectivity(self, value): if not self.admin_user: return - instance = self.admin_user.get_asset_user(self) - instance.set_asset_connectivity(self, value) + self.admin_user.set_asset_connectivity(self, value) def get_auth_info(self): if not self.admin_user: diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 8c5c156a6..bd85d144e 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -35,8 +35,8 @@ class AssetUser(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) - CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_{}_ASSET_CONNECTIVITY" - CONNECTIVITY_AMOUNT_CACHE_KEY = "ASSET_USER_{}_CONNECTIVITY_AMOUNT" + CONNECTIVITY_ASSET_CACHE_KEY = "ASSET_USER_{}_{}_ASSET_CONNECTIVITY" + CONNECTIVITY_AMOUNT_CACHE_KEY = "ASSET_USER_{}_{}_CONNECTIVITY_AMOUNT" ASSETS_AMOUNT_CACHE_KEY = "ASSET_USER_{}_ASSETS_AMOUNT" ASSET_USER_CACHE_TIME = 3600 * 24 @@ -99,21 +99,24 @@ class AssetUser(OrgModelMixin): unreachable = summary.get('dark', {}).keys() reachable = summary.get('contacted', {}).keys() - for asset in self.get_related_assets(): + assets = self.get_related_assets() + if not isinstance(assets, list): + assets = assets.only('id', 'hostname', 'admin_user__id') + for asset in assets: if asset.hostname in unreachable: self.set_asset_connectivity(asset, Connectivity.unreachable()) elif asset.hostname in reachable: self.set_asset_connectivity(asset, Connectivity.reachable()) else: self.set_asset_connectivity(asset, Connectivity.unknown()) - cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id) + cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.username, self.part_id) cache.delete(cache_key) @property def connectivity(self): - assets = self.get_related_assets()\ - .select_related('admin_user')\ - .only('id', 'hostname', 'admin_user') + assets = self.get_related_assets() + if not isinstance(assets, list): + assets = assets.only('id', 'hostname', 'admin_user__id') data = { 'unreachable': [], 'reachable': [], @@ -131,11 +134,11 @@ class AssetUser(OrgModelMixin): @property def connectivity_amount(self): - cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id) + cache_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.username, self.part_id) amount = cache.get(cache_key) if not amount: - connectivity = {k: len(v) for k, v in self.connectivity.items()} - cache.set(cache_key, connectivity, self.ASSET_USER_CACHE_TIME) + amount = {k: len(v) for k, v in self.connectivity.items()} + cache.set(cache_key, amount, self.ASSET_USER_CACHE_TIME) return amount @property @@ -152,17 +155,18 @@ class AssetUser(OrgModelMixin): cache.delete(cache_key) def get_asset_connectivity(self, asset): - i = self.generate_id_with_asset(asset) - key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i) + key = self.get_asset_connectivity_key(asset) return Connectivity.get(key) + def get_asset_connectivity_key(self, asset): + return self.CONNECTIVITY_ASSET_CACHE_KEY.format(self.username, asset.id) + def set_asset_connectivity(self, asset, c): - i = self.generate_id_with_asset(asset) - key = self.CONNECTIVITY_ASSET_CACHE_KEY.format(i) + key = self.get_asset_connectivity_key(asset) Connectivity.set(key, c) # 当为某个系统用户或管理用户设置的的时候，失效掉他们的连接数量 - amount_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.part_id) - cache.delete(amount_key) + amount_key = self.CONNECTIVITY_AMOUNT_CACHE_KEY.format(self.username, '*') + cache.delete_pattern(amount_key) def get_asset_user(self, asset): from ..backends import AssetUserManager diff --git a/apps/assets/models/utils.py b/apps/assets/models/utils.py index 0373267b8..d010f95f2 100644 --- a/apps/assets/models/utils.py +++ b/apps/assets/models/utils.py @@ -98,5 +98,11 @@ class Connectivity: def __eq__(self, other): return self.status == other.status + def __gt__(self, other): + return self.status > other.status + + def __lt__(self, other): + return not self.__gt__(other) + def __str__(self): return self.display() diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index 6cda758b8..24e19c82d 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -2,12 +2,12 @@ # from rest_framework import serializers from rest_framework.validators import ValidationError - +from django.db.models import Prefetch from django.utils.translation import ugettext_lazy as _ from orgs.mixins import BulkOrgResourceModelSerializer from common.serializers import AdaptedBulkListSerializer -from ..models import Asset, Protocol +from ..models import Asset, Protocol, Node, Label from .base import ConnectivitySerializer __all__ = [ @@ -58,7 +58,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer): 'cpu_model', 'cpu_count', 'cpu_cores', 'cpu_vcpus', 'memory', 'disk_total', 'disk_info', 'os', 'os_version', 'os_arch', 'hostname_raw', 'comment', 'created_by', 'date_created', - 'hardware_info', 'connectivity' + 'hardware_info', 'connectivity', ] read_only_fields = ( 'vendor', 'model', 'sn', 'cpu_model', 'cpu_count', @@ -76,8 +76,11 @@ class AssetSerializer(BulkOrgResourceModelSerializer): @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('labels', 'nodes', 'protocols')\ - .select_related('admin_user', 'domain') + queryset = queryset.prefetch_related( + Prefetch('nodes', queryset=Node.objects.all().only('id')), + Prefetch('labels', queryset=Label.objects.all().only('id')), + 'protocols' + ).select_related('admin_user', 'domain') return queryset @staticmethod diff --git a/apps/assets/tasks.py b/apps/assets/tasks.py index 54fb7068d..dbd6808f1 100644 --- a/apps/assets/tasks.py +++ b/apps/assets/tasks.py @@ -576,6 +576,7 @@ def test_asset_user_connectivity_util(asset_user, task_name, run_as_admin=False) tasks = get_test_asset_user_connectivity_tasks(asset_user.asset) if not tasks: + logger.debug("No tasks ") return args = (task_name,) diff --git a/apps/assets/templates/assets/_asset_user_list.html b/apps/assets/templates/assets/_asset_user_list.html index fda69e5d8..381aec13d 100644 --- a/apps/assets/templates/assets/_asset_user_list.html +++ b/apps/assets/templates/assets/_asset_user_list.html @@ -92,8 +92,8 @@ function initAssetUserTable() { ajax_url: assetUserListUrl, columns: [ {data: "id"}, {data: "hostname"}, {data: "ip"}, - {data: "username", orderable: false}, {data: "version", orderable: false}, - {data: "connectivity", orderable: false}, + {data: "username"}, {data: "version", orderable: false}, + {data: "connectivity"}, {data: "date_created", orderable: false}, {data: "asset", orderable: false} ], diff --git a/apps/assets/templates/assets/admin_user_list.html b/apps/assets/templates/assets/admin_user_list.html index a930a6ce0..0a17eccaa 100644 --- a/apps/assets/templates/assets/admin_user_list.html +++ b/apps/assets/templates/assets/admin_user_list.html @@ -75,7 +75,7 @@ function initTable() { }}, {targets: 4, createdCell: function (td, cellData) { var innerHtml = ""; - var data = cellData['reachable']; + var data = cellData.reachable; if (data !== 0) { innerHtml = "" + data + ""; } else { diff --git a/apps/common/mixins/api.py b/apps/common/mixins/api.py index 2bd1b5a16..ae92e3e97 100644 --- a/apps/common/mixins/api.py +++ b/apps/common/mixins/api.py @@ -83,4 +83,4 @@ class ApiMessageMixin: message = self.get_success_message(resp.data) if message: messages.success(request, message) - return resp \ No newline at end of file + return resp diff --git a/apps/common/renders/csv.py b/apps/common/renders/csv.py index f917edd4c..9bd60cfbc 100644 --- a/apps/common/renders/csv.py +++ b/apps/common/renders/csv.py @@ -57,10 +57,15 @@ class JMSCSVRender(BaseRenderer): request = renderer_context['request'] template = request.query_params.get('template', 'export') view = renderer_context['view'] - data = json.loads(json.dumps(data, cls=encoders.JSONEncoder)) + + if isinstance(data, dict) and data.get("count"): + data = data["results"] + if template == 'import': data = [data[0]] if data else data + data = json.loads(json.dumps(data, cls=encoders.JSONEncoder)) + try: serializer = view.get_serializer() self.set_response_disposition(serializer, renderer_context) diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index a8afee59b..de7602f6b 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -47,7 +47,7 @@ def run_command_execution(cid, **kwargs): try: execution.run() except SoftTimeLimitExceeded: - print("HLLL") + logger.error("Run time out") else: logger.error("Not found the execution id: {}".format(cid)) diff --git a/apps/orgs/mixins/api.py b/apps/orgs/mixins/api.py index 180e9770b..5bdaf2342 100644 --- a/apps/orgs/mixins/api.py +++ b/apps/orgs/mixins/api.py @@ -28,7 +28,8 @@ class OrgModelViewSet(IDInCacheFilterMixin, ModelViewSet): class OrgBulkModelViewSet(IDInCacheFilterMixin, BulkModelViewSet): def get_queryset(self): queryset = super().get_queryset().all() - if hasattr(self, 'serializer_class') and \ + if hasattr(self, 'action') and self.action == 'list' and \ + hasattr(self, 'serializer_class') and \ hasattr(self.serializer_class, 'setup_eager_loading'): queryset = self.serializer_class.setup_eager_loading(queryset) return queryset diff --git a/apps/orgs/mixins/models.py b/apps/orgs/mixins/models.py index a928ea0f3..3f54e14a7 100644 --- a/apps/orgs/mixins/models.py +++ b/apps/orgs/mixins/models.py @@ -32,10 +32,10 @@ class OrgManager(models.Manager): kwargs['org_id'] = _current_org.id elif _current_org.is_default(): queryset = queryset.filter(org_id="") - + # # lines = traceback.format_stack() # print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>") - # for line in lines[-10:-5]: + # for line in lines[-10:-1]: # print(line) # print("<<<<<<<<<<<<<<<<<<<<<<<<<<<<") diff --git a/apps/perms/api/user_permission.py b/apps/perms/api/user_permission.py index e072fed95..14439703e 100644 --- a/apps/perms/api/user_permission.py +++ b/apps/perms/api/user_permission.py @@ -262,7 +262,6 @@ class UserGrantedNodesWithAssetsAsTreeApi(UserPermissionCacheMixin, ListAPIView) system_users=self.system_user_id ) nodes = util.get_nodes_with_assets() - print(list(nodes.keys())) for node, assets in nodes.items(): data = parse_node_to_tree_node(node) queryset.append(data) diff --git a/apps/templates/_import_modal.html b/apps/templates/_import_modal.html index 01a1cdf72..9211bdcb9 100644 --- a/apps/templates/_import_modal.html +++ b/apps/templates/_import_modal.html @@ -10,7 +10,7 @@ {% csrf_token %}
- {% trans 'Download the import template' %} + {% trans 'Download the import template' %}