From 28da8197358545a1fcf0ee97786e7d41ec9a6942 Mon Sep 17 00:00:00 2001 From: xinwen Date: Sun, 16 Aug 2020 23:08:58 +0800 Subject: [PATCH] =?UTF-8?q?perf(assets):=20=E4=BC=98=E5=8C=96=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E6=A0=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改树策略，做读优化，写的速度降低 --- apps/assets/api/__init__.py | 1 + apps/assets/api/asset.py | 8 +- apps/assets/api/mixin.py | 100 ++++ apps/assets/api/node.py | 98 ++-- apps/assets/exceptions.py | 6 + apps/assets/filters.py | 46 +- .../migrations/0056_auto_20200904_1751.py | 23 + ...node_value_assets_amount_and_parent_key.py | 38 ++ apps/assets/models/node.py | 197 ++------ apps/assets/pagination.py | 42 ++ apps/assets/serializers/asset.py | 1 - apps/assets/signals_handler.py | 290 +++++++---- apps/assets/tasks/nodes_amount.py | 0 apps/assets/tasks/push_system_user.py | 7 +- apps/assets/urls/api_urls.py | 7 +- apps/assets/utils.py | 196 +------- apps/common/const/distributed_lock_key.py | 2 + apps/common/const/signals.py | 14 + apps/common/db/utils.py | 27 + apps/common/exceptions.py | 15 + apps/common/mixins/api.py | 3 +- apps/common/thread_pools.py | 17 + apps/locale/zh/LC_MESSAGES/django.mo | Bin 57154 -> 57168 bytes apps/locale/zh/LC_MESSAGES/django.po | 201 ++++---- apps/ops/api/command.py | 34 +- apps/ops/celery/__init__.py | 1 + apps/orgs/lock.py | 131 +++++ apps/perms/api/asset_permission_relation.py | 28 +- apps/perms/api/system_user_permission.py | 21 +- apps/perms/api/user_group_permission.py | 167 ++++++- apps/perms/api/user_permission/common.py | 55 +- apps/perms/api/user_permission/mixin.py | 110 ++-- .../user_permission/user_permission_assets.py | 160 ++++-- .../user_permission/user_permission_nodes.py | 185 +++++-- .../user_permission_nodes_with_assets.py | 169 +++++-- apps/perms/async_tasks/__init__.py | 0 apps/perms/async_tasks/mapping_node_task.py | 47 ++ apps/perms/exceptions.py | 14 + ...uildusertreetask_usergrantedmappingnode.py | 53 ++ .../migrations/0014_build_users_perm_tree.py | 27 + apps/perms/models/asset_permission.py | 21 +- apps/perms/pagination.py | 26 + .../serializers/asset_permission_relation.py | 12 +- apps/perms/serializers/user_permission.py | 11 - apps/perms/signals_handler.py | 163 ++++-- apps/perms/tasks.py | 17 +- apps/perms/urls/asset_permission.py | 71 ++- apps/perms/utils/__init__.py | 3 +- apps/perms/utils/asset_permission.py | 469 ++---------------- apps/perms/utils/user_node_tree.py | 334 +++++++++++++ .../css/plugins/ztree/awesomeStyle/fa.less | 4 +- apps/static/fonts/fontawesome-webfont.svg | 2 +- jms | 10 +- 53 files changed, 2318 insertions(+), 1366 deletions(-) create mode 100644 apps/assets/api/mixin.py create mode 100644 apps/assets/exceptions.py create mode 100644 apps/assets/migrations/0056_auto_20200904_1751.py create mode 100644 apps/assets/migrations/0057_fill_node_value_assets_amount_and_parent_key.py create mode 100644 apps/assets/pagination.py create mode 100644 apps/assets/tasks/nodes_amount.py create mode 100644 apps/common/const/distributed_lock_key.py create mode 100644 apps/common/const/signals.py create mode 100644 apps/common/db/utils.py create mode 100644 apps/common/thread_pools.py create mode 100644 apps/orgs/lock.py create mode 100644 apps/perms/async_tasks/__init__.py create mode 100644 apps/perms/async_tasks/mapping_node_task.py create mode 100644 apps/perms/exceptions.py create mode 100644 apps/perms/migrations/0013_rebuildusertreetask_usergrantedmappingnode.py create mode 100644 apps/perms/migrations/0014_build_users_perm_tree.py create mode 100644 apps/perms/pagination.py create mode 100644 apps/perms/utils/user_node_tree.py diff --git a/apps/assets/api/__init__.py b/apps/assets/api/__init__.py index 342f71bd4..59cb8e602 100644 --- a/apps/assets/api/__init__.py +++ b/apps/assets/api/__init__.py @@ -1,3 +1,4 @@ +from .mixin import * from .admin_user import * from .asset import * from .label import * diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index 7877c5b90..ee28a52bf 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # - +from assets.api import FilterAssetByNodeMixin from rest_framework.viewsets import ModelViewSet from rest_framework.generics import RetrieveAPIView from django.shortcuts import get_object_or_404 @@ -14,7 +14,7 @@ from .. import serializers from ..tasks import ( update_asset_hardware_info_manual, test_asset_connectivity_manual ) -from ..filters import AssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend +from ..filters import FilterAssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend logger = get_logger(__file__) @@ -25,7 +25,7 @@ __all__ = [ ] -class AssetViewSet(OrgBulkModelViewSet): +class AssetViewSet(FilterAssetByNodeMixin, OrgBulkModelViewSet): """ API endpoint that allows Asset to be viewed or edited. """ @@ -41,7 +41,7 @@ class AssetViewSet(OrgBulkModelViewSet): 'display': serializers.AssetDisplaySerializer, } permission_classes = (IsOrgAdminOrAppUser,) - extra_filter_backends = [AssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend] + extra_filter_backends = [FilterAssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend] def set_assets_node(self, assets): if not isinstance(assets, list): diff --git a/apps/assets/api/mixin.py b/apps/assets/api/mixin.py new file mode 100644 index 000000000..2876fd0e3 --- /dev/null +++ b/apps/assets/api/mixin.py @@ -0,0 +1,100 @@ +from typing import List + +from assets.models import Node, Asset +from assets.pagination import AssetLimitOffsetPagination +from common.utils import lazyproperty, dict_get_any, is_uuid, get_object_or_none + + +class SerializeToTreeNodeMixin: + permission_classes = () + + def serialize_nodes(self, nodes: List[Node], with_asset_amount=False): + if with_asset_amount: + def _name(node: Node): + return '{} ({})'.format(node.value, node.assets_amount) + else: + def _name(node: Node): + return node.value + data = [ + { + 'id': node.key, + 'name': _name(node), + 'title': _name(node), + 'pId': node.parent_key, + 'isParent': True, + 'open': node.is_org_root(), + 'meta': { + 'node': { + "id": node.id, + "key": node.key, + "value": node.value, + }, + 'type': 'node' + } + } + for node in nodes + ] + return data + + def get_platform(self, asset: Asset): + default = 'file' + icon = {'windows', 'linux'} + platform = asset.platform_base.lower() + if platform in icon: + return platform + return default + + def serialize_assets(self, assets, node_key=None): + if node_key is None: + get_pid = lambda asset: getattr(asset, 'parent_key', '') + else: + get_pid = lambda asset: node_key + + data = [ + { + 'id': str(asset.id), + 'name': asset.hostname, + 'title': asset.ip, + 'pId': get_pid(asset), + 'isParent': False, + 'open': False, + 'iconSkin': self.get_platform(asset), + 'meta': { + 'type': 'asset', + 'asset': { + 'id': asset.id, + 'hostname': asset.hostname, + 'ip': asset.ip, + 'protocols': asset.protocols_as_list, + 'platform': asset.platform_base, + }, + } + } + for asset in assets + ] + return data + + +class FilterAssetByNodeMixin: + pagination_class = AssetLimitOffsetPagination + + @lazyproperty + def is_query_node_all_assets(self): + request = self.request + query_all_arg = request.query_params.get('all') + show_current_asset_arg = request.query_params.get('show_current_asset') + if show_current_asset_arg is not None: + return show_current_asset_arg != '1' + return query_all_arg == '1' + + @lazyproperty + def node(self): + node_id = dict_get_any(self.request.query_params, ['node', 'node_id']) + if not node_id: + return None + + if is_uuid(node_id): + node = get_object_or_none(Node, id=node_id) + else: + node = get_object_or_none(Node, key=node_id) + return node diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 2d959e9ea..5eb91e575 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -1,16 +1,24 @@ # ~*~ coding: utf-8 ~*~ +from functools import partial +from collections import namedtuple, defaultdict -from collections import namedtuple from rest_framework import status from rest_framework.serializers import ValidationError from rest_framework.response import Response from django.utils.translation import ugettext_lazy as _ from django.shortcuts import get_object_or_404, Http404 +from django.utils.decorators import method_decorator +from django.db.models.signals import m2m_changed +from common.exceptions import SomeoneIsDoingThis +from common.const.signals import PRE_REMOVE, POST_REMOVE +from assets.models import Asset from common.utils import get_logger, get_object_or_none from common.tree import TreeNodeSerializer +from common.const.distributed_lock_key import UPDATE_NODE_TREE_LOCK_KEY from orgs.mixins.api import OrgModelViewSet from orgs.mixins import generics +from orgs.lock import org_level_transaction_lock from ..hands import IsOrgAdmin from ..models import Node from ..tasks import ( @@ -18,12 +26,13 @@ from ..tasks import ( test_node_assets_connectivity_manual, ) from .. import serializers +from .mixin import SerializeToTreeNodeMixin logger = get_logger(__file__) __all__ = [ 'NodeViewSet', 'NodeChildrenApi', 'NodeAssetsApi', - 'NodeAddAssetsApi', 'NodeRemoveAssetsApi', 'NodeReplaceAssetsApi', + 'NodeAddAssetsApi', 'NodeRemoveAssetsApi', 'MoveAssetsToNodeApi', 'NodeAddChildrenApi', 'NodeListAsTreeApi', 'NodeChildrenAsTreeApi', 'NodeTaskCreateApi', @@ -136,7 +145,7 @@ class NodeChildrenApi(generics.ListCreateAPIView): return queryset -class NodeChildrenAsTreeApi(NodeChildrenApi): +class NodeChildrenAsTreeApi(SerializeToTreeNodeMixin, NodeChildrenApi): """ 节点子节点作为树返回， [ @@ -150,31 +159,23 @@ class NodeChildrenAsTreeApi(NodeChildrenApi): """ model = Node - serializer_class = TreeNodeSerializer - http_method_names = ['get'] - def get_queryset(self): - queryset = super().get_queryset() - queryset = [node.as_tree_node() for node in queryset] - queryset = self.add_assets_if_need(queryset) - queryset = sorted(queryset) - return queryset + def list(self, request, *args, **kwargs): + nodes = self.get_queryset().order_by('value') + nodes = self.serialize_nodes(nodes, with_asset_amount=True) + assets = self.get_assets() + data = [*nodes, *assets] + return Response(data=data) - def add_assets_if_need(self, queryset): + def get_assets(self): include_assets = self.request.query_params.get('assets', '0') == '1' if not include_assets: - return queryset + return [] assets = self.instance.get_assets().only( "id", "hostname", "ip", "os", "org_id", "protocols", ) - for asset in assets: - queryset.append(asset.as_tree_node(self.instance)) - return queryset - - def check_need_refresh_nodes(self): - if self.request.query_params.get('refresh', '0') == '1': - Node.refresh_nodes() + return self.serialize_assets(assets, self.instance.key) class NodeAssetsApi(generics.ListAPIView): @@ -208,6 +209,8 @@ class NodeAddChildrenApi(generics.UpdateAPIView): return Response("OK") +@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='patch') +@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='put') class NodeAddAssetsApi(generics.UpdateAPIView): model = Node serializer_class = serializers.NodeAssetsSerializer @@ -220,6 +223,8 @@ class NodeAddAssetsApi(generics.UpdateAPIView): instance.assets.add(*tuple(assets)) +@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='patch') +@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='put') class NodeRemoveAssetsApi(generics.UpdateAPIView): model = Node serializer_class = serializers.NodeAssetsSerializer @@ -228,15 +233,17 @@ class NodeRemoveAssetsApi(generics.UpdateAPIView): def perform_update(self, serializer): assets = serializer.validated_data.get('assets') - instance = self.get_object() - if instance != Node.org_root(): - instance.assets.remove(*tuple(assets)) - else: - assets = [asset for asset in assets if asset.nodes.count() > 1] - instance.assets.remove(*tuple(assets)) + node = self.get_object() + node.assets.remove(*assets) + + # 把孤儿资产添加到 root 节点 + orphan_assets = Asset.objects.filter(id__in=[a.id for a in assets], nodes__isnull=True).distinct() + Node.org_root().assets.add(*orphan_assets) -class NodeReplaceAssetsApi(generics.UpdateAPIView): +@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='patch') +@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='put') +class MoveAssetsToNodeApi(generics.UpdateAPIView): model = Node serializer_class = serializers.NodeAssetsSerializer permission_classes = (IsOrgAdmin,) @@ -244,9 +251,39 @@ class NodeReplaceAssetsApi(generics.UpdateAPIView): def perform_update(self, serializer): assets = serializer.validated_data.get('assets') - instance = self.get_object() - for asset in assets: - asset.nodes.set([instance]) + node = self.get_object() + self.remove_old_nodes(assets) + node.assets.add(*assets) + + def remove_old_nodes(self, assets): + m2m_model = Asset.nodes.through + + # 查询资产与节点关系表，查出要移动资产与节点的所有关系 + relates = m2m_model.objects.filter(asset__in=assets).values_list('asset_id', 'node_id') + if relates: + # 对关系以资产进行分组，用来发 `reverse=False` 信号 + asset_nodes_mapper = defaultdict(set) + for asset_id, node_id in relates: + asset_nodes_mapper[asset_id].add(node_id) + + # 组建一个资产 id -> Asset 的 mapper + asset_mapper = {asset.id: asset for asset in assets} + + # 创建删除关系信号发送函数 + senders = [] + for asset_id, node_id_set in asset_nodes_mapper.items(): + senders.append(partial(m2m_changed.send, sender=m2m_model, instance=asset_mapper[asset_id], + reverse=False, model=Node, pk_set=node_id_set)) + # 发送 pre 信号 + [sender(action=PRE_REMOVE) for sender in senders] + num = len(relates) + asset_ids, node_ids = zip(*relates) + # 删除之前的关系 + rows, _i = m2m_model.objects.filter(asset_id__in=asset_ids, node_id__in=node_ids).delete() + if rows != num: + raise SomeoneIsDoingThis + # 发送 post 信号 + [sender(action=POST_REMOVE) for sender in senders] class NodeTaskCreateApi(generics.CreateAPIView): @@ -267,7 +304,6 @@ class NodeTaskCreateApi(generics.CreateAPIView): @staticmethod def refresh_nodes_cache(): - Node.refresh_nodes() Task = namedtuple('Task', ['id']) task = Task(id="0") return task diff --git a/apps/assets/exceptions.py b/apps/assets/exceptions.py new file mode 100644 index 000000000..099e68f11 --- /dev/null +++ b/apps/assets/exceptions.py @@ -0,0 +1,6 @@ +from rest_framework import status +from common.exceptions import JMSException + + +class NodeIsBeingUpdatedByOthers(JMSException): + status_code = status.HTTP_409_CONFLICT diff --git a/apps/assets/filters.py b/apps/assets/filters.py index 149ed12a8..feed9f9dc 100644 --- a/apps/assets/filters.py +++ b/apps/assets/filters.py @@ -6,7 +6,8 @@ from rest_framework import filters from django.db.models import Q from common.utils import dict_get_any, is_uuid, get_object_or_none -from .models import Node, Label +from .models import Label +from assets.models import Node class AssetByNodeFilterBackend(filters.BaseFilterBackend): @@ -43,10 +44,6 @@ class AssetByNodeFilterBackend(filters.BaseFilterBackend): node = get_object_or_none(Node, key=node_id) return node, True - @staticmethod - def perform_query(pattern, queryset): - return queryset.filter(nodes__key__regex=pattern).distinct() - def filter_queryset(self, request, queryset, view): node, has_query_arg = self.get_query_node(request) if not has_query_arg: @@ -56,12 +53,41 @@ class AssetByNodeFilterBackend(filters.BaseFilterBackend): return queryset query_all = self.is_query_all(request) if query_all: - pattern = node.get_all_children_pattern(with_self=True) + return queryset.filter( + Q(nodes__key__istartswith=f'{node.key}:') | + Q(nodes__key=node.key) + ).distinct() else: - # pattern = node.get_children_key_pattern(with_self=True) - # 只显示当前节点下资产 - pattern = r"^{}$".format(node.key) - return self.perform_query(pattern, queryset) + return queryset.filter(nodes__key=node.key).distinct() + + +class FilterAssetByNodeFilterBackend(filters.BaseFilterBackend): + """ + 需要与 `assets.api.mixin.FilterAssetByNodeMixin` 配合使用 + """ + fields = ['node', 'all'] + + def get_schema_fields(self, view): + return [ + coreapi.Field( + name=field, location='query', required=False, + type='string', example='', description='', schema=None, + ) + for field in self.fields + ] + + def filter_queryset(self, request, queryset, view): + node = view.node + if node is None: + return queryset + query_all = view.is_query_node_all_assets + if query_all: + return queryset.filter( + Q(nodes__key__istartswith=f'{node.key}:') | + Q(nodes__key=node.key) + ).distinct() + else: + return queryset.filter(nodes__key=node.key).distinct() class LabelFilterBackend(filters.BaseFilterBackend): diff --git a/apps/assets/migrations/0056_auto_20200904_1751.py b/apps/assets/migrations/0056_auto_20200904_1751.py new file mode 100644 index 000000000..1a7a34af1 --- /dev/null +++ b/apps/assets/migrations/0056_auto_20200904_1751.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.13 on 2020-09-04 09:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0055_auto_20200811_1845'), + ] + + operations = [ + migrations.AddField( + model_name='node', + name='assets_amount', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='node', + name='parent_key', + field=models.CharField(db_index=True, default='', max_length=64, verbose_name='Parent key'), + ), + ] diff --git a/apps/assets/migrations/0057_fill_node_value_assets_amount_and_parent_key.py b/apps/assets/migrations/0057_fill_node_value_assets_amount_and_parent_key.py new file mode 100644 index 000000000..458975692 --- /dev/null +++ b/apps/assets/migrations/0057_fill_node_value_assets_amount_and_parent_key.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.13 on 2020-08-21 08:20 + +from django.db import migrations +from django.db.models import Q + + +def fill_node_value(apps, schema_editor): + Node = apps.get_model('assets', 'Node') + Asset = apps.get_model('assets', 'Asset') + node_queryset = Node.objects.all() + node_amount = node_queryset.count() + width = len(str(node_amount)) + print('\n') + for i, node in enumerate(node_queryset): + print(f'\t{i+1:0>{width}}/{node_amount} compute node[{node.key}]`s assets_amount ...') + assets_amount = Asset.objects.filter( + Q(nodes__key__istartswith=f'{node.key}:') | Q(nodes=node) + ).distinct().count() + key = node.key + try: + parent_key = key[:key.rindex(':')] + except ValueError: + parent_key = '' + node.assets_amount = assets_amount + node.parent_key = parent_key + node.save() + print(' ' + '.'*65, end='') + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0056_auto_20200904_1751'), + ] + + operations = [ + migrations.RunPython(fill_node_value) + ] diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 461e61e08..dd91cd993 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -2,132 +2,35 @@ # import uuid import re -import time from django.db import models, transaction from django.db.models import Q from django.db.utils import IntegrityError from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext -from django.core.cache import cache +from django.db.transaction import atomic -from common.utils import get_logger, lazyproperty +from common.utils import get_logger +from common.utils.common import lazyproperty from orgs.mixins.models import OrgModelMixin, OrgManager from orgs.utils import get_current_org, tmp_to_org, current_org from orgs.models import Organization -__all__ = ['Node'] +__all__ = ['Node', 'FamilyMixin', 'compute_parent_key'] logger = get_logger(__name__) +def compute_parent_key(key): + try: + return key[:key.rindex(':')] + except ValueError: + return '' + + class NodeQuerySet(models.QuerySet): def delete(self): - raise PermissionError("Bulk delete node deny") - - -class TreeCache: - updated_time_cache_key = 'NODE_TREE_UPDATED_AT_{}' - cache_time = 3600 - assets_updated_time_cache_key = 'NODE_TREE_ASSETS_UPDATED_AT_{}' - - def __init__(self, tree, org_id): - now = time.time() - self.created_time = now - self.assets_created_time = now - self.tree = tree - self.org_id = org_id - - def _has_changed(self, tp="tree"): - if tp == "assets": - key = self.assets_updated_time_cache_key.format(self.org_id) - else: - key = self.updated_time_cache_key.format(self.org_id) - updated_time = cache.get(key, 0) - if updated_time > self.created_time: - return True - else: - return False - - @classmethod - def set_changed(cls, tp="tree", t=None, org_id=None): - if org_id is None: - org_id = current_org.id - if tp == "assets": - key = cls.assets_updated_time_cache_key.format(org_id) - else: - key = cls.updated_time_cache_key.format(org_id) - ttl = cls.cache_time - if not t: - t = time.time() - cache.set(key, t, ttl) - - def tree_has_changed(self): - return self._has_changed("tree") - - def set_tree_changed(self, t=None): - logger.debug("Set tree tree changed") - self.__class__.set_changed(t=t, tp="tree") - - def assets_has_changed(self): - return self._has_changed("assets") - - def set_tree_assets_changed(self, t=None): - logger.debug("Set tree assets changed") - self.__class__.set_changed(t=t, tp="assets") - - def get(self): - if self.tree_has_changed(): - self.renew() - return self.tree - if self.assets_has_changed(): - self.tree.init_assets() - return self.tree - - def renew(self): - new_obj = self.__class__.new(self.org_id) - self.tree = new_obj.tree - self.created_time = new_obj.created_time - self.assets_created_time = new_obj.assets_created_time - - @classmethod - def new(cls, org_id=None): - from ..utils import TreeService - logger.debug("Create node tree") - if not org_id: - org_id = current_org.id - with tmp_to_org(org_id): - tree = TreeService.new() - obj = cls(tree, org_id) - obj.tree = tree - return obj - - -class TreeMixin: - _org_tree_map = {} - - @classmethod - def tree(cls): - org_id = current_org.org_id() - t = cls.get_local_tree_cache(org_id) - - if t is None: - t = TreeCache.new() - cls._org_tree_map[org_id] = t - return t.get() - - @classmethod - def get_local_tree_cache(cls, org_id=None): - t = cls._org_tree_map.get(org_id) - return t - - @classmethod - def refresh_tree(cls, t=None): - TreeCache.set_changed(tp="tree", t=t, org_id=current_org.id) - - @classmethod - def refresh_node_assets(cls, t=None): - TreeCache.set_changed(tp="assets", t=t, org_id=current_org.id) + raise NotImplementedError class FamilyMixin: @@ -175,13 +78,16 @@ class FamilyMixin: return re.match(children_pattern, self.key) def get_children(self, with_self=False): - pattern = self.get_children_key_pattern(with_self=with_self) - return Node.objects.filter(key__regex=pattern) + q = Q(parent_key=self.key) + if with_self: + q |= Q(key=self.key) + return Node.objects.filter(q) def get_all_children(self, with_self=False): - pattern = self.get_all_children_pattern(with_self=with_self) - children = Node.objects.filter(key__regex=pattern) - return children + q = Q(key__istartswith=f'{self.key}:') + if with_self: + q |= Q(key=self.key) + return Node.objects.filter(q) @property def children(self): @@ -192,10 +98,10 @@ class FamilyMixin: return self.get_all_children(with_self=False) def create_child(self, value, _id=None): - with transaction.atomic(): + with atomic(savepoint=False): child_key = self.get_next_child_key() child = self.__class__.objects.create( - id=_id, key=child_key, value=value + id=_id, key=child_key, value=value, parent_key=self.key, ) return child @@ -255,10 +161,13 @@ class FamilyMixin: ancestor_keys = self.get_ancestor_keys(with_self=with_self) return self.__class__.objects.filter(key__in=ancestor_keys) - @property - def parent_key(self): - parent_key = ":".join(self.key.split(":")[:-1]) - return parent_key + # @property + # def parent_key(self): + # parent_key = ":".join(self.key.split(":")[:-1]) + # return parent_key + + def compute_parent_key(self): + return compute_parent_key(self.key) def is_parent(self, other): return other.is_children(self) @@ -300,32 +209,23 @@ class FamilyMixin: return [*tuple(ancestors), self, *tuple(children)] -class FullValueMixin: - key = '' - - @lazyproperty - def full_value(self): - if self.is_org_root(): - return self.value - value = self.tree().get_node_full_tag(self.key) - return value - - class NodeAssetsMixin: key = '' id = None - @lazyproperty - def assets_amount(self): - amount = self.tree().assets_amount(self.key) - return amount + @classmethod + def clear_all_nodes_assets_amount(cls): + nodes = cls.objects.all() + for node in nodes: + count = node.get_all_assets().count() def get_all_assets(self): from .asset import Asset if self.is_org_root(): return Asset.objects.filter(org_id=self.org_id) - pattern = '^{0}$|^{0}:'.format(self.key) - return Asset.objects.filter(nodes__key__regex=pattern).distinct() + + q = Q(nodes__key__startswith=self.key) | Q(nodes__key=self.key) + return Asset.objects.filter(q).distinct() def get_assets(self): from .asset import Asset @@ -496,12 +396,15 @@ class SomeNodesMixin: logger.info('Modify key ( {} > {} )'.format(old_key, new_key)) -class Node(OrgModelMixin, SomeNodesMixin, TreeMixin, FamilyMixin, FullValueMixin, NodeAssetsMixin): +class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) key = models.CharField(unique=True, max_length=64, verbose_name=_("Key")) # '1:1:1:1' value = models.CharField(max_length=128, verbose_name=_("Value")) child_mark = models.IntegerField(default=0) date_create = models.DateTimeField(auto_now_add=True) + parent_key = models.CharField(max_length=64, verbose_name=_("Parent key"), + db_index=True, default='') + assets_amount = models.IntegerField(default=0) objects = OrgManager.from_queryset(NodeQuerySet)() is_node = True @@ -536,18 +439,20 @@ class Node(OrgModelMixin, SomeNodesMixin, TreeMixin, FamilyMixin, FullValueMixin def name(self): return self.value + @lazyproperty + def full_value(self): + # 不要在列表中调用该属性 + values = self.__class__.objects.filter( + key__in=self.get_ancestor_keys() + ).values_list('key', 'value') + values = [v for k, v in sorted(values, key=lambda x: len(x[0]))] + values.append(self.value) + return ' / '.join(values) + @property def level(self): return len(self.key.split(':')) - @classmethod - def refresh_nodes(cls): - cls.refresh_tree() - - @classmethod - def refresh_assets(cls): - cls.refresh_node_assets() - def as_tree_node(self): from common.tree import TreeNode name = '{} ({})'.format(self.value, self.assets_amount) diff --git a/apps/assets/pagination.py b/apps/assets/pagination.py new file mode 100644 index 000000000..0a2562c29 --- /dev/null +++ b/apps/assets/pagination.py @@ -0,0 +1,42 @@ +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.request import Request + +from assets.models import Node + + +class AssetLimitOffsetPagination(LimitOffsetPagination): + """ + 需要与 `assets.api.mixin.FilterAssetByNodeMixin` 配合使用 + """ + def get_count(self, queryset): + """ + 1. 如果查询节点下的所有资产，那 count 使用 Node.assets_amount + 2. 如果有其他过滤条件使用 super + 3. 如果只查询该节点下的资产使用 super + """ + exclude_query_params = { + self.limit_query_param, + self.offset_query_param, + 'node', 'all', 'show_current_asset' + } + + has_filter = False + for k, v in self._request.query_params.items(): + if k not in exclude_query_params and v is not None: + has_filter = True + break + if has_filter: + return super().get_count(queryset) + + is_query_all = self._view.is_query_node_all_assets + if is_query_all: + node = self._view.node + if not node: + node = Node.org_root() + return node.assets_amount + return super().get_count(queryset) + + def paginate_queryset(self, queryset, request: Request, view=None): + self._request = request + self._view = view + return super().paginate_queryset(queryset, request, view=None) diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index 4100c439b..1f03d14c9 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -75,7 +75,6 @@ class AssetSerializer(BulkOrgResourceModelSerializer): """ class Meta: model = Asset - list_serializer_class = AdaptedBulkListSerializer fields_mini = ['id', 'hostname', 'ip'] fields_small = fields_mini + [ 'protocol', 'port', 'protocols', 'is_active', 'public_ip', diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index f3329c286..41b60d30a 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -1,17 +1,21 @@ # -*- coding: utf-8 -*- # -from collections import defaultdict +from operator import add, sub + +from assets.utils import is_asset_exists_in_node from django.db.models.signals import ( - post_save, m2m_changed, post_delete + post_save, m2m_changed, pre_delete, post_delete ) -from django.db.models.aggregates import Count +from django.db.models import Q, F from django.dispatch import receiver +from common.local import thread_local +from common.exceptions import M2MReverseNotAllowed +from common.const.signals import PRE_ADD, POST_ADD, POST_REMOVE, PRE_CLEAR from common.utils import get_logger from common.decorator import on_transaction_commit -from orgs.utils import tmp_to_root_org -from .models import Asset, SystemUser, Node, AuthBook -from .utils import TreeService +from .models import Asset, SystemUser, Node, compute_parent_key +from users.models import User from .tasks import ( update_assets_hardware_info_util, test_asset_connectivity_util, @@ -54,15 +58,6 @@ def on_asset_created_or_update(sender, instance=None, created=False, **kwargs): instance.nodes.add(Node.org_root()) -@receiver(post_delete, sender=Asset) -def on_asset_delete(sender, instance=None, **kwargs): - """ - 当资产删除时，刷新节点，节点中存在节点和资产的关系 - """ - logger.debug("Asset delete signal recv: {}".format(instance)) - Node.refresh_assets() - - @receiver(post_save, sender=SystemUser, dispatch_uid="jms") def on_system_user_update(sender, instance=None, created=True, **kwargs): """ @@ -82,7 +77,7 @@ def on_system_user_assets_change(sender, instance=None, action='', model=None, p """ 当系统用户和资产关系发生变化时，应该重新推送系统用户到新添加的资产中 """ - if action != "post_add": + if action != POST_ADD: return logger.debug("System user assets change signal recv: {}".format(instance)) queryset = model.objects.filter(pk__in=pk_set) @@ -101,7 +96,7 @@ def on_system_user_users_change(sender, instance=None, action='', model=None, pk """ 当系统用户和用户关系发生变化时，应该重新推送系统用户资产中 """ - if action != "post_add": + if action != POST_ADD: return if not instance.username_same_with_user: return @@ -120,7 +115,7 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None, """ 当系统用户和节点关系发生变化时，应该将节点下资产关联到新的系统用户上 """ - if action != "post_add": + if action != POST_ADD: return logger.info("System user nodes update signal recv: {}".format(instance)) @@ -135,104 +130,217 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None, @receiver(m2m_changed, sender=SystemUser.groups.through) -def on_system_user_groups_change(sender, instance=None, action=None, model=None, - pk_set=None, reverse=False, **kwargs): +def on_system_user_groups_change(instance, action, pk_set, reverse, **kwargs): """ 当系统用户和用户组关系发生变化时，应该将组下用户关联到新的系统用户上 """ - if action != "post_add" or reverse: + if action != POST_ADD: return + if reverse: + raise M2MReverseNotAllowed logger.info("System user groups update signal recv: {}".format(instance)) - groups = model.objects.filter(pk__in=pk_set).annotate(users_count=Count("users")) - users = groups.filter(users_count__gt=0).values_list('users', flat=True) - instance.users.add(*tuple(users)) + + users = User.objects.filter(groups__id__in=pk_set).distinct() + instance.users.add(users) @receiver(m2m_changed, sender=Asset.nodes.through) -def on_asset_nodes_change(sender, instance=None, action='', **kwargs): +def on_asset_nodes_add(instance, action, reverse, pk_set, **kwargs): """ - 资产节点发生变化时，刷新节点 - """ - if action.startswith('post'): - logger.debug("Asset nodes change signal recv: {}".format(instance)) - Node.refresh_assets() - with tmp_to_root_org(): - Node.refresh_assets() + 本操作共访问 4 次数据库 - -@receiver(m2m_changed, sender=Asset.nodes.through) -def on_asset_nodes_add(sender, instance=None, action='', model=None, pk_set=None, **kwargs): - """ 当资产的节点发生变化时，或者 当节点的资产关系发生变化时， 节点下新增的资产，添加到节点关联的系统用户中 """ - if action != "post_add": + if action != POST_ADD: return logger.debug("Assets node add signal recv: {}".format(action)) - if model == Node: - nodes = model.objects.filter(pk__in=pk_set).values_list('key', flat=True) - assets = [instance.id] - else: + if reverse: nodes = [instance.key] - assets = model.objects.filter(pk__in=pk_set).values_list('id', flat=True) + asset_ids = pk_set + else: + nodes = Node.objects.filter(pk__in=pk_set).values_list('key', flat=True) + asset_ids = [instance.id] + # 节点资产发生变化时，将资产关联到节点及祖先节点关联的系统用户, 只关注新增的 nodes_ancestors_keys = set() - node_tree = TreeService.new() for node in nodes: - ancestors_keys = node_tree.ancestors_ids(nid=node) - nodes_ancestors_keys.update(ancestors_keys) - system_users = SystemUser.objects.filter(nodes__key__in=nodes_ancestors_keys) + nodes_ancestors_keys.update(Node.get_node_ancestor_keys(node, with_self=True)) - system_users_assets = defaultdict(set) - for system_user in system_users: - assets_has_set = system_user.assets.all().filter(id__in=assets).values_list('id', flat=True) - assets_remain = set(assets) - set(assets_has_set) - system_users_assets[system_user].update(assets_remain) - for system_user, _assets in system_users_assets.items(): - system_user.assets.add(*tuple(_assets)) + # 查询所有祖先节点关联的系统用户，都是要跟资产建立关系的 + system_user_ids = SystemUser.objects.filter( + nodes__key__in=nodes_ancestors_keys + ).distinct().values_list('id', flat=True) + + # 查询所有已存在的关系 + m2m_model = SystemUser.assets.through + exist = set(m2m_model.objects.filter( + systemuser_id__in=system_user_ids, asset_id__in=asset_ids + ).values_list('systemuser_id', 'asset_id')) + # TODO 优化 + to_create = [] + for system_user_id in system_user_ids: + asset_ids_to_push = [] + for asset_id in asset_ids: + if (system_user_id, asset_id) in exist: + continue + asset_ids_to_push.append(asset_id) + to_create.append(m2m_model( + systemuser_id=system_user_id, + asset_id=asset_id + )) + push_system_user_to_assets.delay(system_user_id, asset_ids_to_push) + m2m_model.objects.bulk_create(to_create) + + +def _update_node_assets_amount(node: Node, asset_pk_set: set, operator=add): + """ + 一个节点与多个资产关系变化时，更新计数 + + :param node: 节点实例 + :param asset_pk_set: 资产的`id`集合, 内部不会修改该值 + :param operator: 操作 + * -> Node + # -> Asset + + * [3] + / \ + * * [2] + / \ + * * [1] + / / \ + * [a] # # [b] + + """ + # 获取节点[1]祖先节点的 `key` 含自己，也就是[1, 2, 3]节点的`key` + ancestor_keys = node.get_ancestor_keys(with_self=True) + ancestors = Node.objects.filter(key__in=ancestor_keys).order_by('-key') + to_update = [] + for ancestor in ancestors: + # 迭代祖先节点的`key`，顺序是 [1] -> [2] -> [3] + # 查询该节点及其后代节点是否包含要操作的资产，将包含的从要操作的 + # 资产集合中去掉，他们是重复节点，无论增加或删除都不会影响节点的资产数量 + + asset_pk_set -= set(Asset.objects.filter( + id__in=asset_pk_set + ).filter( + Q(nodes__key__istartswith=f'{ancestor.key}:') | + Q(nodes__key=ancestor.key) + ).distinct().values_list('id', flat=True)) + if not asset_pk_set: + # 要操作的资产集合为空，说明都是重复资产，不用改变节点资产数量 + # 而且既然它包含了，它的祖先节点肯定也包含了，所以祖先节点都不用 + # 处理了 + break + ancestor.assets_amount = operator(F('assets_amount'), len(asset_pk_set)) + to_update.append(ancestor) + Node.objects.bulk_update(to_update, fields=('assets_amount', 'parent_key')) + + +def _remove_ancestor_keys(ancestor_key, tree_set): + # 这里判断 `ancestor_key` 不能是空，防止数据错误导致的死循环 + # 判断是否在集合里，来区分是否已被处理过 + while ancestor_key and ancestor_key in tree_set: + tree_set.remove(ancestor_key) + ancestor_key = compute_parent_key(ancestor_key) + + +def _update_nodes_asset_amount(node_keys, asset_pk, operator): + """ + 一个资产与多个节点关系变化时，更新计数 + + :param node_keys: 节点 id 的集合 + :param asset_pk: 资产 id + :param operator: 操作 + """ + + # 所有相关节点的祖先节点，组成一棵局部树 + ancestor_keys = set() + for key in node_keys: + ancestor_keys.update(Node.get_node_ancestor_keys(key)) + + # 相关节点可能是其他相关节点的祖先节点，如果是从相关节点里干掉 + node_keys -= ancestor_keys + + to_update_keys = [] + for key in node_keys: + # 遍历相关节点，处理它及其祖先节点 + # 查询该节点是否包含待处理资产 + exists = is_asset_exists_in_node(asset_pk, key) + parent_key = compute_parent_key(key) + + if exists: + # 如果资产在该节点，那么他及其祖先节点都不用处理 + _remove_ancestor_keys(parent_key, ancestor_keys) + continue + else: + # 不存在，要更新本节点 + to_update_keys.append(key) + # 这里判断 `parent_key` 不能是空，防止数据错误导致的死循环 + # 判断是否在集合里，来区分是否已被处理过 + while parent_key and parent_key in ancestor_keys: + exists = is_asset_exists_in_node(asset_pk, parent_key) + if exists: + _remove_ancestor_keys(parent_key, ancestor_keys) + break + else: + to_update_keys.append(parent_key) + ancestor_keys.remove(parent_key) + parent_key = compute_parent_key(parent_key) + + Node.objects.filter(key__in=to_update_keys).update( + assets_amount=operator(F('assets_amount'), 1) + ) @receiver(m2m_changed, sender=Asset.nodes.through) -def on_asset_nodes_remove(sender, instance=None, action='', model=None, - pk_set=None, **kwargs): +def update_nodes_assets_amount(action, instance, reverse, pk_set, **kwargs): + # 不允许 `pre_clear` ，因为该信号没有 `pk_set` + # [官网](https://docs.djangoproject.com/en/3.1/ref/signals/#m2m-changed) + refused = (PRE_CLEAR,) + if action in refused: + raise ValueError - """ - 监控资产删除节点关系, 或节点删除资产，避免产生游离资产 - """ - if action not in ["post_remove", "pre_clear", "post_clear"]: + mapper = { + PRE_ADD: add, + POST_REMOVE: sub + } + if action not in mapper: return - if action == "pre_clear": - if model == Node: - instance._nodes = list(instance.nodes.all()) - else: - instance._assets = list(instance.assets.all()) - return - logger.debug("Assets node remove signal recv: {}".format(action)) - if action == "post_remove": - queryset = model.objects.filter(pk__in=pk_set) + + operator = mapper[action] + + if reverse: + node: Node = instance + asset_pk_set = set(pk_set) + _update_node_assets_amount(node, asset_pk_set, operator) else: - if model == Node: - queryset = instance._nodes - else: - queryset = instance._assets - if model == Node: - assets = [instance] - else: - assets = queryset - if isinstance(assets, list): - assets_not_has_node = [] - for asset in assets: - if asset.nodes.all().count() == 0: - assets_not_has_node.append(asset.id) - else: - assets_not_has_node = assets.annotate(nodes_count=Count('nodes'))\ - .filter(nodes_count=0).values_list('id', flat=True) - Node.org_root().assets.add(*tuple(assets_not_has_node)) + asset_pk = instance.id + # 与资产直接关联的节点 + node_keys = set(Node.objects.filter(id__in=pk_set).values_list('key', flat=True)) + _update_nodes_asset_amount(node_keys, asset_pk, operator) -@receiver([post_save, post_delete], sender=Node) -def on_node_update_or_created(sender, **kwargs): - # 刷新节点 - Node.refresh_nodes() - with tmp_to_root_org(): - Node.refresh_nodes() +ASSET_DELETE_SIGNAL_FOR_NODE_TREE_PARAMS = 'asset_delete_signal_for_node_tree_params' + + +@receiver(pre_delete, sender=Asset) +def on_asset_delete(instance: Asset, **kwargs): + node_keys = Node.objects.filter( + assets=instance + ).distinct().values_list('key', flat=True) + + params = { + 'node_keys': set(node_keys), + 'asset_pk': instance.id, + 'operator': sub + } + + setattr(thread_local, ASSET_DELETE_SIGNAL_FOR_NODE_TREE_PARAMS, params) + + +@receiver(post_delete, sender=Asset) +def on_asset_post_delete(instance: Asset, **kwargs): + params = getattr(thread_local, ASSET_DELETE_SIGNAL_FOR_NODE_TREE_PARAMS, None) + if params and params.get('asset_pk') == instance.id: + _update_nodes_asset_amount(**params) diff --git a/apps/assets/tasks/nodes_amount.py b/apps/assets/tasks/nodes_amount.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/assets/tasks/push_system_user.py b/apps/assets/tasks/push_system_user.py index d947be77d..7fc7bdab2 100644 --- a/apps/assets/tasks/push_system_user.py +++ b/apps/assets/tasks/push_system_user.py @@ -2,10 +2,12 @@ from itertools import groupby from celery import shared_task +from common.db.utils import get_object_if_need, get_objects_if_need from django.utils.translation import ugettext as _ from django.db.models import Empty from common.utils import encrypt_password, get_logger +from assets.models import SystemUser, Asset from orgs.utils import org_aware_func from . import const from .utils import clean_ansible_task_hosts, group_asset_by_platform @@ -221,6 +223,7 @@ def push_system_user_util(system_user, assets, task_name, username=None): @shared_task(queue="ansible") def push_system_user_to_assets_manual(system_user, username=None): + system_user = get_object_if_need(SystemUser, system_user) assets = system_user.get_related_assets() task_name = _("Push system users to assets: {}").format(system_user.name) return push_system_user_util(system_user, assets, task_name=task_name, username=username) @@ -239,10 +242,10 @@ def push_system_user_a_asset_manual(system_user, asset, username=None): @shared_task(queue="ansible") def push_system_user_to_assets(system_user, assets, username=None): task_name = _("Push system users to assets: {}").format(system_user.name) + system_user = get_object_if_need(SystemUser, system_user) + assets = get_objects_if_need(Asset, assets) return push_system_user_util(system_user, assets, task_name, username=username) - - # @shared_task # @register_as_period_task(interval=3600) # @after_app_ready_start diff --git a/apps/assets/urls/api_urls.py b/apps/assets/urls/api_urls.py index d70accc23..3df73d219 100644 --- a/apps/assets/urls/api_urls.py +++ b/apps/assets/urls/api_urls.py @@ -2,6 +2,7 @@ from django.urls import path, re_path from rest_framework_nested import routers from rest_framework_bulk.routes import BulkRouter +from django.db.transaction import non_atomic_requests from common import api as capi @@ -54,9 +55,9 @@ urlpatterns = [ path('nodes/children/', api.NodeChildrenApi.as_view(), name='node-children-2'), path('nodes//children/add/', api.NodeAddChildrenApi.as_view(), name='node-add-children'), path('nodes//assets/', api.NodeAssetsApi.as_view(), name='node-assets'), - path('nodes//assets/add/', api.NodeAddAssetsApi.as_view(), name='node-add-assets'), - path('nodes//assets/replace/', api.NodeReplaceAssetsApi.as_view(), name='node-replace-assets'), - path('nodes//assets/remove/', api.NodeRemoveAssetsApi.as_view(), name='node-remove-assets'), + path('nodes//assets/add/', non_atomic_requests(api.NodeAddAssetsApi.as_view()), name='node-add-assets'), + path('nodes//assets/replace/', non_atomic_requests(api.MoveAssetsToNodeApi.as_view()), name='node-replace-assets'), + path('nodes//assets/remove/', non_atomic_requests(api.NodeRemoveAssetsApi.as_view()), name='node-remove-assets'), path('nodes//tasks/', api.NodeTaskCreateApi.as_view(), name='node-task-create'), path('gateways//test-connective/', api.GatewayTestConnectionApi.as_view(), name='test-gateway-connective'), diff --git a/apps/assets/utils.py b/apps/assets/utils.py index 6b4d8111a..ac514ba49 100644 --- a/apps/assets/utils.py +++ b/apps/assets/utils.py @@ -5,6 +5,8 @@ from treelib.exceptions import NodeIDAbsentError from collections import defaultdict from copy import deepcopy +from django.db.models import Q + from common.utils import get_logger, timeit, lazyproperty from .models import Asset, Node @@ -12,184 +14,22 @@ from .models import Asset, Node logger = get_logger(__file__) -class TreeService(Tree): - tag_sep = ' / ' +def check_node_assets_amount(): + for node in Node.objects.all(): + assets_amount = Asset.objects.filter( + Q(nodes__key__istartswith=f'{node.key}:') | Q(nodes=node) + ).distinct().count() - @staticmethod - @timeit - def get_nodes_assets_map(): - nodes_assets_map = defaultdict(set) - asset_node_list = Node.assets.through.objects.values_list( - 'asset', 'node__key' - ) - for asset_id, key in asset_node_list: - nodes_assets_map[key].add(asset_id) - return nodes_assets_map + if node.assets_amount != assets_amount: + print(f' wrong assets amount ' + f'{node.assets_amount} right is {assets_amount}') + node.assets_amount = assets_amount + node.save() - @classmethod - @timeit - def new(cls): - from .models import Node - all_nodes = list(Node.objects.all().values("key", "value")) - all_nodes.sort(key=lambda x: len(x["key"].split(":"))) - tree = cls() - tree.create_node(tag='', identifier='', data={}) - for node in all_nodes: - key = node["key"] - value = node["value"] - parent_key = ":".join(key.split(":")[:-1]) - tree.safe_create_node( - tag=value, identifier=key, - parent=parent_key, - ) - tree.init_assets() - return tree - def init_assets(self): - node_assets_map = self.get_nodes_assets_map() - for node in self.all_nodes_itr(): - key = node.identifier - assets = node_assets_map.get(key, set()) - data = {"assets": assets, "all_assets": None} - node.data = data - - def safe_create_node(self, **kwargs): - parent = kwargs.get("parent") - if not self.contains(parent): - kwargs['parent'] = self.root - self.create_node(**kwargs) - - def all_children_ids(self, nid, with_self=True): - children_ids = self.expand_tree(nid) - if not with_self: - next(children_ids) - return list(children_ids) - - def all_children(self, nid, with_self=True, deep=False): - children_ids = self.all_children_ids(nid, with_self=with_self) - return [self.get_node(i, deep=deep) for i in children_ids] - - def ancestors_ids(self, nid, with_self=True): - ancestor_ids = list(self.rsearch(nid)) - ancestor_ids.pop() - if not with_self: - ancestor_ids.pop(0) - return ancestor_ids - - def ancestors(self, nid, with_self=False, deep=False): - ancestor_ids = self.ancestors_ids(nid, with_self=with_self) - ancestors = [self.get_node(i, deep=deep) for i in ancestor_ids] - return ancestors - - def get_node_full_tag(self, nid): - ancestors = self.ancestors(nid, with_self=True) - ancestors.reverse() - return self.tag_sep.join([n.tag for n in ancestors]) - - def get_family(self, nid, deep=False): - ancestors = self.ancestors(nid, with_self=False, deep=deep) - children = self.all_children(nid, with_self=False) - return ancestors + [self[nid]] + children - - @staticmethod - def is_parent(child, parent): - parent_id = child.bpointer - return parent_id == parent.identifier - - def root_node(self): - return self.get_node(self.root) - - def get_node(self, nid, deep=False): - node = super().get_node(nid) - if deep: - node = self.copy_node(node) - node.data = {} - return node - - def parent(self, nid, deep=False): - parent = super().parent(nid) - if deep: - parent = self.copy_node(parent) - return parent - - @lazyproperty - def invalid_assets(self): - assets = Asset.objects.filter(is_active=False).values_list('id', flat=True) - return assets - - def set_assets(self, nid, assets): - node = self.get_node(nid) - if node.data is None: - node.data = {} - node.data["assets"] = assets - - def assets(self, nid): - node = self.get_node(nid) - return node.data.get("assets", set()) - - def valid_assets(self, nid): - return set(self.assets(nid)) - set(self.invalid_assets) - - def all_assets(self, nid): - node = self.get_node(nid) - if node.data is None: - node.data = {} - all_assets = node.data.get("all_assets") - if all_assets is not None: - return all_assets - all_assets = set(self.assets(nid)) - try: - children = self.children(nid) - except NodeIDAbsentError: - children = [] - for child in children: - all_assets.update(self.all_assets(child.identifier)) - node.data["all_assets"] = all_assets - return all_assets - - def all_valid_assets(self, nid): - return set(self.all_assets(nid)) - set(self.invalid_assets) - - def assets_amount(self, nid): - return len(self.all_assets(nid)) - - def valid_assets_amount(self, nid): - return len(self.all_valid_assets(nid)) - - @staticmethod - def copy_node(node): - new_node = deepcopy(node) - new_node.fpointer = None - return new_node - - def safe_add_ancestors(self, node, ancestors): - # 如果没有祖先节点，那么添加该节点, 父节点是root node - if len(ancestors) == 0: - parent = self.root_node() - else: - parent = ancestors[0] - - # 如果当前节点已再树中，则移动当前节点到父节点中 - # 这个是由于 当前节点放到了二级节点中 - if not self.contains(parent.identifier): - # logger.debug('Add parent: {}'.format(parent.identifier)) - self.safe_add_ancestors(parent, ancestors[1:]) - - if self.contains(node.identifier): - # msg = 'Move node to parent: {} => {}'.format( - # node.identifier, parent.identifier - # ) - # logger.debug(msg) - self.move_node(node.identifier, parent.identifier) - else: - # logger.debug('Add node: {}'.format(node.identifier)) - self.add_node(node, parent) - # - # def __getstate__(self): - # self.mutex = None - # self.all_nodes_assets_map = {} - # self.nodes_assets_map = {} - # return self.__dict__ - - # def __setstate__(self, state): - # self.__dict__ = state +def is_asset_exists_in_node(asset_pk, node_key): + return Asset.objects.filter( + id=asset_pk + ).filter( + Q(nodes__key__istartswith=f'{node_key}:') | Q(nodes__key=node_key) + ).exists() diff --git a/apps/common/const/distributed_lock_key.py b/apps/common/const/distributed_lock_key.py new file mode 100644 index 000000000..735781841 --- /dev/null +++ b/apps/common/const/distributed_lock_key.py @@ -0,0 +1,2 @@ +UPDATE_NODE_TREE_LOCK_KEY = 'org_level_transaction_lock_{org_id}_assets_update_node_tree' +UPDATE_MAPPING_NODE_TASK_LOCK_KEY = 'org_level_transaction_lock_{user_id}_update_mapping_node_task' diff --git a/apps/common/const/signals.py b/apps/common/const/signals.py new file mode 100644 index 000000000..b28c1310b --- /dev/null +++ b/apps/common/const/signals.py @@ -0,0 +1,14 @@ +""" +`m2m_changed` + +``` +def m2m_signals_handler(action, instance, reverse, model, pk_set, using): + pass +``` +""" +PRE_ADD = 'pre_add' +POST_ADD = 'post_add' +PRE_REMOVE = 'pre_remove' +POST_REMOVE = 'post_remove' +PRE_CLEAR = 'pre_clear' +POST_CLEAR = 'post_clear' diff --git a/apps/common/db/utils.py b/apps/common/db/utils.py new file mode 100644 index 000000000..b71b27906 --- /dev/null +++ b/apps/common/db/utils.py @@ -0,0 +1,27 @@ +from common.utils import get_logger + +logger = get_logger(__file__) + + +def get_object_if_need(model, pk): + if not isinstance(pk, model): + try: + return model.objects.get(id=pk) + except model.DoesNotExist as e: + logger.error(f'DoesNotExist: <{model.__name__}:{pk}> not exist') + raise e + return pk + + +def get_objects_if_need(model, pks): + if not pks: + return pks + if not isinstance(pks[0], model): + objs = list(model.objects.filter(id__in=pks)) + if len(objs) != len(pks): + pks = set(pks) + exists_pks = {o.id for o in objs} + not_found_pks = ','.join(pks - exists_pks) + logger.error(f'DoesNotExist: <{model.__name__}: {not_found_pks}>') + return objs + return pks diff --git a/apps/common/exceptions.py b/apps/common/exceptions.py index ded24374a..396430df3 100644 --- a/apps/common/exceptions.py +++ b/apps/common/exceptions.py @@ -18,3 +18,18 @@ class JMSObjectDoesNotExist(APIException): if detail is None and object_name: detail = self.default_detail % object_name super(JMSObjectDoesNotExist, self).__init__(detail=detail, code=code) + + +class SomeoneIsDoingThis(JMSException): + status_code = status.HTTP_409_CONFLICT + default_detail = _('Someone else is doing this. Please wait for complete') + + +class Timeout(JMSException): + status_code = status.HTTP_408_REQUEST_TIMEOUT + default_detail = _('Your request timeout') + + +class M2MReverseNotAllowed(JMSException): + status_code = status.HTTP_400_BAD_REQUEST + default_detail = _('M2M reverse not allowed') diff --git a/apps/common/mixins/api.py b/apps/common/mixins/api.py index 3e9aea665..6754bf178 100644 --- a/apps/common/mixins/api.py +++ b/apps/common/mixins/api.py @@ -11,8 +11,6 @@ from django.core.cache import cache from django.http import JsonResponse from rest_framework.response import Response from rest_framework.settings import api_settings -from rest_framework import status -from rest_framework_bulk.drf3.mixins import BulkDestroyModelMixin from common.drf.filters import IDSpmFilter, CustomFilter, IDInFilter from ..utils import lazyproperty @@ -237,6 +235,7 @@ class RelationMixin: for i in instances: to_id = getattr(i, self.to_field).id + # TODO 优化，不应该每次都查询数据库 from_obj = getattr(i, self.from_field) from_to_mapper[from_obj].append(to_id) diff --git a/apps/common/thread_pools.py b/apps/common/thread_pools.py new file mode 100644 index 000000000..4fcf499a7 --- /dev/null +++ b/apps/common/thread_pools.py @@ -0,0 +1,17 @@ +from concurrent.futures import ThreadPoolExecutor + + +class SingletonThreadPoolExecutor(ThreadPoolExecutor): + """ + 该类不要直接实例化 + """ + + def __new__(cls, max_workers=None, thread_name_prefix=None): + if cls is SingletonThreadPoolExecutor: + raise NotImplementedError + if getattr(cls, '_object', None) is None: + cls._object = ThreadPoolExecutor( + max_workers=max_workers, + thread_name_prefix=thread_name_prefix + ) + return cls._object diff --git a/apps/locale/zh/LC_MESSAGES/django.mo b/apps/locale/zh/LC_MESSAGES/django.mo index 0a5f32daffcece8b72731cf2c4986bf1073a6259..9435b2b0e1afb9dbf7e0aa6132b2ae82ea64c5e5 100644 GIT binary patch delta 16850 zcmZA82Xs&O|HttwF=E6@A_+mP*cw#q6hI2zs5^=0I%bM z=8lsEM}FWq8E^__!a3$ra~*2IyD$KcVIW?|bodN)-)jtDe#h@aZ^vP%4h1YOjyi!d zsEKP}Fg8UE&=YxT&S3PzVW|7ZVpg1tf%p|_!E4R!sBsUW8$sn9m6RjFIO0Mr9H%_a zMs+-an&>JP!5df@{aQLs8jM8si$a}XZOn%8sD&k%(@-Zf2epAEEjfR!cr6J%!z~zq zmrw&fLOsjBP!s)Q`82J(fda4~`Mjtd*2a?91k>Xv)I<|d{br#~W-IC)JJyQxSHo!% zTImZ6!hcW$1hw{Fy1dBC?nI$(tcY4z4b%j6umCp4a`-Wp#Z_1dZ(>f&#Yai^MWM#8 z>{8LQt%H%+!WuqBO*9>~vpJ}Ki%}=G9yP!=)PM(2Cw3Hd-$nB}>b^Ue8J}ZrOx@Po zxSOAf2B?P`@O=!%mZ+oZhkBN-Im29HZbUt@!8OP)Ks}0YQ43mY`8}AQ_z3Ex9-@x=E$U=4wfFjmn~|uE z)WCH5{5PPYqid!FcEdpIgBoZIYRA)2J6&Y)b_^jtj@tPh%#E*6+g>`p+MA07DU}w z#;k(j#C1Dz{yK^tBs5@u)RB$E95@~I%)UeIU=^zWR*UzcCOC<@{~BuI`>2!q6SXkE zPTso`fQn0@7F4m5>kV9sga&Sk+EE+SQN^MLj7KeO9O^{opz>d%CR&X}a3>bWyI29U zclI{$9_}D+iW=9yi#KnOYnA+{1-yfrr~+z*@1s`S67ykaY>4CVDgKB$q1|1*g&jbh z#1E+PE+QWU=O*f;s&w;y%s0enVz-ASR$vVhhfz2B#dx2JY*>^y*y5U~6R3yUNn=$1 zmZ+DwBkEllh!Hpq^WsJ50Mvkqs3Tm6 zTHtEbJFo}!MRW?&;X~9Te1`hirt9fF!VuK`1yKD;V<_`G6{+ZL{{X|VH)@3wQ13t@ z>c$1A6IzL>ahv6LqwYIl`3I=`o>=@RYP`QO95eLtHe4KC?d)ADX|N`0NA*zoR_KG> z%pT}R+#9u^c+^hEppJMp=E3>60=J_cP50j3j>nr*Q1i{|&H2AWWgZEw@EB@>3#gsm z!1VYS^>V$k_KdM!zd$oL7NtD`b>vMk19nC|nm(v;CRlz3W+a{;%lT`f<<_tnwSawC z9DlINsPRUlKTbl;GY575YScpAV^kujTtMCM z7Ijp9{k)UNj*4@k+6$p3j>Jq@%koW8??OA&kJmWVPA6byoQC=|EkK>zZseq0XFnA^ zyW^-8y+94*6Xz`~2WrBqSO}Y<7VMb8@F2IJwTTmZg-+|t{ zlNL2`X4FDL(alaJf{Ko;B5H^AF&nl=?JN#;gNxxf1+}mwEQ{;065d5EFkiekVG-1k zN1{%kiskEIcH&0yoWB}6S%+Szi3Xx}(wWh{<*o9kP9bBo)djywj-V1LvJEy3ov z5sRSTQ19n{Y1AWahC0~}sB!xukI;37P|-?9qmJlPufdsO`9#zNUs*m0wWAHF0e7Ny zbR0GCB~<@g7>IwM-j(#jyhoQ6byDG&^7FrJN`<3E4OkOHuqoQT6A*XMsK6|HckxgIsqR@71NM=jt-)U!Q@ zdbuv6?t6eb;@?sC{cZY<@FvQRnm8Qw@)kucxFWhr)Tg3>TALkFN7x3TgqF zMtSXFsGSr?-S;kP;wlz5u=ZxC6KjiFP$$$o5`*vJ0Mv$;jpF>(kVHbyd@Jf`&tOsf z1*6b^w72uxs2iK025xC_JF^RFVZBh}4X}7J79pOAx^FvbV|!fda0+!Kmr!r#UDQeZ ziF&4iEL0QcF$I)cw6tCzODic&5d3P$%gwrJ@OvEV0=->_I*2 z6R4xSgqq+w>c&S{7@uP}=KjR98fxbqF${-dZk&xe*<>t-2aw;4T<0$;I*Qz5ydx@! z`G{MhCK!tCaUu4?zcCtnj`d!`l{l360_Mdg;~eKBjKc^#j#|JojKpl?`L{ug#^TKH zY^I{O@)ulVJikNMJ_w3(89sLKW6YFP=#316y zm<|_W%FqAhRP@s-3AOUQs3Xra(aRS^9ccyBLYtw!P!@uRXIHPLbNHtJ*Y26eP$rh4y0GptVB3U%Kc z%P&No*fP{Yl2FfjCu)HQQ42bav3L~=qFZm8_l463b(AhjyMjr(|pstXIuv>5w}IXE3;9LXsy@oI)|ue;;R^z%5iR@-u|~UycPS- z^mgix8XyN|#nPyqMx#DG%`r1}N9}kpYR6+x<9&vDq%%IV{Km?qVZ4 zWSph1C;lx6yWwRF_u({Vd;h_r;~X9X{Ug8RO{RU~T;6>AWF9}^@GsnnqrTz?8vQEI zCx@>W@QEP*;u}t06Rce5oxm>C%J-w5?Mc*%FQJa?f$1#rzUgwH+M`hY>tiu&hGlRh z>ZM(S5qJW1f`4E*W?9Sz)gh8f9c+w8a26(G`z78NO}3@pH(3!g3iSwTTigQm&U7{V zo1;+g%w&sam|t6di)&22>nu<}nLddkM25 zYW%vW6KaY@aKyKqe_JX`N$9Qq$2wMA<_%m63zKhX4z&C{)Q%USKAx*lFX0Kx|7<=s zUz>j4dE*A5`iHqzi9~-AwXH*QvkmGk?}XYxf6Rx&Q45%l+R678pD}Ns7WT;EjLW_L zA*g&I)Oc=fDw?3Nb?AWEiDS_hM_Ya@rX~K|;xAAy(-JI>yD&dKKz%RxfA75`RZ#ae zLM`}1voo>~*Xd^+rkb;?!&l~V%WpJynES2$n8g=S3%+joTjmqgH|rbJ&I4C?3o475 z^Z~C#MIWzvsGYY*eL?g$U2~eb0QIO=ncFOX1k;nhX#Q+IG@qOQpiU(HO8&)FpZ}~> zlxT(;pf&1|bg}$E)X@z?E$~xwuDJ?z-)=03M^WQFHlL$DMQ_ZYRbG23O!@g=j*14T zj+w9lYJip&x5o^`y)8c&BZx;@`x4ZG*P<4<$MR>*o91t*asRS7Rg!)FGbedB24N;T z6fw)8RvwL-pq<6Bs7E#)^-SlX2HuL=$R3N2S$x{!tElmBnNRdvZ($N|NYulEtG$8x zpmq|E8t_xpjnh#JSz!4jbF=05qWYaiE$F(n-$5<(iN%@Mc;ke)RJ6k)s2$Zr-Pj5> zPzN*C9BL+*Gf@lu#$0dtgQ#)OSp3j@Wu{r{&EsaLqKQH+5rO{1B{2(Dw)XeT)@BUq zi)f%Z3riAjLT%tS@&nO%XmP?iZ=NaUT%@1tETf`{lB{8;c?|W7#6{Fs^K0ZMs#9*g zH^C|MB5DKI&4>6t@hgk#@RKY%adV8s7>mD1iFy9tQ_+sEpjLhlwX-LvFO+QjuvR|E z3`0#=0QGJ}qCR$&umpBQO*94dcgLL=fq$TWZO^sQyRQsp(&xX5D%ikmiJGvh*&8*{ z0Ele(w@W6hzczrQC~ zya+>ySED94f_m9bq84@!b^mMAZ?o4g2=)6$80u$7rOiD5C@L*T=!Pk%FPga)FGal@ z>rfNzwfqs(gy&H^er)Zp%zsehrQPEFrWAncR~9v1Rn&Nmws8JgseEW1V$g>;&K!to zhzDb49D&;5WYho)P$#hz`K8I(g+bVPtJgln9Bn3GP1>iR7WkuU4OcC3%i<>%zd)_r zcbhk$zZrWs4n_p8w|uQs09u|z1*WQ<%G?-rn`)a21rJ|^@ptC9%_K+*8T?d zX~?qOJFx)FL0k|6urh{X6AZ^*);<|E@gmebt5N;-dU@BmY7O_ig!3BpRq40G+j(Ks zk(NdcP|xyh&2FfL_CrlH*c^-M|2gWJFSh)8EJD0HCFc3xqoN~zjyl4BQW~i5^acz< z4V2&Fa%ME@-DzTWvity4|Iw&NGy}DO9Tj@;Sn z#o16VQ$cKq)ldtYWPV{TG1sET*@@cNslBe3xM_)J*3oyLH&9m0N5|Z#9aYDo*Z|+f zc+`TIn`=-D*o5l8-|{Cg7x6iZpQ8G`bS;s3zqg`{W-ipp6i2PNA!?%bW;fIXu@(0RRar=y}H3qjpj9M!QL>iZzt^6#M@NegQqgsq9kUR&-i`o&=WOq9O}^|puV|gqZW7^b>C&w=l&7uod`ba zJ-SHLM9oqC+n~nlZuyZ%Ie&E+M1r7j6g+{kQ;ny`m;9FE%A7;_$~f3mgjvHW4|N&cqA4Sw+M z>w;R)2-JKDsQah5RJ5YGSP+w};f#61e1tl>KP}F3+#4tiHBbRegOQex!VJV!QNNbg z$4vM!>V(FjHsmH!(aKh#K5m<^I$pv|7;?g!sE}C#eaV-#xV%{n^;KNQ;*OZ|tT8qD zk*E_LgId5WWPI0IY7Og9D>~pcIM+}Mdw@E!KTtQmHZ%U{wdX=Dv=D}28H<~s?)%8% zu4aF8REj+R$(ER7E=AqA4z++Smfwd(h)<$E=da9MC%uJNLoJ{c=EOm$A6B2E?pug? z@hED+_myXU=LHoFnD>;|F&gU<$6`fXj}`C{>f>4HG?QRW%#CwVAIr_CBY%upF!dSl zSFu1WNn8%~h`XXrIsx68RKBJXkEPFg&*)oo3+i{gAIw`=pZHI#gVE=_6Pk#ccqVG% z<(Lt-V0Ju!+VMry$=*ht)WdT;|Eg5{&U+1YQ8zTgKVL=DUs>Mg0_UHVhV&P_8*`ckQ3IC3AS{R4*?VR?)DB`%k8UDrVJpoo z=0Vg^pRxEM<{*A$aTfQYH*pwhfC#g!Sp&6@M)(1KXz_A$4QgkbEIw-SDbz{)g8E+g z4fPH^GfQ0Z=5u4HXlDaa1C7N1oNe*?Pg2gw@r>G4$m%W$GAN8_TM&;X^ zy;0*0LFRLvF;qfG%)%;|gnFwVpl-s0pWIB+kJ?cm%bx$G8Nu{^WfuH=qVSg=O&< z%LiZc@?}x)QeD)zA7KtR6`w~c+R^aq-aFxvKbh`dM6=xHyePD7_L-m(Jy$v>vhasB5=vvECwz z_HmRf)~_bH!IX#eElG~&>1ewrDK)LnkJRsyA4btNop=rMAM7-d|F(>4CXv{^DA6R zdry*nI$*9YlmQHwouaF}nUg+ish6|ZYvtc~t#X$9Bx`$>*thIEb+X%t`lh}~%Q?zo z>v0Xgw>$aG!ug!?#+DXC+fwSS@Jiy(WrN-A){>db3(Bt)T?@@Kw6~*tL&-roNIs6z zjIx*bTMVPLq15HR0x5U10^(RoWBPO^H-h?9j3Pcvy#R9MbeJO;vzoETP{4V8v;wZ{E$|y>1 za^tZQ>2CFZ# zJ{5`UQ~#gEb%~2n9#B5v-v8lbimt58A5XuR#9ItGzO%DREK!3%|e$l=RexQ{pK*xF?j7 zHx)0?=Oi*x^m9$WTl1C3)k?pj!zvqC`D98kaRsVUQmTTsr^Cm3}NPW-t- zfP0Tj6{3NZ`}AB#$xlvKH|otOzY$;ZQqBqLy~%xtL&*8ze&P+(YvI3F9`XTp6z`DV zNQtCeq{9>IcR&wyF2NC!wnL6=p#bEa?Q5m}GYG{_AeuMk~>zT#&b=6|^ z-9!CvJZ-s?)NfMHXnoaZ4CCoHlpWZG^4j|H3ym|xtGmuED*Z`RqKo zy%stBkDL`KP04Mi&me_C51>f;j6SI*17Tk(U{vR~UBjksenr4#uql!lZfN<6s_ z=u@Avns^Itw|joaHRRGUUTgZTrfequnG!_XOzU?Gr_rWsOe&|bbDP9dg0U3+&%5<; zJh_G37)E^q^(IH7PPB<1@QQFZw37=Dbr{00K!dMljQ6`f=PW%ssC$_H|?CzvCjq>mH9=(qK z8{jEoU4PJLwAH-~zZnvTTELE-mh@Rlt}vw-aU7)!xgEHb^6!pl4os6WGJIEAv6`V2}1$|_=iN+RVn@qe#V77S-hU3+MMMty!t zKj!acH|(VG0QKzn8VBG6Tuqt44KIm%Q`hy9I5VXZrM;JOX5l5;H&OOd&tdt!#A(>z zB;voR_r!dZ#?+6v3~-F1>kx?$>bWWNiNCh|aOz>yQ!~hy_$TpUN;UfH>h0kSqiqyr zmF1OwLHr9v*M7?T_N03HHZ^FU`k(R6+5Jb!t)=c_I{bigkW!MOtB1LU_NBz1 zS)9_yAEy$V)d+AuqV+RM9a^80ornFb=SbpAv}GgiNcn@f5wQ;?kmFu&{YX38s9zvo z8_y_VS0K64l#Tx;up{LUa$FglP%77`-AVkdMqziimv-9P(C29xNPU;(3t}4Lk=WXD zkMI+6ks6Sz7NruoBx_Ge^2baYGlaUDe&LVW*pG6N`cFRm2kP|fs}vnGQVvlD+8}<^ zBk0(M@&n~D@qe!a1jiCj*UZ~sA9rl{Pit3N+LC)fuC|TVgj_n+$^8m5C_&78n>c~| zZ`e1vVsu#H=Lw`o_fdiwTeE6CV@a zD?YqiT(5pT!Uy&26<;d6X`h(RWQTO_H7LA$+`#azaee#ui5V1=ynR%@fPgk}g9nBW zj2SRECVo)(pk53!IQh!-;f0gCZCRWux%jpoX_M#g$>N(nKECI#-;KFH^V{SV`_K86 zypyo$_VzDsZ`*b^X~~`W%kL&my}fh(lO0p;t=ar*(xf}P61>(stEb$ZI6ZmP@iSQi f$1eYM-gw&Yt{Z!AQ^MUjn{IFWG`Z2W=c)b=xSWC! delta 16811 zcmZA82Y62B|HttYBSeshkyuIWncCFeYR29(wkD`it9b0KRBF_2QEC;jsjW3ydzGR; zwYMs*8vVaNxxc-xUjK7lx7Ypsp8MSEob%)f?WsLU59~?myB3mcw#ShK7D+sBIi4qeUDNYg`FY;yTAufexL|G1n}XTu zc-|;niB~XxUC;Z8cFF2_UWO!|&pT7!^Gd!?upB+(4tLm%eB^>_)-;Wb>> z#Pfo1T2s$Ug$pq?t~58B`%x1A2 zPy;u@Q0#;ncnEUWyz!V4C!?;Pg~2!qXwFbYPTGAo|hdn zpcd{cPDLHGM0MB>L$Nz*t45;kHJx@8wJ1Rr2Ve23~ULtAIKncpn#^m!Gi zXzOcXS!{?}!DQ4*XQEax57llJYGNBP3m(ACcojA9pQwS;v~v$*D5{@YsD(zO7T6Rs z>-~?VqNjK~YT&7;t(=9rw@XnStV7+3EvO0Yv-~N{Nqh;lQ?F55ou<9pnQW-`Ma?Rx zg*3txdjDHf(bmN%fderE4o7wL8EVB#Q7heO@o@|z{sp!27nmIbJGg!dpx%;VsD)It zd|k5%`m|MTtU*`QM`S1B~jN^ zGwWj{af^=ZzqVp933WIM^)O9Cy>?4c_iP(#1-nu0k6L^NHNbV$^^Z^kze4R?Kqoh` zFx0b>%i>C?3DxQ3a~(G&p^iJDR@4i1D?UMWI2JXrc+?74qVnre1MR{5_!Abwm-s&B zjByKSi@S+Cq593<*^QgWXO-fp2~3S=<=411(W2>40kA z9rY0RMLjEHFfT5~2t16%@D^%^Lc1p})aO;FqK;akwz4DY7WA?FDD)?ugc@ju<>y&` zDe9?TiTeCFjCxl7LA6ia!(E>dHBL6v4&_eFbN`D{QNuEJq6(HIu8-RC(WnlWp|)@X zYJz)E&%i0v!}Tksz}Ki-n6#&RZ9`GFFhA=05~y}nF*Da4(TW;la_oXySufQ2 zp_l|Gn3GW*OhZjy#9IPSsKcolVvCVl8u{KiKL(~9& zp;qh{>jnx&tu!11usrIatBGpg%8W5T!~*08pguttVJcjYx@9|J*?)C(&Q4szK;nC- zfnQiYS)3an2n&%9Lv>UGbt@X8255#!u^Vby>^pOJF?vJ+fX~a*F22s?>MGG-vuff=q9Rz z*Ql9>_HplTF4P5eQ9IQfwbdOh?rG=yqXr&JKo`9Qa|MNk9P zLM^BPCVoC(dg2(=!}#$)_Ft9hB=k_NKyBHN=2eU&evI0IppV^`%5c;UG%!1&cA_8Z zEgEeuLVac1fm-odRR34ZhdwI0cW=$4gWOC5Q1535YNe5=9Vu^dG-{w;m>b8V9?I3I z0S{snUPWCOIN05iaMVO{p?1Vqh>8X(iMkcltURy=Fn3*9)DG6h!q^e@IWkRoz5lDJXy*G-x8j&JxP;n?m*!j4K*>IF zTb&j)fy}7aEdup$FDEJ^D0oEEVm*K-5adSsah*XqmYhwS^n) z{6Wl2d=_=@A7MfK3$?;r!`%d`q9z!P+QFt+1LKCX|5>SQBoT@~p=Ndqb?={}CiEJ! zV5$*rYxAHc+!A%Z2WlaMP}liTE1zib=XQP}YR6WfCbVV*`>&^d0|~8kH!41en(0Z@ zmfpnz_zFv7u90rVZBW;BLv3(^Ojt{;WknFXkEeXA_74z-0lQ3D*b z_$NDm0d)Bt~=u1hk?{hd%CMiQ4Wo1s?zF=oS0F*~k7cGl+|rBaH-Wz;vC z;L&a;N}#r=I_AJQ%!i+1d)$h>FvA%4o6Jbm!?z!Y;6seSZe#huhT}0W-at(tV4VKO zh1ut$!pFPU3Jc*+sE6?tuE%ua-M83%sHZ)4g4@dRs0l4ab$kTN;1ev1`6jx1AA{QZ zUZ@=#XU@WmjPEU_k^;A4GTejuDRmGv^NXl0FFDEO%b~V38a2@#s1Fz)>fX*lJxj|_ z_dWsD?_Siye8}?W(U*e6|ETD~d#HQ=5|#HRyZ1dMs-rCEk9o}~)H6{MGh$`bN}FRi zeuTOuGf)%SiTeCFX7MkR+5boq4@v0hO*6%Zld!d`}F(ff$P; zQ7gS~=bxY^_S)ifQ(c@LHPI4N+5c2jqDd6O)~L_!@fe8PP_NTL)XdMIw)`gQA$o(l zKFc(BJ`V;GmqAUmI%>Y z@ZD!_rpacx6{bbqyHE_q;;0o?N4+&oF)j8$t#lA-rDIV2O+($vS(sMu|8gohNF>;a z^O&0n+`$IqQ-7{$^0#Ez6)$6C61IJ|`-h88bNRTXeX)7`=wRR#4f7iKbN>IU%S-aQ`F24q3-D^)Qo>d?bHL)?@RXy7m7Mx2Gza} z7Q`l497muY)(w~!PoQ@21^ObX1b^ik6vLXt4eXtJ%*SiF#J1T0F~KvYd_8iM7@s0d=o-T7yI8Y1Aj)HH-g5U6*Wy8!!+RhnW#( zUOOLUmP7Ub0cwXDuVDZ4QyEU84X!{v&ED6pVL4RCH83C6GY43H0cyoxp+51xMLmQk zEPum%V!km0R=R#O`KaiE>}D}cLtN7uG&S3xp7I#f3i@FV`~)?DMW}_Ww)m|1D{5ko zEKa@3wa<*o`|?szhc!_HG_(dCF@!h{ld7TR$D%*+OpE8D9;W432=`)6e1Q5qNc)X@ zMk=DNi$+bjxzp!$rlOhjwFaM>vt0vkq4|yFx0>IXhwS`ui!Y%jeBJWDnNLxlsBcl@ zWn8TZasNwF(TvKYUa#7ym3KgWkn}T0nKR7As9Uwp+-3Qr7(o7#dBc2YzBIixOhE5{ zDk@blof6mt)j@02E$L$U0jP-%Lrrj^Ip17|x^5rl#$%}do|rFDZ_!&b(^_t;8Wf?T z3rm|-Fg0;q)cKYccfeG{v6df%d5K5Z`Q@kyZ$wRSzva)F|1%$>`hC5Y{ZCCL$vT$^ zLS2{%l`mkHM$NoBYJheY$DwZ7c+@>zfa-WBYC-!gK5p?Di+@4&|C_$`s`8XXK75NG zV4n4^qu!_q4McS~5!L=P)I=6re!aQf@&{1u&Y&iA-Ok@eP4ub7LB4NYN10J8EPz^3 zHPnTzP#twNVs&2IUA#hx1knr3;BWQJ+yejMmNrMr_Y;DMGaS?23l_o_L#>}Umz}_KAPVk zKT*BXo7{D$%}b~STsI$Led1RZ*WxEx2ys&^iro`q?*Cj%tVXT)3Ti_4P%C?i`alWU z?DCn+?5F{Aqn?dosMoGM7RFAffu^H=XWWB%@dfJZdbTZGr}w`E6?Ih6tZTMJO{A+C ziyElE#lul68jF>14yvE?sCGBZhp2vDns3cyTkZW1q@oK$P;Ws5Y9$|7KF%D9`gMJR z#b04&;%`v{97R2Br%)5ShwAr@8L-WE5cP#5`!?Qx{qQJHqBOQZT`(Q>K{Mat6{u%p z6KbFXmOqLb@B(VZPwf0F)7$R)OMwxb&wy%I64hU&?d-oQ(InJy2Moq;)^HFeAs%Ip z#pJ{jFfC3)t#kqE`t_(C*@Ap|@=jt#?7zdEpJdK3=lG~prNJW9%CFmrhZa9Ky`Ao- zR~pnnc~I>OqgGVb^3~0{Sde^U)C7lPN}Pcia2~23-#RLK>Nlf0JYb$Qf5F6iiFzB} zSU!B0>!1+od~wuUQWLdyQ0RMv5);%qflF4#p1@OXR0gK#gV899^LPAl?#^m&3uaL=q)C$@PLaWQE>@OT%p+r zbC7S3T96M5;AAX`38)>oYd%3;|56QgLDGY+K^oLO%V2R)RKqeB*Fa66p4k?)GqI=% zPeBc|$Xsr&LG`l<)ow4UAKw`&8t{fC-k~1Cw1?b^@}VxQgUYwCxG$=M;TBIr_49?f z1=apEYC`u>{X8?@Ir;aW!>&OXszHQV81)pFwYVW_z;Hi5IchE z?4925)r5i3ZOrhMmYf#cCo0>gCUk5jk*<6?fhzN zMZ66QWAG2|H=~NE>pNjM<9ofSsN-qqk8{jLs1+}_c(b_+)xiOaPol2BVEJFoN9NzC z9ZGS;EhIZ?oT8Zc{+FSm4r`z~YHSVKTD~)C2l}G!@fbV57}fC#)I@h-X*`bVFUe7N zT@dQM&xv{_8lY}n_oKZ3n#m**x?l#X!-c5)CUXY{5br~^J7!))wY!O$-~-FQGXsyg z_Fc?#A3o}GVb z`M21Ue5T{Bz0YTrd8mnOL=C(P)xiPOgic~^{LS(ye{|xjGDkg)BtNOzXLU)A1!|iHL>4OD|~~xF6jwpder$`sD7d_8+^ z^XvUzMI{1Hqh|a}C-5Dr!+fV)To-E*55Tgx6W_-_P_JdwX?|75I+z{5M7@rCQCt2J zgD~|O_Z2J~`l3iwq@sHqi<f1!3D<snfF{lrqo~VA}Q0)fU`LX5{ z^D~TNd~Y5V4V3(f8#o-*Q883UrBEGJ#(G#EHQ+ofic2sLohD|Z&hk~uD(A^hwR3;Rw=YqoW_s&UAJ8{AbAhtl zT3p9(>`J~-crz&f*wlJ(ZYA|Lcs2fXiBO;Z(Md--GJjF-QFMG|Uf_HO%2yOUe?O4# zM`=bmOuP~!C~YaVxGq=X)%+$++?&#nHZkNzP@j&aiH}mxg&bMD-w9?>DibARS+^4N z{G5m%T{6AzDra6&quKA1NN8gvp&!D8Gu44^lA?-@y9(@Aos7-_SNWP_hocbDyKczOgl$0gp z^{@G5=vU){~jl-vtKy3%<2`Wm#0xv;sW>v=ivL4)YOMl zKBnxWU1myz9}m!HB+^piXvCN5#G@U5uAspN{1ubieeV>%Qtu?5QzY&+hE0ia+d4QWxM^?@) zqU@zSbg9Ju(l!rm+ER3^a(J0&^TcvWO`@JXNg!Y8ESVzyQQ1)6J)%-H)zQE#O#KG= zfz~qE)^*KdwLL`r9iF$`N$S5*Pi<}0W-R^bo5>z*toGKHFEZXRSND0hsC-1CEM*=I zuM>YoJqsl#^;+ak*#7uG(RBsjPna9 zb@bn^d_tmrd_eiE?E^^ZIAu*$&rN+ce#yBqmajtGllq6`##8_Im`_~I`cN(n(@aIX z1JuXGk1ih?RgAOxzf^TB`A^F})IX)Xv2*#UPl!KUKEjuTKJHs~sa?^4tEW)9kPo8N zqim!MAlHI6^(dQ&cj9+;&2vm3r=t~RJ!Kd1O-dN&KDTx^@l!S880Y6jd$&nEB^X1| z|IAw(Cy-mlh1sZYr5=P8DVNAC!=K1)rktZRA=jPUZt7R!H&zG@Z%wT*XKw zPtZ#ef4f4#ia&GFuY}{V4W%t-rr}?dXVg1!E+1CGnUv||f7I2~bHz8P80yJ=S{u|&KVjX|dW|Y-eTl+7FGrQA?UvOKK|C(Gu74m&4-N^069o9~y=lm@4o%R18 z>j-4Oe#JYS(9x8W*D*RV;^89Sl==&Nj?*Z+sK-;vQ`Qovp~O?p694x&Wx)vg)Nz3G zFR3q1Y{&SqcENX?>`FZZ-(o+Uj2kFZxZqFXIO;n7CQeH!M``a;-sgCk^V=!=sb{kM zLE;oFaI#|T2{|Z@$^WRI@+T-dj*tkWo}IFg_)E(Vqn?erKNl~+SHwS1s?uJ^hYoKz z=SET1TVCl`#J4Fr4qJX5^(@p+QGEI}UdJ#yIhn>?tiIeX%xtl1>Hd8BpWJ(-3Rv6! z;pbMLuRQJgTifi`K_@Ijxyv29UOAw3bxpc&xDSr?*B2G@p$aZhFcBH*M z)Gv{*iD#6sBa~c8%9ei<*pc#*97ifI8pn^?A;GOnsl_^I>x0 zkrt<88jq=uC0A4(a#W{OB=@bgNlfy`Z0j?Ox=!8Vj~dvA@-y{oN%)8ARCM?r4Ff61 zCZ2!;ME}Usc?h&~Utha{bQmIbv4tz$* z#K?DuCz5}PA0-s1mMveJJ~6$!MGhF)KW1RJ9zzqV^xEo|Gg3LPPM-k-WBPRImKa6G z^ywPeD5n3wUNLczbs8l+`tVWmWKD5_2%qasNYf(CXCNLV!@Jbl8InM3jg+?qN4 z&bm2wX0J%7zx^A(aD^*3c(>LszPV?r3vSJtc5m#eTazZx*W(?1{QYmuoOXNNiiB4C Y(kD&$_~6-;34?w-9h}hq>T|#U1HaN}{Qv*} diff --git a/apps/locale/zh/LC_MESSAGES/django.po b/apps/locale/zh/LC_MESSAGES/django.po index 3b5624384..b41f381cc 100644 --- a/apps/locale/zh/LC_MESSAGES/django.po +++ b/apps/locale/zh/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: JumpServer 0.3.3\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-09-07 16:23+0800\n" +"POT-Creation-Date: 2020-08-25 16:18+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: ibuler \n" "Language-Team: JumpServer team\n" @@ -42,7 +42,7 @@ msgstr "自定义" #: users/templates/users/user_profile.html:51 #: users/templates/users/user_pubkey_update.html:57 #: users/templates/users/user_remote_app_permission.html:36 -#: xpack/plugins/cloud/models.py:36 +#: xpack/plugins/cloud/models.py:35 msgid "Name" msgstr "名称" @@ -85,8 +85,8 @@ msgstr "数据库" #: users/templates/users/user_group_detail.html:62 #: users/templates/users/user_group_list.html:16 #: users/templates/users/user_profile.html:138 -#: xpack/plugins/change_auth_plan/models.py:77 xpack/plugins/cloud/models.py:54 -#: xpack/plugins/cloud/models.py:149 xpack/plugins/gathered_user/models.py:26 +#: xpack/plugins/change_auth_plan/models.py:77 xpack/plugins/cloud/models.py:53 +#: xpack/plugins/cloud/models.py:140 xpack/plugins/gathered_user/models.py:26 msgid "Comment" msgstr "备注" @@ -125,7 +125,7 @@ msgstr "Kubernetes应用" #: users/templates/users/user_asset_permission.html:70 #: users/templates/users/user_granted_remote_app.html:36 #: xpack/plugins/change_auth_plan/models.py:283 -#: xpack/plugins/cloud/models.py:275 +#: xpack/plugins/cloud/models.py:266 msgid "Asset" msgstr "资产" @@ -149,8 +149,8 @@ msgstr "参数" #: orgs/models.py:23 orgs/models.py:389 perms/models/base.py:54 #: users/models/user.py:542 users/serializers/group.py:35 #: users/templates/users/user_detail.html:97 -#: xpack/plugins/change_auth_plan/models.py:81 xpack/plugins/cloud/models.py:57 -#: xpack/plugins/cloud/models.py:155 xpack/plugins/gathered_user/models.py:30 +#: xpack/plugins/change_auth_plan/models.py:81 xpack/plugins/cloud/models.py:56 +#: xpack/plugins/cloud/models.py:146 xpack/plugins/gathered_user/models.py:30 msgid "Created by" msgstr "创建者" @@ -163,7 +163,7 @@ msgstr "创建者" #: common/mixins/models.py:50 ops/models/adhoc.py:38 ops/models/command.py:27 #: orgs/models.py:24 orgs/models.py:387 perms/models/base.py:55 #: users/models/group.py:18 users/templates/users/user_group_detail.html:58 -#: xpack/plugins/cloud/models.py:60 xpack/plugins/cloud/models.py:158 +#: xpack/plugins/cloud/models.py:59 xpack/plugins/cloud/models.py:149 msgid "Date created" msgstr "创建日期" @@ -180,11 +180,11 @@ msgstr "远程应用" msgid "Deleted failed, There are related assets" msgstr "删除失败，存在关联资产" -#: assets/api/node.py:49 +#: assets/api/node.py:52 msgid "You can't update the root node name" msgstr "不能修改根节点名称" -#: assets/api/node.py:56 +#: assets/api/node.py:59 msgid "Deletion failed and the node contains children or assets" msgstr "删除失败，节点包含子节点或资产" @@ -196,7 +196,7 @@ msgstr "不能移除资产的管理用户账号" msgid "Latest version could not be delete" msgstr "最新版本的不能被删除" -#: assets/models/asset.py:146 xpack/plugins/cloud/providers/base.py:17 +#: assets/models/asset.py:146 xpack/plugins/cloud/providers/base.py:16 msgid "Base" msgstr "基础" @@ -262,7 +262,7 @@ msgstr "激活" #: assets/models/asset.py:199 assets/models/cluster.py:19 #: assets/models/user.py:66 templates/_nav.html:44 -#: xpack/plugins/cloud/models.py:142 xpack/plugins/cloud/serializers.py:84 +#: xpack/plugins/cloud/models.py:133 xpack/plugins/cloud/serializers.py:83 msgid "Admin user" msgstr "管理用户" @@ -433,7 +433,7 @@ msgid "Default" msgstr "默认" #: assets/models/cluster.py:36 assets/models/label.py:14 -#: users/models/user.py:661 +#: users/models/user.py:667 msgid "System" msgstr "系统" @@ -555,7 +555,7 @@ msgstr "默认资产组" #: tickets/models/ticket.py:30 tickets/models/ticket.py:137 #: tickets/serializers/request_asset_perm.py:65 #: tickets/serializers/ticket.py:31 users/forms/group.py:15 -#: users/models/user.py:159 users/models/user.py:649 +#: users/models/user.py:159 users/models/user.py:655 #: users/serializers/group.py:20 #: users/templates/users/user_asset_permission.html:38 #: users/templates/users/user_asset_permission.html:64 @@ -569,7 +569,7 @@ msgstr "默认资产组" msgid "User" msgstr "用户" -#: assets/models/label.py:19 assets/models/node.py:502 settings/models.py:28 +#: assets/models/label.py:19 assets/models/node.py:515 settings/models.py:28 msgid "Value" msgstr "值" @@ -577,33 +577,37 @@ msgstr "值" msgid "Category" msgstr "分类" -#: assets/models/node.py:223 +#: assets/models/node.py:233 msgid "New node" msgstr "新节点" -#: assets/models/node.py:384 +#: assets/models/node.py:397 msgid "ungrouped" msgstr "未分组" -#: assets/models/node.py:386 users/templates/users/_granted_assets.html:130 +#: assets/models/node.py:399 users/templates/users/_granted_assets.html:130 msgid "empty" msgstr "空" -#: assets/models/node.py:388 +#: assets/models/node.py:401 msgid "favorite" msgstr "收藏夹" -#: assets/models/node.py:501 +#: assets/models/node.py:514 perms/models/asset_permission.py:179 msgid "Key" msgstr "键" -#: assets/models/node.py:511 assets/serializers/system_user.py:45 +#: assets/models/node.py:518 +msgid "Parent key" +msgstr "ssh私钥" + +#: assets/models/node.py:527 assets/serializers/system_user.py:45 #: assets/serializers/system_user.py:178 perms/forms/asset_permission.py:92 #: perms/forms/asset_permission.py:99 #: users/templates/users/user_asset_permission.html:41 #: users/templates/users/user_asset_permission.html:73 #: users/templates/users/user_asset_permission.html:158 -#: xpack/plugins/cloud/models.py:138 xpack/plugins/cloud/serializers.py:85 +#: xpack/plugins/cloud/models.py:129 xpack/plugins/cloud/serializers.py:84 msgid "Node" msgstr "节点" @@ -829,25 +833,25 @@ msgstr "更新节点资产硬件信息: {}" msgid "Gather assets users" msgstr "收集资产上的用户" -#: assets/tasks/push_system_user.py:176 +#: assets/tasks/push_system_user.py:177 #: assets/tasks/system_user_connectivity.py:89 msgid "System user is dynamic: {}" msgstr "系统用户是动态的: {}" -#: assets/tasks/push_system_user.py:207 +#: assets/tasks/push_system_user.py:208 msgid "Start push system user for platform: [{}]" msgstr "推送系统用户到平台: [{}]" -#: assets/tasks/push_system_user.py:208 +#: assets/tasks/push_system_user.py:209 #: assets/tasks/system_user_connectivity.py:81 msgid "Hosts count: {}" msgstr "主机数量: {}" -#: assets/tasks/push_system_user.py:225 assets/tasks/push_system_user.py:241 +#: assets/tasks/push_system_user.py:251 assets/tasks/push_system_user.py:267 msgid "Push system users to assets: {}" msgstr "推送系统用户到入资产: {}" -#: assets/tasks/push_system_user.py:233 +#: assets/tasks/push_system_user.py:259 msgid "Push system users to asset: {}({}) => {}" msgstr "推送系统用户到入资产: {}({}) => {}" @@ -1002,7 +1006,7 @@ msgstr "启用" msgid "-" msgstr "" -#: audits/models.py:96 xpack/plugins/cloud/models.py:210 +#: audits/models.py:96 xpack/plugins/cloud/models.py:201 msgid "Failed" msgstr "失败" @@ -1026,19 +1030,19 @@ msgstr "Agent" #: authentication/templates/authentication/_mfa_confirm_modal.html:14 #: authentication/templates/authentication/login_otp.html:6 #: users/forms/profile.py:52 users/models/user.py:523 -#: users/serializers/user.py:229 users/templates/users/user_detail.html:77 +#: users/serializers/user.py:232 users/templates/users/user_detail.html:77 #: users/templates/users/user_profile.html:87 msgid "MFA" msgstr "多因子认证" #: audits/models.py:105 xpack/plugins/change_auth_plan/models.py:304 -#: xpack/plugins/cloud/models.py:223 +#: xpack/plugins/cloud/models.py:214 msgid "Reason" msgstr "原因" #: audits/models.py:106 tickets/serializers/request_asset_perm.py:63 -#: tickets/serializers/ticket.py:29 xpack/plugins/cloud/models.py:220 -#: xpack/plugins/cloud/models.py:278 +#: tickets/serializers/ticket.py:29 xpack/plugins/cloud/models.py:211 +#: xpack/plugins/cloud/models.py:269 msgid "Status" msgstr "状态" @@ -1051,7 +1055,7 @@ msgid "Is success" msgstr "是否成功" #: audits/serializers.py:73 ops/models/command.py:24 -#: xpack/plugins/cloud/models.py:218 +#: xpack/plugins/cloud/models.py:209 msgid "Result" msgstr "结果" @@ -1265,7 +1269,7 @@ msgid "Show" msgstr "显示" #: authentication/templates/authentication/_access_key_modal.html:66 -#: users/models/user.py:421 users/serializers/user.py:226 +#: users/models/user.py:421 users/serializers/user.py:229 #: users/templates/users/user_profile.html:94 #: users/templates/users/user_profile.html:163 #: users/templates/users/user_profile.html:166 @@ -1274,7 +1278,7 @@ msgid "Disable" msgstr "禁用" #: authentication/templates/authentication/_access_key_modal.html:67 -#: users/models/user.py:422 users/serializers/user.py:227 +#: users/models/user.py:422 users/serializers/user.py:230 #: users/templates/users/user_profile.html:92 #: users/templates/users/user_profile.html:170 msgid "Enable" @@ -1434,6 +1438,14 @@ msgstr "CSV 文件最大为 %d 字节" msgid "%s object does not exist." msgstr "%s对象不存在" +#: common/exceptions.py:25 +msgid "Someone else is doing this. Please wait for complete" +msgstr "其他人正在操作，请等待他人完成" + +#: common/exceptions.py:30 +msgid "Your request timeout" +msgstr "您的请求超时了" + #: common/fields/form.py:33 msgid "Not a valid json" msgstr "不是合法json" @@ -1466,7 +1478,7 @@ msgstr "" msgid "Marshal data to text field" msgstr "" -#: common/fields/model.py:165 +#: common/fields/model.py:157 msgid "Encrypt field using Secret Key" msgstr "" @@ -1722,7 +1734,8 @@ msgstr "提示：RDP 协议不支持单独控制上传或下载文件" #: perms/forms/asset_permission.py:86 perms/forms/database_app_permission.py:41 #: perms/forms/remote_app_permission.py:43 perms/models/base.py:50 #: templates/_nav.html:21 users/forms/user.py:168 users/models/group.py:31 -#: users/models/user.py:507 users/templates/users/_select_user_modal.html:16 +#: users/models/user.py:507 users/serializers/user.py:48 +#: users/templates/users/_select_user_modal.html:16 #: users/templates/users/user_asset_permission.html:39 #: users/templates/users/user_asset_permission.html:67 #: users/templates/users/user_database_app_permission.html:38 @@ -2558,7 +2571,7 @@ msgid "Confirmed system-user changed" msgstr "确认的系统用户变更了" #: tickets/api/request_asset_perm.py:107 tickets/api/request_asset_perm.py:114 -#: xpack/plugins/cloud/models.py:211 +#: xpack/plugins/cloud/models.py:202 msgid "Succeed" msgstr "成功" @@ -2749,7 +2762,7 @@ msgstr "" " \n" " " -#: users/api/user.py:156 +#: users/api/user.py:158 msgid "Could not reset self otp, use profile reset instead" msgstr "不能在该页面重置多因子认证, 请去个人信息页面重置" @@ -2831,8 +2844,8 @@ msgid "Public key should not be the same as your old one." msgstr "不能和原来的密钥相同" #: users/forms/profile.py:137 users/forms/user.py:90 -#: users/serializers/user.py:189 users/serializers/user.py:271 -#: users/serializers/user.py:329 +#: users/serializers/user.py:192 users/serializers/user.py:274 +#: users/serializers/user.py:332 msgid "Not a valid ssh public key" msgstr "SSH密钥不合法" @@ -2902,63 +2915,63 @@ msgstr "微信" msgid "Date password last updated" msgstr "最后更新密码日期" -#: users/models/user.py:657 +#: users/models/user.py:663 msgid "Administrator" msgstr "管理员" -#: users/models/user.py:660 +#: users/models/user.py:666 msgid "Administrator is the super user of system" msgstr "Administrator是初始的超级管理员" -#: users/serializers/user.py:50 users/serializers/user.py:84 +#: users/serializers/user.py:53 users/serializers/user.py:87 msgid "Organization role name" msgstr "组织角色名称" -#: users/serializers/user.py:75 users/serializers/user.py:242 +#: users/serializers/user.py:78 users/serializers/user.py:245 msgid "Is first login" msgstr "首次登录" -#: users/serializers/user.py:76 +#: users/serializers/user.py:79 msgid "Is valid" msgstr "账户是否有效" -#: users/serializers/user.py:77 +#: users/serializers/user.py:80 msgid "Is expired" msgstr " 是否过期" -#: users/serializers/user.py:78 +#: users/serializers/user.py:81 msgid "Avatar url" msgstr "头像路径" -#: users/serializers/user.py:82 +#: users/serializers/user.py:85 msgid "Groups name" msgstr "用户组名" -#: users/serializers/user.py:83 +#: users/serializers/user.py:86 msgid "Source name" msgstr "用户来源名" -#: users/serializers/user.py:85 +#: users/serializers/user.py:88 msgid "Super role name" msgstr "超级角色名称" -#: users/serializers/user.py:86 +#: users/serializers/user.py:89 msgid "Total role name" msgstr "汇总角色名称" -#: users/serializers/user.py:109 +#: users/serializers/user.py:112 msgid "Role limit to {}" msgstr "角色只能为 {}" -#: users/serializers/user.py:121 users/serializers/user.py:295 +#: users/serializers/user.py:124 users/serializers/user.py:298 msgid "Password does not match security rules" msgstr "密码不满足安全规则" -#: users/serializers/user.py:287 +#: users/serializers/user.py:290 msgid "The old password is incorrect" msgstr "旧密码错误" -#: users/serializers/user.py:301 +#: users/serializers/user.py:304 msgid "The newly set password is inconsistent" msgstr "两次密码不一致" @@ -2972,7 +2985,7 @@ msgstr "安全令牌验证" #: users/templates/users/_base_otp.html:14 users/templates/users/_user.html:13 #: users/templates/users/user_profile_update.html:55 -#: xpack/plugins/cloud/models.py:124 xpack/plugins/cloud/serializers.py:83 +#: xpack/plugins/cloud/models.py:119 xpack/plugins/cloud/serializers.py:82 msgid "Account" msgstr "账户" @@ -3136,7 +3149,7 @@ msgstr "很强" #: users/templates/users/user_database_app_permission.html:41 #: users/templates/users/user_list.html:19 #: users/templates/users/user_remote_app_permission.html:41 -#: xpack/plugins/cloud/models.py:51 +#: xpack/plugins/cloud/models.py:50 msgid "Validity" msgstr "有效" @@ -3835,95 +3848,79 @@ msgstr "无法将数据发送到远程" msgid "Cloud center" msgstr "云管中心" -#: xpack/plugins/cloud/models.py:30 +#: xpack/plugins/cloud/models.py:29 msgid "Available" msgstr "有效" -#: xpack/plugins/cloud/models.py:31 +#: xpack/plugins/cloud/models.py:30 msgid "Unavailable" msgstr "无效" -#: xpack/plugins/cloud/models.py:40 +#: xpack/plugins/cloud/models.py:39 msgid "Provider" msgstr "云服务商" -#: xpack/plugins/cloud/models.py:43 +#: xpack/plugins/cloud/models.py:42 msgid "Access key id" msgstr "" -#: xpack/plugins/cloud/models.py:47 +#: xpack/plugins/cloud/models.py:46 msgid "Access key secret" msgstr "" -#: xpack/plugins/cloud/models.py:65 +#: xpack/plugins/cloud/models.py:64 msgid "Cloud account" msgstr "云账号" -#: xpack/plugins/cloud/models.py:120 -msgid "Instance name" -msgstr "实例名称" - -#: xpack/plugins/cloud/models.py:121 -msgid "Instance name and Partial IP" -msgstr "实例名称和部分IP" - -#: xpack/plugins/cloud/models.py:127 xpack/plugins/cloud/serializers.py:59 +#: xpack/plugins/cloud/models.py:122 xpack/plugins/cloud/serializers.py:59 msgid "Regions" msgstr "地域" -#: xpack/plugins/cloud/models.py:130 +#: xpack/plugins/cloud/models.py:125 msgid "Instances" msgstr "实例" -#: xpack/plugins/cloud/models.py:134 -msgid "Hostname strategy" -msgstr "主机名策略" - -#: xpack/plugins/cloud/models.py:146 xpack/plugins/cloud/serializers.py:87 +#: xpack/plugins/cloud/models.py:137 xpack/plugins/cloud/serializers.py:86 msgid "Always update" msgstr "总是更新" -#: xpack/plugins/cloud/models.py:152 +#: xpack/plugins/cloud/models.py:143 msgid "Date last sync" msgstr "最后同步日期" -#: xpack/plugins/cloud/models.py:163 xpack/plugins/cloud/models.py:216 +#: xpack/plugins/cloud/models.py:154 xpack/plugins/cloud/models.py:207 msgid "Sync instance task" msgstr "同步实例任务" -#: xpack/plugins/cloud/models.py:226 xpack/plugins/cloud/models.py:281 +#: xpack/plugins/cloud/models.py:217 xpack/plugins/cloud/models.py:272 msgid "Date sync" msgstr "同步日期" -#: xpack/plugins/cloud/models.py:254 +#: xpack/plugins/cloud/models.py:245 msgid "Unsync" msgstr "未同步" -#: xpack/plugins/cloud/models.py:255 -msgid "New Sync" -msgstr "新同步" - -#: xpack/plugins/cloud/models.py:256 +#: xpack/plugins/cloud/models.py:246 xpack/plugins/cloud/models.py:247 msgid "Synced" msgstr "已同步" -#: xpack/plugins/cloud/models.py:257 +#: xpack/plugins/cloud/models.py:248 msgid "Released" msgstr "已释放" -#: xpack/plugins/cloud/models.py:262 +#: xpack/plugins/cloud/models.py:253 msgid "Sync task" msgstr "同步任务" -#: xpack/plugins/cloud/models.py:266 +#: xpack/plugins/cloud/models.py:257 msgid "Sync instance task history" msgstr "同步实例任务历史" -#: xpack/plugins/cloud/models.py:269 +#: xpack/plugins/cloud/models.py:260 msgid "Instance" msgstr "实例" -#: xpack/plugins/cloud/models.py:272 +#: xpack/plugins/cloud/models.py:263 msgid "Region" msgstr "地域" @@ -4007,7 +4004,7 @@ msgstr "执行次数" msgid "Instance count" msgstr "实例个数" -#: xpack/plugins/cloud/serializers.py:86 +#: xpack/plugins/cloud/serializers.py:85 #: xpack/plugins/gathered_user/serializers.py:20 msgid "Periodic display" msgstr "定时执行" @@ -4072,34 +4069,30 @@ msgstr "管理页面logo" msgid "Logo of logout page" msgstr "退出页面logo" -#: xpack/plugins/license/api.py:37 +#: xpack/plugins/license/api.py:46 msgid "License import successfully" msgstr "许可证导入成功" -#: xpack/plugins/license/api.py:38 +#: xpack/plugins/license/api.py:47 msgid "License is invalid" msgstr "无效的许可证" -#: xpack/plugins/license/meta.py:11 xpack/plugins/license/models.py:124 +#: xpack/plugins/license/meta.py:11 xpack/plugins/license/models.py:94 msgid "License" msgstr "许可证" -#: xpack/plugins/license/models.py:71 +#: xpack/plugins/license/models.py:74 msgid "Standard edition" msgstr "标准版" -#: xpack/plugins/license/models.py:73 +#: xpack/plugins/license/models.py:76 msgid "Enterprise edition" msgstr "企业版" -#: xpack/plugins/license/models.py:75 +#: xpack/plugins/license/models.py:78 msgid "Ultimate edition" msgstr "旗舰版" -#: xpack/plugins/license/models.py:77 -msgid "Community edition" -msgstr "社区版" - #~ msgid "Organization User" #~ msgstr "组织用户" diff --git a/apps/ops/api/command.py b/apps/ops/api/command.py index 48ead8f30..f9a82a767 100644 --- a/apps/ops/api/command.py +++ b/apps/ops/api/command.py @@ -3,12 +3,13 @@ from rest_framework import viewsets from rest_framework.exceptions import ValidationError from django.db import transaction +from django.db.models import Q from django.utils.translation import ugettext as _ from django.conf import settings +from assets.models import Asset, Node from orgs.mixins.api import RootOrgViewMixin from common.permissions import IsValidUser -from perms.utils import AssetPermissionUtil from ..models import CommandExecution from ..serializers import CommandExecutionSerializer from ..tasks import run_command_execution @@ -27,9 +28,34 @@ class CommandExecutionViewSet(RootOrgViewMixin, viewsets.ModelViewSet): data = serializer.validated_data assets = data["hosts"] system_user = data["run_as"] - util = AssetPermissionUtil(self.request.user) - util.filter_permissions(system_users=system_user.id) - permed_assets = util.get_assets().filter(id__in=[a.id for a in assets]) + user = self.request.user + + q = Q(granted_by_permissions__system_users__id=system_user.id) & ( + Q(granted_by_permissions__users=user) | + Q(granted_by_permissions__user_groups__users=user) + ) + + permed_assets = set() + permed_assets.update( + Asset.objects.filter( + id__in=[a.id for a in assets] + ).filter(q).distinct() + ) + node_keys = Node.objects.filter(q).distinct().values_list('key', flat=True) + + nodes_assets_q = Q() + for _key in node_keys: + nodes_assets_q |= Q(nodes__key__startswith=f'{_key}:') + nodes_assets_q |= Q(nodes__key=_key) + + permed_assets.update( + Asset.objects.filter( + id__in=[a.id for a in assets] + ).filter( + nodes_assets_q + ).distinct() + ) + invalid_assets = set(assets) - set(permed_assets) if invalid_assets: msg = _("Not has host {} permission").format( diff --git a/apps/ops/celery/__init__.py b/apps/ops/celery/__init__.py index a9866780b..e0c97816a 100644 --- a/apps/ops/celery/__init__.py +++ b/apps/ops/celery/__init__.py @@ -19,6 +19,7 @@ configs = {k: v for k, v in settings.__dict__.items() if k.startswith('CELERY')} configs["CELERY_QUEUES"] = [ Queue("celery", Exchange("celery"), routing_key="celery"), Queue("ansible", Exchange("ansible"), routing_key="ansible"), + Queue("celery_node_tree", Exchange("celery_node_tree"), routing_key="celery_node_tree") ] configs["CELERY_ROUTES"] = { "ops.tasks.run_ansible_task": {'exchange': 'ansible', 'routing_key': 'ansible'}, diff --git a/apps/orgs/lock.py b/apps/orgs/lock.py new file mode 100644 index 000000000..c129b8bcd --- /dev/null +++ b/apps/orgs/lock.py @@ -0,0 +1,131 @@ +from uuid import uuid4 +from functools import wraps + +from django.core.cache import cache +from django.db.transaction import atomic +from rest_framework.request import Request +from rest_framework.exceptions import NotAuthenticated + +from orgs.utils import current_org +from common.exceptions import SomeoneIsDoingThis, Timeout +from common.utils.timezone import dt_formater, now + +# Redis 中锁值得模板，该模板提供了很强的可读性，方便调试与排错 +VALUE_TEMPLATE = '{stage}:{username}:{user_id}:{now}:{rand_str}' + +# 锁的状态 +DOING = 'doing' # 处理中，此状态的锁可以被干掉 +COMMITING = 'commiting' # 提交事务中，此状态很重要，要确保事务在锁消失之前返回了，不要轻易删除该锁 + +client = cache.client.get_client(write=True) + + +""" +将锁的状态从 `doing` 切换到 `commiting` +KEYS[1]： key +ARGV[1]: doingvalue +ARGV[2]: commitingvalue +ARGV[3]: timeout +""" +change_lock_state_to_commiting_lua = ''' +if (redis.call("get", KEYS[1]) == ARGV[1]) +then + return redis.call("set", KEYS[1], ARGV[2], "EX", ARGV[3], "XX") +else + return 0 +end +''' +change_lock_state_to_commiting_lua_obj = client.register_script(change_lock_state_to_commiting_lua) + + +""" +释放锁，两种`value`都要检查`doing`和`commiting` +KEYS[1]： key +ARGV[1]: 两个 `value` 中的其中一个 +ARGV[2]: 两个 `value` 中的其中一个 +""" +release_lua = ''' +if (redis.call("get",KEYS[1]) == ARGV[1] or redis.call("get",KEYS[1]) == ARGV[2]) +then + return redis.call("del",KEYS[1]) +else + return 0 +end +''' +release_lua_obj = client.register_script(release_lua) + + +def acquire(key, value, timeout): + return client.set(key, value, ex=timeout, nx=True) + + +def get(key): + return client.get(key) + + +def change_lock_state_to_commiting(key, doingvalue, commitingvalue, timeout=600): + # 将锁的状态从 `doing` 切换到 `commiting` + return bool(change_lock_state_to_commiting_lua_obj(keys=(key,), args=(doingvalue, commitingvalue, timeout))) + + +def release(key, value1, value2): + # 释放锁，两种`value` `doing`和`commiting` 都要检查 + return release_lua_obj(keys=(key,), args=(value1, value2)) + + +def _generate_value(request: Request, stage=DOING): + # 不支持匿名用户 + user = request.user + if user.is_anonymous: + raise NotAuthenticated + + return VALUE_TEMPLATE.format( + stage=stage, username=user.username, user_id=user.id, + now=dt_formater(now()), rand_str=uuid4() + ) + + +default_wait_msg = SomeoneIsDoingThis.default_detail + + +def org_level_transaction_lock(key, timeout=300, wait_msg=default_wait_msg): + """ + 被装饰的 `View` 必须取消自身的 `ATOMIC_REQUESTS`，因为该装饰器要有事务的完全控制权 + [官网](https://docs.djangoproject.com/en/3.1/topics/db/transactions/#tying-transactions-to-http-requests) + + 1. 获取锁：只有当锁对应的 `key` 不存在时成功获取，`value` 设置为 `doing` + 2. 开启事务：本次请求的事务必须确保在这里开启 + 3. 执行 `View` 体 + 4. `View` 体执行结束未异常，此时事务还未提交 + 5. 检查锁是否过时，过时事务回滚，不过时，重新设置`key`延长`key`有效期，已确保足够时间提交事务，同时把`key`的状态改为`commiting` + 6. 提交事务 + 7. 释放锁，释放的时候会检查`doing`与`commiting`的值，因为删除或者更改锁必须提供与当前锁的`value`相同的值，确保不误删 + [锁参考文章](http://doc.redisfans.com/string/set.html#id2) + """ + + def decorator(fun): + @wraps(fun) + def wrapper(request, *args, **kwargs): + # `key`可能是组织相关的，如果是把组织`id`加上 + _key = key.format(org_id=current_org.id) + doing_value = _generate_value(request) + commiting_value = _generate_value(request, stage=COMMITING) + try: + lock = acquire(_key, doing_value, timeout) + if not lock: + raise SomeoneIsDoingThis(detail=wait_msg) + with atomic(savepoint=False): + ret = fun(request, *args, **kwargs) + # 提交事务前，检查一下锁是否还在 + # 锁在的话，更新锁的状态为 `commiting`，延长锁时间，确保事务提交 + # 锁不在的话回滚 + ok = change_lock_state_to_commiting(_key, doing_value, commiting_value) + if not ok: + # 超时或者被中断了 + raise Timeout + return ret + finally: + # 释放锁，锁的两个值都要尝试，不确定异常是从什么位置抛出的 + release(_key, commiting_value, doing_value) + return wrapper + return decorator diff --git a/apps/perms/api/asset_permission_relation.py b/apps/perms/api/asset_permission_relation.py index 4207a995c..172391dc8 100644 --- a/apps/perms/api/asset_permission_relation.py +++ b/apps/perms/api/asset_permission_relation.py @@ -2,9 +2,12 @@ # from rest_framework import generics from django.db.models import F, Value +from django.db.models import Q from django.db.models.functions import Concat from django.shortcuts import get_object_or_404 +from assets.models import Node, Asset +from orgs.mixins.api import OrgRelationMixin from orgs.mixins.api import OrgBulkModelViewSet from orgs.utils import current_org from common.permissions import IsOrgAdmin @@ -19,9 +22,9 @@ __all__ = [ ] -class RelationMixin(OrgBulkModelViewSet): +class RelationMixin(OrgRelationMixin, OrgBulkModelViewSet): def get_queryset(self): - queryset = self.model.objects.all() + queryset = super().get_queryset() org_id = current_org.org_id() if org_id is not None: queryset = queryset.filter(assetpermission__org_id=org_id) @@ -31,7 +34,7 @@ class RelationMixin(OrgBulkModelViewSet): class AssetPermissionUserRelationViewSet(RelationMixin): serializer_class = serializers.AssetPermissionUserRelationSerializer - model = models.AssetPermission.users.through + m2m_field = models.AssetPermission.users.field permission_classes = (IsOrgAdmin,) filter_fields = [ 'id', "user", "assetpermission", @@ -62,7 +65,7 @@ class AssetPermissionAllUserListApi(generics.ListAPIView): class AssetPermissionUserGroupRelationViewSet(RelationMixin): serializer_class = serializers.AssetPermissionUserGroupRelationSerializer - model = models.AssetPermission.user_groups.through + m2m_field = models.AssetPermission.user_groups.field permission_classes = (IsOrgAdmin,) filter_fields = [ 'id', "usergroup", "assetpermission" @@ -78,7 +81,7 @@ class AssetPermissionUserGroupRelationViewSet(RelationMixin): class AssetPermissionAssetRelationViewSet(RelationMixin): serializer_class = serializers.AssetPermissionAssetRelationSerializer - model = models.AssetPermission.assets.through + m2m_field = models.AssetPermission.assets.field permission_classes = (IsOrgAdmin,) filter_fields = [ 'id', 'asset', 'assetpermission', @@ -101,15 +104,20 @@ class AssetPermissionAllAssetListApi(generics.ListAPIView): def get_queryset(self): pk = self.kwargs.get("pk") perm = get_object_or_404(models.AssetPermission, pk=pk) - assets = perm.get_all_assets().only( - *self.serializer_class.Meta.only_fields - ) + + asset_q = Q(granted_by_permissions=perm) + granted_node_keys = Node.objects.filter(granted_by_permissions=perm).distinct().values_list('key', flat=True) + for key in granted_node_keys: + asset_q |= Q(nodes__key__startswith=f'{key}:') + asset_q |= Q(nodes__key=key) + + assets = Asset.objects.filter(asset_q).only(*self.serializer_class.Meta.only_fields) return assets class AssetPermissionNodeRelationViewSet(RelationMixin): serializer_class = serializers.AssetPermissionNodeRelationSerializer - model = models.AssetPermission.nodes.through + m2m_field = models.AssetPermission.nodes.field permission_classes = (IsOrgAdmin,) filter_fields = [ 'id', 'node', 'assetpermission', @@ -125,7 +133,7 @@ class AssetPermissionNodeRelationViewSet(RelationMixin): class AssetPermissionSystemUserRelationViewSet(RelationMixin): serializer_class = serializers.AssetPermissionSystemUserRelationSerializer - model = models.AssetPermission.system_users.through + m2m_field = models.AssetPermission.system_users.field permission_classes = (IsOrgAdmin,) filter_fields = [ 'id', 'systemuser', 'assetpermission', diff --git a/apps/perms/api/system_user_permission.py b/apps/perms/api/system_user_permission.py index aa0ceda39..0c026b54d 100644 --- a/apps/perms/api/system_user_permission.py +++ b/apps/perms/api/system_user_permission.py @@ -1,21 +1,24 @@ - - from rest_framework import generics +from django.db.models import Q +from django.utils.decorators import method_decorator + +from assets.models import SystemUser from common.permissions import IsValidUser from orgs.utils import tmp_to_root_org from .. import serializers +@method_decorator(tmp_to_root_org(), name='list') class SystemUserPermission(generics.ListAPIView): permission_classes = (IsValidUser,) serializer_class = serializers.SystemUserSerializer def get_queryset(self): - return self.get_user_system_users() - - def get_user_system_users(self): - from perms.utils import AssetPermissionUtil user = self.request.user - with tmp_to_root_org(): - util = AssetPermissionUtil(user) - return util.get_system_users() + + queryset = SystemUser.objects.filter( + Q(granted_by_permissions__users=user) | + Q(granted_by_permissions__user_groups__users=user) + ).distinct() + + return queryset diff --git a/apps/perms/api/user_group_permission.py b/apps/perms/api/user_group_permission.py index 73e191bc7..8c1b3071d 100644 --- a/apps/perms/api/user_group_permission.py +++ b/apps/perms/api/user_group_permission.py @@ -1,44 +1,169 @@ # -*- coding: utf-8 -*- # +from itertools import chain +from django.db.models import Q +from rest_framework.generics import ListAPIView +from rest_framework.response import Response +from common.permissions import IsOrgAdminOrAppUser +from common.utils import lazyproperty +from perms.models import AssetPermission +from assets.models import Asset, Node from . import user_permission as uapi -from .mixin import UserGroupPermissionMixin +from perms import serializers +from perms.utils.asset_permission import get_asset_system_users_id_with_actions_by_group +from assets.api.mixin import SerializeToTreeNodeMixin +from users.models import UserGroup __all__ = [ 'UserGroupGrantedAssetsApi', 'UserGroupGrantedNodesApi', - 'UserGroupGrantedNodeAssetsApi', 'UserGroupGrantedNodeChildrenApi', + 'UserGroupGrantedNodeAssetsApi', 'UserGroupGrantedNodeChildrenAsTreeApi', - 'UserGroupGrantedNodeChildrenWithAssetsAsTreeApi', 'UserGroupGrantedAssetSystemUsersApi', - # 'UserGroupGrantedNodeChildrenWithAssetsAsTreeApi', ] -class UserGroupGrantedAssetsApi(UserGroupPermissionMixin, uapi.UserGrantedAssetsApi): - pass +class UserGroupMixin: + @lazyproperty + def group(self): + group_id = self.kwargs.get('pk') + return UserGroup.objects.get(id=group_id) -class UserGroupGrantedNodeAssetsApi(UserGroupPermissionMixin, uapi.UserGrantedNodeAssetsApi): - pass +class UserGroupGrantedAssetsApi(ListAPIView): + """ + 获取用户组直接授权的资产 + """ + permission_classes = (IsOrgAdminOrAppUser,) + serializer_class = serializers.AssetGrantedSerializer + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + filter_fields = ['hostname', 'ip', 'id', 'comment'] + search_fields = ['hostname', 'ip', 'comment'] + + def get_queryset(self): + user_group_id = self.kwargs.get('pk', '') + + return Asset.objects.filter( + Q(granted_by_permissions__user_groups__id=user_group_id) + ).distinct().only( + *self.only_fields + ) -class UserGroupGrantedNodesApi(UserGroupPermissionMixin, uapi.UserGrantedNodesApi): - pass +class UserGroupGrantedNodeAssetsApi(ListAPIView): + permission_classes = (IsOrgAdminOrAppUser,) + serializer_class = serializers.AssetGrantedSerializer + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + filter_fields = ['hostname', 'ip', 'id', 'comment'] + search_fields = ['hostname', 'ip', 'comment'] + + def get_queryset(self): + user_group_id = self.kwargs.get('pk', '') + node_id = self.kwargs.get("node_id") + node = Node.objects.get(id=node_id) + + granted = AssetPermission.objects.filter( + user_groups__id=user_group_id, + nodes__id=node_id + ).exists() + if granted: + assets = Asset.objects.filter( + Q(nodes__key__startswith=f'{node.key}:') | + Q(nodes__key=node.key) + ) + return assets + else: + granted_node_keys = Node.objects.filter( + granted_by_permissions__user_groups__id=user_group_id, + key__startswith=f'{node.key}:' + ).distinct().values_list('key', flat=True) + + granted_node_q = Q() + for _key in granted_node_keys: + granted_node_q |= Q(nodes__key__startswith=f'{_key}:') + granted_node_q |= Q(nodes__key=_key) + + granted_asset_q = ( + Q(granted_by_permissions__user_groups__id=user_group_id) & + ( + Q(nodes__key__startswith=f'{node.key}:') | + Q(nodes__key=node.key) + ) + ) + + assets = Asset.objects.filter( + granted_node_q | granted_asset_q + ).distinct() + return assets -class UserGroupGrantedNodeChildrenApi(UserGroupPermissionMixin, uapi.UserGrantedNodeChildrenApi): - pass +class UserGroupGrantedNodesApi(ListAPIView): + serializer_class = serializers.NodeGrantedSerializer + permission_classes = (IsOrgAdminOrAppUser,) + + def get_queryset(self): + user_group_id = self.kwargs.get('pk', '') + nodes = Node.objects.filter( + Q(granted_by_permissions__user_groups__id=user_group_id) | + Q(assets__granted_by_permissions__user_groups__id=user_group_id) + ) + return nodes -class UserGroupGrantedNodeChildrenAsTreeApi(UserGroupPermissionMixin, uapi.UserGrantedNodeChildrenAsTreeApi): - pass +class UserGroupGrantedNodeChildrenAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): + permission_classes = (IsOrgAdminOrAppUser,) + + def get_children_nodes(self, parent_key): + return Node.objects.filter(parent_key=parent_key) + + def add_children_key(self, node_key, key, key_set): + if key.startswith(f'{node_key}:'): + try: + end = key.index(':', len(node_key) + 1) + key_set.add(key[:end]) + except ValueError: + key_set.add(key) + + def get_nodes(self): + group_id = self.kwargs.get('pk') + node_key = self.request.query_params.get('key', None) + + granted_keys = Node.objects.filter( + granted_by_permissions__user_groups__id=group_id + ).values_list('key', flat=True) + + asset_granted_keys = Node.objects.filter( + assets__granted_by_permissions__user_groups__id=group_id + ).values_list('key', flat=True) + + if node_key is None: + root_keys = set() + for _key in chain(granted_keys, asset_granted_keys): + root_keys.add(_key.split(':', 1)[0]) + return Node.objects.filter(key__in=root_keys) + else: + children_keys = set() + for _key in granted_keys: + # 判断当前节点是否是授权节点 + if node_key == _key: + return self.get_children_nodes(node_key) + # 判断当前节点有没有授权的父节点 + if node_key.startswith(f'{_key}:'): + return self.get_children_nodes(node_key) + self.add_children_key(node_key, _key, children_keys) + + for _key in asset_granted_keys: + self.add_children_key(node_key, _key, children_keys) + + return Node.objects.filter(key__in=children_keys) + + def list(self, request, *args, **kwargs): + nodes = self.get_nodes() + nodes = self.serialize_nodes(nodes) + return Response(data=nodes) -class UserGroupGrantedNodeChildrenWithAssetsAsTreeApi(UserGroupPermissionMixin, uapi.UserGrantedNodeChildrenWithAssetsAsTreeApi): - pass - - -class UserGroupGrantedAssetSystemUsersApi(UserGroupPermissionMixin, uapi.UserGrantedAssetSystemUsersApi): - pass - +class UserGroupGrantedAssetSystemUsersApi(UserGroupMixin, uapi.UserGrantedAssetSystemUsersForAdminApi): + def get_asset_system_users_id_with_actions(self, asset): + return get_asset_system_users_id_with_actions_by_group(self.group, asset) diff --git a/apps/perms/api/user_permission/common.py b/apps/perms/api/user_permission/common.py index 17900a6bc..2f8a4a56a 100644 --- a/apps/perms/api/user_permission/common.py +++ b/apps/perms/api/user_permission/common.py @@ -3,38 +3,38 @@ import uuid from django.shortcuts import get_object_or_404 +from django.utils.decorators import method_decorator from rest_framework.views import APIView, Response from rest_framework.generics import ( ListAPIView, get_object_or_404, RetrieveAPIView, DestroyAPIView ) -from common.permissions import IsOrgAdminOrAppUser, IsOrgAdmin -from common.utils import get_logger -from ...utils import ( - AssetPermissionUtil -) +from orgs.utils import tmp_to_root_org +from perms.utils.asset_permission import get_asset_system_users_id_with_actions_by_user +from common.permissions import IsOrgAdminOrAppUser, IsOrgAdmin, IsValidUser +from common.utils import get_logger, lazyproperty + from ...hands import User, Asset, SystemUser from ... import serializers from ...models import Action -from .mixin import UserAssetPermissionMixin logger = get_logger(__name__) __all__ = [ 'RefreshAssetPermissionCacheApi', - 'UserGrantedAssetSystemUsersApi', + 'UserGrantedAssetSystemUsersForAdminApi', 'ValidateUserAssetPermissionApi', 'GetUserAssetPermissionActionsApi', 'UserAssetPermissionsCacheApi', + 'MyGrantedAssetSystemUsersApi', ] -class GetUserAssetPermissionActionsApi(UserAssetPermissionMixin, - RetrieveAPIView): +class GetUserAssetPermissionActionsApi(RetrieveAPIView): permission_classes = (IsOrgAdminOrAppUser,) serializer_class = serializers.ActionsSerializer - def get_obj(self): + def get_user(self): user_id = self.request.query_params.get('user_id', '') user = get_object_or_404(User, id=user_id) return user @@ -52,18 +52,18 @@ class GetUserAssetPermissionActionsApi(UserAssetPermissionMixin, asset = get_object_or_404(Asset, id=asset_id) system_user = get_object_or_404(SystemUser, id=system_id) - system_users_actions = self.util.get_asset_system_users_id_with_actions(asset) + system_users_actions = get_asset_system_users_id_with_actions_by_user(self.get_user(), asset) actions = system_users_actions.get(system_user.id) return {"actions": actions} -class ValidateUserAssetPermissionApi(UserAssetPermissionMixin, APIView): +class ValidateUserAssetPermissionApi(APIView): permission_classes = (IsOrgAdminOrAppUser,) def get_cache_policy(self): return 0 - def get_obj(self): + def get_user(self): user_id = self.request.query_params.get('user_id', '') user = get_object_or_404(User, id=user_id) return user @@ -82,7 +82,7 @@ class ValidateUserAssetPermissionApi(UserAssetPermissionMixin, APIView): asset = get_object_or_404(Asset, id=asset_id) system_user = get_object_or_404(SystemUser, id=system_id) - system_users_actions = self.util.get_asset_system_users_id_with_actions(asset) + system_users_actions = get_asset_system_users_id_with_actions_by_user(self.get_user(), asset) actions = system_users_actions.get(system_user.id) if actions is None: return Response({'msg': False}, status=403) @@ -91,23 +91,31 @@ class ValidateUserAssetPermissionApi(UserAssetPermissionMixin, APIView): return Response({'msg': False}, status=403) +# TODO 删除 class RefreshAssetPermissionCacheApi(RetrieveAPIView): permission_classes = (IsOrgAdmin,) def retrieve(self, request, *args, **kwargs): - AssetPermissionUtil.expire_all_user_tree_cache() return Response({'msg': True}, status=200) -class UserGrantedAssetSystemUsersApi(UserAssetPermissionMixin, ListAPIView): +class UserGrantedAssetSystemUsersForAdminApi(ListAPIView): permission_classes = (IsOrgAdminOrAppUser,) serializer_class = serializers.AssetSystemUserSerializer only_fields = serializers.AssetSystemUserSerializer.Meta.only_fields + @lazyproperty + def user(self): + user_id = self.kwargs.get('pk') + return User.objects.get(id=user_id) + + def get_asset_system_users_id_with_actions(self, asset): + return get_asset_system_users_id_with_actions_by_user(self.user, asset) + def get_queryset(self): asset_id = self.kwargs.get('asset_id') asset = get_object_or_404(Asset, id=asset_id) - system_users_with_actions = self.util.get_asset_system_users_id_with_actions(asset) + system_users_with_actions = self.get_asset_system_users_id_with_actions(asset) system_users_id = system_users_with_actions.keys() system_users = SystemUser.objects.filter(id__in=system_users_id)\ .only(*self.serializer_class.Meta.only_fields) \ @@ -119,9 +127,18 @@ class UserGrantedAssetSystemUsersApi(UserAssetPermissionMixin, ListAPIView): return system_users -class UserAssetPermissionsCacheApi(UserAssetPermissionMixin, DestroyAPIView): +@method_decorator(tmp_to_root_org(), name='list') +class MyGrantedAssetSystemUsersApi(UserGrantedAssetSystemUsersForAdminApi): + permission_classes = (IsValidUser,) + + @lazyproperty + def user(self): + return self.request.user + + +# TODO 删除 +class UserAssetPermissionsCacheApi(DestroyAPIView): permission_classes = (IsOrgAdmin,) def destroy(self, request, *args, **kwargs): - self.util.expire_user_tree_cache() return Response(status=204) diff --git a/apps/perms/api/user_permission/mixin.py b/apps/perms/api/user_permission/mixin.py index 426f7d34d..9466aa78c 100644 --- a/apps/perms/api/user_permission/mixin.py +++ b/apps/perms/api/user_permission/mixin.py @@ -1,83 +1,55 @@ # -*- coding: utf-8 -*- # +from common.permissions import IsOrgAdminOrAppUser, IsValidUser from common.utils import lazyproperty -from common.tree import TreeNodeSerializer -from django.db.models import QuerySet -from ..mixin import UserPermissionMixin -from ...utils import AssetPermissionUtil, ParserNode -from ...hands import Node, Asset +from rest_framework.generics import get_object_or_404 + +from users.models import User +from perms.models import UserGrantedMappingNode +from common.exceptions import JMSObjectDoesNotExist +from perms.async_tasks.mapping_node_task import submit_update_mapping_node_task_for_user +from ...hands import Node -class UserAssetPermissionMixin(UserPermissionMixin): - util = None +class UserGrantedNodeDispatchMixin: - def get_cache_policy(self): - return self.request.query_params.get('cache_policy', '0') + def submit_update_mapping_node_task(self, user): + submit_update_mapping_node_task_for_user(user) - @lazyproperty - def util(self): - cache_policy = self.get_cache_policy() - system_user_id = self.request.query_params.get("system_user") - util = AssetPermissionUtil(self.obj, cache_policy=cache_policy) - if system_user_id: - util.filter_permissions(system_users=system_user_id) - return util - - @lazyproperty - def tree(self): - return self.util.get_user_tree() - - -class UserNodeTreeMixin: - serializer_class = TreeNodeSerializer - nodes_only_fields = ParserNode.nodes_only_fields - - def parse_nodes_to_queryset(self, nodes): - if isinstance(nodes, QuerySet): - nodes = nodes.only(*self.nodes_only_fields) - _queryset = [] - - for node in nodes: - assets_amount = self.tree.valid_assets_amount(node.key) - if assets_amount == 0 and not node.key.startswith('-'): - continue - node.assets_amount = assets_amount - data = ParserNode.parse_node_to_tree_node(node) - _queryset.append(data) - return _queryset - - def get_serializer_queryset(self, queryset): - queryset = self.parse_nodes_to_queryset(queryset) + def dispatch_node_process(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + if mapping_node is None: + ancestor_keys = Node.get_node_ancestor_keys(key) + granted = UserGrantedMappingNode.objects.filter(key__in=ancestor_keys, granted=True).exists() + if not granted: + raise JMSObjectDoesNotExist(object_name=Node._meta.object_name) + queryset = self.on_granted_node(key, mapping_node, node) + else: + if mapping_node.granted: + # granted_node + queryset = self.on_granted_node(key, mapping_node, node) + else: + queryset = self.on_ungranted_node(key, mapping_node, node) return queryset - def get_serializer(self, queryset=None, many=True, **kwargs): - if queryset is None: - queryset = Node.objects.none() - queryset = self.get_serializer_queryset(queryset) - queryset.sort() - return super().get_serializer(queryset, many=many, **kwargs) + def on_granted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + raise NotImplementedError + + def on_ungranted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + raise NotImplementedError -class UserAssetTreeMixin: - serializer_class = TreeNodeSerializer - nodes_only_fields = ParserNode.assets_only_fields +class ForAdminMixin: + permission_classes = (IsOrgAdminOrAppUser,) - @staticmethod - def parse_assets_to_queryset(assets, node): - _queryset = [] - for asset in assets: - data = ParserNode.parse_asset_to_tree_node(node, asset) - _queryset.append(data) - return _queryset + @lazyproperty + def user(self): + user_id = self.kwargs.get('pk') + return User.objects.get(id=user_id) - def get_serializer_queryset(self, queryset): - queryset = queryset.only(*self.nodes_only_fields) - _queryset = self.parse_assets_to_queryset(queryset, None) - return _queryset - def get_serializer(self, queryset=None, many=True, **kwargs): - if queryset is None: - queryset = Asset.objects.none() - queryset = self.get_serializer_queryset(queryset) - queryset.sort() - return super().get_serializer(queryset, many=many, **kwargs) +class ForUserMixin: + permission_classes = (IsValidUser,) + + @lazyproperty + def user(self): + return self.request.user diff --git a/apps/perms/api/user_permission/user_permission_assets.py b/apps/perms/api/user_permission/user_permission_assets.py index 389c06bdc..cba95dd53 100644 --- a/apps/perms/api/user_permission/user_permission_assets.py +++ b/apps/perms/api/user_permission/user_permission_assets.py @@ -1,66 +1,150 @@ # -*- coding: utf-8 -*- # - -from django.shortcuts import get_object_or_404 -from django.conf import settings +from django.db.models import Q +from django.utils.decorators import method_decorator +from perms.api.user_permission.mixin import UserGrantedNodeDispatchMixin from rest_framework.generics import ListAPIView +from rest_framework.response import Response +from django.conf import settings -from common.permissions import IsOrgAdminOrAppUser -from common.utils import get_logger, timeit +from assets.api.mixin import SerializeToTreeNodeMixin +from common.utils import get_object_or_none +from users.models import User +from common.permissions import IsOrgAdminOrAppUser, IsValidUser +from common.utils import get_logger from ...hands import Node from ... import serializers -from .mixin import UserAssetPermissionMixin, UserAssetTreeMixin +from perms.models import UserGrantedMappingNode +from perms.utils.user_node_tree import get_node_all_granted_assets +from perms.pagination import GrantedAssetLimitOffsetPagination +from assets.models import Asset +from orgs.utils import tmp_to_root_org +from .mixin import ForAdminMixin, ForUserMixin logger = get_logger(__name__) __all__ = [ - 'UserGrantedAssetsApi', 'UserGrantedAssetsAsTreeApi', - 'UserGrantedNodeAssetsApi', + 'UserDirectGrantedAssetsForAdminApi', 'MyAllAssetsAsTreeApi', + 'UserGrantedNodeAssetsForAdminApi', 'MyDirectGrantedAssetsApi', + 'UserDirectGrantedAssetsAsTreeForAdminApi', 'MyGrantedNodeAssetsApi', + 'MyUngroupAssetsAsTreeApi', ] -class UserGrantedAssetsApi(UserAssetPermissionMixin, ListAPIView): - permission_classes = (IsOrgAdminOrAppUser,) +@method_decorator(tmp_to_root_org(), name='list') +class UserDirectGrantedAssetsApi(ListAPIView): serializer_class = serializers.AssetGrantedSerializer only_fields = serializers.AssetGrantedSerializer.Meta.only_fields filter_fields = ['hostname', 'ip', 'id', 'comment'] search_fields = ['hostname', 'ip', 'comment'] - def filter_by_nodes(self, queryset): - node_id = self.request.query_params.get("node") - if not node_id: - return queryset - node = get_object_or_404(Node, pk=node_id) - query_all = self.request.query_params.get("all", "0") in ["1", "true"] - if query_all: - pattern = '^{0}$|^{0}:'.format(node.key) - queryset = queryset.filter(nodes__key__regex=pattern).distinct() - else: - queryset = queryset.filter(nodes=node) - return queryset - - def filter_queryset(self, queryset): - queryset = super().filter_queryset(queryset) - queryset = self.filter_by_nodes(queryset) - return queryset - def get_queryset(self): - queryset = self.util.get_assets().only(*self.only_fields).order_by( - settings.TERMINAL_ASSET_LIST_SORT_BY + user = self.user + + return Asset.objects.filter( + Q(granted_by_permissions__users=user) | + Q(granted_by_permissions__user_groups__users=user) + ).distinct().only( + *self.only_fields ) - return queryset -class UserGrantedAssetsAsTreeApi(UserAssetTreeMixin, UserGrantedAssetsApi): +@method_decorator(tmp_to_root_org(), name='list') +class AssetsAsTreeMixin(SerializeToTreeNodeMixin): + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + data = self.serialize_assets(queryset, None) + return Response(data=data) + + +class UserDirectGrantedAssetsForAdminApi(ForAdminMixin, UserDirectGrantedAssetsApi): pass -class UserGrantedNodeAssetsApi(UserGrantedAssetsApi): +class MyDirectGrantedAssetsApi(ForUserMixin, UserDirectGrantedAssetsApi): + pass + + +@method_decorator(tmp_to_root_org(), name='list') +class UserDirectGrantedAssetsAsTreeForAdminApi(ForAdminMixin, AssetsAsTreeMixin, UserDirectGrantedAssetsApi): + pass + + +@method_decorator(tmp_to_root_org(), name='list') +class MyUngroupAssetsAsTreeApi(ForUserMixin, AssetsAsTreeMixin, UserDirectGrantedAssetsApi): + def get_queryset(self): + queryset = super().get_queryset() + if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + queryset = queryset.none() + return queryset + + +@method_decorator(tmp_to_root_org(), name='list') +class UserAllGrantedAssetsApi(ListAPIView): + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + + def get_queryset(self): + user = self.user + + granted_node_keys = Node.objects.filter( + Q(granted_by_permissions__users=user) | + Q(granted_by_permissions__user_groups__users=user) + ).distinct().values_list('key', flat=True) + + granted_node_q = Q() + for _key in granted_node_keys: + granted_node_q |= Q(nodes__key__startswith=f'{_key}:') + granted_node_q |= Q(nodes__key=_key) + + q = Q(granted_by_permissions__users=user) | \ + Q(granted_by_permissions__user_groups__users=user) + + if granted_node_q: + q |= granted_node_q + + return Asset.objects.filter(q).distinct().only( + *self.only_fields + ) + + +class MyAllAssetsAsTreeApi(ForUserMixin, AssetsAsTreeMixin, UserAllGrantedAssetsApi): + pass + + +@method_decorator(tmp_to_root_org(), name='list') +class UserGrantedNodeAssetsApi(UserGrantedNodeDispatchMixin, ListAPIView): + serializer_class = serializers.AssetGrantedSerializer + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + filter_fields = ['hostname', 'ip', 'id', 'comment'] + search_fields = ['hostname', 'ip', 'comment'] + pagination_class = GrantedAssetLimitOffsetPagination + def get_queryset(self): node_id = self.kwargs.get("node_id") - node = get_object_or_404(Node, pk=node_id) - deep = self.request.query_params.get("all", "0") == "1" - queryset = self.util.get_nodes_assets(node, deep=deep)\ - .only(*self.only_fields) - return queryset + user = self.user + + mapping_node: UserGrantedMappingNode = get_object_or_none( + UserGrantedMappingNode, user=user, node_id=node_id) + node = Node.objects.get(id=node_id) + return self.dispatch_node_process(node.key, mapping_node, node) + + def on_granted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + self.node = node + return Asset.objects.filter( + Q(nodes__key__startswith=f'{node.key}:') | + Q(nodes__id=node.id) + ).distinct() + + def on_ungranted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + self.node = mapping_node + user = self.user + return get_node_all_granted_assets(user, node.key) + + +class UserGrantedNodeAssetsForAdminApi(ForAdminMixin, UserGrantedNodeAssetsApi): + pass + + +class MyGrantedNodeAssetsApi(ForUserMixin, UserGrantedNodeAssetsApi): + pass diff --git a/apps/perms/api/user_permission/user_permission_nodes.py b/apps/perms/api/user_permission/user_permission_nodes.py index 311f2a048..39bc8dfe8 100644 --- a/apps/perms/api/user_permission/user_permission_nodes.py +++ b/apps/perms/api/user_permission/user_permission_nodes.py @@ -1,82 +1,159 @@ # -*- coding: utf-8 -*- # - -from django.shortcuts import get_object_or_404 +from django.db.models import Q, F +from perms.api.user_permission.mixin import ForAdminMixin, ForUserMixin from rest_framework.generics import ( - ListAPIView, get_object_or_404 + ListAPIView ) +from rest_framework.response import Response -from common.permissions import IsOrgAdminOrAppUser +from perms.utils.user_node_tree import ( + node_annotate_mapping_node, get_ungranted_node_children, + is_granted, get_granted_assets_amount, node_annotate_set_granted, +) +from common.utils.django import get_object_or_none +from common.utils import lazyproperty +from perms.models import UserGrantedMappingNode +from orgs.utils import tmp_to_root_org +from assets.api.mixin import SerializeToTreeNodeMixin from common.utils import get_logger -from ...hands import Node, NodeSerializer +from ...hands import Node +from .mixin import UserGrantedNodeDispatchMixin from ... import serializers -from .mixin import UserNodeTreeMixin, UserAssetPermissionMixin logger = get_logger(__name__) __all__ = [ - 'UserGrantedNodesApi', - 'UserGrantedNodesAsTreeApi', - 'UserGrantedNodeChildrenApi', - 'UserGrantedNodeChildrenAsTreeApi', + 'UserGrantedNodesForAdminApi', + 'MyGrantedNodesApi', + 'MyGrantedNodesAsTreeApi', + 'UserGrantedNodeChildrenForAdminApi', + 'MyGrantedNodeChildrenApi', + 'UserGrantedNodeChildrenAsTreeForAdminApi', + 'MyGrantedNodeChildrenAsTreeApi', + 'NodeChildrenAsTreeApi', ] -class UserGrantedNodesApi(UserAssetPermissionMixin, ListAPIView): - """ - 查询用户授权的所有节点的API - """ - permission_classes = (IsOrgAdminOrAppUser,) +class GrantedNodeBaseApi(ListAPIView): + @lazyproperty + def user(self): + raise NotImplementedError + + def get_nodes(self): + # 不使用 `get_queryset` 单独定义 `get_nodes` 的原因是 + # `get_nodes` 返回的不一定是 `queryset` + raise NotImplementedError + + +class NodeChildrenApi(GrantedNodeBaseApi): serializer_class = serializers.NodeGrantedSerializer - nodes_only_fields = NodeSerializer.Meta.only_fields - def get_serializer_context(self): - context = super().get_serializer_context() - if self.serializer_class == serializers.NodeGrantedSerializer: - context["tree"] = self.tree - return context - - def get_queryset(self): - node_keys = self.util.get_nodes() - queryset = Node.objects.filter(key__in=node_keys)\ - .only(*self.nodes_only_fields) - return queryset + @tmp_to_root_org() + def list(self, request, *args, **kwargs): + nodes = self.get_nodes() + serializer = self.get_serializer(nodes, many=True) + return Response(serializer.data) -class UserGrantedNodesAsTreeApi(UserNodeTreeMixin, UserGrantedNodesApi): - pass +class NodeChildrenAsTreeApi(SerializeToTreeNodeMixin, GrantedNodeBaseApi): + @tmp_to_root_org() + def list(self, request, *args, **kwargs): + nodes = self.get_nodes() + nodes = self.serialize_nodes(nodes, with_asset_amount=True) + return Response(data=nodes) -class UserGrantedNodeChildrenApi(UserGrantedNodesApi): - node = None - root_keys = None # 如果是第一次访问，则需要把二级节点添加进去，这个 roots_keys +class UserGrantedNodeChildrenMixin(UserGrantedNodeDispatchMixin): - def get(self, request, *args, **kwargs): - key = self.request.query_params.get("key") - pk = self.request.query_params.get("id") + def get_nodes(self): + user = self.user + key = self.request.query_params.get('key') - node = None - if pk is not None: - node = get_object_or_404(Node, id=pk) - elif key is not None: - node = get_object_or_404(Node, key=key) - self.node = node - return super().get(request, *args, **kwargs) + self.submit_update_mapping_node_task(user) - def get_queryset(self): - if self.node: - children = self.tree.children(self.node.key) + if not key: + nodes = get_ungranted_node_children(user) else: - children = self.tree.children(self.tree.root) - # 默认打开组织节点下的节点 - self.root_keys = [child.identifier for child in children] - for key in self.root_keys: - children.extend(self.tree.children(key)) - node_keys = [n.identifier for n in children] - queryset = Node.objects.filter(key__in=node_keys) - return queryset + mapping_node = get_object_or_none( + UserGrantedMappingNode, user=user, key=key + ) + nodes = self.dispatch_node_process(key, mapping_node, None) + return nodes + + def on_granted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + return Node.objects.filter(parent_key=key) + + def on_ungranted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + user = self.user + nodes = get_ungranted_node_children(user, key) + return nodes -class UserGrantedNodeChildrenAsTreeApi(UserNodeTreeMixin, UserGrantedNodeChildrenApi): +class UserGrantedNodesMixin: + """ + 查询用户授权的所有节点 直接授权节点 + 授权资产关联的节点 + """ + + def get_nodes(self): + user = self.user + + # 获取 `UserGrantedMappingNode` 中对应的 `Node` + nodes = Node.objects.filter( + mapping_nodes__user=user, + ).annotate(**node_annotate_mapping_node).distinct() + + key2nodes_mapper = {} + descendant_q = Q() + + for _node in nodes: + if not is_granted(_node): + # 未授权的节点资产数量设置为 `UserGrantedMappingNode` 中的数量 + _node.assets_amount = get_granted_assets_amount(_node) + else: + # 直接授权的节点 + # 增加查询后代节点的过滤条件 + descendant_q |= Q(key__startswith=f'{_node.key}:') + + key2nodes_mapper[_node.key] = _node + + if descendant_q: + descendant_nodes = Node.objects.filter(descendant_q).annotate(**node_annotate_set_granted) + for _node in descendant_nodes: + key2nodes_mapper[_node.key] = _node + + all_nodes = key2nodes_mapper.values() + return all_nodes + + +# ------------------------------------------ +# 最终的 api +class UserGrantedNodeChildrenForAdminApi(ForAdminMixin, UserGrantedNodeChildrenMixin, NodeChildrenApi): pass + + +class MyGrantedNodeChildrenApi(ForUserMixin, UserGrantedNodeChildrenMixin, NodeChildrenApi): + pass + + +class UserGrantedNodeChildrenAsTreeForAdminApi(ForAdminMixin, UserGrantedNodeChildrenMixin, NodeChildrenAsTreeApi): + pass + + +class MyGrantedNodeChildrenAsTreeApi(ForUserMixin, UserGrantedNodeChildrenMixin, NodeChildrenAsTreeApi): + pass + + +class UserGrantedNodesForAdminApi(ForAdminMixin, UserGrantedNodesMixin, NodeChildrenApi): + pass + + +class MyGrantedNodesApi(ForUserMixin, UserGrantedNodesMixin, NodeChildrenApi): + pass + + +class MyGrantedNodesAsTreeApi(ForUserMixin, UserGrantedNodesMixin, NodeChildrenAsTreeApi): + pass + +# ------------------------------------------ diff --git a/apps/perms/api/user_permission/user_permission_nodes_with_assets.py b/apps/perms/api/user_permission/user_permission_nodes_with_assets.py index 974705c16..88f0202c5 100644 --- a/apps/perms/api/user_permission/user_permission_nodes_with_assets.py +++ b/apps/perms/api/user_permission/user_permission_nodes_with_assets.py @@ -1,56 +1,145 @@ # -*- coding: utf-8 -*- # -from common.utils import get_logger -from ...utils import ParserNode -from .mixin import UserAssetTreeMixin -from ...hands import Node -from .user_permission_nodes import UserGrantedNodesAsTreeApi -from .user_permission_nodes import UserGrantedNodeChildrenAsTreeApi +from rest_framework.generics import ListAPIView +from rest_framework.request import Request +from rest_framework.response import Response +from django.db.models import Q, F +from users.models import User +from common.permissions import IsValidUser, IsOrgAdminOrAppUser +from common.utils.django import get_object_or_none +from common.utils import get_logger +from .user_permission_nodes import MyGrantedNodesAsTreeApi +from .mixin import UserGrantedNodeDispatchMixin +from perms.models import UserGrantedMappingNode +from perms.utils.user_node_tree import ( + TMP_GRANTED_FIELD, TMP_GRANTED_ASSETS_AMOUNT_FIELD, node_annotate_mapping_node, + is_granted, get_granted_assets_amount, node_annotate_set_granted, + get_granted_q, get_ungranted_node_children +) + +from assets.models import Asset +from assets.api import SerializeToTreeNodeMixin +from orgs.utils import tmp_to_root_org +from ...hands import Node logger = get_logger(__name__) __all__ = [ - 'UserGrantedNodesAsTreeApi', - 'UserGrantedNodesWithAssetsAsTreeApi', - 'UserGrantedNodeChildrenAsTreeApi', - 'UserGrantedNodeChildrenWithAssetsAsTreeApi', + 'MyGrantedNodesAsTreeApi', + 'UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi', + 'MyGrantedNodesWithAssetsAsTreeApi', + 'MyGrantedNodeChildrenWithAssetsAsTreeApi', ] -class UserGrantedNodesWithAssetsAsTreeApi(UserGrantedNodesAsTreeApi): - assets_only_fields = ParserNode.assets_only_fields +class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): + permission_classes = (IsValidUser,) - def get_serializer_queryset(self, queryset): - _queryset = super().get_serializer_queryset(queryset) - _all_assets = self.util.get_assets().only(*self.assets_only_fields) - _all_assets_map = {a.id: a for a in _all_assets} - for node in queryset: - assets_ids = self.tree.assets(node.key) - assets = [_all_assets_map[_id] for _id in assets_ids if _id in _all_assets_map] - _queryset.extend( - UserAssetTreeMixin.parse_assets_to_queryset(assets, node) - ) - return _queryset + @tmp_to_root_org() + def list(self, request: Request, *args, **kwargs): + user = request.user + + # 获取 `UserGrantedMappingNode` 中对应的 `Node` + nodes = Node.objects.filter( + mapping_nodes__user=user, + ).annotate(**node_annotate_mapping_node).distinct() + + key2nodes_mapper = {} + descendant_q = Q() + granted_q = Q() + + for _node in nodes: + if not is_granted(_node): + # 未授权的节点资产数量设置为 `UserGrantedMappingNode` 中的数量 + _node.assets_amount = get_granted_assets_amount(_node) + else: + # 直接授权的节点 + + # 增加查询该节点及其后代节点资产的过滤条件 + granted_q |= Q(nodes__key__startswith=f'{_node.key}:') + granted_q |= Q(nodes__key=_node.key) + + # 增加查询后代节点的过滤条件 + descendant_q |= Q(key__startswith=f'{_node.key}:') + + key2nodes_mapper[_node.key] = _node + + if descendant_q: + descendant_nodes = Node.objects.filter(descendant_q).annotate(**node_annotate_set_granted) + for _node in descendant_nodes: + key2nodes_mapper[_node.key] = _node + + all_nodes = key2nodes_mapper.values() + + # 查询出所有资产 + all_assets = Asset.objects.filter( + get_granted_q(user) | + granted_q + ).annotate(parent_key=F('nodes__key')).distinct() + + data = [ + *self.serialize_nodes(all_nodes, with_asset_amount=True), + *self.serialize_assets(all_assets) + ] + return Response(data=data) -class UserGrantedNodeChildrenWithAssetsAsTreeApi(UserGrantedNodeChildrenAsTreeApi): - nodes_only_fields = ParserNode.nodes_only_fields - assets_only_fields = ParserNode.assets_only_fields +class UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi(UserGrantedNodeDispatchMixin, SerializeToTreeNodeMixin, ListAPIView): + permission_classes = (IsOrgAdminOrAppUser, ) + + def on_granted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + nodes = Node.objects.filter(parent_key=key) + assets = Asset.objects.filter(nodes__key=key).distinct() + return nodes, assets + + def on_ungranted_node(self, key, mapping_node: UserGrantedMappingNode, node: Node = None): + user = self.get_user() + assets = Asset.objects.none() + nodes = Node.objects.filter( + parent_key=key, + mapping_nodes__user=user, + ).annotate( + **node_annotate_mapping_node + ).distinct() + + # TODO 可配置 + for _node in nodes: + if not is_granted(_node): + _node.assets_amount = get_granted_assets_amount(_node) + + if mapping_node.asset_granted: + assets = Asset.objects.filter( + nodes__key=key, + ).filter(get_granted_q(user)) + return nodes, assets + + def get_user(self): + user_id = self.kwargs.get('pk') + return User.objects.get(id=user_id) + + @tmp_to_root_org() + def list(self, request: Request, *args, **kwargs): + user = self.get_user() + key = request.query_params.get('key') + self.submit_update_mapping_node_task(user) - def get_serializer_queryset(self, queryset): - _queryset = super().get_serializer_queryset(queryset) nodes = [] - if self.node: - nodes.append(self.node) - elif self.root_keys: - nodes = Node.objects.filter(key__in=self.root_keys) + assets = [] + if not key: + root_nodes = get_ungranted_node_children(user) + nodes.extend(root_nodes) + else: + mapping_node: UserGrantedMappingNode = get_object_or_none( + UserGrantedMappingNode, user=user, key=key) + nodes, assets = self.dispatch_node_process(key, mapping_node) + nodes = self.serialize_nodes(nodes, with_asset_amount=True) + assets = self.serialize_assets(assets, key) + return Response(data=[*nodes, *assets]) - for node in nodes: - assets = self.util.get_nodes_assets(node).only( - *self.assets_only_fields - ) - _queryset.extend( - UserAssetTreeMixin.parse_assets_to_queryset(assets, node) - ) - return _queryset + +class MyGrantedNodeChildrenWithAssetsAsTreeApi(UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi): + permission_classes = (IsValidUser, ) + + def get_user(self): + return self.request.user diff --git a/apps/perms/async_tasks/__init__.py b/apps/perms/async_tasks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/perms/async_tasks/mapping_node_task.py b/apps/perms/async_tasks/mapping_node_task.py new file mode 100644 index 000000000..c45527ab7 --- /dev/null +++ b/apps/perms/async_tasks/mapping_node_task.py @@ -0,0 +1,47 @@ +from django.utils.crypto import get_random_string +from perms.utils import rebuild_user_mapping_nodes_if_need_with_lock + +from common.thread_pools import SingletonThreadPoolExecutor +from common.utils import get_logger +from perms.models import RebuildUserTreeTask + +logger = get_logger(__name__) + + +class Executor(SingletonThreadPoolExecutor): + pass + + +executor = Executor() + + +def run_mapping_node_tasks(): + failed_user_ids = [] + + ident = get_random_string() + logger.debug(f'[{ident}]mapping_node_tasks running') + + while True: + task = RebuildUserTreeTask.objects.exclude( + user_id__in=failed_user_ids + ).first() + + if task is None: + break + + user = task.user + try: + rebuild_user_mapping_nodes_if_need_with_lock(user) + except: + logger.exception(f'[{ident}]mapping_node_tasks_exception') + failed_user_ids.append(user.id) + + logger.debug(f'[{ident}]mapping_node_tasks finished') + + +def submit_update_mapping_node_task(): + executor.submit(run_mapping_node_tasks) + + +def submit_update_mapping_node_task_for_user(user): + executor.submit(rebuild_user_mapping_nodes_if_need_with_lock, user) diff --git a/apps/perms/exceptions.py b/apps/perms/exceptions.py new file mode 100644 index 000000000..684a5da88 --- /dev/null +++ b/apps/perms/exceptions.py @@ -0,0 +1,14 @@ +from rest_framework import status +from django.utils.translation import gettext_lazy as _ + +from common.exceptions import JMSException + + +class AdminIsModifyingPerm(JMSException): + status_code = status.HTTP_409_CONFLICT + default_detail = _('The administrator is modifying permissions. Please wait') + + +class CanNotRemoveAssetPermNow(JMSException): + status_code = status.HTTP_409_CONFLICT + default_detail = _('The authorization cannot be revoked for the time being') diff --git a/apps/perms/migrations/0013_rebuildusertreetask_usergrantedmappingnode.py b/apps/perms/migrations/0013_rebuildusertreetask_usergrantedmappingnode.py new file mode 100644 index 000000000..09efa093c --- /dev/null +++ b/apps/perms/migrations/0013_rebuildusertreetask_usergrantedmappingnode.py @@ -0,0 +1,53 @@ +# Generated by Django 2.2.13 on 2020-09-07 08:40 + +import assets.models.node +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('assets', '0057_fill_node_value_assets_amount_and_parent_key'), + ('perms', '0012_k8sapppermission'), + ] + + operations = [ + migrations.CreateModel( + name='UserGrantedMappingNode', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('key', models.CharField(db_index=True, max_length=64, verbose_name='Key')), + ('granted', models.BooleanField(db_index=True, default=False)), + ('asset_granted', models.BooleanField(db_index=True, default=False)), + ('parent_key', models.CharField(db_index=True, default='', max_length=64, verbose_name='Parent key')), + ('assets_amount', models.IntegerField(default=0)), + ('node', models.ForeignKey(db_constraint=False, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='mapping_nodes', to='assets.Node')), + ('user', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + bases=(assets.models.node.FamilyMixin, models.Model), + ), + migrations.CreateModel( + name='RebuildUserTreeTask', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/apps/perms/migrations/0014_build_users_perm_tree.py b/apps/perms/migrations/0014_build_users_perm_tree.py new file mode 100644 index 000000000..85df89b35 --- /dev/null +++ b/apps/perms/migrations/0014_build_users_perm_tree.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.13 on 2020-08-21 08:20 + +from django.db import migrations +from perms.tasks import dispatch_mapping_node_tasks + + +def start_build_users_perm_tree_task(apps, schema_editor): + User = apps.get_model('users', 'User') + RebuildUserTreeTask = apps.get_model('perms', 'RebuildUserTreeTask') + + user_ids = User.objects.all().values_list('id', flat=True).distinct() + RebuildUserTreeTask.objects.bulk_create( + [RebuildUserTreeTask(user_id=i) for i in user_ids] + ) + + dispatch_mapping_node_tasks.delay() + + +class Migration(migrations.Migration): + + dependencies = [ + ('perms', '0013_rebuildusertreetask_usergrantedmappingnode'), + ] + + operations = [ + migrations.RunPython(start_build_users_perm_tree_task) + ] diff --git a/apps/perms/models/asset_permission.py b/apps/perms/models/asset_permission.py index f2755a568..8ef52d6b9 100644 --- a/apps/perms/models/asset_permission.py +++ b/apps/perms/models/asset_permission.py @@ -2,19 +2,20 @@ import uuid import logging from functools import reduce -from django.db import models from django.utils.translation import ugettext_lazy as _ +from common.db import models +from common.fields.model import JsonListTextField from common.utils import lazyproperty from orgs.models import Organization from orgs.utils import get_current_org -from assets.models import Asset, SystemUser, Node +from assets.models import Asset, SystemUser, Node, FamilyMixin from .base import BasePermission __all__ = [ - 'AssetPermission', 'Action', + 'AssetPermission', 'Action', 'UserGrantedMappingNode', 'RebuildUserTreeTask', ] logger = logging.getLogger(__name__) @@ -174,3 +175,17 @@ class AssetPermission(BasePermission): print('Error continue') continue + +class UserGrantedMappingNode(FamilyMixin, models.JMSBaseModel): + node = models.ForeignKey('assets.Node', default=None, on_delete=models.CASCADE, + db_constraint=False, null=True, related_name='mapping_nodes') + key = models.CharField(max_length=64, verbose_name=_("Key"), db_index=True) # '1:1:1:1' + user = models.ForeignKey('users.User', db_constraint=False, on_delete=models.CASCADE) + granted = models.BooleanField(default=False, db_index=True) + asset_granted = models.BooleanField(default=False, db_index=True) + parent_key = models.CharField(max_length=64, default='', verbose_name=_('Parent key'), db_index=True) # '1:1:1:1' + assets_amount = models.IntegerField(default=0) + + +class RebuildUserTreeTask(models.JMSBaseModel): + user = models.ForeignKey('users.User', on_delete=models.CASCADE, verbose_name=_('User')) diff --git a/apps/perms/pagination.py b/apps/perms/pagination.py new file mode 100644 index 000000000..6c794d193 --- /dev/null +++ b/apps/perms/pagination.py @@ -0,0 +1,26 @@ +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.request import Request + + +class GrantedAssetLimitOffsetPagination(LimitOffsetPagination): + def get_count(self, queryset): + exclude_query_params = { + self.limit_query_param, + self.offset_query_param, + 'key', 'all', 'show_current_asset', + 'cache_policy', 'display', 'draw' + } + has_filter = False + for k, v in self._request.query_params.items(): + if k not in exclude_query_params and v is not None: + has_filter = True + break + if has_filter: + return super().get_count(queryset) + node = self._view.node + return node.assets_amount + + def paginate_queryset(self, queryset, request: Request, view=None): + self._request = request + self._view = view + return super().paginate_queryset(queryset, request, view=None) diff --git a/apps/perms/serializers/asset_permission_relation.py b/apps/perms/serializers/asset_permission_relation.py index 808f40468..e466ad338 100644 --- a/apps/perms/serializers/asset_permission_relation.py +++ b/apps/perms/serializers/asset_permission_relation.py @@ -96,7 +96,7 @@ class AssetPermissionAllAssetSerializer(serializers.Serializer): class AssetPermissionNodeRelationSerializer(RelationMixin, serializers.ModelSerializer): - node_display = serializers.SerializerMethodField() + node_display = serializers.CharField(source='node.full_value', read_only=True) class Meta(RelationMixin.Meta): model = AssetPermission.nodes.through @@ -104,16 +104,6 @@ class AssetPermissionNodeRelationSerializer(RelationMixin, serializers.ModelSeri 'id', 'node', "node_display", ] - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.tree = Node.tree() - - def get_node_display(self, obj): - if hasattr(obj, 'node_key'): - return self.tree.get_node_full_tag(obj.node_key) - else: - return obj.node.full_value - class AssetPermissionSystemUserRelationSerializer(RelationMixin, serializers.ModelSerializer): systemuser_display = serializers.ReadOnlyField() diff --git a/apps/perms/serializers/user_permission.py b/apps/perms/serializers/user_permission.py index 11d1bcfc7..40334786f 100644 --- a/apps/perms/serializers/user_permission.py +++ b/apps/perms/serializers/user_permission.py @@ -82,8 +82,6 @@ class AssetGrantedSerializer(serializers.ModelSerializer): class NodeGrantedSerializer(serializers.ModelSerializer): - assets_amount = serializers.SerializerMethodField() - class Meta: model = Node fields = [ @@ -91,15 +89,6 @@ class NodeGrantedSerializer(serializers.ModelSerializer): ] read_only_fields = fields - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.tree = self.context.get("tree") - - def get_assets_amount(self, obj): - if not self.tree: - return 0 - return self.tree.assets_amount(obj.key) - class ActionsSerializer(serializers.Serializer): actions = ActionsField(read_only=True) diff --git a/apps/perms/signals_handler.py b/apps/perms/signals_handler.py index 7a40bcb66..84fa02561 100644 --- a/apps/perms/signals_handler.py +++ b/apps/perms/signals_handler.py @@ -1,52 +1,98 @@ # -*- coding: utf-8 -*- # -from django.db.models.signals import m2m_changed, post_save, post_delete -from django.dispatch import receiver +from itertools import chain +from django.db.models.signals import m2m_changed, pre_delete +from django.dispatch import receiver +from django.db import transaction +from django.db.models import Q + +from perms.tasks import dispatch_mapping_node_tasks +from users.models import User +from assets.models import Asset from common.utils import get_logger -from common.decorator import on_transaction_commit -from .models import AssetPermission, RemoteAppPermission -from .utils.asset_permission import AssetPermissionUtil +from common.exceptions import M2MReverseNotAllowed +from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR +from .models import AssetPermission, RemoteAppPermission, RebuildUserTreeTask logger = get_logger(__file__) -@receiver([post_save, post_delete], sender=AssetPermission) -@on_transaction_commit -def on_permission_change(sender, action='', **kwargs): - logger.debug('Asset permission changed, refresh user tree cache') - AssetPermissionUtil.expire_all_user_tree_cache() - # Todo: 检查授权规则到期，从而修改授权规则 +@receiver([pre_delete], sender=AssetPermission) +def on_asset_permission_delete(instance, **kwargs): + # 授权删除之前，查出所有相关用户 + create_rebuild_user_tree_task_by_asset_perm(instance) + + +def create_rebuild_user_tree_task(user_ids): + RebuildUserTreeTask.objects.bulk_create( + [RebuildUserTreeTask(user_id=i) for i in user_ids] + ) + transaction.on_commit(dispatch_mapping_node_tasks) + + +def create_rebuild_user_tree_task_by_asset_perm(asset_perm: AssetPermission): + user_ap_query_name = AssetPermission.users.field.related_query_name() + group_ap_query_name = AssetPermission.user_groups.field.related_query_name() + + user_ap_q = Q(**{f'{user_ap_query_name}': asset_perm}) + group_ap_q = Q(**{f'groups__{group_ap_query_name}': asset_perm}) + user_ids = User.objects.filter(user_ap_q | group_ap_q).distinct().values_list('id', flat=True) + create_rebuild_user_tree_task(user_ids) + + +def need_rebuild_mapping_node(action): + return action in (POST_REMOVE, POST_ADD, POST_CLEAR) + + @receiver(m2m_changed, sender=AssetPermission.nodes.through) -def on_permission_nodes_changed(sender, instance=None, action='', reverse=None, **kwargs): - if action != 'post_add' and reverse: +def on_permission_nodes_changed(instance, action, reverse, pk_set, model, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + create_rebuild_user_tree_task_by_asset_perm(instance) + + if action != POST_ADD: return logger.debug("Asset permission nodes change signal received") - nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + nodes = model.objects.filter(pk__in=pk_set) system_users = instance.system_users.all() + + # TODO 待优化 for system_user in system_users: - system_user.nodes.add(*tuple(nodes)) + system_user.nodes.add(*nodes) @receiver(m2m_changed, sender=AssetPermission.assets.through) -def on_permission_assets_changed(sender, instance=None, action='', reverse=None, **kwargs): - if action != 'post_add' and reverse: +def on_permission_assets_changed(instance, action, reverse, pk_set, model, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + create_rebuild_user_tree_task_by_asset_perm(instance) + + if action != POST_ADD: return logger.debug("Asset permission assets change signal received") - assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + assets = model.objects.filter(pk__in=pk_set) + + # TODO 待优化 system_users = instance.system_users.all() for system_user in system_users: system_user.assets.add(*tuple(assets)) @receiver(m2m_changed, sender=AssetPermission.system_users.through) -def on_asset_permission_system_users_changed(sender, instance=None, action='', - reverse=False, **kwargs): - if action != 'post_add' and reverse: +def on_asset_permission_system_users_changed(instance, action, reverse, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if action != POST_ADD: return logger.debug("Asset permission system_users change signal received") system_users = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) @@ -63,28 +109,42 @@ def on_asset_permission_system_users_changed(sender, instance=None, action='', @receiver(m2m_changed, sender=AssetPermission.users.through) -def on_asset_permission_users_changed(sender, instance=None, action='', - reverse=False, **kwargs): - if action != 'post_add' and reverse: +def on_asset_permission_users_changed(instance, action, reverse, pk_set, model, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + create_rebuild_user_tree_task(pk_set) + + if action != POST_ADD: return logger.debug("Asset permission users change signal received") - users = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + users = model.objects.filter(pk__in=pk_set) system_users = instance.system_users.all() + # TODO 待优化 for system_user in system_users: if system_user.username_same_with_user: system_user.users.add(*tuple(users)) @receiver(m2m_changed, sender=AssetPermission.user_groups.through) -def on_asset_permission_user_groups_changed(sender, instance=None, action='', - reverse=False, **kwargs): - if action != 'post_add' and reverse: +def on_asset_permission_user_groups_changed(instance, action, pk_set, model, + reverse, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + user_ids = User.objects.filter(groups__id__in=pk_set).distinct().values_list('id', flat=True) + create_rebuild_user_tree_task(user_ids) + + if action != POST_ADD: return logger.debug("Asset permission user groups change signal received") - groups = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + groups = model.objects.filter(pk__in=pk_set) system_users = instance.system_users.all() + # TODO 待优化 for system_user in system_users: if system_user.username_same_with_user: system_user.groups.add(*tuple(groups)) @@ -93,7 +153,7 @@ def on_asset_permission_user_groups_changed(sender, instance=None, action='', @receiver(m2m_changed, sender=RemoteAppPermission.system_users.through) def on_remote_app_permission_system_users_changed(sender, instance=None, action='', reverse=False, **kwargs): - if action != 'post_add' or reverse: + if action != POST_ADD or reverse: return system_users = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) logger.debug("Remote app permission system_users change signal received") @@ -110,12 +170,47 @@ def on_remote_app_permission_system_users_changed(sender, instance=None, @receiver(m2m_changed, sender=RemoteAppPermission.users.through) def on_remoteapps_permission_users_changed(sender, instance=None, action='', reverse=False, **kwargs): - on_asset_permission_users_changed(sender, instance=instance, action=action, - reverse=reverse, **kwargs) + if action != POST_ADD and reverse: + return + logger.debug("Asset permission users change signal received") + users = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + system_users = instance.system_users.all() + + for system_user in system_users: + if system_user.username_same_with_user: + system_user.users.add(*tuple(users)) @receiver(m2m_changed, sender=RemoteAppPermission.user_groups.through) def on_remoteapps_permission_user_groups_changed(sender, instance=None, action='', reverse=False, **kwargs): - on_asset_permission_user_groups_changed(sender, instance=instance, - action=action, reverse=reverse, **kwargs) + if action != POST_ADD and reverse: + return + logger.debug("Asset permission user groups change signal received") + groups = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + system_users = instance.system_users.all() + + for system_user in system_users: + if system_user.username_same_with_user: + system_user.groups.add(*tuple(groups)) + + +@receiver(m2m_changed, sender=Asset.nodes.through) +def on_node_asset_change(action, instance, reverse, pk_set, **kwargs): + if not need_rebuild_mapping_node(action): + return + + if reverse: + asset_pk_set = pk_set + else: + asset_pk_set = [instance.id] + + user_ap_query_name = AssetPermission.users.field.related_query_name() + group_ap_query_name = AssetPermission.user_groups.field.related_query_name() + + user_ap_q = Q(**{f'{user_ap_query_name}__assets__id__in': asset_pk_set}) + group_ap_q = Q(**{f'groups__{group_ap_query_name}__assets__id__in': asset_pk_set}) + + from_user_ids = User.objects.filter(user_ap_q).values_list('id', flat=True) + from_group_ids = User.objects.filter(group_ap_q).values_list('id', flat=True) + create_rebuild_user_tree_task(chain(from_user_ids, from_group_ids)) diff --git a/apps/perms/tasks.py b/apps/perms/tasks.py index b8696e7c8..973553dc4 100644 --- a/apps/perms/tasks.py +++ b/apps/perms/tasks.py @@ -2,9 +2,24 @@ from __future__ import absolute_import, unicode_literals from celery import shared_task -from common.utils import get_logger, encrypt_password +from common.utils import get_logger +from users.models import User +from perms.models import RebuildUserTreeTask +from perms.utils.user_node_tree import rebuild_user_mapping_nodes_if_need_with_lock logger = get_logger(__file__) +@shared_task(queue='node_tree') +def rebuild_user_mapping_nodes_celery_task(user_id): + logger.info(f'rebuild user[{user_id}] mapping nodes') + user = User.objects.get(id=user_id) + rebuild_user_mapping_nodes_if_need_with_lock(user) + +@shared_task(queue='node_tree') +def dispatch_mapping_node_tasks(): + user_ids = RebuildUserTreeTask.objects.all().values_list('user_id', flat=True).distinct() + for id in user_ids: + logger.info(f'dispatch mapping node task for user[{id}]') + rebuild_user_mapping_nodes_celery_task.delay(id) diff --git a/apps/perms/urls/asset_permission.py b/apps/perms/urls/asset_permission.py index ff8ac0c48..43970af87 100644 --- a/apps/perms/urls/asset_permission.py +++ b/apps/perms/urls/asset_permission.py @@ -2,6 +2,7 @@ from django.urls import path, include from rest_framework_bulk.routes import BulkRouter + from .. import api router = BulkRouter() @@ -13,46 +14,62 @@ router.register('asset-permissions-nodes-relations', api.AssetPermissionNodeRela router.register('asset-permissions-system-users-relations', api.AssetPermissionSystemUserRelationViewSet, 'asset-permissions-system-users-relation') user_permission_urlpatterns = [ - path('/assets/', api.UserGrantedAssetsApi.as_view(), name='user-assets'), - path('assets/', api.UserGrantedAssetsApi.as_view(), name='my-assets'), + # 统一说明： + # ``: `User.pk` + # 直接授权：在 `AssetPermission` 中关联的对象 - # Assets as tree - path('/assets/tree/', api.UserGrantedAssetsAsTreeApi.as_view(), name='user-assets-as-tree'), - path('assets/tree/', api.UserGrantedAssetsAsTreeApi.as_view(), name='my-assets-as-tree'), + # --------------------------------------------------------- + # 获取用户所有直接授权的资产 - # Nodes - path('/nodes/', api.UserGrantedNodesApi.as_view(), name='user-nodes'), - path('nodes/', api.UserGrantedNodesApi.as_view(), name='my-nodes'), + # 以 serializer 格式返回 + path('/assets/', api.UserDirectGrantedAssetsForAdminApi.as_view(), name='user-assets'), + path('assets/', api.MyDirectGrantedAssetsApi.as_view(), name='my-assets'), - # Node children - path('/nodes/children/', api.UserGrantedNodeChildrenApi.as_view(), name='user-nodes-children'), - path('nodes/children/', api.UserGrantedNodesApi.as_view(), name='my-nodes-children'), + # Tree Node 的数据格式返回 + path('/assets/tree/', api.UserDirectGrantedAssetsAsTreeForAdminApi.as_view(), name='user-assets-as-tree'), + path('assets/tree/', api.MyAllAssetsAsTreeApi.as_view(), name='my-assets-as-tree'), + path('ungroup/assets/tree/', api.MyUngroupAssetsAsTreeApi.as_view(), name='my-ungroup-assets-as-tree'), + # ^--------------------------------------------------------^ - # Node as tree - path('/nodes/tree/', api.UserGrantedNodesAsTreeApi.as_view(), name='user-nodes-as-tree'), - path('nodes/tree/', api.UserGrantedNodesAsTreeApi.as_view(), name='my-nodes-as-tree'), + # 获取用户所有`直接授权的节点`与`直接授权资产`关联的节点 + # 以 serializer 格式返回 + path('/nodes/', api.UserGrantedNodesForAdminApi.as_view(), name='user-nodes'), + path('nodes/', api.MyGrantedNodesApi.as_view(), name='my-nodes'), - # Node with assets as tree - path('/nodes-with-assets/tree/', api.UserGrantedNodesWithAssetsAsTreeApi.as_view(), name='user-nodes-with-assets-as-tree'), - path('nodes-with-assets/tree/', api.UserGrantedNodesWithAssetsAsTreeApi.as_view(), name='my-nodes-with-assets-as-tree'), + # 以 Tree Node 的数据格式返回 + path('/nodes/tree/', api.MyGrantedNodesAsTreeApi.as_view(), name='user-nodes-as-tree'), + path('nodes/tree/', api.MyGrantedNodesAsTreeApi.as_view(), name='my-nodes-as-tree'), + # ^--------------------------------------------------------^ - # Node children as tree - path('/nodes/children/tree/', api.UserGrantedNodeChildrenAsTreeApi.as_view(), name='user-nodes-children-as-tree'), - path('nodes/children/tree/', api.UserGrantedNodeChildrenAsTreeApi.as_view(), name='my-nodes-children-as-tree'), + # 一层一层的获取用户授权的节点， + # 以 Serializer 的数据格式返回 + path('/nodes/children/', api.UserGrantedNodeChildrenForAdminApi.as_view(), name='user-nodes-children'), + path('nodes/children/', api.MyGrantedNodeChildrenApi.as_view(), name='my-nodes-children'), + + # 以 Tree Node 的数据格式返回 + path('/nodes/children/tree/', api.UserGrantedNodeChildrenAsTreeForAdminApi.as_view(), name='user-nodes-children-as-tree'), + # 部分调用位置 + # - 普通用户 -> 我的资产 -> 展开节点 时调用 + path('nodes/children/tree/', api.MyGrantedNodeChildrenAsTreeApi.as_view(), name='my-nodes-children-as-tree'), + # ^--------------------------------------------------------^ + + # 此接口会返回整棵树 + # 普通用户 -> 命令执行 -> 左侧树 + path('nodes-with-assets/tree/', api.MyGrantedNodesWithAssetsAsTreeApi.as_view(), name='my-nodes-with-assets-as-tree'), # Node children with assets as tree - path('/nodes/children-with-assets/tree/', api.UserGrantedNodeChildrenWithAssetsAsTreeApi.as_view(), name='user-nodes-children-with-assets-as-tree'), - path('nodes/children-with-assets/tree/', api.UserGrantedNodeChildrenWithAssetsAsTreeApi.as_view(), name='my-nodes-children-with-assets-as-tree'), + path('/nodes/children-with-assets/tree/', api.UserGrantedNodeChildrenWithAssetsAsTreeForAdminApi.as_view(), name='user-nodes-children-with-assets-as-tree'), + path('nodes/children-with-assets/tree/', api.MyGrantedNodeChildrenWithAssetsAsTreeApi.as_view(), name='my-nodes-children-with-assets-as-tree'), # Node assets - path('/nodes//assets/', api.UserGrantedNodeAssetsApi.as_view(), name='user-node-assets'), - path('nodes//assets/', api.UserGrantedNodeAssetsApi.as_view(), name='my-node-assets'), + path('/nodes//assets/', api.UserGrantedNodeAssetsForAdminApi.as_view(), name='user-node-assets'), + path('nodes//assets/', api.MyGrantedNodeAssetsApi.as_view(), name='my-node-assets'), # Asset System users - path('/assets//system-users/', api.UserGrantedAssetSystemUsersApi.as_view(), name='user-asset-system-users'), - path('assets//system-users/', api.UserGrantedAssetSystemUsersApi.as_view(), name='my-asset-system-users'), + path('/assets//system-users/', api.UserGrantedAssetSystemUsersForAdminApi.as_view(), name='user-asset-system-users'), + path('assets//system-users/', api.MyGrantedAssetSystemUsersApi.as_view(), name='my-asset-system-users'), - # Expire user permission cache + # TODO 要废弃 Expire user permission cache path('/asset-permissions/cache/', api.UserAssetPermissionsCacheApi.as_view(), name='user-asset-permission-cache'), path('asset-permissions/cache/', api.UserAssetPermissionsCacheApi.as_view(), name='my-asset-permission-cache'), diff --git a/apps/perms/utils/__init__.py b/apps/perms/utils/__init__.py index 35e29adb6..5ff0a8dfc 100644 --- a/apps/perms/utils/__init__.py +++ b/apps/perms/utils/__init__.py @@ -4,4 +4,5 @@ from .asset_permission import * from .remote_app_permission import * from .database_app_permission import * -from .k8s_app_permission import * \ No newline at end of file +from .k8s_app_permission import * +from .user_node_tree import * diff --git a/apps/perms/utils/asset_permission.py b/apps/perms/utils/asset_permission.py index b78d5f930..66c997895 100644 --- a/apps/perms/utils/asset_permission.py +++ b/apps/perms/utils/asset_permission.py @@ -1,28 +1,16 @@ -# coding: utf-8 -import time -import pickle from collections import defaultdict -from functools import reduce -from django.core.cache import cache from django.db.models import Q -from django.conf import settings -from orgs.utils import current_org -from common.utils import get_logger, timeit, lazyproperty -from common.tree import TreeNode -from assets.utils import TreeService +from common.utils import get_logger from ..models import AssetPermission -from ..hands import Node, Asset, SystemUser, User, FavoriteAsset +from ..hands import Asset, User +from users.models import UserGroup +from perms.models.base import BasePermissionQuerySet logger = get_logger(__file__) -__all__ = [ - 'ParserNode', 'AssetPermissionUtil', -] - - def get_user_permissions(user, include_group=True): if include_group: groups = user.groups.all() @@ -57,432 +45,39 @@ def get_system_user_permissions(system_user): ) -class AssetPermissionUtilCacheMixin: - user_tree_cache_key = 'USER_PERM_TREE_{}_{}_{}' - user_tree_cache_ttl = settings.ASSETS_PERM_CACHE_TIME - user_tree_cache_enable = settings.ASSETS_PERM_CACHE_ENABLE - user_tree_map = {} - cache_policy = '0' - obj_id = '' - _filter_id = 'None' +def get_asset_system_users_id_with_actions(asset_perm_queryset: BasePermissionQuerySet, asset: Asset): + nodes = asset.get_nodes() + node_keys = set() + for node in nodes: + ancestor_keys = node.get_ancestor_keys(with_self=True) + node_keys.update(ancestor_keys) - @property - def cache_key(self): - return self.get_cache_key() - - def get_cache_key(self, org_id=None): - if org_id is None: - org_id = current_org.org_id() - - key = self.user_tree_cache_key.format( - org_id, self.obj_id, self._filter_id - ) - return key - - def expire_user_tree_cache(self): - cache.delete(self.cache_key) - - @classmethod - def expire_all_user_tree_cache(cls): - expire_cache_key = "USER_TREE_EXPIRED_AT" - latest_expired = cache.get(expire_cache_key, 0) - now = time.time() - if now - latest_expired < 60: - return - key = cls.user_tree_cache_key.format('*', '1', '1') - key = key.replace('_1', '') - cache.delete_pattern(key) - cache.set(expire_cache_key, now) - - @classmethod - def expire_org_tree_cache(cls, org_id=None): - if org_id is None: - org_id = current_org.org_id() - key = cls.user_tree_cache_key.format(org_id, '*', '1') - key = key.replace('_1', '') - cache.delete_pattern(key) - - def set_user_tree_to_cache(self, user_tree): - data = pickle.dumps(user_tree) - cache.set(self.cache_key, data, self.user_tree_cache_ttl) - - def get_user_tree_from_cache(self): - data = cache.get(self.cache_key) - if not data: - return None - user_tree = pickle.loads(data) - return user_tree - - @timeit - def get_user_tree_from_cache_if_need(self): - if not self.user_tree_cache_enable: - return None - if self.cache_policy == '1': - return self.get_user_tree_from_cache() - elif self.cache_policy == '2': - self.expire_user_tree_cache() - return None - else: - return None - - def set_user_tree_to_cache_if_need(self, user_tree): - if self.cache_policy == '0': - return - if not self.user_tree_cache_enable: - return None - self.set_user_tree_to_cache(user_tree) - - -class AssetPermissionUtil(AssetPermissionUtilCacheMixin): - get_permissions_map = { - "User": get_user_permissions, - "UserGroup": get_user_group_permissions, - "Asset": get_asset_permissions, - "Node": get_node_permissions, - "SystemUser": get_system_user_permissions, - } - assets_only = ( - 'id', 'hostname', 'ip', "platform", "domain_id", - 'comment', 'is_active', 'os', 'org_id' + queryset = asset_perm_queryset.filter( + Q(assets=asset) | + Q(nodes__key__in=node_keys) ) + asset_protocols = asset.protocols_as_dict.keys() + values = queryset.filter( + system_users__protocol__in=asset_protocols + ).distinct().values_list('system_users', 'actions') + system_users_actions = defaultdict(int) - def __init__(self, obj=None, cache_policy='0'): - self.object = obj - self.cache_policy = cache_policy - self.obj_id = str(obj.id) if obj else None - self._permissions = None - self._filter_id = 'None' # 当通过filter更改 permission是标记 - self.change_org_if_need() - self._user_tree = None - self._user_tree_filter_id = 'None' - - if not isinstance(obj, User): - self.cache_policy = '0' - - @staticmethod - def change_org_if_need(): - pass - - @lazyproperty - def full_tree(self): - return Node.tree() - - @property - def permissions(self): - if self._permissions is not None: - return self._permissions - if self.object is None: - return AssetPermission.objects.none() - object_cls = self.object.__class__.__name__ - func = self.get_permissions_map[object_cls] - permissions = func(self.object) - self._permissions = permissions - return permissions - - @timeit - def filter_permissions(self, **filters): - self.cache_policy = '0' - self._permissions = self.permissions.filter(**filters) - - @lazyproperty - def user_tree(self): - return self.get_user_tree() - - @timeit - def get_assets_direct(self): - """ - 返回直接授权的资产， - 并添加到tree.assets中 - :return: - {asset.id: {system_user.id: actions, }, } - """ - assets_ids = self.permissions.values_list('assets', flat=True) - return Asset.objects.filter(id__in=assets_ids) - - @timeit - def get_nodes_direct(self): - """ - 返回直接授权的节点， - 并将节点添加到tree.nodes中，并将节点下的资产添加到tree.assets中 - :return: - {node.key: {system_user.id: actions,}, } - """ - nodes_ids = self.permissions.values_list('nodes', flat=True) - return Node.objects.filter(id__in=nodes_ids) - - @timeit - def add_direct_nodes_to_user_tree(self, user_tree): - """ - 将授权规则的节点放到用户树上, 从full tree中粘贴子树 - """ - nodes_direct_keys = self.permissions \ - .exclude(nodes__isnull=True) \ - .values_list('nodes__key', flat=True) \ - .distinct() - nodes_direct_keys = list(nodes_direct_keys) - # 排序，保证从上层节点开始加 - nodes_direct_keys.sort(key=lambda x: len(x)) - for key in nodes_direct_keys: - # 如果树上已经有这个节点，代表子树已经存在 - if user_tree.contains(key): - continue - # 找到这个节点的父节点，如果父节点不在树上，则挂到ROOT上 - parent = self.full_tree.parent(key) - if not user_tree.contains(parent.identifier): - parent = user_tree.root_node() - subtree = self.full_tree.subtree(key) - user_tree.paste(parent.identifier, subtree, deep=True) - - for node in user_tree.all_nodes_itr(): - assets = list(self.full_tree.assets(node.identifier)) - user_tree.set_assets(node.identifier, assets) - - @timeit - def add_single_assets_node_to_user_tree(self, user_tree): - """ - 将单独授权的资产放到树上，如果设置了单独资产到 未分组中，则放到未分组中 - 如果没有，则查询资产属于的资产组，放到树上 - """ - # 添加单独授权资产的节点 - nodes_single_assets = defaultdict(set) - queryset = self.permissions.exclude(assets__isnull=True) \ - .values_list('assets', 'assets__nodes__key') \ - .distinct() - - for item in queryset: - nodes_single_assets[item[1]].add(item[0]) - nodes_single_assets.pop(None, None) - - for key in tuple(nodes_single_assets.keys()): - if user_tree.contains(key): - nodes_single_assets.pop(key) - - if not nodes_single_assets: - return - - # 如果要设置到ungroup中 - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - node_key = Node.ungrouped_key - node_value = Node.ungrouped_value - user_tree.create_node( - identifier=node_key, tag=node_value, - parent=user_tree.root, - ) - assets = set() - for _assets in nodes_single_assets.values(): - assets.update(set(_assets)) - user_tree.set_assets(node_key, assets) - return - - # 获取单独授权资产，并没有在授权的节点上 - for key, assets in nodes_single_assets.items(): - if not self.full_tree.contains(key): - continue - node = self.full_tree.get_node(key, deep=True) - parent_id = self.full_tree.parent(key).identifier - parent = user_tree.get_node(parent_id) - if not parent: - parent = user_tree.root_node() - user_tree.add_node(node, parent) - user_tree.set_assets(node.identifier, assets) - - @timeit - def parse_user_tree_to_full_tree(self, user_tree): - """ - 经过前面两个动作，用户授权的节点已放到树上，但是树不是完整的， - 这里要将树构造成一个完整的树 - """ - # 开始修正user_tree，保证父节点都在树上 - root_children = user_tree.children('') - for child in root_children: - # print("child: {}".format(child.identifier)) - if child.identifier.isdigit(): - continue - if child.identifier.startswith('-'): - continue - ancestors = self.full_tree.ancestors( - child.identifier, with_self=False, deep=True, - ) - # print("Get ancestors: {}".format(len(ancestors))) - if not ancestors: - continue - user_tree.safe_add_ancestors(child, ancestors) - - def add_favorite_node_if_need(self, user_tree): - if not isinstance(self.object, User): - return - node_key = Node.favorite_key - node_value = Node.favorite_value - user_tree.create_node( - identifier=node_key, tag=node_value, - parent=user_tree.root, - ) - node = user_tree.get_node(node_key) - assets_id = FavoriteAsset.get_user_favorite_assets_id(self.object) - all_valid_assets = user_tree.all_valid_assets(user_tree.root) - valid_assets_id = set(assets_id) & all_valid_assets - user_tree.set_assets(node_key, valid_assets_id) - # 必须设置这个，否则看不到个数 - node.data['all_assets'] = None - - def set_user_tree_to_local(self, user_tree): - self._user_tree = user_tree - self._user_tree_filter_id = self._filter_id - - def get_user_tree_from_local(self): - if self._user_tree and self._user_tree_filter_id == self._filter_id: - return self._user_tree - return None - - @timeit - def get_user_tree(self): - user_tree = self.get_user_tree_from_cache_if_need() - if user_tree: - return user_tree - user_tree = TreeService() - full_tree_root = self.full_tree.root_node() - user_tree.create_node( - tag=full_tree_root.tag, - identifier=full_tree_root.identifier - ) - self.add_direct_nodes_to_user_tree(user_tree) - self.add_single_assets_node_to_user_tree(user_tree) - self.parse_user_tree_to_full_tree(user_tree) - self.add_favorite_node_if_need(user_tree) - self.set_user_tree_to_cache_if_need(user_tree) - self.set_user_tree_to_local(user_tree) - # print(user_tree) - return user_tree - - # Todo: 是否可以获取多个资产的系统用户 - def get_asset_system_users_id_with_actions(self, asset): - nodes = asset.get_nodes() - nodes_keys_related = set() - for node in nodes: - ancestor_keys = node.get_ancestor_keys(with_self=True) - nodes_keys_related.update(set(ancestor_keys)) - kwargs = {"assets": asset} - - if nodes_keys_related: - kwargs["nodes__key__in"] = nodes_keys_related - - queryset = self.permissions - if kwargs == 1: - queryset = queryset.filter(**kwargs) - elif len(kwargs) > 1: - kwargs = [{k: v} for k, v in kwargs.items()] - args = [Q(**kw) for kw in kwargs] - args = reduce(lambda x, y: x | y, args) - queryset = queryset.filter(args) - else: - queryset = queryset.none() - asset_protocols = asset.protocols_as_dict.keys() - values = queryset.filter(system_users__protocol__in=asset_protocols).distinct()\ - .values_list('system_users', 'actions') - system_users_actions = defaultdict(int) - for system_user_id, actions in values: - if None in (system_user_id, actions): - continue - for i, action in values: - system_users_actions[i] |= actions - return system_users_actions - - def get_permissions_nodes_and_assets(self): - from assets.models import Node - permissions = self.permissions - nodes_keys = permissions.exclude(nodes__isnull=True)\ - .values_list('nodes__key', flat=True) - assets_ids = permissions.exclude(assets__isnull=True)\ - .values_list('assets', flat=True) - nodes_keys = set(nodes_keys) - assets_ids = set(assets_ids) - nodes_keys = Node.clean_children_keys(nodes_keys) - return nodes_keys, assets_ids - - @timeit - def get_assets(self): - nodes_keys, assets_ids = self.get_permissions_nodes_and_assets() - queryset = Node.get_nodes_all_assets( - nodes_keys, extra_assets_ids=assets_ids - ) - return queryset.valid() - - def get_nodes_assets(self, node, deep=False): - if deep: - assets_ids = self.user_tree.all_assets(node.key) - else: - assets_ids = self.user_tree.assets(node.key) - queryset = Asset.objects.filter(id__in=assets_ids) - return queryset.valid() - - def get_nodes(self): - return [n.identifier for n in self.user_tree.all_nodes_itr()] - - def get_system_users(self): - system_users_id = self.permissions.values_list('system_users', flat=True).distinct() - return SystemUser.objects.filter(id__in=system_users_id) + for system_user_id, actions in values: + if None in (system_user_id, actions): + continue + system_users_actions[system_user_id] |= actions + return system_users_actions -class ParserNode: - nodes_only_fields = ("key", "value", "id") - assets_only_fields = ("hostname", "id", "ip", "protocols", "domain", "org_id") - system_users_only_fields = ( - "id", "name", "username", "protocol", "priority", "login_mode", +def get_asset_system_users_id_with_actions_by_user(user: User, asset: Asset): + queryset = AssetPermission.objects.filter( + Q(users=user) | Q(user_groups__users=user) ) + return get_asset_system_users_id_with_actions(queryset, asset) - @staticmethod - def parse_node_to_tree_node(node): - name = '{} ({})'.format(node.value, node.assets_amount) - data = { - 'id': node.key, - 'name': name, - 'title': name, - 'pId': node.parent_key, - 'isParent': True, - 'open': node.is_org_root(), - 'meta': { - 'node': { - "id": node.id, - "key": node.key, - "value": node.value, - }, - 'type': 'node' - } - } - tree_node = TreeNode(**data) - return tree_node - @staticmethod - def parse_asset_to_tree_node(node, asset): - icon_skin = 'file' - platform = asset.platform_base.lower() - if platform == 'windows': - icon_skin = 'windows' - elif platform == 'linux': - icon_skin = 'linux' - parent_id = node.key if node else '' - data = { - 'id': str(asset.id), - 'name': asset.hostname, - 'title': asset.ip, - 'pId': parent_id, - 'isParent': False, - 'open': False, - 'iconSkin': icon_skin, - 'nocheck': not asset.has_protocol('ssh'), - 'meta': { - 'type': 'asset', - 'asset': { - 'id': asset.id, - 'hostname': asset.hostname, - 'ip': asset.ip, - 'protocols': asset.protocols_as_list, - 'platform': asset.platform_base, - 'domain': asset.domain_id, - 'org_name': asset.org_name, - 'org_id': asset.org_id - }, - } - } - tree_node = TreeNode(**data) - return tree_node +def get_asset_system_users_id_with_actions_by_group(group: UserGroup, asset: Asset): + queryset = AssetPermission.objects.filter( + user_groups=group + ) + return get_asset_system_users_id_with_actions(queryset, asset) diff --git a/apps/perms/utils/user_node_tree.py b/apps/perms/utils/user_node_tree.py new file mode 100644 index 000000000..8612bc3e5 --- /dev/null +++ b/apps/perms/utils/user_node_tree.py @@ -0,0 +1,334 @@ +from functools import reduce, wraps +from operator import or_, and_ +from uuid import uuid4 +import threading +import inspect + +from django.conf import settings +from django.db.models import F, Q, Value, BooleanField + +from common.utils import get_logger +from common.const.distributed_lock_key import UPDATE_MAPPING_NODE_TASK_LOCK_KEY +from orgs.utils import tmp_to_root_org +from common.utils.timezone import dt_formater, now +from assets.models import Node, Asset +from django.db.transaction import atomic +from orgs import lock +from perms.models import UserGrantedMappingNode, RebuildUserTreeTask +from users.models import User + +logger = get_logger(__name__) + +ADD = 'add' +REMOVE = 'remove' + + +# 使用场景 +# Asset.objects.filter(get_granted_q(user)) +def get_granted_q(user: User): + _now = now() + return reduce(and_, ( + Q(granted_by_permissions__date_start__lt=_now), + Q(granted_by_permissions__date_expired__gt=_now), + Q(granted_by_permissions__is_active=True), + (Q(granted_by_permissions__users=user) | Q(granted_by_permissions__user_groups__users=user)) + )) + + +TMP_GRANTED_FIELD = '_granted' +TMP_ASSET_GRANTED_FIELD = '_asset_granted' +TMP_GRANTED_ASSETS_AMOUNT_FIELD = '_granted_assets_amount' + + +# 使用场景 +# `Node.objects.annotate(**node_annotate_mapping_node)` +node_annotate_mapping_node = { + TMP_GRANTED_FIELD: F('mapping_nodes__granted'), + TMP_ASSET_GRANTED_FIELD: F('mapping_nodes__asset_granted'), + TMP_GRANTED_ASSETS_AMOUNT_FIELD: F('mapping_nodes__assets_amount') +} + + +# 使用场景 +# `Node.objects.annotate(**node_annotate_set_granted)` +node_annotate_set_granted = { + TMP_GRANTED_FIELD: Value(True, output_field=BooleanField()), +} + + +def is_granted(node): + return getattr(node, TMP_GRANTED_FIELD, False) + + +def is_asset_granted(node): + return getattr(node, TMP_ASSET_GRANTED_FIELD, False) + + +def get_granted_assets_amount(node): + return getattr(node, TMP_GRANTED_ASSETS_AMOUNT_FIELD, 0) + + +def set_granted(obj): + setattr(obj, TMP_GRANTED_FIELD, True) + + +def set_asset_granted(obj): + setattr(obj, TMP_ASSET_GRANTED_FIELD, True) + + +VALUE_TEMPLATE = '{stage}:{rand_str}:thread:{thread_name}:{thread_id}:{now}' + + +def _generate_value(stage=lock.DOING): + cur_thread = threading.current_thread() + + return VALUE_TEMPLATE.format( + stage=stage, + thread_name=cur_thread.name, + thread_id=cur_thread.ident, + now=dt_formater(now()), + rand_str=uuid4() + ) + + +def build_user_mapping_node_lock(func): + @wraps(func) + def wrapper(*args, **kwargs): + call_args = inspect.getcallargs(func, *args, **kwargs) + user = call_args.get('user') + if user is None or not isinstance(user, User): + raise ValueError('You function must have `user` argument') + + key = UPDATE_MAPPING_NODE_TASK_LOCK_KEY.format(user_id=user.id) + doing_value = _generate_value() + commiting_value = _generate_value(stage=lock.COMMITING) + + try: + locked = lock.acquire(key, doing_value, timeout=600) + if not locked: + logger.error(f'update_mapping_node_task_locked_failed for user: {user.id}') + raise lock.SomeoneIsDoingThis + + with atomic(savepoint=False): + func(*args, **kwargs) + ok = lock.change_lock_state_to_commiting(key, doing_value, commiting_value) + if not ok: + logger.error(f'update_mapping_node_task_timeout for user: {user.id}') + raise lock.Timeout + finally: + lock.release(key, commiting_value, doing_value) + return wrapper + + +@build_user_mapping_node_lock +def rebuild_user_mapping_nodes_if_need_with_lock(user: User): + tasks = RebuildUserTreeTask.objects.filter(user=user) + if tasks: + tasks.delete() + rebuild_user_mapping_nodes(user) + + +@build_user_mapping_node_lock +def rebuild_user_mapping_nodes_with_lock(user: User): + rebuild_user_mapping_nodes(user) + + +@tmp_to_root_org() +def compute_tmp_mapping_node_from_perm(user: User): + node_only_fields = ('id', 'key', 'parent_key', 'assets_amount') + + # 查询直接授权节点 + nodes = Node.objects.filter( + get_granted_q(user) + ).distinct().only(*node_only_fields) + granted_key_set = {_node.key for _node in nodes} + + def _has_ancestor_granted(node): + """ + 判断一个节点是否有授权过的祖先节点 + """ + ancestor_keys = set(node.get_ancestor_keys()) + return ancestor_keys & granted_key_set + + key2leaf_nodes_mapper = {} + + # 给授权节点设置 _granted 标识，同时去重 + for _node in nodes: + if _has_ancestor_granted(_node): + continue + + if _node.key not in key2leaf_nodes_mapper: + set_granted(_node) + key2leaf_nodes_mapper[_node.key] = _node + + # 查询授权资产关联的节点设置 + def process_direct_granted_assets(): + # 查询直接授权资产 + asset_ids = Asset.objects.filter( + get_granted_q(user) + ).distinct().values_list('id', flat=True) + # 查询授权资产关联的节点设置 + granted_asset_nodes = Node.objects.filter( + assets__id__in=asset_ids + ).distinct().only(*node_only_fields) + + # 给资产授权关联的节点设置 _asset_granted 标识，同时去重 + for _node in granted_asset_nodes: + if _has_ancestor_granted(_node): + continue + + if _node.key not in key2leaf_nodes_mapper: + key2leaf_nodes_mapper[_node.key] = _node + set_asset_granted(key2leaf_nodes_mapper[_node.key]) + + if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + process_direct_granted_assets() + + leaf_nodes = key2leaf_nodes_mapper.values() + + # 计算所有祖先节点 + ancestor_keys = set() + for _node in leaf_nodes: + ancestor_keys.update(_node.get_ancestor_keys()) + + # 从祖先节点 key 中去掉同时也是叶子节点的 key + ancestor_keys -= key2leaf_nodes_mapper.keys() + # 查出祖先节点 + ancestors = Node.objects.filter(key__in=ancestor_keys).only(*node_only_fields) + return [*leaf_nodes, *ancestors] + + +def create_mapping_nodes(user, nodes, clear=True): + to_create = [] + for node in nodes: + _granted = getattr(node, TMP_GRANTED_FIELD, False) + _asset_granted = getattr(node, TMP_ASSET_GRANTED_FIELD, False) + _granted_assets_amount = getattr(node, TMP_GRANTED_ASSETS_AMOUNT_FIELD, 0) + to_create.append(UserGrantedMappingNode( + user=user, + node=node, + key=node.key, + parent_key=node.parent_key, + granted=_granted, + asset_granted=_asset_granted, + assets_amount=_granted_assets_amount, + )) + + if clear: + UserGrantedMappingNode.objects.filter(user=user).delete() + UserGrantedMappingNode.objects.bulk_create(to_create) + + +def set_node_granted_assets_amount(user, node): + """ + 不依赖`UserGrantedMappingNode`直接查询授权计算资产数量 + """ + _granted = getattr(node, TMP_GRANTED_FIELD, False) + if _granted: + assets_amount = node.assets_amount + else: + assets_amount = count_node_all_granted_assets(user, node.key) + setattr(node, TMP_GRANTED_ASSETS_AMOUNT_FIELD, assets_amount) + + +def rebuild_user_mapping_nodes(user): + tmp_nodes = compute_tmp_mapping_node_from_perm(user) + for _node in tmp_nodes: + set_node_granted_assets_amount(user, _node) + create_mapping_nodes(user, tmp_nodes) + + +def get_node_all_granted_assets(user: User, key): + """ + 此算法依据 `UserGrantedMappingNode` 的数据查询 + 1. 查询该节点下的直接授权节点 + 2. 查询该节点下授权资产关联的节点 + """ + + assets = Asset.objects.none() + + # 查询该节点下的授权节点 + granted_mapping_nodes = UserGrantedMappingNode.objects.filter( + user=user, + granted=True, + ).filter(Q(key__startswith=f'{key}:') | Q(key=key)) + + # 根据授权节点构建资产查询条件 + granted_nodes_qs = [] + for _node in granted_mapping_nodes: + granted_nodes_qs.append(Q(nodes__key__startswith=f'{_node.key}:')) + granted_nodes_qs.append(Q(nodes__key=_node.key)) + + # 查询该节点下的资产授权节点 + only_asset_granted_mapping_nodes = UserGrantedMappingNode.objects.filter( + user=user, + asset_granted=True, + granted=False, + ).filter(Q(key__startswith=f'{key}:') | Q(key=key)) + + # 根据资产授权节点构建查询 + only_asset_granted_nodes_qs = [] + for _node in only_asset_granted_mapping_nodes: + only_asset_granted_nodes_qs.append(Q(nodes__id=_node.node_id)) + + q = [] + if granted_nodes_qs: + q.append(reduce(or_, granted_nodes_qs)) + + if only_asset_granted_nodes_qs: + only_asset_granted_nodes_q = reduce(or_, only_asset_granted_nodes_qs) + only_asset_granted_nodes_q &= get_granted_q(user) + q.append(only_asset_granted_nodes_q) + + if q: + assets = Asset.objects.filter(reduce(or_, q)).distinct() + return assets + + +def get_node_all_granted_assets_from_perm(user: User, key): + """ + 此算法依据 `AssetPermission` 的数据查询 + 1. 查询该节点下的直接授权节点 + 2. 查询该节点下授权资产关联的节点 + """ + granted_q = get_granted_q(user) + + granted_nodes = Node.objects.filter( + Q(key__startswith=f'{key}:') | Q(key=key) + ).filter(granted_q).distinct() + + # 直接授权资产查询条件 + granted_asset_filter_q = (Q(nodes__key__startswith=f'{key}:') | Q(nodes__key=key)) & granted_q + + # 根据授权节点构建资产查询条件 + q = granted_asset_filter_q + for _node in granted_nodes: + q |= Q(nodes__key__startswith=f'{_node.key}:') + q |= Q(nodes__key=_node.key) + + asset_qs = Asset.objects.filter(q).distinct() + return asset_qs + + +def count_node_all_granted_assets(user: User, key): + return get_node_all_granted_assets_from_perm(user, key).count() + + +def get_ungranted_node_children(user, key=''): + """ + 获取用户授权树中未授权节点的子节点 + 只匹配在 `UserGrantedMappingNode` 中存在的节点 + """ + nodes = Node.objects.filter( + mapping_nodes__user=user, + parent_key=key + ).annotate( + _granted_assets_amount=F('mapping_nodes__assets_amount'), + _granted=F('mapping_nodes__granted') + ).distinct() + + # 设置节点授权资产数量 + for _node in nodes: + if not is_granted(_node): + _node.assets_amount = get_granted_assets_amount(_node) + return nodes diff --git a/apps/static/css/plugins/ztree/awesomeStyle/fa.less b/apps/static/css/plugins/ztree/awesomeStyle/fa.less index 3714884a7..39d46ac1e 100644 --- a/apps/static/css/plugins/ztree/awesomeStyle/fa.less +++ b/apps/static/css/plugins/ztree/awesomeStyle/fa.less @@ -145,7 +145,7 @@ @fa-twitter: "\f099"; @fa-facebook: "\f09a"; @fa-github: "\f09b"; -@fa-unlock: "\f09c"; +@fa-release: "\f09c"; @fa-credit-card: "\f09d"; @fa-rss: "\f09e"; @fa-hdd-o: "\f0a0"; @@ -283,7 +283,7 @@ @fa-html5: "\f13b"; @fa-css3: "\f13c"; @fa-anchor: "\f13d"; -@fa-unlock-alt: "\f13e"; +@fa-release-alt: "\f13e"; @fa-bullseye: "\f140"; @fa-ellipsis-h: "\f141"; @fa-ellipsis-v: "\f142"; diff --git a/apps/static/fonts/fontawesome-webfont.svg b/apps/static/fonts/fontawesome-webfont.svg index 855c845e5..c384a818e 100755 --- a/apps/static/fonts/fontawesome-webfont.svg +++ b/apps/static/fonts/fontawesome-webfont.svg @@ -537,7 +537,7 @@ q-93 26 -192 26t-192 -26q-16 11 -42.5 27t-83.5 38.5t-85 13.5q-45 -113 -8 -204q-7 t-5 -11.5t9 -14t13 -12l7 -5q22 -10 43.5 -38t31.5 -51l10 -23q13 -38 44 -61.5t67 -30t69.5 -7t55.5 3.5l23 4q0 -38 0.5 -88.5t0.5 -54.5q0 -18 -13 -30t-40 -7q-232 77 -378.5 277.5t-146.5 451.5q0 209 103 385.5t279.5 279.5t385.5 103zM291 305q3 7 -7 12 q-10 3 -13 -2q-3 -7 7 -12q9 -6 13 2zM322 271q7 5 -2 16q-10 9 -16 3q-7 -5 2 -16q10 -10 16 -3zM352 226q9 7 0 19q-8 13 -17 6q-9 -5 0 -18t17 -7zM394 184q8 8 -4 19q-12 12 -20 3q-9 -8 4 -19q12 -12 20 -3zM451 159q3 11 -13 16q-15 4 -19 -7t13 -15q15 -6 19 6z M514 154q0 13 -17 11q-16 0 -16 -11q0 -13 17 -11q16 0 16 11zM572 164q-2 11 -18 9q-16 -3 -14 -15t18 -8t14 14z" /> -