From dbb441dfb6e19043b5648177f34ea6fb128d0a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=80=81=E5=B9=BF?= Date: Wed, 11 Sep 2019 18:52:42 +0800 Subject: [PATCH] Perm cache (#3211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] 修改用户树的cache * [Update] 修改用户树缓存 * [Update] telnet (beta) => telnet --- .../migrations/0038_auto_20190911_1634.py | 23 +++++ apps/assets/models/asset.py | 2 +- apps/assets/models/node.py | 13 ++- apps/assets/models/user.py | 2 +- apps/assets/signals_handler.py | 4 +- .../assets/templates/assets/_system_user.html | 4 +- apps/assets/utils.py | 1 + apps/common/signals_handlers.py | 5 +- apps/perms/api/user_permission.py | 54 ++++++----- apps/perms/signals_handler.py | 39 +++++--- apps/perms/utils/asset_permission.py | 89 +++++++++++-------- 11 files changed, 150 insertions(+), 86 deletions(-) create mode 100644 apps/assets/migrations/0038_auto_20190911_1634.py diff --git a/apps/assets/migrations/0038_auto_20190911_1634.py b/apps/assets/migrations/0038_auto_20190911_1634.py new file mode 100644 index 000000000..0533f5baf --- /dev/null +++ b/apps/assets/migrations/0038_auto_20190911_1634.py @@ -0,0 +1,23 @@ +# Generated by Django 2.1.7 on 2019-09-11 08:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0037_auto_20190724_2002'), + ] + + operations = [ + migrations.AlterField( + model_name='asset', + name='protocol', + field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('telnet', 'telnet'), ('vnc', 'vnc')], default='ssh', max_length=128, verbose_name='Protocol'), + ), + migrations.AlterField( + model_name='systemuser', + name='protocol', + field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('telnet', 'telnet'), ('vnc', 'vnc')], default='ssh', max_length=16, verbose_name='Protocol'), + ), + ] diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 06c3b18d8..2d3b871b5 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -57,7 +57,7 @@ class ProtocolsMixin: PROTOCOL_CHOICES = ( (PROTOCOL_SSH, 'ssh'), (PROTOCOL_RDP, 'rdp'), - (PROTOCOL_TELNET, 'telnet (beta)'), + (PROTOCOL_TELNET, 'telnet'), (PROTOCOL_VNC, 'vnc'), ) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 2594d2763..89a4b6a0f 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -11,7 +11,7 @@ from django.utils.translation import ugettext from django.core.cache import cache from orgs.mixins.models import OrgModelMixin, OrgManager -from orgs.utils import set_current_org, get_current_org, tmp_to_root_org, tmp_to_org +from orgs.utils import set_current_org, get_current_org, tmp_to_org from orgs.models import Organization @@ -398,13 +398,24 @@ class Node(OrgModelMixin, SomeNodesMixin, TreeMixin, FamilyMixin, FullValueMixin def level(self): return len(self.key.split(':')) + @staticmethod + def refresh_user_tree_cache(): + """ + 当节点-节点关系，节点-资产关系发生变化时，应该刷新用户授权树缓存 + :return: + """ + from perms.utils.asset_permission import AssetPermissionUtilV2 + AssetPermissionUtilV2.expire_all_user_tree_cache() + @classmethod def refresh_nodes(cls): cls.refresh_tree() + cls.refresh_user_tree_cache() @classmethod def refresh_assets(cls): cls.refresh_node_assets() + cls.refresh_user_tree_cache() def as_tree_node(self): from common.tree import TreeNode diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index ce2af149b..361d770c5 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -96,7 +96,7 @@ class SystemUser(AssetUser): PROTOCOL_CHOICES = ( (PROTOCOL_SSH, 'ssh'), (PROTOCOL_RDP, 'rdp'), - (PROTOCOL_TELNET, 'telnet (beta)'), + (PROTOCOL_TELNET, 'telnet'), (PROTOCOL_VNC, 'vnc'), ) diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 03b14f79b..f4ff2ce04 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -108,7 +108,7 @@ def on_system_user_nodes_change(sender, instance=None, action=None, model=None, else: nodes_keys = [instance.key] system_users = queryset - assets = Node.get_nodes_all_assets(nodes_keys) + assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True) for system_user in system_users: system_user.assets.add(*tuple(assets)) @@ -132,7 +132,7 @@ def on_asset_nodes_add(sender, instance=None, action='', model=None, pk_set=None if action != "post_add": return logger.debug("Assets node add signal recv: {}".format(action)) - queryset = model.objects.filter(pk__in=pk_set) + queryset = model.objects.filter(pk__in=pk_set).values_list('id', flat=True) if model == Node: nodes = queryset assets = [instance] diff --git a/apps/assets/templates/assets/_system_user.html b/apps/assets/templates/assets/_system_user.html index 1f34f3f55..078877cfa 100644 --- a/apps/assets/templates/assets/_system_user.html +++ b/apps/assets/templates/assets/_system_user.html @@ -119,7 +119,7 @@ function autoLoginModeProtocol() { $(sudo_id).closest('.form-group').addClass('hidden'); $(shell_id).closest('.form-group').addClass('hidden'); } - else if (protocol === 'telnet (beta)') { + else if (protocol === 'telnet') { $('.auth-fields').removeClass('hidden'); $(auto_generate_key).closest('.form-group').addClass('hidden'); $(private_key_id).closest('.form-group').addClass('hidden'); @@ -165,7 +165,7 @@ function manualLoginModeProtocol() { $(sudo_id).closest('.form-group').addClass('hidden'); $(shell_id).closest('.form-group').addClass('hidden'); } - else if (protocol === 'telnet (beta)') { + else if (protocol === 'telnet') { $('.auth-fields').addClass('hidden'); $(auto_generate_key).closest('.form-group').addClass('hidden'); $(password_id).closest('.form-group').addClass('hidden'); diff --git a/apps/assets/utils.py b/apps/assets/utils.py index b94e84f26..0e73ea088 100644 --- a/apps/assets/utils.py +++ b/apps/assets/utils.py @@ -70,6 +70,7 @@ class TreeService(Tree): @classmethod @timeit def new(cls): + print("Call new") from .models import Node from orgs.utils import tmp_to_root_org diff --git a/apps/common/signals_handlers.py b/apps/common/signals_handlers.py index 8a8463006..91901676d 100644 --- a/apps/common/signals_handlers.py +++ b/apps/common/signals_handlers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # import re +import os from collections import defaultdict from django.conf import settings from django.dispatch import receiver @@ -12,8 +13,8 @@ from common.utils import get_logger from .local import thread_local pattern = re.compile(r'FROM `(\w+)`') -# logger = logging.getLogger('jmsdb') logger = get_logger(__name__) +DEBUG_DB_QUERY = os.environ.get('DEBUG_DB_QUERY', '0') == '1' class Counter: @@ -57,7 +58,7 @@ def on_request_finished_release_local(sender, **kwargs): thread_local.__release_local__() -if settings.DEBUG: +if settings.DEBUG and DEBUG_DB_QUERY: request_finished.connect(on_request_finished_logging_db_query) diff --git a/apps/perms/api/user_permission.py b/apps/perms/api/user_permission.py index f19fc56d8..7c7e513cf 100644 --- a/apps/perms/api/user_permission.py +++ b/apps/perms/api/user_permission.py @@ -45,13 +45,13 @@ class UserPermissionMixin: def initial(self, *args, **kwargs): super().initial(*args, *kwargs) - self.obj = self.get_object() + self.obj = self.get_obj() def get(self, request, *args, **kwargs): set_to_root_org() return super().get(request, *args, **kwargs) - def get_object(self): + def get_obj(self): user_id = self.kwargs.get('pk', '') if user_id: user = get_object_or_404(User, id=user_id) @@ -65,12 +65,13 @@ class UserPermissionMixin: return super().get_permissions() -class UserNodePermissionMixin(UserPermissionMixin): +class UserAssetPermissionMixin(UserPermissionMixin): util = None def initial(self, *args, **kwargs): super().initial(*args, *kwargs) - self.util = AssetPermissionUtilV2(self.obj) + cache_policy = self.request.query_params.get('cache_policy', '0') + self.util = AssetPermissionUtilV2(self.obj, cache_policy=cache_policy) class UserNodeTreeMixin: @@ -125,7 +126,7 @@ class UserAssetTreeMixin: return super().get_serializer(queryset, many=many, **kwargs) -class UserGrantedAssetsApi(UserPermissionMixin, ListAPIView): +class UserGrantedAssetsApi(UserAssetPermissionMixin, ListAPIView): permission_classes = (IsOrgAdminOrAppUser,) serializer_class = serializers.AssetGrantedSerializer only_fields = serializers.AssetGrantedSerializer.Meta.only_fields @@ -151,8 +152,7 @@ class UserGrantedAssetsApi(UserPermissionMixin, ListAPIView): return queryset def get_queryset(self): - util = AssetPermissionUtilV2(self.obj) - queryset = util.get_assets().only(*self.only_fields) + queryset = self.util.get_assets().only(*self.only_fields) return queryset @@ -165,13 +165,12 @@ class UserGrantedNodeAssetsApi(UserGrantedAssetsApi): node_id = self.kwargs.get("node_id") node = get_object_or_404(Node, pk=node_id) deep = self.request.query_params.get("all", "0") == "1" - util = AssetPermissionUtilV2(self.obj) - queryset = util.get_nodes_assets(node, deep=deep)\ + queryset = self.util.get_nodes_assets(node, deep=deep)\ .only(*self.only_fields) return queryset -class UserGrantedNodesApi(UserNodePermissionMixin, ListAPIView): +class UserGrantedNodesApi(UserAssetPermissionMixin, ListAPIView): """ 查询用户授权的所有节点的API """ @@ -267,55 +266,56 @@ class UserGrantedNodeChildrenWithAssetsAsTreeApi(UserGrantedNodeChildrenAsTreeAp return _queryset -class GetUserAssetPermissionActionsApi(RetrieveAPIView): +class GetUserAssetPermissionActionsApi(UserAssetPermissionMixin, RetrieveAPIView): permission_classes = (IsOrgAdminOrAppUser,) serializer_class = serializers.ActionsSerializer - def get_object(self): + def get_obj(self): user_id = self.request.query_params.get('user_id', '') + user = get_object_or_404(User, id=user_id) + return user + + def get_object(self): asset_id = self.request.query_params.get('asset_id', '') system_id = self.request.query_params.get('system_user_id', '') try: - user_id = uuid.UUID(user_id) asset_id = uuid.UUID(asset_id) system_id = uuid.UUID(system_id) except ValueError: return Response({'msg': False}, status=403) - user = get_object_or_404(User, id=user_id) asset = get_object_or_404(Asset, id=asset_id) system_user = get_object_or_404(SystemUser, id=system_id) - util = AssetPermissionUtilV2(user) - system_users_actions = util.get_asset_system_users_with_actions(asset) + system_users_actions = self.util.get_asset_system_users_with_actions(asset) actions = system_users_actions.get(system_user) return {"actions": actions} -class ValidateUserAssetPermissionApi(APIView): +class ValidateUserAssetPermissionApi(UserAssetPermissionMixin, APIView): permission_classes = (IsOrgAdminOrAppUser,) + + def get_obj(self): + user_id = self.request.query_params.get('user_id', '') + user = get_object_or_404(User, id=user_id) + return user def get(self, request, *args, **kwargs): - user_id = request.query_params.get('user_id', '') asset_id = request.query_params.get('asset_id', '') system_id = request.query_params.get('system_user_id', '') action_name = request.query_params.get('action_name', '') - cache_policy = self.request.query_params.get("cache_policy", '0') try: - user_id = uuid.UUID(user_id) asset_id = uuid.UUID(asset_id) system_id = uuid.UUID(system_id) except ValueError: return Response({'msg': False}, status=403) - user = get_object_or_404(User, id=user_id) asset = get_object_or_404(Asset, id=asset_id) system_user = get_object_or_404(SystemUser, id=system_id) - util = AssetPermissionUtilV2(user) - system_users_actions = util.get_asset_system_users_with_actions(asset) + system_users_actions = self.util.get_asset_system_users_with_actions(asset) actions = system_users_actions.get(system_user) if action_name in Action.value_to_choices(actions): return Response({'msg': True}, status=200) @@ -326,21 +326,19 @@ class RefreshAssetPermissionCacheApi(RetrieveAPIView): permission_classes = (IsOrgAdmin,) def retrieve(self, request, *args, **kwargs): - # expire all cache - # AssetPermissionUtil.expire_all_cache() + AssetPermissionUtilV2.expire_all_user_tree_cache() return Response({'msg': True}, status=200) -class UserGrantedAssetSystemUsersApi(UserPermissionMixin, ListAPIView): +class UserGrantedAssetSystemUsersApi(UserAssetPermissionMixin, ListAPIView): permission_classes = (IsOrgAdminOrAppUser,) serializer_class = serializers.AssetSystemUserSerializer only_fields = serializers.AssetSystemUserSerializer.Meta.only_fields def get_queryset(self): - util = AssetPermissionUtilV2(self.obj) asset_id = self.kwargs.get('asset_id') asset = get_object_or_404(Asset, id=asset_id) - system_users_with_actions = util.get_asset_system_users_with_actions(asset) + system_users_with_actions = self.util.get_asset_system_users_with_actions(asset) system_users = [] for system_user, actions in system_users_with_actions.items(): system_user.actions = actions diff --git a/apps/perms/signals_handler.py b/apps/perms/signals_handler.py index 3736e3a84..e23bc1f8c 100644 --- a/apps/perms/signals_handler.py +++ b/apps/perms/signals_handler.py @@ -6,25 +6,38 @@ from django.dispatch import receiver from common.utils import get_logger from common.decorator import on_transaction_commit from .models import AssetPermission +from .utils.asset_permission import AssetPermissionUtilV2 logger = get_logger(__file__) +permission_m2m_senders = ( + AssetPermission.nodes.through, + AssetPermission.assets.through, + AssetPermission.users.through, + AssetPermission.user_groups.through, +) + -@receiver(post_save, sender=AssetPermission, dispatch_uid="my_unique_identifier") @on_transaction_commit -def on_permission_created(sender, instance=None, created=False, **kwargs): - pass +def on_permission_m2m_change(sender, action='', **kwargs): + if not action.startswith('post'): + return + logger.debug('Asset permission m2m changed, refresh user tree cache') + AssetPermissionUtilV2.expire_all_user_tree_cache() -@receiver(post_save, sender=AssetPermission) -def on_permission_update(sender, **kwargs): - pass +for sender in permission_m2m_senders: + m2m_changed.connect(on_permission_m2m_change, sender=sender) -@receiver(post_delete, sender=AssetPermission) -def on_permission_delete(sender, **kwargs): - pass +@receiver([post_save, post_delete], sender=AssetPermission) +@on_transaction_commit +def on_permission_change(sender, action='', **kwargs): + logger.debug('Asset permission changed, refresh user tree cache') + AssetPermissionUtilV2.expire_all_user_tree_cache() + +# Todo: 检查授权规则到期，从而修改授权规则 @receiver(m2m_changed, sender=AssetPermission.nodes.through) @@ -34,7 +47,7 @@ def on_permission_nodes_changed(sender, instance=None, action='', **kwargs): if isinstance(instance, AssetPermission): logger.debug("Asset permission nodes change signal received") nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) - system_users = instance.system_users.all() + system_users = instance.system_users.all().values_list('id', flat=True) for system_user in system_users: system_user.nodes.add(*tuple(nodes)) @@ -46,7 +59,7 @@ def on_permission_assets_changed(sender, instance=None, action='', **kwargs): if isinstance(instance, AssetPermission): logger.debug("Asset permission assets change signal received") assets = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) - system_users = instance.system_users.all() + system_users = instance.system_users.all().values_list('id', flat=True) for system_user in system_users: system_user.assets.add(*tuple(assets)) @@ -58,8 +71,8 @@ def on_permission_system_users_changed(sender, instance=None, action='', **kwarg if isinstance(instance, AssetPermission): system_users = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) logger.debug("Asset permission system_users change signal received") - assets = instance.assets.all() - nodes = instance.nodes.all() + assets = instance.assets.all().values_list('id', flat=True) + nodes = instance.nodes.all().values_list('id', flat=True) for system_user in system_users: system_user.nodes.add(*tuple(nodes)) system_user.assets.add(*tuple(assets)) diff --git a/apps/perms/utils/asset_permission.py b/apps/perms/utils/asset_permission.py index bf1842c57..8e9f4a6f3 100644 --- a/apps/perms/utils/asset_permission.py +++ b/apps/perms/utils/asset_permission.py @@ -59,7 +59,55 @@ def get_system_user_permissions(system_user): ) -class AssetPermissionUtilV2: +class AssetPermissionUtilCacheMixin: + user_tree_cache_key = 'USER_PERM_TREE_{}' + user_tree_cache_ttl = settings.ASSETS_PERM_CACHE_TIME + user_tree_cache_enable = settings.ASSETS_PERM_CACHE_ENABLE + cache_policy = '0' + obj_id = '' + + def expire_user_tree_cache(self): + key = self.user_tree_cache_key.format(self.obj_id) + cache.delete(key) + + @classmethod + def expire_all_user_tree_cache(cls): + key = cls.user_tree_cache_key.format('*') + cache.delete_pattern(key) + + def set_user_tree_to_cache(self, user_tree): + data = pickle.dumps(user_tree) + key = self.user_tree_cache_key.format(self.obj_id) + cache.set(key, data, self.user_tree_cache_ttl) + + def get_user_tree_from_cache(self): + key = self.user_tree_cache_key.format(self.obj_id) + data = cache.get(key) + if not data: + return None + user_tree = pickle.loads(data) + return user_tree + + def get_user_tree_from_cache_if_need(self): + if not self.user_tree_cache_enable: + return None + if self.cache_policy == '1': + return self.get_user_tree_from_cache() + elif self.cache_policy == '2': + self.expire_user_tree_cache() + return None + else: + return None + + def set_user_tree_to_cache_if_need(self, user_tree): + if self.cache_policy == '0': + return + if not self.user_tree_cache_enable: + return None + self.set_user_tree_to_cache(user_tree) + + +class AssetPermissionUtilV2(AssetPermissionUtilCacheMixin): get_permissions_map = { "User": get_user_permissions, "UserGroup": get_user_group_permissions, @@ -71,8 +119,6 @@ class AssetPermissionUtilV2: 'id', 'hostname', 'ip', "platform", "domain_id", 'comment', 'is_active', 'os', 'org_id' ) - user_tree_cache_key = 'USER_PERM_TREE_{}' - user_tree_cache_ttl = 3600 def __init__(self, obj, cache_policy='0'): self.object = obj @@ -80,13 +126,8 @@ class AssetPermissionUtilV2: self.obj_id = str(obj.id) self._permissions = None self._permissions_id = None # 标记_permission的唯一值 - self._assets = None self._filter_id = 'None' # 当通过filter更改 permission是标记 self.change_org_if_need() - self.nodes = None - self._nodes = None - self._assets_direct = None - self._nodes_direct = None self._user_tree = None self.full_tree = Node.tree() self.mutex = threading.Lock() @@ -109,31 +150,6 @@ class AssetPermissionUtilV2: def filter_permissions(self, **filters): self._permissions = self.permissions.filter(**filters) - @classmethod - def get_user_tree_from_cache(cls, obj_id): - return None - key = cls.user_tree_cache_key.format(obj_id) - data = cache.get(key) - if not data: - return None - user_tree = pickle.loads(data) - return user_tree - - @classmethod - def expire_user_tree_cache(cls, obj_id): - if obj_id == 'all': - key = cls.user_tree_cache_key.format('*') - cache.delete_pattern(key) - else: - key = cls.user_tree_cache_key.format(obj_id) - cache.delete(key) - - @classmethod - def set_user_tree_to_cache(cls, obj_id, user_tree): - data = pickle.dumps(user_tree) - key = cls.user_tree_cache_key.format(obj_id) - cache.set(key, data, cls.user_tree_cache_ttl) - @property def user_tree(self): return self.get_user_tree() @@ -268,11 +284,11 @@ class AssetPermissionUtilV2: @timeit def get_user_tree(self): + # 使用锁，保证多次获取tree的时候顺序执行，可以使用缓存 with self.mutex: if self._user_tree: return self._user_tree - print(id(self), self._user_tree) - user_tree = self.__class__.get_user_tree_from_cache(self.obj_id) + user_tree = self.get_user_tree_from_cache_if_need() if user_tree: self._user_tree = user_tree return user_tree @@ -286,7 +302,8 @@ class AssetPermissionUtilV2: self.add_single_assets_node_to_user_tree(user_tree) self.parse_user_tree_to_full_tree(user_tree) self.add_empty_node_if_need(user_tree) - self.__class__.set_user_tree_to_cache(self.obj_id, user_tree) + self.set_user_tree_to_cache_if_need(user_tree) + self._user_tree = user_tree return user_tree # Todo: 是否可以获取多个资产的系统用户