From 109db8886b20d40df9ab546b5dfad76376ffc8fb Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 28 Jul 2022 19:27:42 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E8=BF=98=E5=8E=9F=E5=9B=9E=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/applications/models/application.py | 2 +- apps/assets/api/__init__.py | 2 - apps/assets/api/asset.py | 4 +- apps/assets/api/cmd_filter.py | 85 --------- apps/assets/models/__init__.py | 1 + apps/assets/models/cmd_filter.py | 226 ++++++++++++++++++++++++ apps/assets/serializers/__init__.py | 1 - apps/assets/serializers/cmd_filter.py | 110 ------------ apps/assets/signal_handlers/asset.py | 17 +- apps/assets/tasks/__init__.py | 2 - apps/assets/tasks/common.py | 36 ---- apps/assets/urls/api_urls.py | 9 +- apps/orgs/signal_handlers/common.py | 3 +- 13 files changed, 240 insertions(+), 258 deletions(-) delete mode 100644 apps/assets/api/cmd_filter.py create mode 100644 apps/assets/models/cmd_filter.py delete mode 100644 apps/assets/serializers/cmd_filter.py diff --git a/apps/applications/models/application.py b/apps/applications/models/application.py index 000094b4f..af1e27c2d 100644 --- a/apps/applications/models/application.py +++ b/apps/applications/models/application.py @@ -9,7 +9,7 @@ from orgs.mixins.models import OrgModelMixin from common.mixins import CommonModelMixin from common.tree import TreeNode from common.utils import is_uuid -from assets.models import Asset +from assets.models import Asset, SystemUser from ..utils import KubernetesTree from .. import const diff --git a/apps/assets/api/__init__.py b/apps/assets/api/__init__.py index e8b06b537..b8cc9e7d8 100644 --- a/apps/assets/api/__init__.py +++ b/apps/assets/api/__init__.py @@ -1,11 +1,9 @@ from .mixin import * from .asset import * from .label import * -from .system_user import * from .accounts import * from .node import * from .domain import * -from .cmd_filter import * from .gathered_user import * from .favorite_asset import * from .account_backup import * diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index a930bfd41..d8ff1547f 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -19,8 +19,8 @@ from assets.api import FilterAssetByNodeMixin from ..models import Asset, Node, Platform, Gateway from .. import serializers from ..tasks import ( - update_assets_hardware_info_manual, test_assets_connectivity_manual, - test_system_users_connectivity_a_asset, push_system_users_a_asset + update_assets_hardware_info_manual, + test_assets_connectivity_manual, ) from ..filters import FilterAssetByNodeFilterBackend, LabelFilterBackend, IpInFilterBackend diff --git a/apps/assets/api/cmd_filter.py b/apps/assets/api/cmd_filter.py deleted file mode 100644 index 0e09d5c73..000000000 --- a/apps/assets/api/cmd_filter.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from rest_framework.response import Response -from rest_framework.generics import CreateAPIView -from django.shortcuts import get_object_or_404 - -from common.utils import reverse -from common.utils import lazyproperty -from orgs.mixins.api import OrgBulkModelViewSet -from ..models import CommandFilter, CommandFilterRule -from .. import serializers - -__all__ = [ - 'CommandFilterViewSet', 'CommandFilterRuleViewSet', 'CommandConfirmAPI', -] - - -class CommandFilterViewSet(OrgBulkModelViewSet): - model = CommandFilter - filterset_fields = ("name",) - search_fields = filterset_fields - serializer_class = serializers.CommandFilterSerializer - - -class CommandFilterRuleViewSet(OrgBulkModelViewSet): - model = CommandFilterRule - filterset_fields = ('content',) - search_fields = filterset_fields - serializer_class = serializers.CommandFilterRuleSerializer - - def get_queryset(self): - fpk = self.kwargs.get('filter_pk') - if not fpk: - return CommandFilterRule.objects.none() - cmd_filter = get_object_or_404(CommandFilter, pk=fpk) - return cmd_filter.rules.all() - - -class CommandConfirmAPI(CreateAPIView): - serializer_class = serializers.CommandConfirmSerializer - rbac_perms = { - 'POST': 'tickets.add_superticket' - } - - def create(self, request, *args, **kwargs): - ticket = self.create_command_confirm_ticket() - response_data = self.get_response_data(ticket) - return Response(data=response_data, status=200) - - def create_command_confirm_ticket(self): - ticket = self.serializer.cmd_filter_rule.create_command_confirm_ticket( - run_command=self.serializer.data.get('run_command'), - session=self.serializer.session, - cmd_filter_rule=self.serializer.cmd_filter_rule, - org_id=self.serializer.org.id, - ) - return ticket - - @staticmethod - def get_response_data(ticket): - confirm_status_url = reverse( - view_name='api-tickets:super-ticket-status', - kwargs={'pk': str(ticket.id)} - ) - ticket_detail_url = reverse( - view_name='api-tickets:ticket-detail', - kwargs={'pk': str(ticket.id)}, - external=True, api_to_ui=True - ) - ticket_detail_url = '{url}?type={type}'.format(url=ticket_detail_url, type=ticket.type) - ticket_assignees = ticket.current_step.ticket_assignees.all() - return { - 'check_confirm_status': {'method': 'GET', 'url': confirm_status_url}, - 'close_confirm': {'method': 'DELETE', 'url': confirm_status_url}, - 'ticket_detail_url': ticket_detail_url, - 'reviewers': [str(ticket_assignee.assignee) for ticket_assignee in ticket_assignees] - } - - @lazyproperty - def serializer(self): - serializer = self.get_serializer(data=self.request.data) - serializer.is_valid(raise_exception=True) - return serializer - diff --git a/apps/assets/models/__init__.py b/apps/assets/models/__init__.py index 4eeab7b0b..7719ca63a 100644 --- a/apps/assets/models/__init__.py +++ b/apps/assets/models/__init__.py @@ -10,3 +10,4 @@ from .favorite_asset import * from .account import * from .backup import * from .user import * +from .cmd_filter import * diff --git a/apps/assets/models/cmd_filter.py b/apps/assets/models/cmd_filter.py new file mode 100644 index 000000000..c7fa33aae --- /dev/null +++ b/apps/assets/models/cmd_filter.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +# +import uuid +import re + +from django.db import models +from django.db.models import Q +from django.core.validators import MinValueValidator, MaxValueValidator +from django.utils.translation import ugettext_lazy as _ + +from users.models import User, UserGroup +from applications.models import Application +from ..models import SystemUser, Asset + +from common.utils import lazyproperty, get_logger, get_object_or_none +from orgs.mixins.models import OrgModelMixin + +logger = get_logger(__file__) + +__all__ = [ + 'CommandFilter', 'CommandFilterRule' +] + + +class CommandFilter(OrgModelMixin): + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + name = models.CharField(max_length=64, verbose_name=_("Name")) + users = models.ManyToManyField( + 'users.User', related_name='cmd_filters', blank=True, + verbose_name=_("User") + ) + user_groups = models.ManyToManyField( + 'users.UserGroup', related_name='cmd_filters', blank=True, + verbose_name=_("User group"), + ) + assets = models.ManyToManyField( + 'assets.Asset', related_name='cmd_filters', blank=True, + verbose_name=_("Asset") + ) + system_users = models.ManyToManyField( + 'assets.SystemUser', related_name='cmd_filters', blank=True, + verbose_name=_("System user")) + applications = models.ManyToManyField( + 'applications.Application', related_name='cmd_filters', blank=True, + verbose_name=_("Application") + ) + is_active = models.BooleanField(default=True, verbose_name=_('Is active')) + comment = models.TextField(blank=True, default='', verbose_name=_("Comment")) + date_created = models.DateTimeField(auto_now_add=True) + date_updated = models.DateTimeField(auto_now=True) + created_by = models.CharField( + max_length=128, blank=True, default='', verbose_name=_('Created by') + ) + + def __str__(self): + return self.name + + class Meta: + unique_together = [('org_id', 'name')] + verbose_name = _("Command filter") + + +class CommandFilterRule(OrgModelMixin): + TYPE_REGEX = 'regex' + TYPE_COMMAND = 'command' + TYPE_CHOICES = ( + (TYPE_REGEX, _('Regex')), + (TYPE_COMMAND, _('Command')), + ) + + ACTION_UNKNOWN = 10 + + class ActionChoices(models.IntegerChoices): + deny = 0, _('Deny') + allow = 9, _('Allow') + confirm = 2, _('Reconfirm') + + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + filter = models.ForeignKey( + 'CommandFilter', on_delete=models.CASCADE, verbose_name=_("Filter"), related_name='rules' + ) + type = models.CharField(max_length=16, default=TYPE_COMMAND, choices=TYPE_CHOICES, verbose_name=_("Type")) + priority = models.IntegerField( + default=50, verbose_name=_("Priority"), help_text=_("1-100, the lower the value will be match first"), + validators=[MinValueValidator(1), MaxValueValidator(100)] + ) + content = models.TextField(verbose_name=_("Content"), help_text=_("One line one command")) + ignore_case = models.BooleanField(default=True, verbose_name=_('Ignore case')) + action = models.IntegerField(default=ActionChoices.deny, choices=ActionChoices.choices, verbose_name=_("Action")) + # 动作: 附加字段 + # - confirm: 命令复核人 + reviewers = models.ManyToManyField( + 'users.User', related_name='review_cmd_filter_rules', blank=True, + verbose_name=_("Reviewers") + ) + comment = models.CharField(max_length=64, blank=True, default='', verbose_name=_("Comment")) + date_created = models.DateTimeField(auto_now_add=True) + date_updated = models.DateTimeField(auto_now=True) + created_by = models.CharField(max_length=128, blank=True, default='', verbose_name=_('Created by')) + + class Meta: + ordering = ('priority', 'action') + verbose_name = _("Command filter rule") + + @lazyproperty + def pattern(self): + if self.type == 'command': + s = self.construct_command_regex(content=self.content) + else: + s = r'{0}'.format(self.content) + + return s + + @classmethod + def construct_command_regex(cls, content): + regex = [] + content = content.replace('\r\n', '\n') + for _cmd in content.split('\n'): + cmd = re.sub(r'\s+', ' ', _cmd) + cmd = re.escape(cmd) + cmd = cmd.replace('\\ ', '\s+') + + # 有空格就不能 铆钉单词了 + if ' ' in _cmd: + regex.append(cmd) + continue + + if not cmd: + continue + + # 如果是单个字符 + if cmd[-1].isalpha(): + regex.append(r'\b{0}\b'.format(cmd)) + else: + regex.append(r'\b{0}'.format(cmd)) + s = r'{}'.format('|'.join(regex)) + return s + + @staticmethod + def compile_regex(regex, ignore_case): + try: + if ignore_case: + pattern = re.compile(regex, re.IGNORECASE) + else: + pattern = re.compile(regex) + except Exception as e: + error = _('The generated regular expression is incorrect: {}').format(str(e)) + logger.error(error) + return False, error, None + return True, '', pattern + + def match(self, data): + succeed, error, pattern = self.compile_regex(self.pattern, self.ignore_case) + if not succeed: + return self.ACTION_UNKNOWN, '' + + found = pattern.search(data) + if not found: + return self.ACTION_UNKNOWN, '' + + if self.action == self.ActionChoices.allow: + return self.ActionChoices.allow, found.group() + else: + return self.ActionChoices.deny, found.group() + + def __str__(self): + return '{} % {}'.format(self.type, self.content) + + def create_command_confirm_ticket(self, run_command, session, cmd_filter_rule, org_id): + from tickets.const import TicketType + from tickets.models import ApplyCommandTicket + data = { + 'title': _('Command confirm') + ' ({})'.format(session.user), + 'type': TicketType.command_confirm, + 'applicant': session.user_obj, + 'apply_run_user_id': session.user_id, + 'apply_run_asset': str(session.asset), + 'apply_run_system_user_id': session.system_user_id, + 'apply_run_command': run_command[:4090], + 'apply_from_session_id': str(session.id), + 'apply_from_cmd_filter_rule_id': str(cmd_filter_rule.id), + 'apply_from_cmd_filter_id': str(cmd_filter_rule.filter.id), + 'org_id': org_id, + } + ticket = ApplyCommandTicket.objects.create(**data) + assignees = self.reviewers.all() + ticket.open_by_system(assignees) + return ticket + + @classmethod + def get_queryset(cls, user_id=None, user_group_id=None, system_user_id=None, + asset_id=None, application_id=None, org_id=None): + user_groups = [] + user = get_object_or_none(User, pk=user_id) + if user: + user_groups.extend(list(user.groups.all())) + user_group = get_object_or_none(UserGroup, pk=user_group_id) + if user_group: + org_id = user_group.org_id + user_groups.append(user_group) + system_user = get_object_or_none(SystemUser, pk=system_user_id) + asset = get_object_or_none(Asset, pk=asset_id) + application = get_object_or_none(Application, pk=application_id) + q = Q() + if user: + q |= Q(users=user) + if user_groups: + q |= Q(user_groups__in=set(user_groups)) + if system_user: + org_id = system_user.org_id + q |= Q(system_users=system_user) + if asset: + org_id = asset.org_id + q |= Q(assets=asset) + if application: + org_id = application.org_id + q |= Q(applications=application) + if q: + cmd_filters = CommandFilter.objects.filter(q).filter(is_active=True) + if org_id: + cmd_filters = cmd_filters.filter(org_id=org_id) + rule_ids = cmd_filters.values_list('rules', flat=True) + rules = cls.objects.filter(id__in=rule_ids) + else: + rules = cls.objects.none() + return rules diff --git a/apps/assets/serializers/__init__.py b/apps/assets/serializers/__init__.py index 5c684652c..9f41d1020 100644 --- a/apps/assets/serializers/__init__.py +++ b/apps/assets/serializers/__init__.py @@ -6,7 +6,6 @@ from .label import * from .system_user import * from .node import * from .domain import * -from .cmd_filter import * from .gathered_user import * from .favorite_asset import * from .account import * diff --git a/apps/assets/serializers/cmd_filter.py b/apps/assets/serializers/cmd_filter.py deleted file mode 100644 index 9a33dd6fa..000000000 --- a/apps/assets/serializers/cmd_filter.py +++ /dev/null @@ -1,110 +0,0 @@ -# -*- coding: utf-8 -*- -# -import re -from rest_framework import serializers - -from django.utils.translation import ugettext_lazy as _ -from ..models import CommandFilter, CommandFilterRule -from orgs.mixins.serializers import BulkOrgResourceModelSerializer -from orgs.utils import tmp_to_root_org -from common.utils import get_object_or_none, lazyproperty -from terminal.models import Session - - -class CommandFilterSerializer(BulkOrgResourceModelSerializer): - class Meta: - model = CommandFilter - fields_mini = ['id', 'name'] - fields_small = fields_mini + [ - 'org_id', 'org_name', 'is_active', - 'date_created', 'date_updated', - 'comment', 'created_by', - ] - fields_fk = ['rules'] - fields_m2m = ['users', 'user_groups', 'system_users', 'assets', 'applications'] - fields = fields_small + fields_fk + fields_m2m - extra_kwargs = { - 'rules': {'read_only': True}, - 'date_created': {'label': _("Date created")}, - 'date_updated': {'label': _("Date updated")}, - } - - -class CommandFilterRuleSerializer(BulkOrgResourceModelSerializer): - type_display = serializers.ReadOnlyField(source='get_type_display', label=_("Type display")) - action_display = serializers.ReadOnlyField(source='get_action_display', label=_("Action display")) - - class Meta: - model = CommandFilterRule - fields_mini = ['id'] - fields_small = fields_mini + [ - 'type', 'type_display', 'content', 'ignore_case', 'pattern', - 'priority', 'action', 'action_display', 'reviewers', - 'date_created', 'date_updated', 'comment', 'created_by', - ] - fields_fk = ['filter'] - fields = fields_small + fields_fk - extra_kwargs = { - 'date_created': {'label': _("Date created")}, - 'date_updated': {'label': _("Date updated")}, - 'action_display': {'label': _("Action display")}, - 'pattern': {'label': _("Pattern")} - } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.set_action_choices() - - def set_action_choices(self): - from django.conf import settings - action = self.fields.get('action') - if not action: - return - choices = action._choices - if not settings.XPACK_ENABLED: - choices.pop(CommandFilterRule.ActionChoices.confirm, None) - action._choices = choices - - def validate_content(self, content): - tp = self.initial_data.get("type") - if tp == CommandFilterRule.TYPE_COMMAND: - regex = CommandFilterRule.construct_command_regex(content) - else: - regex = content - ignore_case = self.initial_data.get('ignore_case') - succeed, error, pattern = CommandFilterRule.compile_regex(regex, ignore_case) - if not succeed: - raise serializers.ValidationError(error) - return content - - -class CommandConfirmSerializer(serializers.Serializer): - session_id = serializers.UUIDField(required=True, allow_null=False) - cmd_filter_rule_id = serializers.UUIDField(required=True, allow_null=False) - run_command = serializers.CharField(required=True, allow_null=False) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.session = None - self.cmd_filter_rule = None - - def validate_session_id(self, session_id): - self.session = self.validate_object_exist(Session, session_id) - return session_id - - def validate_cmd_filter_rule_id(self, cmd_filter_rule_id): - self.cmd_filter_rule = self.validate_object_exist(CommandFilterRule, cmd_filter_rule_id) - return cmd_filter_rule_id - - @staticmethod - def validate_object_exist(model, field_id): - with tmp_to_root_org(): - obj = get_object_or_none(model, id=field_id) - if not obj: - error = '{} Model object does not exist'.format(model.__name__) - raise serializers.ValidationError(error) - return obj - - @lazyproperty - def org(self): - return self.session.org diff --git a/apps/assets/signal_handlers/asset.py b/apps/assets/signal_handlers/asset.py index a7e466c9b..5aac26319 100644 --- a/apps/assets/signal_handlers/asset.py +++ b/apps/assets/signal_handlers/asset.py @@ -8,11 +8,10 @@ from django.dispatch import receiver from common.const.signals import POST_ADD, POST_REMOVE, PRE_REMOVE from common.utils import get_logger from common.decorator import on_transaction_commit -from assets.models import Asset, SystemUser, Node +from assets.models import Asset, Node from assets.tasks import ( update_assets_hardware_info_util, test_asset_connectivity_util, - push_system_user_to_assets, ) logger = get_logger(__file__) @@ -77,15 +76,15 @@ def on_asset_nodes_add(instance, action, reverse, pk_set, **kwargs): nodes_ancestors_keys.update(Node.get_node_ancestor_keys(node, with_self=True)) # 查询所有祖先节点关联的系统用户，都是要跟资产建立关系的 - system_user_ids = SystemUser.objects.filter( - nodes__key__in=nodes_ancestors_keys - ).distinct().values_list('id', flat=True) + # system_user_ids = SystemUser.objects.filter( + # nodes__key__in=nodes_ancestors_keys + # ).distinct().values_list('id', flat=True) # 查询所有已存在的关系 - m2m_model = SystemUser.assets.through - exist = set(m2m_model.objects.filter( - systemuser_id__in=system_user_ids, asset_id__in=asset_ids - ).values_list('systemuser_id', 'asset_id')) + # m2m_model = SystemUser.assets.through + # exist = set(m2m_model.objects.filter( + # systemuser_id__in=system_user_ids, asset_id__in=asset_ids + # ).values_list('systemuser_id', 'asset_id')) # TODO 优化 # to_create = [] # for system_user_id in system_user_ids: diff --git a/apps/assets/tasks/__init__.py b/apps/assets/tasks/__init__.py index 22ccbf503..fcd5fbe46 100644 --- a/apps/assets/tasks/__init__.py +++ b/apps/assets/tasks/__init__.py @@ -6,7 +6,5 @@ from .asset_connectivity import * from .account_connectivity import * from .gather_asset_users import * from .gather_asset_hardware_info import * -from .push_system_user import * -from .system_user_connectivity import * from .nodes_amount import * from .backup import * diff --git a/apps/assets/tasks/common.py b/apps/assets/tasks/common.py index 6ad17b4c3..ec51c5a2b 100644 --- a/apps/assets/tasks/common.py +++ b/apps/assets/tasks/common.py @@ -1,38 +1,2 @@ # -*- coding: utf-8 -*- # - -from celery import shared_task - -from orgs.utils import tmp_to_root_org -from assets.models import AuthBook - -__all__ = ['add_nodes_assets_to_system_users'] - - -# Todo: 等待优化 -@shared_task -@tmp_to_root_org() -def add_nodes_assets_to_system_users(nodes_keys, system_users): - from ..models import Node - from assets.tasks import push_system_user_to_assets - - nodes = Node.objects.filter(key__in=nodes_keys) - assets = Node.get_nodes_all_assets(*nodes) - - for system_user in system_users: - """ 解决资产和节点进行关联时，已经关联过的节点不会触发 authbook post_save 信号， - 无法更新节点下所有资产的管理用户的问题 """ - need_push_asset_ids = [] - for asset in assets: - defaults = {'asset': asset, 'systemuser': system_user, 'org_id': asset.org_id} - instance, created = AuthBook.objects.update_or_create( - defaults=defaults, asset=asset, systemuser=system_user - ) - if created: - need_push_asset_ids.append(asset.id) - # 不再自动更新资产管理用户，只允许用户手动指定。 - # 只要关联都需要更新资产的管理用户 - # instance.update_asset_admin_user_if_need() - - if need_push_asset_ids: - push_system_user_to_assets.delay(system_user.id, need_push_asset_ids) diff --git a/apps/assets/urls/api_urls.py b/apps/assets/urls/api_urls.py index b312a41b7..d3262bd96 100644 --- a/apps/assets/urls/api_urls.py +++ b/apps/assets/urls/api_urls.py @@ -20,15 +20,11 @@ router.register(r'labels', api.LabelViewSet, 'label') router.register(r'nodes', api.NodeViewSet, 'node') router.register(r'domains', api.DomainViewSet, 'domain') router.register(r'gateways', api.GatewayViewSet, 'gateway') -router.register(r'cmd-filters', api.CommandFilterViewSet, 'cmd-filter') router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user') router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset') router.register(r'account-backup-plans', api.AccountBackupPlanViewSet, 'account-backup') router.register(r'account-backup-plan-executions', api.AccountBackupPlanExecutionViewSet, 'account-backup-execution') -cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter') -cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule') - urlpatterns = [ path('assets//gateways/', api.AssetGatewayListApi.as_view(), name='asset-gateway-list'), @@ -54,10 +50,7 @@ urlpatterns = [ path('nodes//tasks/', api.NodeTaskCreateApi.as_view(), name='node-task-create'), path('gateways//test-connective/', api.GatewayTestConnectionApi.as_view(), name='test-gateway-connective'), - - path('cmd-filters/command-confirm/', api.CommandConfirmAPI.as_view(), name='command-confirm'), - ] -urlpatterns += router.urls + cmd_filter_router.urls +urlpatterns += router.urls diff --git a/apps/orgs/signal_handlers/common.py b/apps/orgs/signal_handlers/common.py index 48576a828..fc8b4af12 100644 --- a/apps/orgs/signal_handlers/common.py +++ b/apps/orgs/signal_handlers/common.py @@ -20,7 +20,6 @@ from common.decorator import on_transaction_commit from common.signals import django_ready from common.utils import get_logger from common.utils.connection import RedisPubSub -from assets.models import CommandFilterRule from users.signals import post_user_leave_org @@ -141,7 +140,7 @@ def _clear_users_from_org(org, users): for m in models: _remove_users(m, users, org) - _remove_users(CommandFilterRule, users, org, user_field_name='reviewers') + # _remove_users(CommandFilterRule, users, org, user_field_name='reviewers') @receiver(post_save, sender=User)