From e67a8765136bc304ebb19e7529e6f7591574b965 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 21 Feb 2023 13:00:04 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/api/account/__init__.py | 3 +- apps/accounts/api/account/account.py | 42 ++----------------- apps/accounts/api/account/gathered_account.py | 42 ------------------- apps/accounts/api/account/task.py | 37 ++++++++++++++++ apps/accounts/api/automations/__init__.py | 1 + .../api/{account => automations}/backup.py | 0 .../api/automations/gather_accounts.py | 40 +++++++++++++++++- apps/accounts/automations/base/manager.py | 8 ++-- .../automations/change_secret/manager.py | 2 +- .../automations/push_account/manager.py | 2 +- apps/accounts/tasks/common.py | 2 +- apps/accounts/tasks/push_account.py | 10 +---- apps/accounts/tasks/verify_account.py | 23 ++++++---- apps/assets/api/node.py | 6 +-- apps/assets/automations/base/manager.py | 2 + .../automations/ping_gateway/manager.py | 2 +- .../migrations/0111_alter_asset_options.py | 17 ++++++++ apps/assets/models/asset/common.py | 4 +- apps/assets/models/domain.py | 4 +- apps/ops/ansible/inventory.py | 2 +- apps/ops/signal_handlers.py | 16 +++++++ 21 files changed, 147 insertions(+), 118 deletions(-) delete mode 100644 apps/accounts/api/account/gathered_account.py create mode 100644 apps/accounts/api/account/task.py rename apps/accounts/api/{account => automations}/backup.py (100%) create mode 100644 apps/assets/migrations/0111_alter_asset_options.py diff --git a/apps/accounts/api/account/__init__.py b/apps/accounts/api/account/__init__.py index ea5ab0cd1..f6d8376a6 100644 --- a/apps/accounts/api/account/__init__.py +++ b/apps/accounts/api/account/__init__.py @@ -1,4 +1,3 @@ from .account import * -from .backup import * +from .task import * from .template import * -from .gathered_account import * diff --git a/apps/accounts/api/account/account.py b/apps/accounts/api/account/account.py index dd57761c2..631669acc 100644 --- a/apps/accounts/api/account/account.py +++ b/apps/accounts/api/account/account.py @@ -1,12 +1,11 @@ from django.shortcuts import get_object_or_404 from rest_framework.decorators import action -from rest_framework.generics import CreateAPIView, ListAPIView +from rest_framework.generics import ListAPIView from rest_framework.response import Response from accounts import serializers from accounts.filters import AccountFilterSet from accounts.models import Account -from accounts.tasks import verify_accounts_connectivity_task, push_accounts_to_assets_task from assets.models import Asset from authentication.const import ConfirmType from common.permissions import UserConfirmation @@ -15,7 +14,7 @@ from orgs.mixins.api import OrgBulkModelViewSet __all__ = [ 'AccountViewSet', 'AccountSecretsViewSet', - 'AccountsTaskCreateAPI', 'AccountHistoriesSecretAPI' + 'AccountHistoriesSecretAPI' ] from rbac.permissions import RBACPermission @@ -37,6 +36,7 @@ class AccountViewSet(OrgBulkModelViewSet): def su_from_accounts(self, request, *args, **kwargs): account_id = request.query_params.get('account') asset_id = request.query_params.get('asset') + if account_id: account = get_object_or_404(Account, pk=account_id) accounts = account.get_su_from_accounts() @@ -75,39 +75,3 @@ class AccountHistoriesSecretAPI(RecordViewLogMixin, ListAPIView): def get_queryset(self): return self.model.objects.filter(id=self.kwargs.get('pk')) - - -class AccountsTaskCreateAPI(CreateAPIView): - serializer_class = serializers.AccountTaskSerializer - search_fields = AccountViewSet.search_fields - filterset_class = AccountViewSet.filterset_class - - def check_permissions(self, request): - return request.user.has_perm('assets.test_assetconnectivity') - - def get_accounts(self): - queryset = Account.objects.all() - queryset = self.filter_queryset(queryset) - return queryset - - def perform_create(self, serializer): - data = serializer.validated_data - accounts = data.get('accounts') - account_ids = accounts.values_list('id', flat=True) - asset_ids = [account.asset_id for account in accounts] - - if data['action'] == 'push': - task = push_accounts_to_assets_task.delay(account_ids, asset_ids) - else: - task = verify_accounts_connectivity_task.delay(account_ids, asset_ids) - - data = getattr(serializer, '_data', {}) - data["task"] = task.id - setattr(serializer, '_data', data) - return task - - def get_exception_handler(self): - def handler(e, context): - return Response({"error": str(e)}, status=400) - - return handler diff --git a/apps/accounts/api/account/gathered_account.py b/apps/accounts/api/account/gathered_account.py deleted file mode 100644 index e47befac4..000000000 --- a/apps/accounts/api/account/gathered_account.py +++ /dev/null @@ -1,42 +0,0 @@ -from rest_framework import status -from rest_framework.decorators import action -from rest_framework.response import Response -from django.utils.translation import ugettext_lazy as _ - -from accounts import serializers -from accounts.const import Source -from accounts.models import GatheredAccount -from accounts.filters import GatheredAccountFilterSet -from orgs.mixins.api import OrgBulkModelViewSet - -__all__ = [ - 'GatheredAccountViewSet', -] - - -class GatheredAccountViewSet(OrgBulkModelViewSet): - model = GatheredAccount - search_fields = ('username',) - filterset_class = GatheredAccountFilterSet - serializer_classes = { - 'default': serializers.GatheredAccountSerializer, - } - rbac_perms = { - 'sync_account': 'assets.add_gatheredaccount', - } - - @action(methods=['post'], detail=True, url_path='sync') - def sync_account(self, request, *args, **kwargs): - gathered_account = super().get_object() - asset = gathered_account.asset - username = gathered_account.username - accounts = asset.accounts.filter(username=username) - if accounts.exists(): - accounts.update(source=Source.COLLECTED) - else: - asset.accounts.model.objects.create( - asset=asset, username=username, - name=f'{username}-{_("Collected")}', - source=Source.COLLECTED - ) - return Response(status=status.HTTP_201_CREATED) diff --git a/apps/accounts/api/account/task.py b/apps/accounts/api/account/task.py new file mode 100644 index 000000000..85a3831e2 --- /dev/null +++ b/apps/accounts/api/account/task.py @@ -0,0 +1,37 @@ +from rest_framework.generics import CreateAPIView +from rest_framework.response import Response + +from accounts import serializers +from accounts.tasks import verify_accounts_connectivity_task, push_accounts_to_assets_task + +__all__ = [ + 'AccountsTaskCreateAPI', +] + + +class AccountsTaskCreateAPI(CreateAPIView): + serializer_class = serializers.AccountTaskSerializer + + def check_permissions(self, request): + return request.user.has_perm('assets.test_assetconnectivity') + + def perform_create(self, serializer): + data = serializer.validated_data + accounts = data.get('accounts', []) + account_ids = [a.id for a in accounts] + + if data['action'] == 'push': + task = push_accounts_to_assets_task.delay(account_ids) + else: + task = verify_accounts_connectivity_task.delay(account_ids) + + data = getattr(serializer, '_data', {}) + data["task"] = task.id + setattr(serializer, '_data', data) + return task + + def get_exception_handler(self): + def handler(e, context): + return Response({"error": str(e)}, status=400) + + return handler diff --git a/apps/accounts/api/automations/__init__.py b/apps/accounts/api/automations/__init__.py index 2b0aa0029..a03da88b0 100644 --- a/apps/accounts/api/automations/__init__.py +++ b/apps/accounts/api/automations/__init__.py @@ -1,3 +1,4 @@ +from .backup import * from .base import * from .change_secret import * from .gather_accounts import * diff --git a/apps/accounts/api/account/backup.py b/apps/accounts/api/automations/backup.py similarity index 100% rename from apps/accounts/api/account/backup.py rename to apps/accounts/api/automations/backup.py diff --git a/apps/accounts/api/automations/gather_accounts.py b/apps/accounts/api/automations/gather_accounts.py index 59f55c60e..3abca94dd 100644 --- a/apps/accounts/api/automations/gather_accounts.py +++ b/apps/accounts/api/automations/gather_accounts.py @@ -1,13 +1,22 @@ # -*- coding: utf-8 -*- # +from django.utils.translation import ugettext_lazy as _ +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.response import Response + from accounts import serializers from accounts.const import AutomationTypes +from accounts.const import Source +from accounts.filters import GatheredAccountFilterSet from accounts.models import GatherAccountsAutomation +from accounts.models import GatheredAccount from orgs.mixins.api import OrgBulkModelViewSet from .base import AutomationExecutionViewSet __all__ = [ - 'GatherAccountsAutomationViewSet', 'GatherAccountsExecutionViewSet' + 'GatherAccountsAutomationViewSet', 'GatherAccountsExecutionViewSet', + 'GatheredAccountViewSet' ] @@ -31,3 +40,32 @@ class GatherAccountsExecutionViewSet(AutomationExecutionViewSet): queryset = super().get_queryset() queryset = queryset.filter(automation__type=self.tp) return queryset + + +class GatheredAccountViewSet(OrgBulkModelViewSet): + model = GatheredAccount + search_fields = ('username',) + filterset_class = GatheredAccountFilterSet + serializer_classes = { + 'default': serializers.GatheredAccountSerializer, + } + rbac_perms = { + 'sync_account': 'assets.add_gatheredaccount', + } + + @action(methods=['post'], detail=True, url_path='sync') + def sync_account(self, request, *args, **kwargs): + gathered_account = super().get_object() + asset = gathered_account.asset + username = gathered_account.username + accounts = asset.accounts.filter(username=username) + + if accounts.exists(): + accounts.update(source=Source.COLLECTED) + else: + asset.accounts.model.objects.create( + asset=asset, username=username, + name=f'{username}-{_("Collected")}', + source=Source.COLLECTED + ) + return Response(status=status.HTTP_201_CREATED) diff --git a/apps/accounts/automations/base/manager.py b/apps/accounts/automations/base/manager.py index 4e06217b3..6251c93a2 100644 --- a/apps/accounts/automations/base/manager.py +++ b/apps/accounts/automations/base/manager.py @@ -17,19 +17,19 @@ class VerifyHostCallbackMixin: def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs): host = super().host_callback( - host, asset=asset, account=account, automation=automation, - path_dir=path_dir, **kwargs + host, asset=asset, account=account, + automation=automation, path_dir=path_dir, **kwargs ) if host.get('error'): return host accounts = asset.accounts.all() accounts = self.get_accounts(account, accounts) - inventory_hosts = [] + for account in accounts: h = deepcopy(host) - h['name'] += '_' + account.username + h['name'] += '(' + account.username + ')' self.host_account_mapper[h['name']] = account secret = account.secret diff --git a/apps/accounts/automations/change_secret/manager.py b/apps/accounts/automations/change_secret/manager.py index 971877c5f..4a8e548a3 100644 --- a/apps/accounts/automations/change_secret/manager.py +++ b/apps/accounts/automations/change_secret/manager.py @@ -93,7 +93,7 @@ class ChangeSecretManager(AccountBasePlaybookManager): host['secret_type'] = self.secret_type for account in accounts: h = deepcopy(host) - h['name'] += '_' + account.username + h['name'] += '(' + account.username + ')' new_secret = self.get_secret() recorder = ChangeSecretRecord( diff --git a/apps/accounts/automations/push_account/manager.py b/apps/accounts/automations/push_account/manager.py index c974106a6..12d89e7ed 100644 --- a/apps/accounts/automations/push_account/manager.py +++ b/apps/accounts/automations/push_account/manager.py @@ -63,7 +63,7 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager): host['secret_type'] = self.secret_type for account in accounts: h = deepcopy(host) - h['name'] += '_' + account.username + h['name'] += '(' + account.username + ')' new_secret = self.get_secret() self.name_recorder_mapper[h['name']] = { diff --git a/apps/accounts/tasks/common.py b/apps/accounts/tasks/common.py index 52f2d13b4..4582322c3 100644 --- a/apps/accounts/tasks/common.py +++ b/apps/accounts/tasks/common.py @@ -9,7 +9,7 @@ def quickstart_automation_by_snapshot(task_name, tp, task_snapshot=None): data = generate_automation_execution_data(task_name, tp, task_snapshot) pk = data['id'] - if AutomationExecution.objects.exists(id=pk): + if AutomationExecution.objects.filter(id=pk).exists(): data['id'] = str(uuid.uuid4()) execution = AutomationExecution.objects.create( diff --git a/apps/accounts/tasks/push_account.py b/apps/accounts/tasks/push_account.py index ab7832b2f..10dd4ce64 100644 --- a/apps/accounts/tasks/push_account.py +++ b/apps/accounts/tasks/push_account.py @@ -1,5 +1,4 @@ from celery import shared_task -from collections import defaultdict from django.utils.translation import gettext_noop, ugettext_lazy as _ from accounts.const import AutomationTypes @@ -21,20 +20,15 @@ def push_accounts_to_assets_task(account_ids): from accounts.models import Account accounts = Account.objects.filter(id__in=account_ids) - task_name = gettext_noop("Push accounts to assets") task_name = PushAccountAutomation.generate_unique_name(task_name) - account_asset_mapper = defaultdict(set) for account in accounts: - account_asset_mapper[account.username].add(account.asset) - - for username, assets in account_asset_mapper.items(): task_snapshot = { 'secret': account.secret, 'secret_type': account.secret_type, 'accounts': [account.username], - 'assets': asset_ids, + 'assets': [str(account.asset_id)], } tp = AutomationTypes.push_account - quickstart_automation_by_snapshot(task_name, tp, task_snapshot) \ No newline at end of file + quickstart_automation_by_snapshot(task_name, tp, task_snapshot) diff --git a/apps/accounts/tasks/verify_account.py b/apps/accounts/tasks/verify_account.py index 8bb656e29..efc6bb09d 100644 --- a/apps/accounts/tasks/verify_account.py +++ b/apps/accounts/tasks/verify_account.py @@ -26,15 +26,22 @@ def verify_connectivity_util(assets, tp, accounts, task_name): @org_aware_func("assets") -def verify_accounts_connectivity_util(accounts, assets, task_name): - gateway_assets = assets.filter(platform__name=GATEWAY_NAME) +def verify_accounts_connectivity_util(accounts, task_name): + from assets.models import Asset + + asset_ids = [a.asset_id for a in accounts] + assets = Asset.objects.filter(id__in=asset_ids) + + gateways = assets.filter(platform__name=GATEWAY_NAME) verify_connectivity_util( - gateway_assets, AutomationTypes.verify_gateway_account, accounts, task_name + gateways, AutomationTypes.verify_gateway_account, + accounts, task_name ) - non_gateway_assets = assets.exclude(platform__name=GATEWAY_NAME) + common_assets = assets.exclude(platform__name=GATEWAY_NAME) verify_connectivity_util( - non_gateway_assets, AutomationTypes.verify_account, accounts, task_name + common_assets, AutomationTypes.verify_account, + accounts, task_name ) @@ -42,11 +49,9 @@ def verify_accounts_connectivity_util(accounts, assets, task_name): queue="ansible", verbose_name=_('Verify asset account availability'), activity_callback=lambda self, account_ids, asset_ids: (account_ids, None) ) -def verify_accounts_connectivity_task(account_ids, asset_ids): - from assets.models import Asset +def verify_accounts_connectivity_task(account_ids): from accounts.models import Account, VerifyAccountAutomation - assets = Asset.objects.filter(id__in=asset_ids) accounts = Account.objects.filter(id__in=account_ids) task_name = gettext_noop("Verify accounts connectivity") task_name = VerifyAccountAutomation.generate_unique_name(task_name) - return verify_accounts_connectivity_util(accounts, assets, task_name) + return verify_accounts_connectivity_util(accounts, task_name) diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 3658a29f1..9a60c636f 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -103,7 +103,7 @@ class NodeAddAssetsApi(generics.UpdateAPIView): instance = None permission_classes = (RBACPermission,) rbac_perms = { - 'PUT': 'assets.add_assettonode', + 'PUT': 'assets.change_assettonode', } def perform_update(self, serializer): @@ -118,7 +118,7 @@ class NodeRemoveAssetsApi(generics.UpdateAPIView): instance = None permission_classes = (RBACPermission,) rbac_perms = { - 'PUT': 'assets.remove_assetfromnode', + 'PUT': 'assets.change_assetfromnode', } def perform_update(self, serializer): @@ -140,7 +140,7 @@ class MoveAssetsToNodeApi(generics.UpdateAPIView): instance = None permission_classes = (RBACPermission,) rbac_perms = { - 'PUT': 'assets.move_assettonode', + 'PUT': 'assets.change_assettonode', } def perform_update(self, serializer): diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 9c7ac4fd1..0f9949dce 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -62,6 +62,8 @@ class BasePlaybookManager: ) if not os.path.exists(path): os.makedirs(path, exist_ok=True, mode=0o755) + if settings.DEBUG_DEV: + logger.debug('Ansible runtime dir: {}'.format(path)) return path @staticmethod diff --git a/apps/assets/automations/ping_gateway/manager.py b/apps/assets/automations/ping_gateway/manager.py index d03e5d22d..b3f243fdb 100644 --- a/apps/assets/automations/ping_gateway/manager.py +++ b/apps/assets/automations/ping_gateway/manager.py @@ -33,7 +33,7 @@ class PingGatewayManager: err = _('No account') return False, err - print('Test account: {}'.format(account)) + print('- ' + _('Asset, {}, using account {}').format(gateway, account)) try: proxy.connect( gateway.address, diff --git a/apps/assets/migrations/0111_alter_asset_options.py b/apps/assets/migrations/0111_alter_asset_options.py new file mode 100644 index 000000000..a1e5c3097 --- /dev/null +++ b/apps/assets/migrations/0111_alter_asset_options.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.14 on 2023-02-21 04:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0110_auto_20230220_1051'), + ] + + operations = [ + migrations.AlterModelOptions( + name='asset', + options={'ordering': ['name'], 'permissions': [('refresh_assethardwareinfo', 'Can refresh asset hardware info'), ('test_assetconnectivity', 'Can test asset connectivity'), ('push_assetaccount', 'Can push account to asset'), ('test_account', 'Can verify account'), ('match_asset', 'Can match asset'), ('change_assettonode', 'Can change asset nodes')], 'verbose_name': 'Asset'}, + ), + ] diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index 8fd2d8a2c..4008da5d4 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -284,7 +284,5 @@ class Asset(NodesRelationMixin, AbsConnectivity, JMSOrgBaseModel): ('push_assetaccount', _('Can push account to asset')), ('test_account', _('Can verify account')), ('match_asset', _('Can match asset')), - ('add_assettonode', _('Add asset to node')), - ('move_assettonode', _('Move asset to node')), - ('remove_assetfromnode', _('Remove asset from node')) + ('change_assettonode', _('Can change asset nodes')), ] diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index dfb0e32fe..6dd5e3c4f 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -30,11 +30,11 @@ class Domain(JMSOrgBaseModel): def random_gateway(self): gateways = [gw for gw in self.active_gateways if gw.is_connective] + if not gateways: - logger.warn(f'Gateway all bad. domain={self}, gateway_num={len(gateways)}.') gateways = self.active_gateways if not gateways: - logger.warn(f'Not active gateway. domain={self}') + logger.warn(f'Not active gateway, domain={self}, pass') return None return random.choice(gateways) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 01cdbd87d..5aec90466 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -101,7 +101,7 @@ class JMSInventory: def asset_to_host(self, asset, account, automation, protocols, platform): host = { - 'name': '{}'.format(asset.name), + 'name': '{}'.format(asset.name.replace(' ', '_')), 'jms_asset': { 'id': str(asset.id), 'name': asset.name, 'address': asset.address, 'type': asset.type, 'category': asset.category, diff --git a/apps/ops/signal_handlers.py b/apps/ops/signal_handlers.py index 7d4e15789..faba5c9eb 100644 --- a/apps/ops/signal_handlers.py +++ b/apps/ops/signal_handlers.py @@ -46,9 +46,25 @@ def sync_registered_tasks(*args, **kwargs): @receiver(django_ready) def check_registered_tasks(*args, **kwargs): attrs = ['verbose_name', 'activity_callback'] + ignores = [ + 'users.tasks.check_user_expired_periodic', 'ops.tasks.clean_celery_periodic_tasks', + 'terminal.tasks.delete_terminal_status_period', 'ops.tasks.check_server_performance_period', + 'settings.tasks.ldap.import_ldap_user', 'users.tasks.check_password_expired', + 'assets.tasks.nodes_amount.check_node_assets_amount_task', 'notifications.notifications.publish_task', + 'perms.tasks.check_asset_permission_will_expired', + 'ops.tasks.create_or_update_registered_periodic_tasks', 'perms.tasks.check_asset_permission_expired', + 'settings.tasks.ldap.import_ldap_user_periodic', 'users.tasks.check_password_expired_periodic', + 'common.utils.verify_code.send_async', 'assets.tasks.nodes_amount.check_node_assets_amount_period_task', + 'users.tasks.check_user_expired', 'orgs.tasks.refresh_org_cache_task', + 'terminal.tasks.upload_session_replay_to_external_storage', 'terminal.tasks.clean_orphan_session', + 'audits.tasks.clean_audits_log_period', 'authentication.tasks.clean_django_sessions' + ] + for name, task in app.tasks.items(): if name.startswith('celery.'): continue + if name in ignores: + continue for attr in attrs: if not hasattr(task, attr): print('>>> Task {} has no attribute {}'.format(name, attr))