From a904e59252efd718e47eb2c4090caef799eec879 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Tue, 10 Dec 2024 16:37:48 +0800 Subject: [PATCH 1/2] perf: Change secret --- apps/accounts/automations/base/manager.py | 108 ++++++++++++++++- .../automations/change_secret/manager.py | 109 ++---------------- .../automations/push_account/manager.py | 25 +++- .../accounts/change_secret_report.html | 5 +- .../accounts/push_account_report.html | 77 +++++++++++++ 5 files changed, 217 insertions(+), 107 deletions(-) create mode 100644 apps/accounts/templates/accounts/push_account_report.html diff --git a/apps/accounts/automations/base/manager.py b/apps/accounts/automations/base/manager.py index 44a2c06c4..b86963e7c 100644 --- a/apps/accounts/automations/base/manager.py +++ b/apps/accounts/automations/base/manager.py @@ -1,7 +1,13 @@ -from django.template.loader import render_to_string +from copy import deepcopy + +from django.conf import settings +from django.utils.translation import gettext_lazy as _ from accounts.automations.methods import platform_automation_methods +from accounts.const import SSHKeyStrategy, SecretStrategy, SecretType +from accounts.models import BaseAccountQuerySet from assets.automations.base.manager import BasePlaybookManager +from assets.const import HostTypes from common.utils import get_logger logger = get_logger(__name__) @@ -13,3 +19,103 @@ class AccountBasePlaybookManager(BasePlaybookManager): @property def platform_automation_methods(self): return platform_automation_methods + + +class BaseChangeSecretPushManager(AccountBasePlaybookManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.secret_type = self.execution.snapshot.get('secret_type') + self.secret_strategy = self.execution.snapshot.get( + 'secret_strategy', SecretStrategy.custom + ) + self.ssh_key_change_strategy = self.execution.snapshot.get( + 'ssh_key_change_strategy', SSHKeyStrategy.set_jms + ) + self.account_ids = self.execution.snapshot['accounts'] + + def gen_account_inventory(self, account, asset, h, path_dir): + raise NotImplementedError + + def get_ssh_params(self, secret, secret_type): + kwargs = {} + if secret_type != SecretType.SSH_KEY: + return kwargs + kwargs['strategy'] = self.ssh_key_change_strategy + kwargs['exclusive'] = 'yes' if kwargs['strategy'] == SSHKeyStrategy.set else 'no' + + if kwargs['strategy'] == SSHKeyStrategy.set_jms: + kwargs['regexp'] = '.*{}$'.format(secret.split()[2].strip()) + return kwargs + + def get_accounts(self, privilege_account) -> BaseAccountQuerySet | None: + if not privilege_account: + print('Not privilege account') + return + + asset = privilege_account.asset + accounts = asset.accounts.all() + accounts = accounts.filter(id__in=self.account_ids) + + if self.secret_type: + accounts = accounts.filter(secret_type=self.secret_type) + + if settings.CHANGE_AUTH_PLAN_SECURE_MODE_ENABLED: + accounts = accounts.filter(privileged=False).exclude( + username__in=['root', 'administrator', privilege_account.username] + ) + return accounts + + def handle_ssh_secret(self, secret_type, new_secret, path_dir): + private_key_path = None + if secret_type == SecretType.SSH_KEY: + private_key_path = self.generate_private_key_path(new_secret, path_dir) + new_secret = self.generate_public_key(new_secret) + return new_secret, private_key_path + + def gen_inventory(self, h, account, new_secret, private_key_path, asset): + secret_type = account.secret_type + h['ssh_params'].update(self.get_ssh_params(new_secret, secret_type)) + h['account'] = { + 'name': account.name, + 'username': account.username, + 'secret_type': secret_type, + 'secret': account.escape_jinja2_syntax(new_secret), + 'private_key_path': private_key_path, + 'become': account.get_ansible_become_auth(), + } + if asset.platform.type == 'oracle': + h['account']['mode'] = 'sysdba' if account.privileged else None + return h + + def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs): + host = super().host_callback( + host, asset=asset, account=account, automation=automation, + path_dir=path_dir, **kwargs + ) + if host.get('error'): + return host + + host['check_conn_after_change'] = self.execution.snapshot.get('check_conn_after_change', True) + host['ssh_params'] = {} + + accounts = self.get_accounts(account) + error_msg = _("No pending accounts found") + if not accounts: + print(f'{asset}: {error_msg}') + return [] + + if asset.type == HostTypes.WINDOWS: + accounts = accounts.filter(secret_type=SecretType.PASSWORD) + + inventory_hosts = [] + if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY: + print(f'Windows {asset} does not support ssh key push') + return inventory_hosts + + for account in accounts: + h = deepcopy(host) + h['name'] += '(' + account.username + ')' # To distinguish different accounts + h = self.gen_account_inventory(account, asset, h, path_dir) + inventory_hosts.append(h) + + return inventory_hosts diff --git a/apps/accounts/automations/change_secret/manager.py b/apps/accounts/automations/change_secret/manager.py index 01078a057..c0ab032b2 100644 --- a/apps/accounts/automations/change_secret/manager.py +++ b/apps/accounts/automations/change_secret/manager.py @@ -1,6 +1,5 @@ import os import time -from copy import deepcopy from django.conf import settings from django.utils import timezone @@ -8,73 +7,34 @@ from django.utils.translation import gettext_lazy as _ from xlsxwriter import Workbook from accounts.const import ( - AutomationTypes, SecretType, SSHKeyStrategy, SecretStrategy, ChangeSecretRecordStatusChoice + AutomationTypes, SecretStrategy, ChangeSecretRecordStatusChoice ) -from accounts.models import ChangeSecretRecord, BaseAccountQuerySet +from accounts.models import ChangeSecretRecord from accounts.notifications import ChangeSecretExecutionTaskMsg, ChangeSecretReportMsg from accounts.serializers import ChangeSecretRecordBackUpSerializer -from assets.const import HostTypes from common.db.utils import safe_db_connection from common.decorators import bulk_create_decorator from common.utils import get_logger from common.utils.file import encrypt_and_compress_zip_file from common.utils.timezone import local_now_filename -from ..base.manager import AccountBasePlaybookManager +from ..base.manager import BaseChangeSecretPushManager from ...utils import SecretGenerator logger = get_logger(__name__) -class ChangeSecretManager(AccountBasePlaybookManager): +class ChangeSecretManager(BaseChangeSecretPushManager): ansible_account_prefer = '' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.record_map = self.execution.snapshot.get('record_map', {}) # 这个是某个失败的记录重试 - self.secret_type = self.execution.snapshot.get('secret_type') - self.secret_strategy = self.execution.snapshot.get( - 'secret_strategy', SecretStrategy.custom - ) - self.ssh_key_change_strategy = self.execution.snapshot.get( - 'ssh_key_change_strategy', SSHKeyStrategy.set_jms - ) - self.account_ids = self.execution.snapshot['accounts'] self.name_recorder_mapper = {} # 做个映射，方便后面处理 - self.pending_add_records = [] @classmethod def method_type(cls): return AutomationTypes.change_secret - def get_ssh_params(self, secret, secret_type): - kwargs = {} - if secret_type != SecretType.SSH_KEY: - return kwargs - kwargs['strategy'] = self.ssh_key_change_strategy - kwargs['exclusive'] = 'yes' if kwargs['strategy'] == SSHKeyStrategy.set else 'no' - - if kwargs['strategy'] == SSHKeyStrategy.set_jms: - kwargs['regexp'] = '.*{}$'.format(secret.split()[2].strip()) - return kwargs - - def get_accounts(self, privilege_account) -> BaseAccountQuerySet | None: - if not privilege_account: - print('Not privilege account') - return - - asset = privilege_account.asset - accounts = asset.accounts.all() - accounts = accounts.filter(id__in=self.account_ids) - - if self.secret_type: - accounts = accounts.filter(secret_type=self.secret_type) - - if settings.CHANGE_AUTH_PLAN_SECURE_MODE_ENABLED: - accounts = accounts.filter(privileged=False).exclude( - username__in=['root', 'administrator', privilege_account.username] - ) - return accounts - def get_secret(self, account): if self.secret_strategy == SecretStrategy.custom: new_secret = self.execution.snapshot['secret'] @@ -86,12 +46,11 @@ class ChangeSecretManager(AccountBasePlaybookManager): new_secret = generator.get_secret() return new_secret - def gen_new_secret(self, account, new_secret, path_dir): - private_key_path = None - if account.secret_type == SecretType.SSH_KEY: - private_key_path = self.generate_private_key_path(new_secret, path_dir) - new_secret = self.generate_public_key(new_secret) - return new_secret, private_key_path + def gen_account_inventory(self, account, asset, h, path_dir): + record = self.get_or_create_record(asset, account, h['name']) + new_secret, private_key_path = self.handle_ssh_secret(account.secret_type, record.new_secret, path_dir) + h = self.gen_inventory(h, account, new_secret, private_key_path, asset) + return h def get_or_create_record(self, asset, account, name): asset_account_id = f'{asset.id}-{account.id}' @@ -115,56 +74,6 @@ class ChangeSecretManager(AccountBasePlaybookManager): ) return recorder - def gen_change_secret_inventory(self, h, account, new_secret, private_key_path, asset): - secret_type = account.secret_type - h['ssh_params'].update(self.get_ssh_params(new_secret, secret_type)) - h['account'] = { - 'name': account.name, - 'username': account.username, - 'secret_type': secret_type, - 'secret': account.escape_jinja2_syntax(new_secret), - 'private_key_path': private_key_path, - 'become': account.get_ansible_become_auth(), - } - if asset.platform.type == 'oracle': - h['account']['mode'] = 'sysdba' if account.privileged else None - return h - - def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs): - host = super().host_callback( - host, asset=asset, account=account, automation=automation, - path_dir=path_dir, **kwargs - ) - if host.get('error'): - return host - - host['check_conn_after_change'] = self.execution.snapshot.get('check_conn_after_change', True) - host['ssh_params'] = {} - - accounts = self.get_accounts(account) - error_msg = _("No pending accounts found") - if not accounts: - print(f'{asset}: {error_msg}') - return [] - - if asset.type == HostTypes.WINDOWS: - accounts = accounts.filter(secret_type=SecretType.PASSWORD) - - inventory_hosts = [] - if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY: - print(f'Windows {asset} does not support ssh key push') - return inventory_hosts - - for account in accounts: - h = deepcopy(host) - h['name'] += '(' + account.username + ')' # To distinguish different accounts - record = self.get_or_create_record(asset, account, h['name']) - new_secret, private_key_path = self.gen_new_secret(account, record.new_secret, path_dir) - h = self.gen_change_secret_inventory(h, account, new_secret, private_key_path, asset) - inventory_hosts.append(h) - - return inventory_hosts - def on_host_success(self, host, result): recorder = self.name_recorder_mapper.get(host) if not recorder: diff --git a/apps/accounts/automations/push_account/manager.py b/apps/accounts/automations/push_account/manager.py index 91463f96b..ab1cc61c5 100644 --- a/apps/accounts/automations/push_account/manager.py +++ b/apps/accounts/automations/push_account/manager.py @@ -1,15 +1,34 @@ +from django.utils.translation import gettext_lazy as _ + from accounts.const import AutomationTypes from common.utils import get_logger -from ..base.manager import AccountBasePlaybookManager -from ..change_secret.manager import ChangeSecretManager +from common.utils.timezone import local_now_filename +from ..base.manager import BaseChangeSecretPushManager logger = get_logger(__name__) -class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager): +class PushAccountManager(BaseChangeSecretPushManager): @classmethod def method_type(cls): return AutomationTypes.push_account def get_secret(self, account): return account.secret + + def gen_account_inventory(self, account, asset, h, path_dir): + secret = self.get_secret(account) + secret_type = account.secret_type + new_secret, private_key_path = self.handle_ssh_secret(secret_type, secret, path_dir) + h = self.gen_inventory(h, account, new_secret, private_key_path, asset) + return h + + def print_summary(self): + print('\n\n' + '-' * 80) + plan_execution_end = _('Plan execution end') + print('{} {}\n'.format(plan_execution_end, local_now_filename())) + time_cost = _('Duration') + print('{}: {}s'.format(time_cost, self.duration)) + + def get_report_template(self): + return "accounts/push_account_report.html" diff --git a/apps/accounts/templates/accounts/change_secret_report.html b/apps/accounts/templates/accounts/change_secret_report.html index 2f836b2f8..a6fa25f8d 100644 --- a/apps/accounts/templates/accounts/change_secret_report.html +++ b/apps/accounts/templates/accounts/change_secret_report.html @@ -1,7 +1,7 @@ {% load i18n %}
-

{% trans 'The following is a summary of account change secret or push tasks, please read and process' %}

+

{% trans 'The following is a summary of account change secret tasks, please read and process' %}

@@ -71,7 +71,7 @@ {% endif %}
- {% if summary.lost_accounts %} + {% if summary.fail_accounts %}

{% trans 'Failed accounts' %}: {{ summary.fail_accounts }}

@@ -130,5 +130,4 @@ width: 10%; } - diff --git a/apps/accounts/templates/accounts/push_account_report.html b/apps/accounts/templates/accounts/push_account_report.html new file mode 100644 index 000000000..c04a55f3f --- /dev/null +++ b/apps/accounts/templates/accounts/push_account_report.html @@ -0,0 +1,77 @@ +{% load i18n %} + +
+

{% trans 'The following is a summary of account push tasks, please read and process' %}

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
任务汇总:
{% trans 'Task name' %}:{{ execution.automation.name }}
{% trans 'Date start' %}:{{ execution.date_start | date:"Y/m/d H:i:s" }}
{% trans 'Date end' %}:{{ execution.date_finished | date:"Y/m/d H:i:s" }}
{% trans 'Time using' %}:{{ execution.duration }}s
{% trans 'Assets count' %}:{{ summary.total_assets }}
{% trans 'Asset success count' %}:{{ summary.ok_assets }}
{% trans 'Asset failed count' %}:{{ summary.fail_assets }}
{% trans 'Asset not support count' %}:{{ summary.error_assets }}
+
+ + + From 792b552e257b360f774b1b4df1fc1291f57aa1f6 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Tue, 10 Dec 2024 19:30:23 +0800 Subject: [PATCH 2/2] perf: Remove push account extra api --- apps/accounts/api/automations/push_account.py | 21 +------------------ .../0023_alter_changesecretrecord_options.py | 17 +++++++++++++++ .../models/automations/change_secret.py | 4 ---- apps/accounts/urls.py | 1 - 4 files changed, 18 insertions(+), 25 deletions(-) create mode 100644 apps/accounts/migrations/0023_alter_changesecretrecord_options.py diff --git a/apps/accounts/api/automations/push_account.py b/apps/accounts/api/automations/push_account.py index b91bad379..76caff498 100644 --- a/apps/accounts/api/automations/push_account.py +++ b/apps/accounts/api/automations/push_account.py @@ -2,19 +2,17 @@ # from accounts import serializers from accounts.const import AutomationTypes -from accounts.models import PushAccountAutomation, ChangeSecretRecord +from accounts.models import PushAccountAutomation from orgs.mixins.api import OrgBulkModelViewSet from .base import ( AutomationAssetsListApi, AutomationRemoveAssetApi, AutomationAddAssetApi, AutomationNodeAddRemoveApi, AutomationExecutionViewSet ) -from .change_secret import ChangeSecretRecordViewSet __all__ = [ 'PushAccountAutomationViewSet', 'PushAccountAssetsListApi', 'PushAccountRemoveAssetApi', 'PushAccountAddAssetApi', 'PushAccountNodeAddRemoveApi', 'PushAccountExecutionViewSet', - 'PushAccountRecordViewSet' ] @@ -41,23 +39,6 @@ class PushAccountExecutionViewSet(AutomationExecutionViewSet): return queryset -class PushAccountRecordViewSet(ChangeSecretRecordViewSet): - serializer_class = serializers.ChangeSecretRecordSerializer - tp = AutomationTypes.push_account - - rbac_perms = { - 'list': 'accounts.view_pushsecretrecord', - 'execute': 'accounts.add_pushsecretexecution', - 'secret': 'accounts.view_pushsecretrecord', - } - - def get_queryset(self): - qs = ChangeSecretRecord.get_valid_records() - return qs.filter( - execution__automation__type=self.tp - ) - - class PushAccountAssetsListApi(AutomationAssetsListApi): model = PushAccountAutomation diff --git a/apps/accounts/migrations/0023_alter_changesecretrecord_options.py b/apps/accounts/migrations/0023_alter_changesecretrecord_options.py new file mode 100644 index 000000000..1fb1addbe --- /dev/null +++ b/apps/accounts/migrations/0023_alter_changesecretrecord_options.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.13 on 2024-12-10 11:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0022_alter_changesecretrecord_options_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='changesecretrecord', + options={'verbose_name': 'Change secret record'}, + ), + ] diff --git a/apps/accounts/models/automations/change_secret.py b/apps/accounts/models/automations/change_secret.py index d79c6660c..a894f9d6d 100644 --- a/apps/accounts/models/automations/change_secret.py +++ b/apps/accounts/models/automations/change_secret.py @@ -46,10 +46,6 @@ class ChangeSecretRecord(JMSBaseModel): class Meta: verbose_name = _("Change secret record") - permissions = [ - ('view_pushsecretrecord', _('Can view change secret execution')), - ('add_pushsecretexecution', _('Can add change secret execution')), - ] def __str__(self): return f'{self.account.username}@{self.asset}' diff --git a/apps/accounts/urls.py b/apps/accounts/urls.py index e388e5a68..b9cfea710 100644 --- a/apps/accounts/urls.py +++ b/apps/accounts/urls.py @@ -23,7 +23,6 @@ router.register(r'gather-account-automations', api.GatherAccountsAutomationViewS router.register(r'gather-account-executions', api.GatherAccountsExecutionViewSet, 'gather-account-execution') router.register(r'push-account-automations', api.PushAccountAutomationViewSet, 'push-account-automation') router.register(r'push-account-executions', api.PushAccountExecutionViewSet, 'push-account-execution') -router.register(r'push-account-records', api.PushAccountRecordViewSet, 'push-account-record') router.register(r'check-account-automations', api.CheckAccountAutomationViewSet, 'check-account-automation') router.register(r'check-account-executions', api.CheckAccountExecutionViewSet, 'check-account-execution') router.register(r'account-check-engines', api.CheckAccountEngineViewSet, 'account-check-engine')