From f407bfd7b26a8cf23e80f729d32b8c76ecd39783 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 22 Feb 2023 12:27:30 +0800 Subject: [PATCH 1/6] =?UTF-8?q?fix:=20=E8=BF=87=E6=BB=A4=E7=A6=BB=E7=BA=BF?= =?UTF-8?q?=E7=9A=84=E5=8F=91=E5=B8=83=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/models/applet/applet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/terminal/models/applet/applet.py b/apps/terminal/models/applet/applet.py index ee83c8e52..22afe39e8 100644 --- a/apps/terminal/models/applet/applet.py +++ b/apps/terminal/models/applet/applet.py @@ -96,7 +96,7 @@ class Applet(JMSBaseModel): manifest = cls.validate_pkg(path) name = manifest['name'] - if not has_valid_xpack_license() and name.lower() in ('navicat', ): + if not has_valid_xpack_license() and name.lower() in ('navicat',): return instance = cls.objects.filter(name=name).first() @@ -112,7 +112,9 @@ class Applet(JMSBaseModel): def select_host_account(self): # 选择激活的发布机 - hosts = list(self.hosts.filter(is_active=True).all()) + hosts = [item for item in self.hosts.filter(is_active=True).all() + if item.load != 'offline'] + if not hosts: return None From 6f765ebef5a6592f8d198b3ce11cfd0a730993ac Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 22 Feb 2023 13:08:32 +0800 Subject: [PATCH 2/6] =?UTF-8?q?perf:=20=E4=BF=AE=E5=A4=8D=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E5=B9=B3=E5=8F=B0=E5=86=85=E7=BD=AE=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/serializers/account/account.py | 7 ++++--- apps/assets/const/types.py | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/accounts/serializers/account/account.py b/apps/accounts/serializers/account/account.py index 8cf92671e..34333cfe4 100644 --- a/apps/accounts/serializers/account/account.py +++ b/apps/accounts/serializers/account/account.py @@ -101,9 +101,10 @@ class AccountSerializer(AccountSerializerCreateMixin, BaseAccountSerializer): class Meta(BaseAccountSerializer.Meta): model = Account - fields = BaseAccountSerializer.Meta.fields \ - + ['su_from', 'asset'] \ - + ['template', 'push_now', 'source'] + fields = BaseAccountSerializer.Meta.fields + [ + 'su_from', 'asset', 'template', + 'push_now', 'source', 'connectivity' + ] extra_kwargs = { **BaseAccountSerializer.Meta.extra_kwargs, 'name': {'required': False, 'allow_null': True}, diff --git a/apps/assets/const/types.py b/apps/assets/const/types.py index 3c0989517..cde7c8aba 100644 --- a/apps/assets/const/types.py +++ b/apps/assets/const/types.py @@ -329,9 +329,8 @@ class AllTypes(ChoicesMixin): internal_platforms.append(d['name']) user_platforms = platform_cls.objects.exclude(name__in=internal_platforms) - user_platforms.update(internal=False) - for platform in user_platforms: print("\t- Update platform: {}".format(platform.name)) platform_data = cls.get_type_default_platform(platform.category, platform.type) cls.create_or_update_by_platform_data(platform.name, platform_data, platform_cls=platform_cls) + user_platforms.update(internal=False) From 312fe690159ea4985917b12bebef129263a8ab5b Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 22 Feb 2023 14:10:13 +0800 Subject: [PATCH 3/6] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E5=BA=94=E7=94=A8=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/api/applet/host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/terminal/api/applet/host.py b/apps/terminal/api/applet/host.py index 38c42cd18..89bc1f96c 100644 --- a/apps/terminal/api/applet/host.py +++ b/apps/terminal/api/applet/host.py @@ -58,7 +58,7 @@ class AppletHostDeploymentViewSet(viewsets.ModelViewSet): def applets(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - applet_id = serializer.validated_data.pop('applet_id') + applet_id = serializer.validated_data.pop('applet_id', '') instance = serializer.save() task = run_applet_host_deployment_install_applet.delay(instance.id, applet_id) instance.save_task(task.id) From 156ad665c340690f5e4ca306b869568ad36ff7c8 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 22 Feb 2023 14:52:39 +0800 Subject: [PATCH 4/6] fix: rbac perms --- apps/rbac/const.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/rbac/const.py b/apps/rbac/const.py index 1c0f594f9..2092cd9ff 100644 --- a/apps/rbac/const.py +++ b/apps/rbac/const.py @@ -60,6 +60,7 @@ exclude_permissions = ( ('accounts', 'automationexecution', '*', 'automationexecution'), ('accounts', 'accountbackupexecution', 'delete,change', 'accountbackupexecution'), ('accounts', 'changesecretrecord', 'add,delete,change', 'changesecretrecord'), + ('accounts', 'account', 'change', 'accountsecret'), ('perms', 'userassetgrantedtreenoderelation', '*', '*'), ('perms', 'permedaccount', '*', '*'), From 33f6c5b25791d42f7d6e6c92c8a8f7bb21019ac5 Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 22 Feb 2023 15:13:51 +0800 Subject: [PATCH 5/6] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20=E6=8E=A2?= =?UTF-8?q?=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/automations/base/manager.py | 46 -------------- .../automations/change_secret/manager.py | 8 ++- .../automations/push_account/manager.py | 3 +- .../verify_account/host/posix/main.yml | 6 +- .../automations/verify_account/manager.py | 55 ++++++++++++++++- apps/assets/automations/base/manager.py | 61 ++++++++----------- 6 files changed, 87 insertions(+), 92 deletions(-) diff --git a/apps/accounts/automations/base/manager.py b/apps/accounts/automations/base/manager.py index 6251c93a2..401304597 100644 --- a/apps/accounts/automations/base/manager.py +++ b/apps/accounts/automations/base/manager.py @@ -1,57 +1,11 @@ -from copy import deepcopy - from accounts.automations.methods import platform_automation_methods -from accounts.const import SecretType from assets.automations.base.manager import BasePlaybookManager from common.utils import get_logger logger = get_logger(__name__) -class VerifyHostCallbackMixin: - execution: callable - get_accounts: callable - host_account_mapper: dict - generate_public_key: callable - generate_private_key_path: callable - - def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs): - host = super().host_callback( - host, asset=asset, account=account, - automation=automation, path_dir=path_dir, **kwargs - ) - if host.get('error'): - return host - - accounts = asset.accounts.all() - accounts = self.get_accounts(account, accounts) - inventory_hosts = [] - - for account in accounts: - h = deepcopy(host) - h['name'] += '(' + account.username + ')' - self.host_account_mapper[h['name']] = account - secret = account.secret - - private_key_path = None - if account.secret_type == SecretType.SSH_KEY: - private_key_path = self.generate_private_key_path(secret, path_dir) - secret = self.generate_public_key(secret) - - h['secret_type'] = account.secret_type - h['account'] = { - 'name': account.name, - 'username': account.username, - 'secret_type': account.secret_type, - 'secret': secret, - 'private_key_path': private_key_path - } - inventory_hosts.append(h) - return inventory_hosts - - class AccountBasePlaybookManager(BasePlaybookManager): - pass @property def platform_automation_methods(self): diff --git a/apps/accounts/automations/change_secret/manager.py b/apps/accounts/automations/change_secret/manager.py index a6c14fa23..fc4d53097 100644 --- a/apps/accounts/automations/change_secret/manager.py +++ b/apps/accounts/automations/change_secret/manager.py @@ -87,7 +87,8 @@ class ChangeSecretManager(AccountBasePlaybookManager): accounts = accounts.filter(secret_type=self.secret_type) if not accounts: - print('没有发现待改密账号: %s 用户名: %s 类型: %s' % (asset.name, account.username, self.secret_type)) + msg = '没有发现待改密账号: %s 用户名: %s 类型: %s' % (asset.name, account.username, self.secret_type) + print(msg) return [] method_attr = getattr(automation, self.method_type() + '_method') @@ -98,7 +99,8 @@ class ChangeSecretManager(AccountBasePlaybookManager): host['secret_type'] = self.secret_type if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY: - print(f'Windows {asset} does not support ssh key push \n') + msg = f'Windows {asset} does not support ssh key push \n' + print(msg) return inventory_hosts for account in accounts: @@ -143,7 +145,7 @@ class ChangeSecretManager(AccountBasePlaybookManager): recorder.save() account = recorder.account if not account: - print("Account not found, deleted ?", recorder) + print("Account not found, deleted ?") return account.secret = recorder.new_secret account.save(update_fields=['secret']) diff --git a/apps/accounts/automations/push_account/manager.py b/apps/accounts/automations/push_account/manager.py index 42a53fcb1..6ea99ecdf 100644 --- a/apps/accounts/automations/push_account/manager.py +++ b/apps/accounts/automations/push_account/manager.py @@ -63,7 +63,8 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager): inventory_hosts = [] host['secret_type'] = self.secret_type if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY: - print(f'Windows {asset} does not support ssh key push \n') + msg = f'Windows {asset} does not support ssh key push \n' + print(msg) return inventory_hosts for account in accounts: diff --git a/apps/accounts/automations/verify_account/host/posix/main.yml b/apps/accounts/automations/verify_account/host/posix/main.yml index 41ae1768d..b096f9d84 100644 --- a/apps/accounts/automations/verify_account/host/posix/main.yml +++ b/apps/accounts/automations/verify_account/host/posix/main.yml @@ -1,11 +1,11 @@ - hosts: demo gather_facts: no tasks: - - name: Verify account - ansible.builtin.ping: + - name: Verify account connectivity become: no + ansible.builtin.ping: vars: + ansible_become: no ansible_user: "{{ account.username }}" ansible_password: "{{ account.secret }}" ansible_ssh_private_key_file: "{{ account.private_key_path }}" - ansible_become: no diff --git a/apps/accounts/automations/verify_account/manager.py b/apps/accounts/automations/verify_account/manager.py index 2b6c831dc..3d07a95da 100644 --- a/apps/accounts/automations/verify_account/manager.py +++ b/apps/accounts/automations/verify_account/manager.py @@ -1,18 +1,67 @@ +import os +from copy import deepcopy + from django.db.models import QuerySet -from accounts.const import AutomationTypes, Connectivity +from accounts.const import AutomationTypes, Connectivity, SecretType from common.utils import get_logger -from ..base.manager import VerifyHostCallbackMixin, AccountBasePlaybookManager +from ..base.manager import AccountBasePlaybookManager logger = get_logger(__name__) -class VerifyAccountManager(VerifyHostCallbackMixin, AccountBasePlaybookManager): +class VerifyAccountManager(AccountBasePlaybookManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.host_account_mapper = {} + def prepare_runtime_dir(self): + path = super().prepare_runtime_dir() + ansible_config_path = os.path.join(path, 'ansible.cfg') + + with open(ansible_config_path, 'w') as f: + f.write('[ssh_connection]\n') + f.write('ssh_args = -o ControlMaster=no -o ControlPersist=no\n') + return path + + def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs): + host = super().host_callback( + host, asset=asset, account=account, + automation=automation, path_dir=path_dir, **kwargs + ) + if host.get('error'): + return host + + # host['ssh_args'] = '-o ControlMaster=no -o ControlPersist=no' + accounts = asset.accounts.all() + accounts = self.get_accounts(account, accounts) + inventory_hosts = [] + + for account in accounts: + h = deepcopy(host) + h['name'] += '(' + account.username + ')' + self.host_account_mapper[h['name']] = account + secret = account.secret + + private_key_path = None + if account.secret_type == SecretType.SSH_KEY: + private_key_path = self.generate_private_key_path(secret, path_dir) + secret = self.generate_public_key(secret) + + h['secret_type'] = account.secret_type + h['account'] = { + 'name': account.name, + 'username': account.username, + 'secret_type': account.secret_type, + 'secret': secret, + 'private_key_path': private_key_path + } + inventory_hosts.append(h) + # print("Host: ") + # print(self.json_dumps(inventory_hosts)) + return inventory_hosts + @classmethod def method_type(cls): return AutomationTypes.verify_account diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 02f84dc22..98c1afb9d 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -1,12 +1,11 @@ import json import os import shutil -import yaml - from collections import defaultdict from hashlib import md5 from socket import gethostname +import yaml from django.conf import settings from django.utils import timezone from django.utils.translation import gettext as _ @@ -55,8 +54,7 @@ class BasePlaybookManager: def get_assets_group_by_platform(self): return self.execution.all_assets_group_by_platform() - @lazyproperty - def runtime_dir(self): + def prepare_runtime_dir(self): ansible_dir = settings.ANSIBLE_DIR task_name = self.execution.snapshot['name'] dir_name = '{}_{}'.format(task_name.replace(' ', '_'), self.execution.id) @@ -66,8 +64,14 @@ class BasePlaybookManager: ) if not os.path.exists(path): os.makedirs(path, exist_ok=True, mode=0o755) + return path + + @lazyproperty + def runtime_dir(self): + path = self.prepare_runtime_dir() if settings.DEBUG_DEV: - print(f'Ansible runtime dir: {path}') + msg = 'Ansible runtime dir: {}'.format(path) + print(msg) return path @staticmethod @@ -158,7 +162,8 @@ class BasePlaybookManager: def get_runners(self): assets_group_by_platform = self.get_assets_group_by_platform() if settings.DEBUG_DEV: - print("assets_group_by_platform: {}".format(assets_group_by_platform)) + msg = 'Assets group by platform: {}'.format(dict(assets_group_by_platform)) + print(msg) runners = [] for platform, assets in assets_group_by_platform.items(): assets_bulked = [assets[i:i + self.bulk_size] for i in range(0, len(assets), self.bulk_size)] @@ -199,8 +204,7 @@ class BasePlaybookManager: if state == 'ok': self.on_host_success(host, result) elif state == 'skipped': - # TODO - print('skipped: ', hosts) + pass else: error = hosts.get(host) self.on_host_error(host, error, result) @@ -214,10 +218,14 @@ class BasePlaybookManager: d = json.load(f) return d + @staticmethod + def json_dumps(data): + return json.dumps(data, indent=4, sort_keys=True) + @staticmethod def json_to_file(path, data): with open(path, 'w') as f: - json.dump(data, f) + json.dump(data, f, indent=4, sort_keys=True) def local_gateway_prepare(self, runner): info = self.file_to_json(runner.inventory) @@ -241,47 +249,27 @@ class BasePlaybookManager: def local_gateway_clean(self, runner): servers = self.gateway_servers.get(runner.id, []) - try: - for s in servers: - print('Server down: %s' % s) + for s in servers: + try: s.stop() - except Exception: - pass + except Exception: + pass def before_runner_start(self, runner): self.local_gateway_prepare(runner) def after_runner_end(self, runner): - self.delete_sensitive_data(runner.inventory) self.local_gateway_clean(runner) - def delete_sensitive_data(self, path): + def delete_runtime_dir(self): if settings.DEBUG_DEV: return - - d = self.file_to_json(path) - - def delete_keys(d, keys_to_delete): - """ - 递归函数：删除嵌套字典中的指定键 - """ - if not isinstance(d, dict): - return d - keys = list(d.keys()) - for key in keys: - if key in keys_to_delete: - del d[key] - else: - delete_keys(d[key], keys_to_delete) - return d - - d = delete_keys(d, ['secret', 'ansible_password']) - self.json_to_file(path, d) + shutil.rmtree(self.runtime_dir) def run(self, *args, **kwargs): runners = self.get_runners() if len(runners) > 1: - print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) + print("### 分次执行任务, 总共 {}\n".format(len(runners))) elif len(runners) == 1: print(">>> 开始执行任务\n") else: @@ -303,3 +291,4 @@ class BasePlaybookManager: self.execution.status = 'success' self.execution.date_finished = timezone.now() self.execution.save() + self.delete_runtime_dir() From 89c4635162c25cc8b69e6e788b4c47fc0354045c Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 22 Feb 2023 15:15:00 +0800 Subject: [PATCH 6/6] =?UTF-8?q?perf:=20change=20secrett=20=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=20(#9686)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: feng <1304903146@qq.com> --- apps/accounts/automations/change_secret/manager.py | 6 ++++-- apps/accounts/automations/push_account/manager.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/accounts/automations/change_secret/manager.py b/apps/accounts/automations/change_secret/manager.py index a6c14fa23..6cd36f080 100644 --- a/apps/accounts/automations/change_secret/manager.py +++ b/apps/accounts/automations/change_secret/manager.py @@ -87,7 +87,9 @@ class ChangeSecretManager(AccountBasePlaybookManager): accounts = accounts.filter(secret_type=self.secret_type) if not accounts: - print('没有发现待改密账号: %s 用户名: %s 类型: %s' % (asset.name, account.username, self.secret_type)) + print('没有发现待改密账号: %s 用户名: %s 类型: %s' % ( + asset.name, self.snapshot_account_usernames, self.secret_type + )) return [] method_attr = getattr(automation, self.method_type() + '_method') @@ -98,7 +100,7 @@ class ChangeSecretManager(AccountBasePlaybookManager): host['secret_type'] = self.secret_type if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY: - print(f'Windows {asset} does not support ssh key push \n') + print(f'Windows {asset} does not support ssh key push') return inventory_hosts for account in accounts: diff --git a/apps/accounts/automations/push_account/manager.py b/apps/accounts/automations/push_account/manager.py index 42a53fcb1..770ca9d25 100644 --- a/apps/accounts/automations/push_account/manager.py +++ b/apps/accounts/automations/push_account/manager.py @@ -59,7 +59,6 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager): accounts = asset.accounts.all() accounts = self.get_accounts(account, accounts) - inventory_hosts = [] host['secret_type'] = self.secret_type if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY: