From 0fdc30bed3f448581cce2d979cab716e23f6b84a Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Wed, 2 Nov 2022 20:36:40 +0800 Subject: [PATCH 01/10] perf: account --- apps/assets/models/base.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index f293b9a93..78fd36cb2 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -3,22 +3,21 @@ import io import os import uuid +import sshpubkeys from hashlib import md5 -import sshpubkeys -from django.core.cache import cache from django.db import models from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from django.conf import settings from django.db.models import QuerySet +from common.db import fields from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, random_string, ssh_pubkey_gen, ) -from common.db import fields -from assets.const import Connectivity +from assets.const import Connectivity, SecretType from orgs.mixins.models import OrgModelMixin logger = get_logger(__file__) @@ -49,12 +48,6 @@ class AbsConnectivity(models.Model): class BaseAccount(OrgModelMixin): - class SecretType(models.TextChoices): - password = 'password', _('Password') - ssh_key = 'ssh_key', _('SSH key') - access_key = 'access_key', _('Access key') - token = 'token', _('Token') - id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField(max_length=128, verbose_name=_("Name")) username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True) @@ -78,7 +71,7 @@ class BaseAccount(OrgModelMixin): @property def private_key(self): - if self.secret_type == self.SecretType.ssh_key: + if self.secret_type == SecretType.ssh_key: return self.secret return None @@ -89,7 +82,7 @@ class BaseAccount(OrgModelMixin): @private_key.setter def private_key(self, value): self.secret = value - self.secret_type = 'private_key' + self.secret_type = SecretType.ssh_key @property def ssh_key_fingerprint(self): From 7087d5a74e182e50f33ea833bbf6b172714957c4 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Thu, 3 Nov 2022 12:42:57 +0800 Subject: [PATCH 02/10] perf: account specific --- apps/assets/models/base.py | 28 +++++++++++++++---------- apps/assets/serializers/account/base.py | 6 +++++- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 78fd36cb2..9f4f05649 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -15,7 +15,7 @@ from django.db.models import QuerySet from common.db import fields from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, - random_string, ssh_pubkey_gen, + random_string, ssh_pubkey_gen, lazyproperty ) from assets.const import Connectivity, SecretType from orgs.mixins.models import OrgModelMixin @@ -61,36 +61,42 @@ class BaseAccount(OrgModelMixin): date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) - @property - def password(self): - return self.secret - @property def has_secret(self): return bool(self.secret) + @property + def specific(self): + data = {} + if self.secret_type != SecretType.ssh_key: + return data + data['ssh_key_fingerprint'] = self.ssh_key_fingerprint + return data + @property def private_key(self): if self.secret_type == SecretType.ssh_key: return self.secret return None - @property - def public_key(self): - return '' - @private_key.setter def private_key(self, value): self.secret = value self.secret_type = SecretType.ssh_key + @lazyproperty + def public_key(self): + if self.secret_type == SecretType.ssh_key: + return ssh_pubkey_gen(private_key=self.private_key) + return None + @property def ssh_key_fingerprint(self): if self.public_key: public_key = self.public_key elif self.private_key: try: - public_key = ssh_pubkey_gen(private_key=self.private_key, password=self.password) + public_key = ssh_pubkey_gen(private_key=self.private_key) except IOError as e: return str(e) else: @@ -103,7 +109,7 @@ class BaseAccount(OrgModelMixin): @property def private_key_obj(self): if self.private_key: - key_obj = ssh_key_string_to_obj(self.private_key, password=self.password) + key_obj = ssh_key_string_to_obj(self.private_key) return key_obj else: return None diff --git a/apps/assets/serializers/account/base.py b/apps/assets/serializers/account/base.py index 262272a0a..5db43257e 100644 --- a/apps/assets/serializers/account/base.py +++ b/apps/assets/serializers/account/base.py @@ -21,9 +21,13 @@ class BaseAccountSerializer(BulkOrgResourceModelSerializer): class Meta: model = BaseAccount fields_mini = ['id', 'name', 'username'] - fields_small = fields_mini + ['privileged', 'secret_type', 'secret', 'has_secret'] + fields_small = fields_mini + ['privileged', 'secret_type', 'secret', 'has_secret', 'specific'] fields_other = ['created_by', 'date_created', 'date_updated', 'comment'] fields = fields_small + fields_other + read_only_fields = [ + 'has_secret', 'specific', + 'date_verified', 'created_by', 'date_created', + ] extra_kwargs = { 'secret': {'write_only': True}, 'passphrase': {'write_only': True}, From 4bf147a93fcc81bf0781417aa003dec04bfe0549 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 3 Nov 2022 15:11:20 +0800 Subject: [PATCH 03/10] perf: add remote app installer --- .../deploy_applet_host/playbook.yml | 49 +++++++++++++++---- apps/terminal/const.py | 1 + 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/apps/terminal/automations/deploy_applet_host/playbook.yml b/apps/terminal/automations/deploy_applet_host/playbook.yml index 0c115e91a..b2c64f0cb 100644 --- a/apps/terminal/automations/deploy_applet_host/playbook.yml +++ b/apps/terminal/automations/deploy_applet_host/playbook.yml @@ -13,6 +13,7 @@ RDS_fSingleSessionPerUser: 1 RDS_MaxDisconnectionTime: 60000 RDS_RemoteAppLogoffTimeLimit: 0 + TinkerInstaller: JumpServer-Remoteapp_v0.0.1.exe tasks: - name: Install RDS-Licensing (RDS) @@ -29,16 +30,26 @@ include_management_tools: yes register: rds_install - - name: Download Jmservisor (jumpserver) + - name: Download JumpServer Remoteapp installer (jumpserver) ansible.windows.win_get_url: - url: "{{ DownloadHost }}/Jmservisor.msi" - dest: "{{ ansible_env.TEMP }}\\Jmservisor.msi" + url: "{{ DownloadHost }}/{{ TinkerInstaller }}" + dest: "{{ ansible_env.TEMP }}\\{{ TinkerInstaller }}" - - name: Install the Jmservisor (jumpserver) + - name: Install JumpServer Remoteapp agent (jumpserver) ansible.windows.win_package: - path: "{{ ansible_env.TEMP }}\\Jmservisor.msi" + path: "{{ ansible_env.TEMP }}\\{{ TinkerInstaller }}" + args: + - /VERYSILENT + - /SUPPRESSMSGBOXES + - /NORESTART state: present + - name: Set remote-server on the global system path (remote-server) + ansible.windows.win_path: + elements: + - '%USERPROFILE%\AppData\Local\Programs\JumpServer-Remoteapp\' + scope: user + - name: Download python-3.10.8 ansible.windows.win_get_url: url: "{{ DownloadHost }}/python-3.10.8-amd64.exe" @@ -116,12 +127,12 @@ - name: Download chromedriver (chrome) ansible.windows.win_get_url: - url: "{{ DownloadHost }}/chromedriver_win32.106.zip" - dest: "{{ ansible_env.TEMP }}\\chromedriver_win32.106.zip" + url: "{{ DownloadHost }}/chromedriver_win32.107.zip" + dest: "{{ ansible_env.TEMP }}\\chromedriver_win32.107.zip" - name: Unzip chromedriver (chrome) community.windows.win_unzip: - src: "{{ ansible_env.TEMP }}\\chromedriver_win32.106.zip" + src: "{{ ansible_env.TEMP }}\\chromedriver_win32.107.zip" dest: C:\Program Files\JumpServer\drivers - name: Set chromedriver on the global system path (chrome) @@ -142,8 +153,26 @@ - /quiet - name: Generate component config - ansible.windows.win_shell: > - echo "Todo: Set config" + ansible.windows.win_shell: + "remoteapp-server config --core_host {{ CORE_HOST }} --token {{ BOOTSTRAP_TOKEN }} --host_id {{ HOST_ID }}" + + - name: Install remoteapp-server service + ansible.windows.win_shell: + "remoteapp-server service install" + + - name: Start remoteapp-server service + ansible.windows.win_shell: + "remoteapp-server service start" + + - name: Wait Tinker api health + ansible.windows.win_uri: + url: http://localhost:6068/api/health/ + status_code: 200 + method: GET + register: _result + until: _result.status_code == 200 + retries: 30 + delay: 5 - name: Sync all remote applets ansible.windows.win_shell: > diff --git a/apps/terminal/const.py b/apps/terminal/const.py index 7289f3180..ef23030df 100644 --- a/apps/terminal/const.py +++ b/apps/terminal/const.py @@ -50,6 +50,7 @@ class TerminalTypeChoices(TextChoices): celery = 'celery', 'Celery' magnus = 'magnus', 'Magnus' razor = 'razor', 'Razor' + tinker = 'tinker', 'Tinker' @classmethod def types(cls): From 340d39d7f7ab2405b65e5bbf632fc4632d4b7e4a Mon Sep 17 00:00:00 2001 From: "Jiangjie.Bai" Date: Thu, 3 Nov 2022 16:41:51 +0800 Subject: [PATCH 04/10] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=8E=88?= =?UTF-8?q?=E6=9D=83=E7=BB=99=E7=94=A8=E6=88=B7=E6=89=80=E6=9C=89=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E5=88=97=E8=A1=A8=E7=9A=84API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/perms/api/user_permission/__init__.py | 1 + apps/perms/api/user_permission/accounts.py | 24 ++++++++++++++++++++++ apps/perms/urls/asset_permission.py | 7 +++++-- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 apps/perms/api/user_permission/accounts.py diff --git a/apps/perms/api/user_permission/__init__.py b/apps/perms/api/user_permission/__init__.py index 47f3e84a3..b0db20ee0 100644 --- a/apps/perms/api/user_permission/__init__.py +++ b/apps/perms/api/user_permission/__init__.py @@ -4,3 +4,4 @@ from .common import * from .nodes import * from .assets import * from .nodes_with_assets import * +from .accounts import * diff --git a/apps/perms/api/user_permission/accounts.py b/apps/perms/api/user_permission/accounts.py new file mode 100644 index 000000000..d504ac8f9 --- /dev/null +++ b/apps/perms/api/user_permission/accounts.py @@ -0,0 +1,24 @@ +from rest_framework import generics +from assets.serializers import AccountSerializer +from perms.utils.account import PermAccountUtil +from .mixin import RoleAdminMixin, RoleUserMixin + + +__all__ = ['UserAllGrantedAccountsApi', 'MyAllGrantedAccountsApi'] + + +class UserAllGrantedAccountsApi(RoleAdminMixin, generics.ListAPIView): + """ 授权给用户的所有账号列表 """ + serializer_class = AccountSerializer + filterset_fields = ("name", "username", "privileged", "version") + search_fields = filterset_fields + + def get_queryset(self): + util = PermAccountUtil() + accounts = util.get_perm_accounts_for_user(self.user) + return accounts + + +class MyAllGrantedAccountsApi(RoleUserMixin, UserAllGrantedAccountsApi): + """ 授权给我的所有账号列表 """ + pass diff --git a/apps/perms/urls/asset_permission.py b/apps/perms/urls/asset_permission.py index 095a67dba..99605372d 100644 --- a/apps/perms/urls/asset_permission.py +++ b/apps/perms/urls/asset_permission.py @@ -58,9 +58,12 @@ user_permission_urlpatterns = [ # 收藏的资产 path('/nodes/favorite/assets/', api.UserFavoriteGrantedAssetsApi.as_view(), name='user-ungrouped-assets'), path('nodes/favorite/assets/', api.MyFavoriteGrantedAssetsApi.as_view(), name='my-ungrouped-assets'), - # v3 中上面的 API 基本不用动 - # 获取所有和资产-用户关联的账号列表 + # 获取授权给用户的所有账号 + path('/accounts/', api.UserAllGrantedAccountsApi.as_view(), name='user-accounts'), + path('accounts/', api.MyAllGrantedAccountsApi.as_view(), name='my-accounts'), + + # 获取授权给用户某个资产的所有账号 path('/assets//accounts/', api.UserGrantedAssetAccountsApi.as_view(), name='user-asset-accounts'), path('assets//accounts/', api.MyGrantedAssetAccountsApi.as_view(), name='my-asset-accounts'), # 用户登录资产的特殊账号, @INPUT, @USER 等 From 54f92e100e32c7a36e213b40607ba9c914062c39 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 3 Nov 2022 16:57:34 +0800 Subject: [PATCH 05/10] perf: account backup (#9013) Co-authored-by: feng <1304903146@qq.com> --- apps/assets/api/account/backup.py | 5 ++--- .../{backup => backup_account}/__init__.py | 0 .../{backup => backup_account}/handlers.py | 7 ++++--- .../{backup => backup_account}/manager.py | 2 +- apps/assets/automations/endpoint.py | 4 +++- .../0109_rename_categories_to_types.py | 18 ++++++++++++++++++ apps/assets/models/backup.py | 19 +++++++++++-------- 7 files changed, 39 insertions(+), 16 deletions(-) rename apps/assets/automations/{backup => backup_account}/__init__.py (100%) rename apps/assets/automations/{backup => backup_account}/handlers.py (97%) rename apps/assets/automations/{backup => backup_account}/manager.py (97%) create mode 100644 apps/assets/migrations/0109_rename_categories_to_types.py diff --git a/apps/assets/api/account/backup.py b/apps/assets/api/account/backup.py index 79ae721f8..ff46c3caf 100644 --- a/apps/assets/api/account/backup.py +++ b/apps/assets/api/account/backup.py @@ -4,6 +4,7 @@ from rest_framework import status, viewsets from rest_framework.response import Response from orgs.mixins.api import OrgBulkModelViewSet +from common.const.choices import Trigger from assets import serializers from assets.tasks import execute_account_backup_plan from assets.models import ( @@ -38,9 +39,7 @@ class AccountBackupPlanExecutionViewSet(viewsets.ModelViewSet): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) pid = serializer.data.get('plan') - task = execute_account_backup_plan.delay( - pid=pid, trigger=AccountBackupPlanExecution.Trigger.manual - ) + task = execute_account_backup_plan.delay(pid=pid, trigger=Trigger.manual) return Response({'task': task.id}, status=status.HTTP_201_CREATED) def filter_queryset(self, queryset): diff --git a/apps/assets/automations/backup/__init__.py b/apps/assets/automations/backup_account/__init__.py similarity index 100% rename from apps/assets/automations/backup/__init__.py rename to apps/assets/automations/backup_account/__init__.py diff --git a/apps/assets/automations/backup/handlers.py b/apps/assets/automations/backup_account/handlers.py similarity index 97% rename from apps/assets/automations/backup/handlers.py rename to apps/assets/automations/backup_account/handlers.py index 2addebb86..c95f823a4 100644 --- a/apps/assets/automations/backup/handlers.py +++ b/apps/assets/automations/backup_account/handlers.py @@ -82,15 +82,16 @@ class AssetAccountHandler(BaseAccountHandler): # TODO 可以优化一下查询 在账号上做 category 的缓存 避免数据量大时连表操作 qs = Account.objects.filter( - asset__platform__category__in=categories - ).annotate(category=F('asset__platform__category')) + asset__platform__type__in=categories + ).annotate(category=F('asset__platform__type')) + print(qs, categories) if not qs.exists(): return data_map category_dict = {} for i in AllTypes.grouped_choices_to_objs(): for j in i['children']: - category_dict[j['value']] = j['label'] + category_dict[j['value']] = j['display_name'] header_fields = cls.get_header_fields(AccountSecretSerializer(qs.first())) account_category_map = defaultdict(list) diff --git a/apps/assets/automations/backup/manager.py b/apps/assets/automations/backup_account/manager.py similarity index 97% rename from apps/assets/automations/backup/manager.py rename to apps/assets/automations/backup_account/manager.py index c9558fea0..311361c69 100644 --- a/apps/assets/automations/backup/manager.py +++ b/apps/assets/automations/backup_account/manager.py @@ -12,7 +12,7 @@ from .handlers import AccountBackupHandler logger = get_logger(__name__) -class AccountBackupExecutionManager: +class AccountBackupManager: def __init__(self, execution): self.execution = execution self.date_start = timezone.now() diff --git a/apps/assets/automations/endpoint.py b/apps/assets/automations/endpoint.py index c6eb04593..eb7b2f4ca 100644 --- a/apps/assets/automations/endpoint.py +++ b/apps/assets/automations/endpoint.py @@ -3,6 +3,7 @@ from .gather_facts.manager import GatherFactsManager from .gather_accounts.manager import GatherAccountsManager from .verify_account.manager import VerifyAccountManager from .push_account.manager import PushAccountManager +from .backup_account.manager import AccountBackupManager from ..const import AutomationTypes @@ -13,6 +14,8 @@ class ExecutionManager: AutomationTypes.gather_accounts: GatherAccountsManager, AutomationTypes.verify_account: VerifyAccountManager, AutomationTypes.push_account: PushAccountManager, + # TODO 后期迁移到自动化策略中 + 'backup_account': AccountBackupManager, } def __init__(self, execution): @@ -21,4 +24,3 @@ class ExecutionManager: def run(self, *args, **kwargs): return self._runner.run(*args, **kwargs) - diff --git a/apps/assets/migrations/0109_rename_categories_to_types.py b/apps/assets/migrations/0109_rename_categories_to_types.py new file mode 100644 index 000000000..aa7fcad8f --- /dev/null +++ b/apps/assets/migrations/0109_rename_categories_to_types.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-03 08:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0108_auto_20221027_1053'), + ] + + operations = [ + migrations.RenameField( + model_name='accountbackupplan', + old_name='categories', + new_name='types', + ), + ] diff --git a/apps/assets/models/backup.py b/apps/assets/models/backup.py index 3a27adb45..3cf49a94d 100644 --- a/apps/assets/models/backup.py +++ b/apps/assets/models/backup.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- # import uuid -from functools import reduce from celery import current_task from django.db import models @@ -11,9 +10,9 @@ from django.utils.translation import ugettext_lazy as _ from orgs.mixins.models import OrgModelMixin from ops.mixin import PeriodTaskModelMixin from common.utils import get_logger +from common.const.choices import Trigger from common.db.encoder import ModelJSONFieldEncoder from common.mixins.models import CommonModelMixin -from common.const.choices import Trigger __all__ = ['AccountBackupPlan', 'AccountBackupPlanExecution'] @@ -22,7 +21,7 @@ logger = get_logger(__file__) class AccountBackupPlan(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) - categories = models.JSONField(default=list) + types = models.JSONField(default=list) recipients = models.ManyToManyField( 'users.User', related_name='recipient_escape_route_plans', blank=True, verbose_name=_("Recipient") @@ -53,7 +52,7 @@ class AccountBackupPlan(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): 'crontab': self.crontab, 'org_id': self.org_id, 'created_by': self.created_by, - 'categories': self.categories, + 'types': self.types, 'recipients': { str(recipient.id): (str(recipient), bool(recipient.secret_key)) for recipient in self.recipients.all() @@ -100,9 +99,9 @@ class AccountBackupPlanExecution(OrgModelMixin): verbose_name = _('Account backup execution') @property - def categories(self): - categories = self.plan_snapshot.get('categories') - return categories + def types(self): + types = self.plan_snapshot.get('types') + return types @property def recipients(self): @@ -111,7 +110,11 @@ class AccountBackupPlanExecution(OrgModelMixin): return [] return recipients.values() + @property + def manager_type(self): + return 'backup_account' + def start(self): - from ..task_handlers import ExecutionManager + from assets.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) return manager.run() From 7560a5cd1f3a1d32f49ea92392420a89e731e623 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 3 Nov 2022 18:03:46 +0800 Subject: [PATCH 06/10] perf: deploy applet host --- .../automations/deploy_applet_host/__init__.py | 15 +++++++++------ .../automations/deploy_applet_host/playbook.yml | 6 ++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/apps/terminal/automations/deploy_applet_host/__init__.py b/apps/terminal/automations/deploy_applet_host/__init__.py index c02aa491a..8b415ece4 100644 --- a/apps/terminal/automations/deploy_applet_host/__init__.py +++ b/apps/terminal/automations/deploy_applet_host/__init__.py @@ -27,13 +27,19 @@ class DeployAppletHostManager: def generate_playbook(self): playbook_src = os.path.join(CURRENT_DIR, 'playbook.yml') + base_site_url = settings.BASE_SITE_URL + bootstrap_token = settings.BOOTSTRAP_TOKEN + host_id = str(self.deployment.host.id) + if not base_site_url: + base_site_url = "localhost:8080" with open(playbook_src) as f: plays = yaml.safe_load(f) for play in plays: play['vars'].update(self.deployment.host.deploy_options) - play['vars']['DownloadHost'] = settings.BASE_URL + '/download/' - play['vars']['CORE_HOST'] = settings.BASE_URL - play['vars']['BOOTSTRAP_TOKEN'] = settings.BOOSTRAP_TOKEN + play['vars']['DownloadHost'] = base_site_url + '/download/' + play['vars']['CORE_HOST'] = base_site_url + play['vars']['BOOTSTRAP_TOKEN'] = bootstrap_token + play['vars']['HOST_ID'] = host_id play['vars']['HOST_NAME'] = self.deployment.host.name playbook_dir = os.path.join(self.run_dir, 'playbook') @@ -70,6 +76,3 @@ class DeployAppletHostManager: self.deployment.date_finished = timezone.now() with safe_db_connection(): self.deployment.save() - - - diff --git a/apps/terminal/automations/deploy_applet_host/playbook.yml b/apps/terminal/automations/deploy_applet_host/playbook.yml index b2c64f0cb..867d58f76 100644 --- a/apps/terminal/automations/deploy_applet_host/playbook.yml +++ b/apps/terminal/automations/deploy_applet_host/playbook.yml @@ -5,6 +5,7 @@ DownloadHost: https://demo.jumpserver.org/download Initial: 0 HOST_NAME: test + HOST_ID: 00000000-0000-0000-0000-000000000000 CORE_HOST: https://demo.jumpserver.org BOOTSTRAP_TOKEN: PleaseChangeMe RDS_Licensing: true @@ -38,7 +39,7 @@ - name: Install JumpServer Remoteapp agent (jumpserver) ansible.windows.win_package: path: "{{ ansible_env.TEMP }}\\{{ TinkerInstaller }}" - args: + arguments: - /VERYSILENT - /SUPPRESSMSGBOXES - /NORESTART @@ -154,7 +155,8 @@ - name: Generate component config ansible.windows.win_shell: - "remoteapp-server config --core_host {{ CORE_HOST }} --token {{ BOOTSTRAP_TOKEN }} --host_id {{ HOST_ID }}" + "remoteapp-server config --hostname {{ HOST_NAME }} --core_host {{ CORE_HOST }} + --token {{ BOOTSTRAP_TOKEN }} --host_id {{ HOST_ID }}" - name: Install remoteapp-server service ansible.windows.win_shell: From ebfc3b7b38e98ee6eca4d0c5ab24f564c05264f7 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 3 Nov 2022 22:39:48 +0800 Subject: [PATCH 07/10] perf: change secret (#9014) Co-authored-by: feng <1304903146@qq.com> --- .../automations/change_secret/manager.py | 64 ++++++++++++++++++- .../0110_changesecretrecord_asset.py | 19 ++++++ apps/assets/models/automations/base.py | 7 ++ .../models/automations/change_secret.py | 1 + apps/assets/models/base.py | 1 - apps/assets/notifications.py | 28 +++++++- apps/assets/serializers/__init__.py | 1 + apps/assets/serializers/automation.py | 35 ++++++++++ 8 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 apps/assets/migrations/0110_changesecretrecord_asset.py create mode 100644 apps/assets/serializers/automation.py diff --git a/apps/assets/automations/change_secret/manager.py b/apps/assets/automations/change_secret/manager.py index 4f77dfe85..fd289b12f 100644 --- a/apps/assets/automations/change_secret/manager.py +++ b/apps/assets/automations/change_secret/manager.py @@ -1,17 +1,28 @@ +import os +import time import random import string from copy import deepcopy +from openpyxl import Workbook from collections import defaultdict from django.utils import timezone +from django.conf import settings -from common.utils import lazyproperty, gen_key_pair +from common.utils.timezone import local_now_display +from common.utils.file import encrypt_and_compress_zip_file +from common.utils import get_logger, lazyproperty, gen_key_pair +from users.models import User from assets.models import ChangeSecretRecord +from assets.notifications import ChangeSecretExecutionTaskMsg +from assets.serializers import ChangeSecretRecordBackUpSerializer from assets.const import ( AutomationTypes, SecretType, SecretStrategy, SSHKeyStrategy, DEFAULT_PASSWORD_RULES ) from ..base.manager import BasePlaybookManager +logger = get_logger(__name__) + class ChangeSecretManager(BasePlaybookManager): def __init__(self, *args, **kwargs): @@ -125,7 +136,7 @@ class ChangeSecretManager(BasePlaybookManager): new_secret = self.get_secret() recorder = ChangeSecretRecord( - account=account, execution=self.execution, + asset=asset, account=account, execution=self.execution, old_secret=account.secret, new_secret=new_secret, ) records.append(recorder) @@ -172,4 +183,51 @@ class ChangeSecretManager(BasePlaybookManager): recorder.save() def on_runner_failed(self, runner, e): - pass + logger.error("Change secret error: ", e) + + def run(self, *args, **kwargs): + super().run(*args, **kwargs) + recorders = self.name_recorder_mapper.values() + recorders = list(recorders) + self.send_recorder_mail(recorders) + + def send_recorder_mail(self, recorders): + if not recorders: + return + recipients = self.execution.recipients + if not recipients: + return + recipients = User.objects.filter(id__in=list(recipients)) + + name = self.execution.snapshot['name'] + path = os.path.join(os.path.dirname(settings.BASE_DIR), 'tmp') + filename = os.path.join(path, f'{name}-{local_now_display()}-{time.time()}.xlsx') + if not self.create_file(recorders, filename): + return + + for user in recipients: + attachments = [] + if user.secret_key: + password = user.secret_key.encode('utf8') + attachment = os.path.join(path, f'{name}-{local_now_display()}-{time.time()}.zip') + encrypt_and_compress_zip_file(attachment, password, [filename]) + attachments = [attachment] + ChangeSecretExecutionTaskMsg(name, user).publish(attachments) + os.remove(filename) + + @staticmethod + def create_file(recorders, filename): + serializer_cls = ChangeSecretRecordBackUpSerializer + serializer = serializer_cls(recorders, many=True) + header = [v.label for v in serializer.child.fields.values()] + rows = [list(row.values()) for row in serializer.data] + if not rows: + return False + + rows.insert(0, header) + wb = Workbook(filename) + ws = wb.create_sheet('Sheet1') + for row in rows: + ws.append(row) + wb.save(filename) + return True diff --git a/apps/assets/migrations/0110_changesecretrecord_asset.py b/apps/assets/migrations/0110_changesecretrecord_asset.py new file mode 100644 index 000000000..7a4e862ff --- /dev/null +++ b/apps/assets/migrations/0110_changesecretrecord_asset.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.14 on 2022-11-03 13:57 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0109_rename_categories_to_types'), + ] + + operations = [ + migrations.AddField( + model_name='changesecretrecord', + name='asset', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='assets.asset'), + ), + ] diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index ac1fdb046..5eadca8c4 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -111,6 +111,13 @@ class AutomationExecution(OrgModelMixin): def manager_type(self): return self.snapshot['type'] + @property + def recipients(self): + recipients = self.snapshot.get('recipients') + if not recipients: + return [] + return recipients.values() + def start(self): from assets.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) diff --git a/apps/assets/models/automations/change_secret.py b/apps/assets/models/automations/change_secret.py index 53ca08aba..81871fb3b 100644 --- a/apps/assets/models/automations/change_secret.py +++ b/apps/assets/models/automations/change_secret.py @@ -51,6 +51,7 @@ class ChangeSecretAutomation(BaseAutomation): class ChangeSecretRecord(JMSBaseModel): execution = models.ForeignKey('assets.AutomationExecution', on_delete=models.CASCADE) + asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, null=True) account = models.ForeignKey('assets.Account', on_delete=models.CASCADE, null=True) old_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Old secret')) new_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret')) diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 9f4f05649..8fa91b2cb 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -158,7 +158,6 @@ class BaseAccount(OrgModelMixin): return { 'name': self.name, 'username': self.username, - 'password': self.password, 'public_key': self.public_key, } diff --git a/apps/assets/notifications.py b/apps/assets/notifications.py index 58c02686c..6a67878c9 100644 --- a/apps/assets/notifications.py +++ b/apps/assets/notifications.py @@ -15,11 +15,35 @@ class AccountBackupExecutionTaskMsg(object): def message(self): name = self.name if self.user.secret_key: - return _('{} - The account backup passage task has been completed. See the attachment for details').format(name) + return _('{} - The account backup passage task has been completed. See the attachment for details').format( + name) return _("{} - The account backup passage task has been completed: the encryption password has not been set - " - "please go to personal information -> file encryption password to set the encryption password").format(name) + "please go to personal information -> file encryption password to set the encryption password").format( + name) def publish(self, attachment_list=None): send_mail_attachment_async.delay( self.subject, self.message, [self.user.email], attachment_list ) + + +class ChangeSecretExecutionTaskMsg(object): + subject = _('Notification of implementation result of encryption change plan') + + def __init__(self, name: str, user: User): + self.name = name + self.user = user + + @property + def message(self): + name = self.name + if self.user.secret_key: + return _('{} - The encryption change task has been completed. See the attachment for details').format(name) + return _("{} - The encryption change task has been completed: the encryption password has not been set - " + "please go to personal information -> file encryption password to set the encryption password").format( + name) + + def publish(self, attachments=None): + send_mail_attachment_async.delay( + self.subject, self.message, [self.user.email], attachments + ) diff --git a/apps/assets/serializers/__init__.py b/apps/assets/serializers/__init__.py index 586c6f2e5..252b2dc64 100644 --- a/apps/assets/serializers/__init__.py +++ b/apps/assets/serializers/__init__.py @@ -11,3 +11,4 @@ from .account import * from assets.serializers.account.backup import * from .platform import * from .cagegory import * +from .automation import * diff --git a/apps/assets/serializers/automation.py b/apps/assets/serializers/automation.py new file mode 100644 index 000000000..482f95fc8 --- /dev/null +++ b/apps/assets/serializers/automation.py @@ -0,0 +1,35 @@ +from django.utils.translation import ugettext as _ +from rest_framework import serializers + +from common.utils import get_logger + +from assets.models import ChangeSecretRecord + +logger = get_logger(__file__) + + +class ChangeSecretRecordBackUpSerializer(serializers.ModelSerializer): + asset = serializers.SerializerMethodField(label=_('Asset')) + account = serializers.SerializerMethodField(label=_('Account')) + is_success = serializers.SerializerMethodField(label=_('Is success')) + + class Meta: + model = ChangeSecretRecord + fields = [ + 'id', 'asset', 'account', 'old_secret', 'new_secret', + 'status', 'error', 'is_success' + ] + + @staticmethod + def get_asset(instance): + return str(instance.asset) + + @staticmethod + def get_account(instance): + return str(instance.account) + + @staticmethod + def get_is_success(obj): + if obj.status == 'success': + return _("Success") + return _("Failed") From e995e3b35ae63066bf359cf5c5e1eb0595b4e11f Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Fri, 4 Nov 2022 11:09:56 +0800 Subject: [PATCH 08/10] perf: change secret adjustment --- apps/assets/automations/change_secret/manager.py | 4 +--- apps/assets/models/base.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/assets/automations/change_secret/manager.py b/apps/assets/automations/change_secret/manager.py index fd289b12f..a8b7dd515 100644 --- a/apps/assets/automations/change_secret/manager.py +++ b/apps/assets/automations/change_secret/manager.py @@ -192,10 +192,8 @@ class ChangeSecretManager(BasePlaybookManager): self.send_recorder_mail(recorders) def send_recorder_mail(self, recorders): - if not recorders: - return recipients = self.execution.recipients - if not recipients: + if not recorders or not recipients: return recipients = User.objects.filter(id__in=list(recipients)) diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 8fa91b2cb..c0bf5b078 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -116,7 +116,7 @@ class BaseAccount(OrgModelMixin): @property def private_key_path(self): - if not self.secret_type != 'ssh_key' or not self.secret: + if not self.secret_type != SecretType.ssh_key or not self.secret: return None project_dir = settings.PROJECT_DIR tmp_dir = os.path.join(project_dir, 'tmp') From 8b05bc4b82154dc93387ef0e2cd5a04c5e158f6f Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Fri, 4 Nov 2022 11:15:34 +0800 Subject: [PATCH 09/10] =?UTF-8?q?perf:=20=E5=A4=9A=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E5=8F=91=E9=80=81=E6=96=87=E4=BB=B6=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/notifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/assets/notifications.py b/apps/assets/notifications.py index 6a67878c9..a797bc845 100644 --- a/apps/assets/notifications.py +++ b/apps/assets/notifications.py @@ -22,7 +22,7 @@ class AccountBackupExecutionTaskMsg(object): name) def publish(self, attachment_list=None): - send_mail_attachment_async.delay( + send_mail_attachment_async( self.subject, self.message, [self.user.email], attachment_list ) @@ -44,6 +44,6 @@ class ChangeSecretExecutionTaskMsg(object): name) def publish(self, attachments=None): - send_mail_attachment_async.delay( + send_mail_attachment_async( self.subject, self.message, [self.user.email], attachments ) From 1981bdd3acb954ab99a2305b5bddb10d43414281 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Fri, 4 Nov 2022 11:39:34 +0800 Subject: [PATCH 10/10] perf: account serializer --- apps/assets/serializers/account/account.py | 4 ++-- apps/assets/serializers/account/template.py | 26 +++++++++------------ apps/assets/tasks/push_account.py | 6 ++--- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/apps/assets/serializers/account/account.py b/apps/assets/serializers/account/account.py index b219211c2..efd8d9060 100644 --- a/apps/assets/serializers/account/account.py +++ b/apps/assets/serializers/account/account.py @@ -3,6 +3,7 @@ from rest_framework import serializers from common.drf.serializers import SecretReadableMixin from common.drf.fields import ObjectRelatedField +from assets.tasks import push_accounts_to_assets from assets.models import Account, AccountTemplate, Asset from .base import BaseAccountSerializer @@ -47,8 +48,7 @@ class AccountSerializerCreateMixin(serializers.ModelSerializer): def create(self, validated_data): instance = super().create(validated_data) if self.push_now: - # Todo: push it - print("Start push account to asset") + push_accounts_to_assets.delay([instance.id], [instance.asset_id]) return instance diff --git a/apps/assets/serializers/account/template.py b/apps/assets/serializers/account/template.py index 09c5b4541..4599ca0e7 100644 --- a/apps/assets/serializers/account/template.py +++ b/apps/assets/serializers/account/template.py @@ -1,6 +1,3 @@ -from django.utils.translation import ugettext_lazy as _ -from rest_framework import serializers - from assets.models import AccountTemplate from .base import BaseAccountSerializer @@ -9,15 +6,14 @@ class AccountTemplateSerializer(BaseAccountSerializer): class Meta(BaseAccountSerializer.Meta): model = AccountTemplate - @classmethod - def validate_required(cls, attrs): - # Todo: why ? - required_field_dict = {} - error = _('This field is required.') - for k, v in cls().fields.items(): - if v.required and k not in attrs: - required_field_dict[k] = error - if not required_field_dict: - return - raise serializers.ValidationError(required_field_dict) - + # @classmethod + # def validate_required(cls, attrs): + # # TODO 选择模版后检查一些必填项 + # required_field_dict = {} + # error = _('This field is required.') + # for k, v in cls().fields.items(): + # if v.required and k not in attrs: + # required_field_dict[k] = error + # if not required_field_dict: + # return + # raise serializers.ValidationError(required_field_dict) diff --git a/apps/assets/tasks/push_account.py b/apps/assets/tasks/push_account.py index 19ebd5045..8af71cd3f 100644 --- a/apps/assets/tasks/push_account.py +++ b/apps/assets/tasks/push_account.py @@ -11,8 +11,9 @@ __all__ = [ @org_aware_func("assets") -def push_accounts_to_assets_util(accounts, assets, task_name): +def push_accounts_to_assets_util(accounts, assets): from assets.models import PushAccountAutomation + task_name = gettext_noop("Push accounts to assets") task_name = PushAccountAutomation.generate_unique_name(task_name) account_usernames = list(accounts.values_list('username', flat=True)) @@ -33,5 +34,4 @@ def push_accounts_to_assets(account_ids, asset_ids): assets = Asset.objects.get(id=asset_ids) accounts = Account.objects.get(id=account_ids) - task_name = gettext_noop("Push accounts to assets") - return push_accounts_to_assets_util(accounts, assets, task_name) + return push_accounts_to_assets_util(accounts, assets)