From 4bf147a93fcc81bf0781417aa003dec04bfe0549 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 3 Nov 2022 15:11:20 +0800 Subject: [PATCH 1/2] perf: add remote app installer --- .../deploy_applet_host/playbook.yml | 49 +++++++++++++++---- apps/terminal/const.py | 1 + 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/apps/terminal/automations/deploy_applet_host/playbook.yml b/apps/terminal/automations/deploy_applet_host/playbook.yml index 0c115e91a..b2c64f0cb 100644 --- a/apps/terminal/automations/deploy_applet_host/playbook.yml +++ b/apps/terminal/automations/deploy_applet_host/playbook.yml @@ -13,6 +13,7 @@ RDS_fSingleSessionPerUser: 1 RDS_MaxDisconnectionTime: 60000 RDS_RemoteAppLogoffTimeLimit: 0 + TinkerInstaller: JumpServer-Remoteapp_v0.0.1.exe tasks: - name: Install RDS-Licensing (RDS) @@ -29,16 +30,26 @@ include_management_tools: yes register: rds_install - - name: Download Jmservisor (jumpserver) + - name: Download JumpServer Remoteapp installer (jumpserver) ansible.windows.win_get_url: - url: "{{ DownloadHost }}/Jmservisor.msi" - dest: "{{ ansible_env.TEMP }}\\Jmservisor.msi" + url: "{{ DownloadHost }}/{{ TinkerInstaller }}" + dest: "{{ ansible_env.TEMP }}\\{{ TinkerInstaller }}" - - name: Install the Jmservisor (jumpserver) + - name: Install JumpServer Remoteapp agent (jumpserver) ansible.windows.win_package: - path: "{{ ansible_env.TEMP }}\\Jmservisor.msi" + path: "{{ ansible_env.TEMP }}\\{{ TinkerInstaller }}" + args: + - /VERYSILENT + - /SUPPRESSMSGBOXES + - /NORESTART state: present + - name: Set remote-server on the global system path (remote-server) + ansible.windows.win_path: + elements: + - '%USERPROFILE%\AppData\Local\Programs\JumpServer-Remoteapp\' + scope: user + - name: Download python-3.10.8 ansible.windows.win_get_url: url: "{{ DownloadHost }}/python-3.10.8-amd64.exe" @@ -116,12 +127,12 @@ - name: Download chromedriver (chrome) ansible.windows.win_get_url: - url: "{{ DownloadHost }}/chromedriver_win32.106.zip" - dest: "{{ ansible_env.TEMP }}\\chromedriver_win32.106.zip" + url: "{{ DownloadHost }}/chromedriver_win32.107.zip" + dest: "{{ ansible_env.TEMP }}\\chromedriver_win32.107.zip" - name: Unzip chromedriver (chrome) community.windows.win_unzip: - src: "{{ ansible_env.TEMP }}\\chromedriver_win32.106.zip" + src: "{{ ansible_env.TEMP }}\\chromedriver_win32.107.zip" dest: C:\Program Files\JumpServer\drivers - name: Set chromedriver on the global system path (chrome) @@ -142,8 +153,26 @@ - /quiet - name: Generate component config - ansible.windows.win_shell: > - echo "Todo: Set config" + ansible.windows.win_shell: + "remoteapp-server config --core_host {{ CORE_HOST }} --token {{ BOOTSTRAP_TOKEN }} --host_id {{ HOST_ID }}" + + - name: Install remoteapp-server service + ansible.windows.win_shell: + "remoteapp-server service install" + + - name: Start remoteapp-server service + ansible.windows.win_shell: + "remoteapp-server service start" + + - name: Wait Tinker api health + ansible.windows.win_uri: + url: http://localhost:6068/api/health/ + status_code: 200 + method: GET + register: _result + until: _result.status_code == 200 + retries: 30 + delay: 5 - name: Sync all remote applets ansible.windows.win_shell: > diff --git a/apps/terminal/const.py b/apps/terminal/const.py index 7289f3180..ef23030df 100644 --- a/apps/terminal/const.py +++ b/apps/terminal/const.py @@ -50,6 +50,7 @@ class TerminalTypeChoices(TextChoices): celery = 'celery', 'Celery' magnus = 'magnus', 'Magnus' razor = 'razor', 'Razor' + tinker = 'tinker', 'Tinker' @classmethod def types(cls): From 54f92e100e32c7a36e213b40607ba9c914062c39 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 3 Nov 2022 16:57:34 +0800 Subject: [PATCH 2/2] perf: account backup (#9013) Co-authored-by: feng <1304903146@qq.com> --- apps/assets/api/account/backup.py | 5 ++--- .../{backup => backup_account}/__init__.py | 0 .../{backup => backup_account}/handlers.py | 7 ++++--- .../{backup => backup_account}/manager.py | 2 +- apps/assets/automations/endpoint.py | 4 +++- .../0109_rename_categories_to_types.py | 18 ++++++++++++++++++ apps/assets/models/backup.py | 19 +++++++++++-------- 7 files changed, 39 insertions(+), 16 deletions(-) rename apps/assets/automations/{backup => backup_account}/__init__.py (100%) rename apps/assets/automations/{backup => backup_account}/handlers.py (97%) rename apps/assets/automations/{backup => backup_account}/manager.py (97%) create mode 100644 apps/assets/migrations/0109_rename_categories_to_types.py diff --git a/apps/assets/api/account/backup.py b/apps/assets/api/account/backup.py index 79ae721f8..ff46c3caf 100644 --- a/apps/assets/api/account/backup.py +++ b/apps/assets/api/account/backup.py @@ -4,6 +4,7 @@ from rest_framework import status, viewsets from rest_framework.response import Response from orgs.mixins.api import OrgBulkModelViewSet +from common.const.choices import Trigger from assets import serializers from assets.tasks import execute_account_backup_plan from assets.models import ( @@ -38,9 +39,7 @@ class AccountBackupPlanExecutionViewSet(viewsets.ModelViewSet): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) pid = serializer.data.get('plan') - task = execute_account_backup_plan.delay( - pid=pid, trigger=AccountBackupPlanExecution.Trigger.manual - ) + task = execute_account_backup_plan.delay(pid=pid, trigger=Trigger.manual) return Response({'task': task.id}, status=status.HTTP_201_CREATED) def filter_queryset(self, queryset): diff --git a/apps/assets/automations/backup/__init__.py b/apps/assets/automations/backup_account/__init__.py similarity index 100% rename from apps/assets/automations/backup/__init__.py rename to apps/assets/automations/backup_account/__init__.py diff --git a/apps/assets/automations/backup/handlers.py b/apps/assets/automations/backup_account/handlers.py similarity index 97% rename from apps/assets/automations/backup/handlers.py rename to apps/assets/automations/backup_account/handlers.py index 2addebb86..c95f823a4 100644 --- a/apps/assets/automations/backup/handlers.py +++ b/apps/assets/automations/backup_account/handlers.py @@ -82,15 +82,16 @@ class AssetAccountHandler(BaseAccountHandler): # TODO 可以优化一下查询 在账号上做 category 的缓存 避免数据量大时连表操作 qs = Account.objects.filter( - asset__platform__category__in=categories - ).annotate(category=F('asset__platform__category')) + asset__platform__type__in=categories + ).annotate(category=F('asset__platform__type')) + print(qs, categories) if not qs.exists(): return data_map category_dict = {} for i in AllTypes.grouped_choices_to_objs(): for j in i['children']: - category_dict[j['value']] = j['label'] + category_dict[j['value']] = j['display_name'] header_fields = cls.get_header_fields(AccountSecretSerializer(qs.first())) account_category_map = defaultdict(list) diff --git a/apps/assets/automations/backup/manager.py b/apps/assets/automations/backup_account/manager.py similarity index 97% rename from apps/assets/automations/backup/manager.py rename to apps/assets/automations/backup_account/manager.py index c9558fea0..311361c69 100644 --- a/apps/assets/automations/backup/manager.py +++ b/apps/assets/automations/backup_account/manager.py @@ -12,7 +12,7 @@ from .handlers import AccountBackupHandler logger = get_logger(__name__) -class AccountBackupExecutionManager: +class AccountBackupManager: def __init__(self, execution): self.execution = execution self.date_start = timezone.now() diff --git a/apps/assets/automations/endpoint.py b/apps/assets/automations/endpoint.py index c6eb04593..eb7b2f4ca 100644 --- a/apps/assets/automations/endpoint.py +++ b/apps/assets/automations/endpoint.py @@ -3,6 +3,7 @@ from .gather_facts.manager import GatherFactsManager from .gather_accounts.manager import GatherAccountsManager from .verify_account.manager import VerifyAccountManager from .push_account.manager import PushAccountManager +from .backup_account.manager import AccountBackupManager from ..const import AutomationTypes @@ -13,6 +14,8 @@ class ExecutionManager: AutomationTypes.gather_accounts: GatherAccountsManager, AutomationTypes.verify_account: VerifyAccountManager, AutomationTypes.push_account: PushAccountManager, + # TODO 后期迁移到自动化策略中 + 'backup_account': AccountBackupManager, } def __init__(self, execution): @@ -21,4 +24,3 @@ class ExecutionManager: def run(self, *args, **kwargs): return self._runner.run(*args, **kwargs) - diff --git a/apps/assets/migrations/0109_rename_categories_to_types.py b/apps/assets/migrations/0109_rename_categories_to_types.py new file mode 100644 index 000000000..aa7fcad8f --- /dev/null +++ b/apps/assets/migrations/0109_rename_categories_to_types.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.14 on 2022-11-03 08:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0108_auto_20221027_1053'), + ] + + operations = [ + migrations.RenameField( + model_name='accountbackupplan', + old_name='categories', + new_name='types', + ), + ] diff --git a/apps/assets/models/backup.py b/apps/assets/models/backup.py index 3a27adb45..3cf49a94d 100644 --- a/apps/assets/models/backup.py +++ b/apps/assets/models/backup.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- # import uuid -from functools import reduce from celery import current_task from django.db import models @@ -11,9 +10,9 @@ from django.utils.translation import ugettext_lazy as _ from orgs.mixins.models import OrgModelMixin from ops.mixin import PeriodTaskModelMixin from common.utils import get_logger +from common.const.choices import Trigger from common.db.encoder import ModelJSONFieldEncoder from common.mixins.models import CommonModelMixin -from common.const.choices import Trigger __all__ = ['AccountBackupPlan', 'AccountBackupPlanExecution'] @@ -22,7 +21,7 @@ logger = get_logger(__file__) class AccountBackupPlan(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) - categories = models.JSONField(default=list) + types = models.JSONField(default=list) recipients = models.ManyToManyField( 'users.User', related_name='recipient_escape_route_plans', blank=True, verbose_name=_("Recipient") @@ -53,7 +52,7 @@ class AccountBackupPlan(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): 'crontab': self.crontab, 'org_id': self.org_id, 'created_by': self.created_by, - 'categories': self.categories, + 'types': self.types, 'recipients': { str(recipient.id): (str(recipient), bool(recipient.secret_key)) for recipient in self.recipients.all() @@ -100,9 +99,9 @@ class AccountBackupPlanExecution(OrgModelMixin): verbose_name = _('Account backup execution') @property - def categories(self): - categories = self.plan_snapshot.get('categories') - return categories + def types(self): + types = self.plan_snapshot.get('types') + return types @property def recipients(self): @@ -111,7 +110,11 @@ class AccountBackupPlanExecution(OrgModelMixin): return [] return recipients.values() + @property + def manager_type(self): + return 'backup_account' + def start(self): - from ..task_handlers import ExecutionManager + from assets.automations.endpoint import ExecutionManager manager = ExecutionManager(execution=self) return manager.run()