From cd93de4c006f89bedf30721bfffa2d1c141ba772 Mon Sep 17 00:00:00 2001 From: "Jiangjie.Bai" Date: Tue, 8 Nov 2022 14:30:07 +0800 Subject: [PATCH 1/3] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20Connection=20T?= =?UTF-8?q?oken=20API=20=E9=80=BB=E8=BE=91=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/cmd_filter.py | 2 +- apps/authentication/api/connection_token.py | 6 ++---- apps/authentication/models/connection_token.py | 2 +- apps/authentication/serializers/connection_token.py | 4 ++-- apps/perms/utils/account.py | 8 +++++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apps/assets/models/cmd_filter.py b/apps/assets/models/cmd_filter.py index be8945c55..7023fdbc6 100644 --- a/apps/assets/models/cmd_filter.py +++ b/apps/assets/models/cmd_filter.py @@ -201,7 +201,7 @@ class CommandFilterRule(OrgModelMixin): q |= Q(user_groups__in=set(user_groups)) if account: org_id = account.org_id - q |= Q(accounts__contains=list(account)) |\ + q |= Q(accounts__contains=account.username) | \ Q(accounts__contains=SpecialAccount.ALL.value) if asset: org_id = asset.org_id diff --git a/apps/authentication/api/connection_token.py b/apps/authentication/api/connection_token.py index 08b59581e..0c04531d5 100644 --- a/apps/authentication/api/connection_token.py +++ b/apps/authentication/api/connection_token.py @@ -178,8 +178,6 @@ class ExtraActionApiMixin(RDPFileClientProtocolURLMixin): get_object: callable get_serializer: callable perform_create: callable - check_token_permission: callable - create_connection_token: callable @action(methods=['POST'], detail=False, url_path='secret-info/detail') def get_secret_detail(self, request, *args, **kwargs): @@ -277,10 +275,10 @@ class ConnectionTokenViewSet(ExtraActionApiMixin, RootOrgViewMixin, JMSModelView from perms.utils.account import PermAccountUtil actions, expire_at = PermAccountUtil().validate_permission(user, asset, account_username) if not actions: - error = '' + error = 'No actions' raise PermissionDenied(error) if expire_at < time.time(): - error = '' + error = 'Expired' raise PermissionDenied(error) diff --git a/apps/authentication/models/connection_token.py b/apps/authentication/models/connection_token.py index 3ed4c2a54..48c61f954 100644 --- a/apps/authentication/models/connection_token.py +++ b/apps/authentication/models/connection_token.py @@ -85,7 +85,7 @@ class ConnectionToken(OrgModelMixin, JMSBaseModel): is_valid = False error = _('No user or invalid user') return is_valid, error - if not self.asset or self.asset.is_active: + if not self.asset or not self.asset.is_active: is_valid = False error = _('No asset or inactive asset') return is_valid, error diff --git a/apps/authentication/serializers/connection_token.py b/apps/authentication/serializers/connection_token.py index e809ed78c..6e1f19be1 100644 --- a/apps/authentication/serializers/connection_token.py +++ b/apps/authentication/serializers/connection_token.py @@ -159,7 +159,7 @@ class ConnectionTokenSecretSerializer(OrgResourceModelSerializerMixin): domain = ConnectionTokenDomainSerializer(read_only=True) cmd_filter_rules = ConnectionTokenCmdFilterRuleSerializer(many=True) actions = ActionsField() - expired_at = serializers.IntegerField() + expire_at = serializers.IntegerField() class Meta: model = ConnectionToken @@ -167,5 +167,5 @@ class ConnectionTokenSecretSerializer(OrgResourceModelSerializerMixin): 'id', 'secret', 'user', 'asset', 'account_username', 'account', 'protocol', 'domain', 'gateway', 'cmd_filter_rules', - 'actions', 'expired_at', + 'actions', 'expire_at', ] diff --git a/apps/perms/utils/account.py b/apps/perms/utils/account.py index 3963e113c..8d8f5e743 100644 --- a/apps/perms/utils/account.py +++ b/apps/perms/utils/account.py @@ -53,7 +53,9 @@ class PermAccountUtil(AssetPermissionUtil): user, asset, with_actions=True, with_perms=True ) perm = perms.first() - account = accounts.filter(username=account_username).first() - actions = account.actions if account else [] - expire_at = perm.date_expired if perm else time.time() + actions = [] + for account in accounts: + if account.username == account_username: + actions = account.actions + expire_at = perm.date_expired.timestamp() if perm else time.time() return actions, expire_at From e69bb9f83e5271fc7692c323110572b0b8776180 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 8 Nov 2022 17:54:04 +0800 Subject: [PATCH 2/3] perf: applet host accounts should be inactive by default --- apps/terminal/models/applet/host.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/terminal/models/applet/host.py b/apps/terminal/models/applet/host.py index 295c65b6a..5a85cbaf8 100644 --- a/apps/terminal/models/applet/host.py +++ b/apps/terminal/models/applet/host.py @@ -11,7 +11,6 @@ from common.db.models import JMSBaseModel from common.utils import random_string from assets.models import Host - __all__ = ['AppletHost', 'AppletHostDeployment'] @@ -26,7 +25,7 @@ class AppletHost(Host): ) applets = models.ManyToManyField( 'Applet', verbose_name=_('Applet'), - through='AppletPublication', through_fields=('host', 'applet'), + through='AppletPublication', through_fields=('host', 'applet'), ) LOCKING_ORG = '00000000-0000-0000-0000-000000000004' @@ -70,8 +69,8 @@ class AppletHost(Host): status_applets['published'].append(applet) for status, applets in status_applets.items(): - self.publications.filter(applet__in=applets)\ - .exclude(status=status)\ + self.publications.filter(applet__in=applets) \ + .exclude(status=status) \ .update(status=status) @staticmethod @@ -95,7 +94,7 @@ class AppletHost(Host): account = account_model( username=username, secret=password, name=username, asset_id=self.id, secret_type='password', version=1, - org_id=self.LOCKING_ORG + org_id=self.LOCKING_ORG, is_active=False, ) accounts.append(account) bulk_create_with_history(accounts, account_model, batch_size=20) From ce9ebd94ecd0bea81565ddae3a9b61efa9af888d Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 8 Nov 2022 17:54:51 +0800 Subject: [PATCH 3/3] perf: change secret automation api (#9028) Co-authored-by: feng <1304903146@qq.com> --- apps/assets/api/__init__.py | 1 + apps/assets/api/automations/__init__.py | 3 + apps/assets/api/automations/base.py | 118 +++++++++++++++ apps/assets/api/automations/change_secret.py | 40 +++++ .../assets/api/automations/gather_accounts.py | 0 apps/assets/automations/base/manager.py | 2 +- .../automations/change_secret/manager.py | 8 +- apps/assets/automations/ping/manager.py | 8 +- .../automations/verify_account/manager.py | 4 +- apps/assets/const/account.py | 14 +- apps/assets/const/automation.py | 16 ++ apps/assets/models/automations/__init__.py | 9 +- apps/assets/models/automations/base.py | 2 +- .../models/automations/change_secret.py | 4 +- apps/assets/models/base.py | 14 +- apps/assets/serializers/__init__.py | 2 +- apps/assets/serializers/automation.py | 35 ----- .../serializers/automations/__init__.py | 3 + apps/assets/serializers/automations/base.py | 76 ++++++++++ .../serializers/automations/change_secret.py | 139 ++++++++++++++++++ .../automations/gather_accounts.py | 0 apps/assets/serializers/base.py | 65 +++----- apps/assets/serializers/utils.py | 15 ++ apps/assets/tasks/automation.py | 4 +- apps/assets/urls/api_urls.py | 22 ++- 25 files changed, 488 insertions(+), 116 deletions(-) create mode 100644 apps/assets/api/automations/__init__.py create mode 100644 apps/assets/api/automations/base.py create mode 100644 apps/assets/api/automations/change_secret.py create mode 100644 apps/assets/api/automations/gather_accounts.py delete mode 100644 apps/assets/serializers/automation.py create mode 100644 apps/assets/serializers/automations/__init__.py create mode 100644 apps/assets/serializers/automations/base.py create mode 100644 apps/assets/serializers/automations/change_secret.py create mode 100644 apps/assets/serializers/automations/gather_accounts.py diff --git a/apps/assets/api/__init__.py b/apps/assets/api/__init__.py index c14d8999f..36f734030 100644 --- a/apps/assets/api/__init__.py +++ b/apps/assets/api/__init__.py @@ -6,5 +6,6 @@ from .label import * from .account import * from .node import * from .domain import * +from .automations import * from .gathered_user import * from .favorite_asset import * diff --git a/apps/assets/api/automations/__init__.py b/apps/assets/api/automations/__init__.py new file mode 100644 index 000000000..e4daeda95 --- /dev/null +++ b/apps/assets/api/automations/__init__.py @@ -0,0 +1,3 @@ +from .base import * +from .change_secret import * +from .gather_accounts import * diff --git a/apps/assets/api/automations/base.py b/apps/assets/api/automations/base.py new file mode 100644 index 000000000..1b551f412 --- /dev/null +++ b/apps/assets/api/automations/base.py @@ -0,0 +1,118 @@ +from django.shortcuts import get_object_or_404 +from django.utils.translation import ugettext_lazy as _ +from rest_framework.response import Response +from rest_framework import status, mixins, viewsets + +from orgs.mixins import generics +from assets import serializers +from assets.const import AutomationTypes +from assets.tasks import execute_automation +from assets.models import BaseAutomation, AutomationExecution +from common.const.choices import Trigger + +__all__ = [ + 'AutomationAssetsListApi', 'AutomationRemoveAssetApi', + 'AutomationAddAssetApi', 'AutomationNodeAddRemoveApi', 'AutomationExecutionViewSet' +] + + +class AutomationAssetsListApi(generics.ListAPIView): + serializer_class = serializers.AutomationAssetsSerializer + filter_fields = ("name", "address") + search_fields = filter_fields + + def get_object(self): + pk = self.kwargs.get('pk') + return get_object_or_404(BaseAutomation, pk=pk) + + def get_queryset(self): + instance = self.get_object() + assets = instance.get_all_assets().only( + *self.serializer_class.Meta.only_fields + ) + return assets + + +class AutomationRemoveAssetApi(generics.RetrieveUpdateAPIView): + model = BaseAutomation + serializer_class = serializers.UpdateAssetSerializer + + def update(self, request, *args, **kwargs): + instance = self.get_object() + serializer = self.serializer_class(data=request.data) + + if not serializer.is_valid(): + return Response({'error': serializer.errors}) + + assets = serializer.validated_data.get('assets') + if assets: + instance.assets.remove(*tuple(assets)) + return Response({'msg': 'ok'}) + + +class AutomationAddAssetApi(generics.RetrieveUpdateAPIView): + model = BaseAutomation + serializer_class = serializers.UpdateAssetSerializer + + def update(self, request, *args, **kwargs): + instance = self.get_object() + serializer = self.serializer_class(data=request.data) + if serializer.is_valid(): + assets = serializer.validated_data.get('assets') + if assets: + instance.assets.add(*tuple(assets)) + return Response({"msg": "ok"}) + else: + return Response({"error": serializer.errors}) + + +class AutomationNodeAddRemoveApi(generics.RetrieveUpdateAPIView): + model = BaseAutomation + serializer_class = serializers.UpdateAssetSerializer + + def update(self, request, *args, **kwargs): + action_params = ['add', 'remove'] + action = request.query_params.get('action') + if action not in action_params: + err_info = _("The parameter 'action' must be [{}]".format(','.join(action_params))) + return Response({"error": err_info}) + + instance = self.get_object() + serializer = self.serializer_class(data=request.data) + if serializer.is_valid(): + nodes = serializer.validated_data.get('nodes') + if nodes: + # eg: plan.nodes.add(*tuple(assets)) + getattr(instance.nodes, action)(*tuple(nodes)) + return Response({"msg": "ok"}) + else: + return Response({"error": serializer.errors}) + + +class AutomationExecutionViewSet( + mixins.CreateModelMixin, mixins.ListModelMixin, + mixins.RetrieveModelMixin, viewsets.GenericViewSet +): + search_fields = ('trigger',) + filterset_fields = ('trigger', 'automation_id') + serializer_class = serializers.AutomationExecutionSerializer + + def get_queryset(self): + queryset = AutomationExecution.objects.all() + return queryset + + def filter_queryset(self, queryset): + queryset = super().filter_queryset(queryset) + queryset = queryset.order_by('-date_start') + return queryset + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + automation = serializer.validated_data.get('automation') + tp = serializer.validated_data.get('type') + model = AutomationTypes.get_model(tp) + task = execute_automation.delay( + pid=automation.ok, trigger=Trigger.manual, model=model + ) + return Response({'task': task.id}, status=status.HTTP_201_CREATED) diff --git a/apps/assets/api/automations/change_secret.py b/apps/assets/api/automations/change_secret.py new file mode 100644 index 000000000..944443914 --- /dev/null +++ b/apps/assets/api/automations/change_secret.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# + +from rest_framework import mixins + +from common.utils import get_object_or_none +from orgs.mixins.api import OrgBulkModelViewSet, OrgGenericViewSet + +from assets.models import ChangeSecretAutomation, ChangeSecretRecord, AutomationExecution +from assets import serializers + +__all__ = [ + 'ChangeSecretAutomationViewSet', 'ChangeSecretRecordViewSet' +] + + +class ChangeSecretAutomationViewSet(OrgBulkModelViewSet): + model = ChangeSecretAutomation + filter_fields = ('name', 'secret_type', 'secret_strategy') + search_fields = filter_fields + ordering_fields = ('name',) + serializer_class = serializers.ChangeSecretAutomationSerializer + + +class ChangeSecretRecordViewSet(mixins.ListModelMixin, OrgGenericViewSet): + serializer_class = serializers.ChangeSecretRecordSerializer + filter_fields = ['username', 'asset', 'reason', 'execution'] + search_fields = ['username', 'reason', 'asset__hostname'] + + def get_queryset(self): + return ChangeSecretRecord.objects.all() + + def filter_queryset(self, queryset): + queryset = super().filter_queryset(queryset) + eid = self.request.GET.get('execution_id') + execution = get_object_or_none(AutomationExecution, pk=eid) + if execution: + queryset = queryset.filter(execution=execution) + queryset = queryset.order_by('is_success', '-date_start') + return queryset diff --git a/apps/assets/api/automations/gather_accounts.py b/apps/assets/api/automations/gather_accounts.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index ab8ea01bf..512454d9f 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -47,7 +47,7 @@ class PushOrVerifyHostCallbackMixin: secret = account.secret private_key_path = None - if account.secret_type == SecretType.ssh_key: + if account.secret_type == SecretType.SSH_KEY: private_key_path = self.generate_private_key_path(secret, path_dir) secret = self.generate_public_key(secret) diff --git a/apps/assets/automations/change_secret/manager.py b/apps/assets/automations/change_secret/manager.py index a8b7dd515..fc2ec60fe 100644 --- a/apps/assets/automations/change_secret/manager.py +++ b/apps/assets/automations/change_secret/manager.py @@ -89,9 +89,9 @@ class ChangeSecretManager(BasePlaybookManager): return self.generate_password() def get_secret(self): - if self.secret_type == SecretType.ssh_key: + if self.secret_type == SecretType.SSH_KEY: secret = self.get_ssh_key() - elif self.secret_type == SecretType.password: + elif self.secret_type == SecretType.PASSWORD: secret = self.get_password() else: raise ValueError("Secret must be set") @@ -99,7 +99,7 @@ class ChangeSecretManager(BasePlaybookManager): def get_kwargs(self, account, secret): kwargs = {} - if self.secret_type != SecretType.ssh_key: + if self.secret_type != SecretType.SSH_KEY: return kwargs kwargs['strategy'] = self.execution.snapshot['ssh_key_change_strategy'] kwargs['exclusive'] = 'yes' if kwargs['strategy'] == SSHKeyStrategy.set else 'no' @@ -143,7 +143,7 @@ class ChangeSecretManager(BasePlaybookManager): self.name_recorder_mapper[h['name']] = recorder private_key_path = None - if self.secret_type == SecretType.ssh_key: + if self.secret_type == SecretType.SSH_KEY: private_key_path = self.generate_private_key_path(new_secret, path_dir) new_secret = self.generate_public_key(new_secret) diff --git a/apps/assets/automations/ping/manager.py b/apps/assets/automations/ping/manager.py index 34c05a8f4..305771f0b 100644 --- a/apps/assets/automations/ping/manager.py +++ b/apps/assets/automations/ping/manager.py @@ -21,14 +21,14 @@ class PingManager(BasePlaybookManager): def on_host_success(self, host, result): asset, account = self.host_asset_and_account_mapper.get(host) - asset.set_connectivity(Connectivity.ok) + asset.set_connectivity(Connectivity.OK) if not account: return - account.set_connectivity(Connectivity.ok) + account.set_connectivity(Connectivity.OK) def on_host_error(self, host, error, result): asset, account = self.host_asset_and_account_mapper.get(host) - asset.set_connectivity(Connectivity.failed) + asset.set_connectivity(Connectivity.FAILED) if not account: return - account.set_connectivity(Connectivity.failed) + account.set_connectivity(Connectivity.FAILED) diff --git a/apps/assets/automations/verify_account/manager.py b/apps/assets/automations/verify_account/manager.py index fe46bc0ff..f261631e5 100644 --- a/apps/assets/automations/verify_account/manager.py +++ b/apps/assets/automations/verify_account/manager.py @@ -18,8 +18,8 @@ class VerifyAccountManager(PushOrVerifyHostCallbackMixin, BasePlaybookManager): def on_host_success(self, host, result): account = self.host_account_mapper.get(host) - account.set_connectivity(Connectivity.ok) + account.set_connectivity(Connectivity.OK) def on_host_error(self, host, error, result): account = self.host_account_mapper.get(host) - account.set_connectivity(Connectivity.failed) + account.set_connectivity(Connectivity.FAILED) diff --git a/apps/assets/const/account.py b/apps/assets/const/account.py index 5ec872134..ebeb855ed 100644 --- a/apps/assets/const/account.py +++ b/apps/assets/const/account.py @@ -3,13 +3,13 @@ from django.utils.translation import ugettext_lazy as _ class Connectivity(TextChoices): - unknown = 'unknown', _('Unknown') - ok = 'ok', _('Ok') - failed = 'failed', _('Failed') + UNKNOWN = 'unknown', _('Unknown') + OK = 'ok', _('Ok') + FAILED = 'failed', _('Failed') class SecretType(TextChoices): - password = 'password', _('Password') - ssh_key = 'ssh_key', _('SSH key') - access_key = 'access_key', _('Access key') - token = 'token', _('Token') + PASSWORD = 'password', _('Password') + SSH_KEY = 'ssh_key', _('SSH key') + ACCESS_KEY = 'access_key', _('Access key') + TOKEN = 'token', _('Token') diff --git a/apps/assets/const/automation.py b/apps/assets/const/automation.py index 6b3b6dbd4..99acefa7a 100644 --- a/apps/assets/const/automation.py +++ b/apps/assets/const/automation.py @@ -17,6 +17,22 @@ class AutomationTypes(TextChoices): verify_account = 'verify_account', _('Verify account') gather_accounts = 'gather_accounts', _('Gather accounts') + @classmethod + def get_type_model(cls, tp): + from assets.models import ( + PingAutomation, GatherFactsAutomation, PushAccountAutomation, + ChangeSecretAutomation, VerifyAccountAutomation, GatherAccountsAutomation, + ) + type_model_dict = { + cls.ping: PingAutomation, + cls.gather_facts: GatherFactsAutomation, + cls.push_account: PushAccountAutomation, + cls.change_secret: ChangeSecretAutomation, + cls.verify_account: VerifyAccountAutomation, + cls.gather_accounts: GatherAccountsAutomation, + } + return type_model_dict.get(tp) + class SecretStrategy(TextChoices): custom = 'specific', _('Specific') diff --git a/apps/assets/models/automations/__init__.py b/apps/assets/models/automations/__init__.py index e579fc10f..82fa19620 100644 --- a/apps/assets/models/automations/__init__.py +++ b/apps/assets/models/automations/__init__.py @@ -1,7 +1,8 @@ -from .change_secret import * -from .discovery_account import * +from .ping import * +from .base import * from .push_account import * from .gather_facts import * -from .gather_accounts import * +from .change_secret import * from .verify_account import * -from .ping import * +from .gather_accounts import * +from .discovery_account import * diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index 5eadca8c4..e814d4128 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -3,7 +3,7 @@ from celery import current_task from django.db import models from django.utils.translation import ugettext_lazy as _ -from common.const.choices import Trigger, Status +from common.const.choices import Trigger from common.mixins.models import CommonModelMixin from common.db.fields import EncryptJsonDictTextField from orgs.mixins.models import OrgModelMixin diff --git a/apps/assets/models/automations/change_secret.py b/apps/assets/models/automations/change_secret.py index 81871fb3b..c22b64f51 100644 --- a/apps/assets/models/automations/change_secret.py +++ b/apps/assets/models/automations/change_secret.py @@ -12,7 +12,7 @@ __all__ = ['ChangeSecretAutomation', 'ChangeSecretRecord'] class ChangeSecretAutomation(BaseAutomation): secret_type = models.CharField( choices=SecretType.choices, max_length=16, - default=SecretType.password, verbose_name=_('Secret type') + default=SecretType.PASSWORD, verbose_name=_('Secret type') ) secret_strategy = models.CharField( choices=SecretStrategy.choices, max_length=16, @@ -24,7 +24,7 @@ class ChangeSecretAutomation(BaseAutomation): choices=SSHKeyStrategy.choices, max_length=16, default=SSHKeyStrategy.add, verbose_name=_('SSH key change strategy') ) - recipients = models.ManyToManyField('users.User', blank=True, verbose_name=_("Recipient")) + recipients = models.ManyToManyField('users.User', verbose_name=_("Recipient"), blank=True) def save(self, *args, **kwargs): self.type = AutomationTypes.change_secret diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 1194b7957..7920d3798 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -24,7 +24,7 @@ logger = get_logger(__file__) class AbsConnectivity(models.Model): connectivity = models.CharField( - choices=Connectivity.choices, default=Connectivity.unknown, + choices=Connectivity.choices, default=Connectivity.UNKNOWN, max_length=16, verbose_name=_('Connectivity') ) date_verified = models.DateTimeField(null=True, verbose_name=_("Date verified")) @@ -50,7 +50,7 @@ class BaseAccount(JMSOrgBaseModel): name = models.CharField(max_length=128, verbose_name=_("Name")) username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True) secret_type = models.CharField( - max_length=16, choices=SecretType.choices, default=SecretType.password, verbose_name=_('Secret type') + max_length=16, choices=SecretType.choices, default=SecretType.PASSWORD, verbose_name=_('Secret type') ) secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret')) privileged = models.BooleanField(verbose_name=_("Privileged"), default=False) @@ -65,25 +65,25 @@ class BaseAccount(JMSOrgBaseModel): @property def specific(self): data = {} - if self.secret_type != SecretType.ssh_key: + if self.secret_type != SecretType.SSH_KEY: return data data['ssh_key_fingerprint'] = self.ssh_key_fingerprint return data @property def private_key(self): - if self.secret_type == SecretType.ssh_key: + if self.secret_type == SecretType.SSH_KEY: return self.secret return None @private_key.setter def private_key(self, value): self.secret = value - self.secret_type = SecretType.ssh_key + self.secret_type = SecretType.SSH_KEY @lazyproperty def public_key(self): - if self.secret_type == SecretType.ssh_key: + if self.secret_type == SecretType.SSH_KEY: return ssh_pubkey_gen(private_key=self.private_key) return None @@ -113,7 +113,7 @@ class BaseAccount(JMSOrgBaseModel): @property def private_key_path(self): - if not self.secret_type != SecretType.ssh_key or not self.secret: + if not self.secret_type != SecretType.SSH_KEY or not self.secret: return None project_dir = settings.PROJECT_DIR tmp_dir = os.path.join(project_dir, 'tmp') diff --git a/apps/assets/serializers/__init__.py b/apps/assets/serializers/__init__.py index 252b2dc64..9876e3aa6 100644 --- a/apps/assets/serializers/__init__.py +++ b/apps/assets/serializers/__init__.py @@ -11,4 +11,4 @@ from .account import * from assets.serializers.account.backup import * from .platform import * from .cagegory import * -from .automation import * +from .automations import * diff --git a/apps/assets/serializers/automation.py b/apps/assets/serializers/automation.py deleted file mode 100644 index 482f95fc8..000000000 --- a/apps/assets/serializers/automation.py +++ /dev/null @@ -1,35 +0,0 @@ -from django.utils.translation import ugettext as _ -from rest_framework import serializers - -from common.utils import get_logger - -from assets.models import ChangeSecretRecord - -logger = get_logger(__file__) - - -class ChangeSecretRecordBackUpSerializer(serializers.ModelSerializer): - asset = serializers.SerializerMethodField(label=_('Asset')) - account = serializers.SerializerMethodField(label=_('Account')) - is_success = serializers.SerializerMethodField(label=_('Is success')) - - class Meta: - model = ChangeSecretRecord - fields = [ - 'id', 'asset', 'account', 'old_secret', 'new_secret', - 'status', 'error', 'is_success' - ] - - @staticmethod - def get_asset(instance): - return str(instance.asset) - - @staticmethod - def get_account(instance): - return str(instance.account) - - @staticmethod - def get_is_success(obj): - if obj.status == 'success': - return _("Success") - return _("Failed") diff --git a/apps/assets/serializers/automations/__init__.py b/apps/assets/serializers/automations/__init__.py new file mode 100644 index 000000000..e4daeda95 --- /dev/null +++ b/apps/assets/serializers/automations/__init__.py @@ -0,0 +1,3 @@ +from .base import * +from .change_secret import * +from .gather_accounts import * diff --git a/apps/assets/serializers/automations/base.py b/apps/assets/serializers/automations/base.py new file mode 100644 index 000000000..58c169c13 --- /dev/null +++ b/apps/assets/serializers/automations/base.py @@ -0,0 +1,76 @@ +from django.utils.translation import ugettext as _ +from rest_framework import serializers + +from ops.mixin import PeriodTaskSerializerMixin +from assets.const import AutomationTypes +from assets.models import Asset, BaseAutomation, AutomationExecution +from orgs.mixins.serializers import BulkOrgResourceModelSerializer +from common.utils import get_logger + +logger = get_logger(__file__) + +__all__ = [ + 'BaseAutomationSerializer', 'AutomationExecutionSerializer', + 'UpdateAssetSerializer', 'UpdateNodeSerializer', 'AutomationAssetsSerializer', +] + + +class BaseAutomationSerializer(PeriodTaskSerializerMixin, BulkOrgResourceModelSerializer): + class Meta: + read_only_fields = [ + 'date_created', 'date_updated', 'created_by', 'periodic_display' + ] + fields = read_only_fields + [ + 'id', 'name', 'is_periodic', 'interval', 'crontab', 'comment', + 'type', 'accounts', 'nodes', 'assets', 'is_active' + ] + extra_kwargs = { + 'name': {'required': True}, + 'periodic_display': {'label': _('Periodic perform')}, + } + + +class AutomationExecutionSerializer(serializers.ModelSerializer): + snapshot = serializers.SerializerMethodField(label=_('Automation snapshot')) + type = serializers.ChoiceField(choices=AutomationTypes.choices, write_only=True, label=_('Type')) + trigger_display = serializers.ReadOnlyField(source='get_trigger_display', label=_('Trigger mode')) + + class Meta: + model = AutomationExecution + fields = [ + 'id', 'automation', 'trigger', 'trigger_display', + 'date_start', 'date_finished', 'snapshot', 'type' + ] + + @staticmethod + def get_snapshot(obj): + tp = obj.snapshot['type'] + snapshot = { + 'type': tp, + 'name': obj.snapshot['name'], + 'comment': obj.snapshot['comment'], + 'accounts': obj.snapshot['accounts'], + 'node_amount': len(obj.snapshot['nodes']), + 'asset_amount': len(obj.snapshot['assets']), + 'type_display': getattr(AutomationTypes, tp).label, + } + return snapshot + + +class UpdateAssetSerializer(serializers.ModelSerializer): + class Meta: + model = BaseAutomation + fields = ['id', 'assets'] + + +class UpdateNodeSerializer(serializers.ModelSerializer): + class Meta: + model = BaseAutomation + fields = ['id', 'nodes'] + + +class AutomationAssetsSerializer(serializers.ModelSerializer): + class Meta: + model = Asset + only_fields = ['id', 'name', 'address'] + fields = tuple(only_fields) diff --git a/apps/assets/serializers/automations/change_secret.py b/apps/assets/serializers/automations/change_secret.py new file mode 100644 index 000000000..104a3837e --- /dev/null +++ b/apps/assets/serializers/automations/change_secret.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# +from django.utils.translation import ugettext as _ +from rest_framework import serializers + +from assets.serializers.base import AuthValidateMixin +from assets.models import ChangeSecretAutomation, ChangeSecretRecord +from assets.const import DEFAULT_PASSWORD_RULES, SecretType, SecretStrategy +from common.utils import get_logger + +from .base import BaseAutomationSerializer + +logger = get_logger(__file__) + +__all__ = [ + 'ChangeSecretAutomationSerializer', + 'ChangeSecretRecordSerializer', + 'ChangeSecretRecordBackUpSerializer' +] + + +class ChangeSecretAutomationSerializer(AuthValidateMixin, BaseAutomationSerializer): + password_rules = serializers.DictField(default=DEFAULT_PASSWORD_RULES) + secret_strategy_display = serializers.ReadOnlyField( + source='get_secret_strategy_display', label=_('Secret strategy') + ) + ssh_key_change_strategy_display = serializers.ReadOnlyField( + source='get_ssh_key_strategy_display', label=_('SSH Key strategy') + ) + + class Meta: + model = ChangeSecretAutomation + read_only_fields = BaseAutomationSerializer.Meta.read_only_fields + [ + 'secret_strategy_display', 'ssh_key_change_strategy_display' + ] + fields = BaseAutomationSerializer.Meta.fields + read_only_fields + [ + 'secret_type', 'secret_strategy', 'secret', 'password_rules', + 'ssh_key_change_strategy', 'passphrase', 'recipients', + ] + extra_kwargs = {**BaseAutomationSerializer.Meta.extra_kwargs, **{ + 'recipients': {'label': _('Recipient'), 'help_text': _( + "Currently only mail sending is supported" + )}, + }} + + def validate_password_rules(self, password_rules): + secret_type = self.initial_secret_type + if secret_type != SecretType.PASSWORD: + return password_rules + + length = password_rules.get('length') + symbol_set = password_rules.get('symbol_set', '') + + try: + length = int(length) + except Exception as e: + logger.error(e) + msg = _("* Please enter the correct password length") + raise serializers.ValidationError(msg) + if length < 6 or length > 30: + msg = _('* Password length range 6-30 bits') + raise serializers.ValidationError(msg) + + if not isinstance(symbol_set, str): + symbol_set = str(symbol_set) + + password_rules = {'length': length, 'symbol_set': ''.join(symbol_set)} + return password_rules + + def validate(self, attrs): + secret_type = attrs.get('secret_type') + secret_strategy = attrs.get('secret_strategy') + if secret_type == SecretType.PASSWORD: + attrs.pop('ssh_key_change_strategy', None) + if secret_strategy == SecretStrategy.custom: + attrs.pop('password_rules', None) + else: + attrs.pop('secret', None) + elif secret_type == SecretType.SSH_KEY: + attrs.pop('password_rules', None) + if secret_strategy != SecretStrategy.custom: + attrs.pop('secret', None) + return attrs + + +class ChangeSecretRecordSerializer(serializers.ModelSerializer): + asset_display = serializers.SerializerMethodField(label=_('Asset display')) + account_display = serializers.SerializerMethodField(label=_('Account display')) + is_success = serializers.SerializerMethodField(label=_('Is success')) + + class Meta: + model = ChangeSecretRecord + fields = [ + 'id', 'asset', 'account', 'date_started', 'date_finished', + 'is_success', 'error', 'execution', 'asset_display', 'account_display' + ] + read_only_fields = fields + + @staticmethod + def get_asset_display(instance): + return str(instance.asset) + + @staticmethod + def get_account_display(instance): + return str(instance.account) + + @staticmethod + def get_is_success(obj): + if obj.status == 'success': + return _("Success") + return _("Failed") + + +class ChangeSecretRecordBackUpSerializer(serializers.ModelSerializer): + asset = serializers.SerializerMethodField(label=_('Asset')) + account = serializers.SerializerMethodField(label=_('Account')) + is_success = serializers.SerializerMethodField(label=_('Is success')) + + class Meta: + model = ChangeSecretRecord + fields = [ + 'id', 'asset', 'account', 'old_secret', 'new_secret', + 'status', 'error', 'is_success' + ] + read_only_fields = fields + + @staticmethod + def get_asset(instance): + return str(instance.asset) + + @staticmethod + def get_account(instance): + return str(instance.account) + + @staticmethod + def get_is_success(obj): + if obj.status == 'success': + return _("Success") + return _("Failed") diff --git a/apps/assets/serializers/automations/gather_accounts.py b/apps/assets/serializers/automations/gather_accounts.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/assets/serializers/base.py b/apps/assets/serializers/base.py index 91aa2213a..7b5b62a16 100644 --- a/apps/assets/serializers/base.py +++ b/apps/assets/serializers/base.py @@ -1,68 +1,51 @@ # -*- coding: utf-8 -*- # -from io import StringIO - from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from common.utils import ssh_pubkey_gen, ssh_private_key_gen, validate_ssh_private_key from common.drf.fields import EncryptedField -from .utils import validate_password_for_ansible +from assets.const import SecretType +from .utils import validate_password_for_ansible, validate_ssh_key class AuthValidateMixin(serializers.Serializer): - password = EncryptedField( - label=_('Password'), required=False, allow_blank=True, allow_null=True, - max_length=1024, validators=[validate_password_for_ansible] - ) - private_key = EncryptedField( - label=_('SSH private key'), required=False, allow_blank=True, - allow_null=True, max_length=16384 + secret_type = serializers.CharField(label=_('Secret type'), max_length=16, required=True) + secret = EncryptedField( + label=_('Secret'), required=False, max_length=16384, allow_blank=True, + allow_null=True, write_only=True, ) passphrase = serializers.CharField( allow_blank=True, allow_null=True, required=False, max_length=512, write_only=True, label=_('Key password') ) - def validate_private_key(self, private_key): - if not private_key: - return - passphrase = self.initial_data.get('passphrase') - passphrase = passphrase if passphrase else None - valid = validate_ssh_private_key(private_key, password=passphrase) - if not valid: - raise serializers.ValidationError(_("private key invalid or passphrase error")) + @property + def initial_secret_type(self): + secret_type = self.initial_data.get('secret_type') + return secret_type - private_key = ssh_private_key_gen(private_key, password=passphrase) - string_io = StringIO() - private_key.write_private_key(string_io) - private_key = string_io.getvalue() - return private_key + def validate_secret(self, secret): + if not secret: + return + secret_type = self.initial_secret_type + if secret_type == SecretType.PASSWORD: + validate_password_for_ansible(secret) + return secret + elif secret_type == SecretType.SSH_KEY: + passphrase = self.initial_data.get('passphrase') + passphrase = passphrase if passphrase else None + return validate_ssh_key(secret, passphrase) + else: + return secret @staticmethod def clean_auth_fields(validated_data): - for field in ('password', 'private_key', 'public_key'): + for field in ('secret',): value = validated_data.get(field) if not value: validated_data.pop(field, None) validated_data.pop('passphrase', None) - @staticmethod - def _validate_gen_key(attrs): - private_key = attrs.get('private_key') - if not private_key: - return attrs - - password = attrs.get('passphrase') - username = attrs.get('username') - public_key = ssh_pubkey_gen(private_key, password=password, username=username) - attrs['public_key'] = public_key - return attrs - - def validate(self, attrs): - attrs = self._validate_gen_key(attrs) - return super().validate(attrs) - def create(self, validated_data): self.clean_auth_fields(validated_data) return super().create(validated_data) diff --git a/apps/assets/serializers/utils.py b/apps/assets/serializers/utils.py index 52527e723..0734bc9f1 100644 --- a/apps/assets/serializers/utils.py +++ b/apps/assets/serializers/utils.py @@ -1,6 +1,10 @@ +from io import StringIO + from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers +from common.utils import ssh_private_key_gen, validate_ssh_private_key + def validate_password_for_ansible(password): """ 校验 Ansible 不支持的特殊字符 """ @@ -15,3 +19,14 @@ def validate_password_for_ansible(password): if '"' in password: raise serializers.ValidationError(_('Password can not contains `"` ')) + +def validate_ssh_key(ssh_key, passphrase=None): + valid = validate_ssh_private_key(ssh_key, password=passphrase) + if not valid: + raise serializers.ValidationError(_("private key invalid or passphrase error")) + + ssh_key = ssh_private_key_gen(ssh_key, password=passphrase) + string_io = StringIO() + ssh_key.write_private_key(string_io) + ssh_key = string_io.getvalue() + return ssh_key diff --git a/apps/assets/tasks/automation.py b/apps/assets/tasks/automation.py index c4d5f5043..873e606b6 100644 --- a/apps/assets/tasks/automation.py +++ b/apps/assets/tasks/automation.py @@ -7,9 +7,9 @@ logger = get_logger(__file__) @shared_task(queue='ansible') -def execute_automation(pid, trigger, mode): +def execute_automation(pid, trigger, model): with tmp_to_root_org(): - instance = get_object_or_none(mode, pk=pid) + instance = get_object_or_none(model, pk=pid) if not instance: logger.error("No automation task found: {}".format(pid)) return diff --git a/apps/assets/urls/api_urls.py b/apps/assets/urls/api_urls.py index f1c286054..d2bf6f258 100644 --- a/apps/assets/urls/api_urls.py +++ b/apps/assets/urls/api_urls.py @@ -27,17 +27,25 @@ router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset') router.register(r'account-backup-plans', api.AccountBackupPlanViewSet, 'account-backup') router.register(r'account-backup-plan-executions', api.AccountBackupPlanExecutionViewSet, 'account-backup-execution') +router.register(r'change-secret-automations', api.ChangeSecretAutomationViewSet, 'change-secret-automations') +router.register(r'automation-executions', api.AutomationExecutionViewSet, 'automation-execution') +router.register(r'change-secret-records', api.ChangeSecretRecordViewSet, 'change-secret-records') + urlpatterns = [ # path('assets//gateways/', api.AssetGatewayListApi.as_view(), name='asset-gateway-list'), path('assets//tasks/', api.AssetTaskCreateApi.as_view(), name='asset-task-create'), path('assets/tasks/', api.AssetsTaskCreateApi.as_view(), name='assets-task-create'), path('assets//perm-users/', api.AssetPermUserListApi.as_view(), name='asset-perm-user-list'), - path('assets//perm-users//permissions/', api.AssetPermUserPermissionsListApi.as_view(), name='asset-perm-user-permission-list'), - path('assets//perm-user-groups/', api.AssetPermUserGroupListApi.as_view(), name='asset-perm-user-group-list'), - path('assets//perm-user-groups//permissions/', api.AssetPermUserGroupPermissionsListApi.as_view(), name='asset-perm-user-group-permission-list'), + path('assets//perm-users//permissions/', api.AssetPermUserPermissionsListApi.as_view(), + name='asset-perm-user-permission-list'), + path('assets//perm-user-groups/', api.AssetPermUserGroupListApi.as_view(), + name='asset-perm-user-group-list'), + path('assets//perm-user-groups//permissions/', + api.AssetPermUserGroupPermissionsListApi.as_view(), name='asset-perm-user-group-permission-list'), path('accounts/tasks/', api.AccountTaskCreateAPI.as_view(), name='account-task-create'), - path('account-secrets//histories/', api.AccountHistoriesSecretAPI.as_view(), name='account-secret-history'), + path('account-secrets//histories/', api.AccountHistoriesSecretAPI.as_view(), + name='account-secret-history'), path('nodes/category/tree/', api.CategoryTreeApi.as_view(), name='asset-category-tree'), path('nodes/tree/', api.NodeListAsTreeApi.as_view(), name='node-tree'), @@ -52,7 +60,11 @@ urlpatterns = [ path('nodes//tasks/', api.NodeTaskCreateApi.as_view(), name='node-task-create'), path('gateways//test-connective/', api.GatewayTestConnectionApi.as_view(), name='test-gateway-connective'), + + path('automation//asset/remove/', api.AutomationRemoveAssetApi.as_view(), name='automation-remove-asset'), + path('automation//asset/add/', api.AutomationAddAssetApi.as_view(), name='automation-add-asset'), + path('automation//nodes/', api.AutomationNodeAddRemoveApi.as_view(), name='automation-add-or-remove-node'), + path('automation//assets/', api.AutomationAssetsListApi.as_view(), name='automation-assets'), ] urlpatterns += router.urls -