From ba3f2099e62ceef16568f7b904828155582587ff Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Thu, 10 Nov 2022 14:44:23 +0800 Subject: [PATCH 1/3] perf: audits --- apps/acls/serializers/login_asset_acl.py | 6 +-- apps/audits/const.py | 35 +++++++++++++ apps/audits/models.py | 63 ++++-------------------- apps/audits/serializers.py | 33 ++++++------- apps/audits/signal_handlers.py | 56 ++++++++++----------- apps/common/db/fields.py | 2 - apps/common/drf/fields.py | 1 + apps/common/mixins/views.py | 3 +- 8 files changed, 93 insertions(+), 106 deletions(-) diff --git a/apps/acls/serializers/login_asset_acl.py b/apps/acls/serializers/login_asset_acl.py index 7282bf1a9..6c77eb546 100644 --- a/apps/acls/serializers/login_asset_acl.py +++ b/apps/acls/serializers/login_asset_acl.py @@ -3,7 +3,7 @@ from django.utils.translation import ugettext_lazy as _ from orgs.mixins.serializers import BulkOrgResourceModelSerializer from orgs.models import Organization -from assets.const import Protocol +from common.drf.fields import LabeledChoiceField from acls import models @@ -59,7 +59,7 @@ class LoginAssetACLSerializer(BulkOrgResourceModelSerializer): assets = LoginAssetACLAssestsSerializer() accounts = LoginAssetACLAccountsSerializer() reviewers_amount = serializers.IntegerField(read_only=True, source='reviewers.count') - action_display = serializers.ReadOnlyField(source='get_action_display', label=_('Action')) + action = LabeledChoiceField(choices=models.LoginAssetACL.ActionChoices.choices, label=_('Action')) class Meta: model = models.LoginAssetACL @@ -67,7 +67,7 @@ class LoginAssetACLSerializer(BulkOrgResourceModelSerializer): fields_small = fields_mini + [ 'users', 'accounts', 'assets', 'is_active', 'date_created', 'date_updated', - 'priority', 'action', 'action_display', 'comment', 'created_by', 'org_id' + 'priority', 'action', 'comment', 'created_by', 'org_id' ] fields_m2m = ['reviewers', 'reviewers_amount'] fields = fields_small + fields_m2m diff --git a/apps/audits/const.py b/apps/audits/const.py index 18033ee78..421e87158 100644 --- a/apps/audits/const.py +++ b/apps/audits/const.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # from django.utils.translation import ugettext_lazy as _ +from django.db.models import TextChoices, IntegerChoices DEFAULT_CITY = _("Unknown") @@ -22,3 +23,37 @@ MODELS_NEED_RECORD = ( # xpack 'License', 'Account', 'SyncInstanceTask', 'ChangeAuthPlan', 'GatherUserTask', ) + + +class OperateChoices(TextChoices): + mkdir = 'mkdir', _('Mkdir') + rmdir = 'rmdir', _('Rmdir') + delete = 'delete', _('Delete') + upload = 'upload', _('Upload') + rename = 'rename', _('Rename') + symlink = 'symlink', _('Symlink') + download = 'download', _('Download') + + +class ActionChoices(TextChoices): + view = 'view', _('View') + update = 'update', _('Update') + delete = 'delete', _('Delete') + create = 'create', _('Create') + + +class LoginTypeChoices(TextChoices): + web = 'W', _('Web') + terminal = 'T', _('Terminal') + unknown = 'U', _('Unknown') + + +class MFAChoices(IntegerChoices): + disabled = 0, _('Disabled') + enabled = 1, _('Enabled') + unknown = 2, _('-') + + +class LoginStatusChoices(IntegerChoices): + success = True, _('Success') + failed = False, _('Failed') diff --git a/apps/audits/models.py b/apps/audits/models.py index 5dd8eb0a2..fa069801e 100644 --- a/apps/audits/models.py +++ b/apps/audits/models.py @@ -8,6 +8,9 @@ from common.utils import lazyproperty from orgs.mixins.models import OrgModelMixin, Organization from orgs.utils import current_org +from .const import ( + OperateChoices, ActionChoices, LoginTypeChoices, MFAChoices, LoginStatusChoices +) __all__ = [ 'FTPLog', 'OperateLog', 'PasswordChangeLog', 'UserLoginLog', @@ -15,30 +18,12 @@ __all__ = [ class FTPLog(OrgModelMixin): - OPERATE_DELETE = 'Delete' - OPERATE_UPLOAD = 'Upload' - OPERATE_DOWNLOAD = 'Download' - OPERATE_RMDIR = 'Rmdir' - OPERATE_RENAME = 'Rename' - OPERATE_MKDIR = 'Mkdir' - OPERATE_SYMLINK = 'Symlink' - - OPERATE_CHOICES = ( - (OPERATE_DELETE, _('Delete')), - (OPERATE_UPLOAD, _('Upload')), - (OPERATE_DOWNLOAD, _('Download')), - (OPERATE_RMDIR, _('Rmdir')), - (OPERATE_RENAME, _('Rename')), - (OPERATE_MKDIR, _('Mkdir')), - (OPERATE_SYMLINK, _('Symlink')) - ) - id = models.UUIDField(default=uuid.uuid4, primary_key=True) user = models.CharField(max_length=128, verbose_name=_('User')) remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True) asset = models.CharField(max_length=1024, verbose_name=_("Asset")) system_user = models.CharField(max_length=128, verbose_name=_("System user")) - operate = models.CharField(max_length=16, verbose_name=_("Operate"), choices=OPERATE_CHOICES) + operate = models.CharField(max_length=16, verbose_name=_("Operate"), choices=OperateChoices.choices) filename = models.CharField(max_length=1024, verbose_name=_("Filename")) is_success = models.BooleanField(default=True, verbose_name=_("Success")) date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Date start')) @@ -48,19 +33,9 @@ class FTPLog(OrgModelMixin): class OperateLog(OrgModelMixin): - ACTION_CREATE = 'create' - ACTION_VIEW = 'view' - ACTION_UPDATE = 'update' - ACTION_DELETE = 'delete' - ACTION_CHOICES = ( - (ACTION_CREATE, _("Create")), - (ACTION_VIEW, _("View")), - (ACTION_UPDATE, _("Update")), - (ACTION_DELETE, _("Delete")) - ) id = models.UUIDField(default=uuid.uuid4, primary_key=True) user = models.CharField(max_length=128, verbose_name=_('User')) - action = models.CharField(max_length=16, choices=ACTION_CHOICES, verbose_name=_("Action")) + action = models.CharField(max_length=16, choices=ActionChoices.choices, verbose_name=_("Action")) resource_type = models.CharField(max_length=64, verbose_name=_("Resource Type")) resource = models.CharField(max_length=128, verbose_name=_("Resource")) remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True) @@ -97,35 +72,17 @@ class PasswordChangeLog(models.Model): class UserLoginLog(models.Model): - LOGIN_TYPE_CHOICE = ( - ('W', 'Web'), - ('T', 'Terminal'), - ('U', 'Unknown'), - ) - - MFA_DISABLED = 0 - MFA_ENABLED = 1 - MFA_UNKNOWN = 2 - - MFA_CHOICE = ( - (MFA_DISABLED, _('Disabled')), - (MFA_ENABLED, _('Enabled')), - (MFA_UNKNOWN, _('-')), - ) - - STATUS_CHOICE = ( - (True, _('Success')), - (False, _('Failed')) - ) id = models.UUIDField(default=uuid.uuid4, primary_key=True) username = models.CharField(max_length=128, verbose_name=_('Username')) - type = models.CharField(choices=LOGIN_TYPE_CHOICE, max_length=2, verbose_name=_('Login type')) + type = models.CharField(choices=LoginTypeChoices.choices, max_length=2, verbose_name=_('Login type')) ip = models.GenericIPAddressField(verbose_name=_('Login ip')) city = models.CharField(max_length=254, blank=True, null=True, verbose_name=_('Login city')) user_agent = models.CharField(max_length=254, blank=True, null=True, verbose_name=_('User agent')) - mfa = models.SmallIntegerField(default=MFA_UNKNOWN, choices=MFA_CHOICE, verbose_name=_('MFA')) + mfa = models.SmallIntegerField(default=MFAChoices.unknown, choices=MFAChoices.choices, verbose_name=_('MFA')) reason = models.CharField(default='', max_length=128, blank=True, verbose_name=_('Reason')) - status = models.BooleanField(max_length=2, default=True, choices=STATUS_CHOICE, verbose_name=_('Status')) + status = models.BooleanField( + default=LoginStatusChoices.success, choices=LoginStatusChoices.choices, verbose_name=_('Status') + ) datetime = models.DateTimeField(default=timezone.now, verbose_name=_('Date login')) backend = models.CharField(max_length=32, default='', verbose_name=_('Authentication backend')) diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index 0f595be25..c262480a5 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -3,39 +3,39 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from common.drf.serializers import BulkSerializerMixin +from common.drf.fields import LabeledChoiceField from terminal.models import Session from . import models +from .const import ( + ActionChoices, OperateChoices, MFAChoices, LoginStatusChoices, LoginTypeChoices +) class FTPLogSerializer(serializers.ModelSerializer): - operate_display = serializers.ReadOnlyField(source='get_operate_display', label=_('Operate display')) + operate = LabeledChoiceField(choices=OperateChoices.choices, label=_('Operate')) class Meta: model = models.FTPLog fields_mini = ['id'] fields_small = fields_mini + [ 'user', 'remote_addr', 'asset', 'system_user', 'org_id', - 'operate', 'filename', 'operate_display', - 'is_success', - 'date_start', + 'operate', 'filename', 'is_success', 'date_start', ] fields = fields_small class UserLoginLogSerializer(serializers.ModelSerializer): - type_display = serializers.ReadOnlyField(source='get_type_display', label=_('Type display')) - status_display = serializers.ReadOnlyField(source='get_status_display', label=_('Status display')) - mfa_display = serializers.ReadOnlyField(source='get_mfa_display', label=_('MFA display')) + mfa = LabeledChoiceField(choices=MFAChoices.choices, label=_('MFA')) + type = LabeledChoiceField(choices=LoginTypeChoices.choices, label=_('Type')) + status = LabeledChoiceField(choices=LoginStatusChoices.choices, label=_('Status')) class Meta: model = models.UserLoginLog fields_mini = ['id'] fields_small = fields_mini + [ - 'username', 'type', 'type_display', 'ip', 'city', 'user_agent', - 'mfa', 'mfa_display', 'reason', 'reason_display', 'backend', 'backend_display', - 'status', 'status_display', - 'datetime', + 'username', 'type', 'ip', 'city', 'user_agent', + 'mfa', 'reason', 'reason_display', 'backend', + 'backend_display', 'status', 'datetime', ] fields = fields_small extra_kwargs = { @@ -46,15 +46,14 @@ class UserLoginLogSerializer(serializers.ModelSerializer): class OperateLogSerializer(serializers.ModelSerializer): - action_display = serializers.CharField(source='get_action_display', label=_('Action')) + action = LabeledChoiceField(choices=ActionChoices.choices, label=_('Action')) class Meta: model = models.OperateLog fields_mini = ['id'] fields_small = fields_mini + [ - 'user', 'action', 'action_display', - 'resource_type', 'resource_type_display', 'resource', - 'remote_addr', 'datetime', 'org_id' + 'user', 'action', 'resource_type', 'resource_type_display', + 'resource', 'remote_addr', 'datetime', 'org_id' ] fields = fields_small extra_kwargs = { @@ -66,7 +65,7 @@ class PasswordChangeLogSerializer(serializers.ModelSerializer): class Meta: model = models.PasswordChangeLog fields = ( - 'id', 'user', 'change_by', 'remote_addr', 'datetime' + 'id', 'user', 'change_by', 'remote_addr', 'datetime' ) diff --git a/apps/audits/signal_handlers.py b/apps/audits/signal_handlers.py index f395965c3..34707d30b 100644 --- a/apps/audits/signal_handlers.py +++ b/apps/audits/signal_handlers.py @@ -1,38 +1,34 @@ # -*- coding: utf-8 -*- # -import time - -from django.db.models.signals import ( - post_save, m2m_changed, pre_delete -) -from django.dispatch import receiver from django.conf import settings from django.db import transaction -from django.utils import timezone +from django.dispatch import receiver +from django.utils import timezone, translation from django.utils.functional import LazyObject from django.contrib.auth import BACKEND_SESSION_KEY from django.utils.translation import ugettext_lazy as _ -from django.utils import translation -from rest_framework.renderers import JSONRenderer +from django.db.models.signals import post_save, m2m_changed, pre_delete from rest_framework.request import Request +from rest_framework.renderers import JSONRenderer -from assets.models import Asset -from authentication.signals import post_auth_failed, post_auth_success -from authentication.utils import check_different_city_login_if_need -from jumpserver.utils import current_request -from users.models import User -from users.signals import post_user_change_password -from terminal.models import Session, Command -from .utils import write_login_log, create_operate_log -from . import models, serializers -from .models import OperateLog from orgs.utils import current_org from perms.models import AssetPermission -from terminal.backends.command.serializers import SessionCommandSerializer +from users.models import User +from users.signals import post_user_change_password +from assets.models import Asset +from jumpserver.utils import current_request +from authentication.signals import post_auth_failed, post_auth_success +from authentication.utils import check_different_city_login_if_need +from terminal.models import Session, Command from terminal.serializers import SessionSerializer +from terminal.backends.command.serializers import SessionCommandSerializer from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR from common.utils import get_request_ip, get_logger, get_syslogger from common.utils.encode import data_to_json +from . import models, serializers +from .const import ActionChoices +from .utils import write_login_log, create_operate_log + logger = get_logger(__name__) sys_logger = get_syslogger(__name__) @@ -97,9 +93,9 @@ M2M_NEED_RECORD = { } M2M_ACTION_MAPER = { - POST_ADD: OperateLog.ACTION_CREATE, - POST_REMOVE: OperateLog.ACTION_DELETE, - POST_CLEAR: OperateLog.ACTION_DELETE, + POST_ADD: ActionChoices.create, + POST_REMOVE: ActionChoices.delete, + POST_CLEAR: ActionChoices.delete, } @@ -120,9 +116,9 @@ def on_m2m_changed(sender, action, instance, model, pk_set, **kwargs): resource_type, resource_tmpl_add, resource_tmpl_remove = M2M_NEED_RECORD[sender_name] action = M2M_ACTION_MAPER[action] - if action == OperateLog.ACTION_CREATE: + if action == ActionChoices.create: resource_tmpl = resource_tmpl_add - elif action == OperateLog.ACTION_DELETE: + elif action == ActionChoices.delete: resource_tmpl = resource_tmpl_remove else: return @@ -144,11 +140,11 @@ def on_m2m_changed(sender, action, instance, model, pk_set, **kwargs): model_name: str(obj) })[:128] # `resource` 字段只有 128 个字符长 😔 - to_create.append(OperateLog( + to_create.append(models.OperateLog( user=user, action=action, resource_type=resource_type, resource=resource, remote_addr=remote_addr, org_id=org_id )) - OperateLog.objects.bulk_create(to_create) + models.OperateLog.objects.bulk_create(to_create) @receiver(post_save) @@ -158,15 +154,15 @@ def on_object_created_or_update(sender, instance=None, created=False, update_fie update_fields and 'last_login' in update_fields: return if created: - action = models.OperateLog.ACTION_CREATE + action = ActionChoices.create else: - action = models.OperateLog.ACTION_UPDATE + action = ActionChoices.update create_operate_log(action, sender, instance) @receiver(pre_delete) def on_object_delete(sender, instance=None, **kwargs): - create_operate_log(models.OperateLog.ACTION_DELETE, sender, instance) + create_operate_log(ActionChoices.delete, sender, instance) @receiver(post_user_change_password, sender=User) diff --git a/apps/common/db/fields.py b/apps/common/db/fields.py index 72c4df898..861eed82f 100644 --- a/apps/common/db/fields.py +++ b/apps/common/db/fields.py @@ -7,7 +7,6 @@ from django.utils.encoding import force_text from django.core.validators import MinValueValidator, MaxValueValidator from common.utils import signer, crypto - __all__ = [ 'JsonMixin', 'JsonDictMixin', 'JsonListMixin', 'JsonTypeMixin', 'JsonCharField', 'JsonTextField', 'JsonListCharField', 'JsonListTextField', @@ -189,4 +188,3 @@ class PortField(models.IntegerField): 'validators': [MinValueValidator(0), MaxValueValidator(65535)] }) super().__init__(*args, **kwargs) - diff --git a/apps/common/drf/fields.py b/apps/common/drf/fields.py index 97f9785f5..c42614a3e 100644 --- a/apps/common/drf/fields.py +++ b/apps/common/drf/fields.py @@ -21,6 +21,7 @@ __all__ = [ class ReadableHiddenField(serializers.HiddenField): """ 可读的 HiddenField """ + def __init__(self, **kwargs): super().__init__(**kwargs) self.write_only = False diff --git a/apps/common/mixins/views.py b/apps/common/mixins/views.py index 9f824e5e2..8ff0b2cdc 100644 --- a/apps/common/mixins/views.py +++ b/apps/common/mixins/views.py @@ -9,6 +9,7 @@ from rest_framework.request import Request from common.exceptions import UserConfirmRequired from audits.utils import create_operate_log from audits.models import OperateLog +from audits.const import ActionChoices __all__ = ["PermissionsMixin", "RecordViewLogMixin", "UserConfirmRequiredExceptionMixin"] @@ -40,7 +41,7 @@ class PermissionsMixin(UserPassesTestMixin): class RecordViewLogMixin: - ACTION = OperateLog.ACTION_VIEW + ACTION = ActionChoices.view @staticmethod def get_resource_display(request): From 5494d2fd605c7b3d3b660a421fb25c169e15aa30 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 10 Nov 2022 18:20:39 +0800 Subject: [PATCH 2/3] perf: update applet host deploy --- apps/terminal/automations/deploy_applet_host/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/terminal/automations/deploy_applet_host/__init__.py b/apps/terminal/automations/deploy_applet_host/__init__.py index 8b415ece4..7845a16d6 100644 --- a/apps/terminal/automations/deploy_applet_host/__init__.py +++ b/apps/terminal/automations/deploy_applet_host/__init__.py @@ -31,12 +31,12 @@ class DeployAppletHostManager: bootstrap_token = settings.BOOTSTRAP_TOKEN host_id = str(self.deployment.host.id) if not base_site_url: - base_site_url = "localhost:8080" + base_site_url = "http://localhost:8080" with open(playbook_src) as f: plays = yaml.safe_load(f) for play in plays: play['vars'].update(self.deployment.host.deploy_options) - play['vars']['DownloadHost'] = base_site_url + '/download/' + play['vars']['DownloadHost'] = base_site_url + '/download' play['vars']['CORE_HOST'] = base_site_url play['vars']['BOOTSTRAP_TOKEN'] = bootstrap_token play['vars']['HOST_ID'] = host_id From cd3c3eeaf2364a0f8c1a2d051ccea526d0e8d981 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Thu, 10 Nov 2022 19:11:56 +0800 Subject: [PATCH 3/3] perf: code (#9044) Co-authored-by: feng <1304903146@qq.com> --- apps/acls/serializers/login_acl.py | 3 --- apps/assets/serializers/mixin.py | 11 ----------- 2 files changed, 14 deletions(-) diff --git a/apps/acls/serializers/login_acl.py b/apps/acls/serializers/login_acl.py index 536061082..ed45c8640 100644 --- a/apps/acls/serializers/login_acl.py +++ b/apps/acls/serializers/login_acl.py @@ -53,6 +53,3 @@ class LoginACLSerializer(BulkModelSerializer): def get_rules_serializer(self): return RuleSerializer() - - def get_reviewers_display(self, obj): - return ','.join([str(user) for user in obj.reviewers.all()]) diff --git a/apps/assets/serializers/mixin.py b/apps/assets/serializers/mixin.py index 45943dc2a..e69de29bb 100644 --- a/apps/assets/serializers/mixin.py +++ b/apps/assets/serializers/mixin.py @@ -1,11 +0,0 @@ -from rest_framework import serializers -from django.utils.translation import gettext_lazy as _ - - -class CategoryDisplayMixin(serializers.Serializer): - category_display = serializers.ReadOnlyField( - source='get_category_display', label=_("Category display") - ) - type_display = serializers.ReadOnlyField( - source='get_type_display', label=_("Type display") - )