From e7af0375131ec1beabe049ddf4149d8aad03e36c Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 30 Mar 2022 19:07:49 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=E5=91=BD=E4=BB=A4com?= =?UTF-8?q?mand=20input=20=E9=95=BF=E5=BA=A6=E9=97=AE=E9=A2=98=20(#7996)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 修改命令command input max_length 1024 * perf: 修改命令command input 长度问题 * perf: 修改命令command input 长度问题 * perf: 修改命令command input 长度问题 * perf: 修改命令command input 长度问题 Co-authored-by: Jiangjie.Bai --- apps/common/utils/common.py | 21 +++++++++ apps/terminal/api/command.py | 2 +- apps/terminal/backends/command/db.py | 5 +- apps/terminal/backends/command/serializers.py | 46 ++++++++++--------- apps/terminal/models/command.py | 29 ++++++++++++ apps/terminal/serializers/__init__.py | 1 - apps/terminal/serializers/command.py | 11 ----- apps/terminal/serializers/terminal.py | 13 +++--- apps/users/models/user.py | 6 ++- apps/users/serializers/user.py | 8 +++- apps/users/utils.py | 16 ++++--- utils/generate_fake_data/generate.py | 2 + .../generate_fake_data/resources/terminal.py | 10 ++++ 13 files changed, 117 insertions(+), 53 deletions(-) delete mode 100644 apps/terminal/serializers/command.py create mode 100644 utils/generate_fake_data/resources/terminal.py diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index e5b9de4fd..d579eacfb 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -338,3 +338,24 @@ def get_file_by_arch(dir, filename): settings.BASE_DIR, dir, platform_name, arch, filename ) return file_path + + +def pretty_string(data: str, max_length=128, ellipsis_str='...'): + """ + params: + data: abcdefgh + max_length: 7 + ellipsis_str: ... + return: + ab...gh + """ + if len(data) < max_length: + return data + remain_length = max_length - len(ellipsis_str) + half = remain_length // 2 + if half <= 1: + return data[:max_length] + start = data[:half] + end = data[-half:] + data = f'{start}{ellipsis_str}{end}' + return data diff --git a/apps/terminal/api/command.py b/apps/terminal/api/command.py index 09ed27e30..22a2bb5c3 100644 --- a/apps/terminal/api/command.py +++ b/apps/terminal/api/command.py @@ -14,7 +14,7 @@ from terminal.filters import CommandFilter from orgs.utils import current_org from common.drf.api import JMSBulkModelViewSet from common.utils import get_logger -from terminal.serializers import InsecureCommandAlertSerializer +from terminal.backends.command.serializers import InsecureCommandAlertSerializer from terminal.exceptions import StorageInvalid from ..backends import ( get_command_storage, get_multi_command_storage, diff --git a/apps/terminal/backends/command/db.py b/apps/terminal/backends/command/db.py index bb4fa957a..8b11569e9 100644 --- a/apps/terminal/backends/command/db.py +++ b/apps/terminal/backends/command/db.py @@ -4,6 +4,7 @@ import datetime from django.db import transaction from django.utils import timezone from django.db.utils import OperationalError +from common.utils.common import pretty_string from .base import CommandBase @@ -32,9 +33,11 @@ class CommandStore(CommandBase): """ _commands = [] for c in commands: + cmd_input = pretty_string(c['input']) + cmd_output = pretty_string(c['output'], max_length=1024) _commands.append(self.model( user=c["user"], asset=c["asset"], system_user=c["system_user"], - input=c["input"], output=c["output"], session=c["session"], + input=cmd_input, output=cmd_output, session=c["session"], risk_level=c.get("risk_level", 0), org_id=c["org_id"], timestamp=c["timestamp"] )) diff --git a/apps/terminal/backends/command/serializers.py b/apps/terminal/backends/command/serializers.py index 8114ddbb1..625c4282e 100644 --- a/apps/terminal/backends/command/serializers.py +++ b/apps/terminal/backends/command/serializers.py @@ -4,27 +4,19 @@ from rest_framework import serializers from .models import AbstractSessionCommand +__all__ = ['SessionCommandSerializer', 'InsecureCommandAlertSerializer'] -class SessionCommandSerializer(serializers.Serializer): - """使用这个类作为基础Command Log Serializer类, 用来序列化""" - id = serializers.UUIDField(read_only=True) +class SimpleSessionCommandSerializer(serializers.Serializer): + """ 简单Session命令序列类, 用来提取公共字段 """ user = serializers.CharField(label=_("User")) # 限制 64 字符，见 validate_user asset = serializers.CharField(max_length=128, label=_("Asset")) - system_user = serializers.CharField(max_length=64, label=_("System user")) - input = serializers.CharField(max_length=128, label=_("Command")) - output = serializers.CharField(max_length=1024, allow_blank=True, label=_("Output")) + input = serializers.CharField(max_length=2048, label=_("Command")) session = serializers.CharField(max_length=36, label=_("Session ID")) - risk_level = serializers.ChoiceField(required=False, label=_("Risk level"), choices=AbstractSessionCommand.RISK_LEVEL_CHOICES) - risk_level_display = serializers.SerializerMethodField(label=_('Risk level display')) + risk_level = serializers.ChoiceField( + required=False, label=_("Risk level"), choices=AbstractSessionCommand.RISK_LEVEL_CHOICES + ) org_id = serializers.CharField(max_length=36, required=False, default='', allow_null=True, allow_blank=True) - timestamp = serializers.IntegerField(label=_('Timestamp')) - remote_addr = serializers.CharField(read_only=True, label=_('Remote Address')) - - @staticmethod - def get_risk_level_display(obj): - risk_mapper = dict(AbstractSessionCommand.RISK_LEVEL_CHOICES) - return risk_mapper.get(obj.risk_level) def validate_user(self, value): if len(value) > 64: @@ -32,9 +24,21 @@ class SessionCommandSerializer(serializers.Serializer): return value -class InsecureCommandAlertSerializer(serializers.Serializer): - input = serializers.CharField() - asset = serializers.CharField() - user = serializers.CharField() - risk_level = serializers.IntegerField() - session = serializers.UUIDField() +class InsecureCommandAlertSerializer(SimpleSessionCommandSerializer): + pass + + +class SessionCommandSerializer(SimpleSessionCommandSerializer): + """使用这个类作为基础Command Log Serializer类, 用来序列化""" + + id = serializers.UUIDField(read_only=True) + system_user = serializers.CharField(max_length=64, label=_("System user")) + output = serializers.CharField(max_length=2048, allow_blank=True, label=_("Output")) + risk_level_display = serializers.SerializerMethodField(label=_('Risk level display')) + timestamp = serializers.IntegerField(label=_('Timestamp')) + remote_addr = serializers.CharField(read_only=True, label=_('Remote Address')) + + @staticmethod + def get_risk_level_display(obj): + risk_mapper = dict(AbstractSessionCommand.RISK_LEVEL_CHOICES) + return risk_mapper.get(obj.risk_level) diff --git a/apps/terminal/models/command.py b/apps/terminal/models/command.py index 7609902f2..3e94740ff 100644 --- a/apps/terminal/models/command.py +++ b/apps/terminal/models/command.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import time + from django.db import models from django.db.models.signals import post_save from django.utils.translation import ugettext_lazy as _ @@ -18,6 +20,33 @@ class CommandManager(models.Manager): class Command(AbstractSessionCommand): objects = CommandManager() + @classmethod + def generate_fake(cls, count=100, org=None): + import uuid + import datetime + from orgs.models import Organization + from common.utils import random_string + + if not org: + org = Organization.default() + + d = datetime.datetime.now() - datetime.timedelta(days=1) + commands = [ + cls(**{ + 'user': random_string(6), + 'asset': random_string(10), + 'system_user': random_string(6), + 'session': str(uuid.uuid4()), + 'input': random_string(16), + 'output': random_string(64), + 'timestamp': int(d.timestamp()), + 'org_id': str(org.id) + }) + for i in range(count) + ] + cls.objects.bulk_create(commands) + print(f'Create {len(commands)} commands of org ({org})') + class Meta: db_table = "terminal_command" ordering = ('-timestamp',) diff --git a/apps/terminal/serializers/__init__.py b/apps/terminal/serializers/__init__.py index a2a5bbf30..4e868cabc 100644 --- a/apps/terminal/serializers/__init__.py +++ b/apps/terminal/serializers/__init__.py @@ -3,5 +3,4 @@ from .terminal import * from .session import * from .storage import * -from .command import * from .sharing import * diff --git a/apps/terminal/serializers/command.py b/apps/terminal/serializers/command.py deleted file mode 100644 index 01343e825..000000000 --- a/apps/terminal/serializers/command.py +++ /dev/null @@ -1,11 +0,0 @@ -# ~*~ coding: utf-8 ~*~ -from rest_framework import serializers - - -class InsecureCommandAlertSerializer(serializers.Serializer): - input = serializers.CharField() - asset = serializers.CharField() - user = serializers.CharField() - risk_level = serializers.IntegerField() - session = serializers.UUIDField() - org_id = serializers.CharField() diff --git a/apps/terminal/serializers/terminal.py b/apps/terminal/serializers/terminal.py index be991d8a8..dd6565348 100644 --- a/apps/terminal/serializers/terminal.py +++ b/apps/terminal/serializers/terminal.py @@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _ from common.drf.serializers import BulkModelSerializer, AdaptedBulkListSerializer from common.utils import is_uuid from users.serializers import ServiceAccountSerializer -from common.utils import get_request_ip +from common.utils import get_request_ip, pretty_string from .. import const from ..models import ( @@ -111,12 +111,11 @@ class TerminalRegistrationSerializer(serializers.ModelSerializer): valid = super().is_valid(raise_exception=raise_exception) if not valid: return valid - name = self.validated_data.get('name') - if len(name) > 128: - self.validated_data['comment'] = name - name = '{}...{}'.format(name[:32], name[-32:]) - self.validated_data['name'] = name - + raw_name = self.validated_data.get('name') + name = pretty_string(raw_name) + self.validated_data['name'] = name + if len(raw_name) > 128: + self.validated_data['comment'] = raw_name data = {'name': name} kwargs = {'data': data} if self.instance and self.instance.user: diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 3aae3223b..1a5ce634c 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -401,10 +401,12 @@ class RoleMixin: def is_staff(self, value): pass + service_account_email_suffix = '@local.domain' + @classmethod - def create_service_account(cls, name, comment): + def create_service_account(cls, name, email, comment): app = cls.objects.create( - username=name, name=name, email='{}@local.domain'.format(name), + username=name, name=name, email=email, comment=comment, is_first_login=False, created_by='System', is_service_account=True, ) diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index b998e8f6d..56409addf 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -6,6 +6,7 @@ from rest_framework import serializers from common.mixins import CommonBulkSerializerMixin from common.validators import PhoneValidator +from common.utils import pretty_string from rbac.builtin import BuiltinRole from rbac.permissions import RBACPermission from rbac.models import OrgRoleBinding, SystemRoleBinding, Role @@ -268,7 +269,9 @@ class ServiceAccountSerializer(serializers.ModelSerializer): def get_email(self): name = self.initial_data.get('name') - return '{}@serviceaccount.local'.format(name) + name_max_length = 128 - len(User.service_account_email_suffix) + name = pretty_string(name, max_length=name_max_length, ellipsis_str='-') + return '{}{}'.format(name, User.service_account_email_suffix) def validate_name(self, name): email = self.get_email() @@ -283,6 +286,7 @@ class ServiceAccountSerializer(serializers.ModelSerializer): def create(self, validated_data): name = validated_data['name'] + email = self.get_email() comment = validated_data.get('comment', '') - user, ak = User.create_service_account(name, comment) + user, ak = User.create_service_account(name, email, comment) return user diff --git a/apps/users/utils.py b/apps/users/utils.py index 3602d67ee..b32b0c0e0 100644 --- a/apps/users/utils.py +++ b/apps/users/utils.py @@ -11,7 +11,7 @@ from django.conf import settings from django.core.cache import cache from common.tasks import send_mail_async -from common.utils import reverse, get_object_or_none, ip +from common.utils import reverse, get_object_or_none, ip, pretty_string from .models import User logger = logging.getLogger('jumpserver') @@ -229,12 +229,14 @@ class LoginIpBlockUtil(BlockGlobalIpUtilBase): BLOCK_KEY_TMPL = "_LOGIN_BLOCK_{}" -def construct_user_email(username, email): - if '@' not in email: - if '@' in username: - email = username - else: - email = '{}@{}'.format(username, settings.EMAIL_SUFFIX) +def construct_user_email(username, email, email_suffix=''): + if '@' in email: + return email + if '@' in username: + return username + if not email_suffix: + email_suffix = settings.EMAIL_SUFFIX + email = f'{username}@{email_suffix}' return email diff --git a/utils/generate_fake_data/generate.py b/utils/generate_fake_data/generate.py index 839295049..fb41d7e1d 100644 --- a/utils/generate_fake_data/generate.py +++ b/utils/generate_fake_data/generate.py @@ -15,6 +15,7 @@ django.setup() from resources.assets import AssetsGenerator, NodesGenerator, SystemUsersGenerator, AdminUsersGenerator from resources.users import UserGroupGenerator, UserGenerator from resources.perms import AssetPermissionGenerator +from resources.terminal import CommandGenerator # from resources.system import StatGenerator @@ -26,6 +27,7 @@ resource_generator_mapper = { 'user': UserGenerator, 'user_group': UserGroupGenerator, 'asset_permission': AssetPermissionGenerator, + 'command': CommandGenerator, # 'stat': StatGenerator } diff --git a/utils/generate_fake_data/resources/terminal.py b/utils/generate_fake_data/resources/terminal.py new file mode 100644 index 000000000..48d825b47 --- /dev/null +++ b/utils/generate_fake_data/resources/terminal.py @@ -0,0 +1,10 @@ +from .base import FakeDataGenerator +from terminal.models import Command + + +class CommandGenerator(FakeDataGenerator): + resource = 'command' + + def do_generate(self, batch, batch_size): + Command.generate_fake(len(batch), self.org) +