From 3755f8f33acd1744b5ae3a62e518a2b8e73a48fd Mon Sep 17 00:00:00 2001 From: "Jiangjie.Bai" Date: Mon, 13 Jun 2022 10:54:34 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=8E=A8=E9=80=81?= =?UTF-8?q?=E5=8A=A8=E6=80=81=E7=94=A8=E6=88=B7=20comment=20=E4=B8=AD?= =?UTF-8?q?=E5=8C=85=E5=90=AB=E7=A9=BA=E6=A0=BC=E5=AF=BC=E8=87=B4=E6=8E=A8?= =?UTF-8?q?=E9=80=81=E5=A4=B1=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/tasks/push_system_user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/assets/tasks/push_system_user.py b/apps/assets/tasks/push_system_user.py index 98d2a0922..ee4af0fd5 100644 --- a/apps/assets/tasks/push_system_user.py +++ b/apps/assets/tasks/push_system_user.py @@ -33,16 +33,17 @@ def _dump_args(args: dict): def get_push_unixlike_system_user_tasks(system_user, username=None, **kwargs): - comment = system_user.name algorithm = kwargs.get('algorithm') if username is None: username = system_user.username + comment = system_user.name if system_user.username_same_with_user: from users.models import User user = User.objects.filter(username=username).only('name', 'username').first() if user: comment = f'{system_user.name}[{str(user)}]' + comment = comment.replace(' ', '') password = system_user.password public_key = system_user.public_key From 0464b1a9e69c2ca407b213a797652460b2620159 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 13 Jun 2022 14:22:07 +0800 Subject: [PATCH 2/5] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=20rbac=20=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: migrate --- apps/rbac/builtin.py | 30 ++++++---- .../migrations/0004_auto_20211201_1901.py | 53 +++++++++------- utils/test_run_migrations.py | 60 +++++++++++++++++++ 3 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 utils/test_run_migrations.py diff --git a/apps/rbac/builtin.py b/apps/rbac/builtin.py index b736ba226..34733dc59 100644 --- a/apps/rbac/builtin.py +++ b/apps/rbac/builtin.py @@ -126,6 +126,8 @@ class BuiltinRole: org_user = PredefineRole( '7', ugettext_noop('OrgUser'), Scope.org, user_perms ) + system_role_mapper = None + org_role_mapper = None @classmethod def get_roles(cls): @@ -138,22 +140,24 @@ class BuiltinRole: @classmethod def get_system_role_by_old_name(cls, name): - mapper = { - 'App': cls.system_component, - 'Admin': cls.system_admin, - 'User': cls.system_user, - 'Auditor': cls.system_auditor - } - return mapper[name].get_role() + if not cls.system_role_mapper: + cls.system_role_mapper = { + 'App': cls.system_component.get_role(), + 'Admin': cls.system_admin.get_role(), + 'User': cls.system_user.get_role(), + 'Auditor': cls.system_auditor.get_role() + } + return cls.system_role_mapper[name] @classmethod def get_org_role_by_old_name(cls, name): - mapper = { - 'Admin': cls.org_admin, - 'User': cls.org_user, - 'Auditor': cls.org_auditor, - } - return mapper[name].get_role() + if not cls.org_role_mapper: + cls.org_role_mapper = { + 'Admin': cls.org_admin.get_role(), + 'User': cls.org_user.get_role(), + 'Auditor': cls.org_auditor.get_role(), + } + return cls.org_role_mapper[name] @classmethod def sync_to_db(cls, show_msg=False): diff --git a/apps/rbac/migrations/0004_auto_20211201_1901.py b/apps/rbac/migrations/0004_auto_20211201_1901.py index 36736f254..eafe37539 100644 --- a/apps/rbac/migrations/0004_auto_20211201_1901.py +++ b/apps/rbac/migrations/0004_auto_20211201_1901.py @@ -1,5 +1,4 @@ -# Generated by Django 3.1.13 on 2021-12-01 11:01 - +import time from django.db import migrations from rbac.builtin import BuiltinRole @@ -11,31 +10,45 @@ def migrate_system_role_binding(apps, schema_editor): role_binding_model = apps.get_model('rbac', 'SystemRoleBinding') users = user_model.objects.using(db_alias).all() - role_bindings = [] - for user in users: - role = BuiltinRole.get_system_role_by_old_name(user.role) - role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id) - role_bindings.append(role_binding) - role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) + grouped_users = [users[i:i+1000] for i in range(0, len(users), 1000)] + for i, group in enumerate(grouped_users): + role_bindings = [] + start = time.time() + for user in group: + role = BuiltinRole.get_system_role_by_old_name(user.role) + role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id) + role_bindings.append(role_binding) + role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) + print("Create role binding: {}-{} using: {:.2f}s".format( + i*1000, i*1000 + len(group), time.time()-start + )) def migrate_org_role_binding(apps, schema_editor): db_alias = schema_editor.connection.alias org_member_model = apps.get_model('orgs', 'OrganizationMember') role_binding_model = apps.get_model('rbac', 'RoleBinding') - members = org_member_model.objects.using(db_alias).all() + members = org_member_model.objects.using(db_alias)\ + .only('role', 'user_id', 'org_id')\ + .all() - role_bindings = [] - for member in members: - role = BuiltinRole.get_org_role_by_old_name(member.role) - role_binding = role_binding_model( - scope='org', - user_id=member.user.id, - role_id=role.id, - org_id=member.org.id - ) - role_bindings.append(role_binding) - role_binding_model.objects.bulk_create(role_bindings) + grouped_members = [members[i:i+1000] for i in range(0, len(members), 1000)] + for i, group in enumerate(grouped_members): + role_bindings = [] + start = time.time() + for member in group: + role = BuiltinRole.get_org_role_by_old_name(member.role) + role_binding = role_binding_model( + scope='org', + user_id=member.user_id, + role_id=role.id, + org_id=member.org_id + ) + role_bindings.append(role_binding) + role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) + print("Create role binding: {}-{} using: {:.2f}s".format( + i*1000, i*1000 + len(group), time.time()-start + )) class Migration(migrations.Migration): diff --git a/utils/test_run_migrations.py b/utils/test_run_migrations.py new file mode 100644 index 000000000..c45c46ec1 --- /dev/null +++ b/utils/test_run_migrations.py @@ -0,0 +1,60 @@ +# Generated by Django 3.1.13 on 2021-12-01 11:01 +import os +import sys +import django +import time + +app_path = '***** Change me *******' +sys.path.insert(0, app_path) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jumpserver.settings") +django.setup() + +from django.apps import apps +from django.db import connection + +# ========================== 添加到需要测试的 migrations 上方 ========================== + + +from django.db import migrations + +from rbac.builtin import BuiltinRole + + +def migrate_some_binding(apps, schema_editor): + db_alias = schema_editor.connection.alias + user_model = apps.get_model('users', 'User') + role_binding_model = apps.get_model('rbac', 'SystemRoleBinding') + users = user_model.objects.using(db_alias).all() + + grouped_users = [users[i:i+1000] for i in range(0, len(users), 1000)] + for i, group in enumerate(grouped_users): + role_bindings = [] + start = time.time() + for user in group: + role = BuiltinRole.get_system_role_by_old_name(user.role) + role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id) + role_bindings.append(role_binding) + role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) + print("Create role binding: {}-{} using: {:.2f}s".format( + i*1000, i*1000 + len(group), time.time()-start + )) + + +class Migration(migrations.Migration): + + dependencies = [ + ('rbac', '0003_auto_20211130_1037'), + ] + + operations = [ + migrations.RunPython(migrate_some_binding), + ] + + +# ================== 添加到下方 ====================== +def main(): + schema_editor = connection.schema_editor() + migrate_some_binding(apps, schema_editor) + + +# main() From 25ae790f7d14927d7f3268e6de772b1f1e35475b Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:45:10 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9client=20=E7=89=88?= =?UTF-8?q?=E6=9C=AC=20(#8375)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: feng626 <1304903146@qq.com> --- apps/templates/resource_download.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/templates/resource_download.html b/apps/templates/resource_download.html index 09215b532..57570eb8e 100644 --- a/apps/templates/resource_download.html +++ b/apps/templates/resource_download.html @@ -15,7 +15,7 @@ p {
-

JumpServer {% trans 'Client' %} v1.1.5

+

JumpServer {% trans 'Client' %} v1.1.6

{% trans 'JumpServer Client, currently used to launch the client, now only support launch RDP SSH client, The Telnet client will next' %}

From 95f8b12912a775104c6da2c5962be0cb908de8cb Mon Sep 17 00:00:00 2001 From: "Jiangjie.Bai" Date: Mon, 13 Jun 2022 15:33:27 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E9=83=A8=E5=88=86?= =?UTF-8?q?=20password=20encrypted=20field=20extra=20kwargs=20=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E4=B8=8D=E7=94=9F=E6=95=88=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/serializers/account.py | 5 ----- apps/assets/serializers/base.py | 4 +++- apps/assets/serializers/system_user.py | 11 +++++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/assets/serializers/account.py b/apps/assets/serializers/account.py index bc4bea563..5cbb26d88 100644 --- a/apps/assets/serializers/account.py +++ b/apps/assets/serializers/account.py @@ -5,7 +5,6 @@ from assets.models import AuthBook from orgs.mixins.serializers import BulkOrgResourceModelSerializer from .base import AuthSerializerMixin -from .utils import validate_password_contains_left_double_curly_bracket from common.utils.encode import ssh_pubkey_gen from common.drf.serializers import SecretReadableMixin @@ -32,10 +31,6 @@ class AccountSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): fields = fields_small + fields_fk extra_kwargs = { 'username': {'required': True}, - 'password': { - 'write_only': True, - "validators": [validate_password_contains_left_double_curly_bracket] - }, 'private_key': {'write_only': True}, 'public_key': {'write_only': True}, 'systemuser_display': {'label': _('System user display')} diff --git a/apps/assets/serializers/base.py b/apps/assets/serializers/base.py index c20b1abb8..92cc65f19 100644 --- a/apps/assets/serializers/base.py +++ b/apps/assets/serializers/base.py @@ -8,6 +8,7 @@ from rest_framework import serializers from common.utils import ssh_pubkey_gen, ssh_private_key_gen, validate_ssh_private_key from common.drf.fields import EncryptedField from assets.models import Type +from .utils import validate_password_contains_left_double_curly_bracket class AuthSerializer(serializers.ModelSerializer): @@ -33,7 +34,8 @@ class AuthSerializer(serializers.ModelSerializer): class AuthSerializerMixin(serializers.ModelSerializer): password = EncryptedField( - label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024 + label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024, + validators=[validate_password_contains_left_double_curly_bracket] ) private_key = EncryptedField( label=_('SSH private key'), required=False, allow_blank=True, allow_null=True, max_length=4096 diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index 278a99d87..68ade0ebd 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -25,6 +25,11 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): """ 系统用户 """ + password = EncryptedField( + label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024, + trim_whitespace=False, validators=[validate_password_contains_left_double_curly_bracket], + write_only=True + ) auto_generate_key = serializers.BooleanField(initial=True, required=False, write_only=True) type_display = serializers.ReadOnlyField(source='get_type_display', label=_('Type display')) ssh_key_fingerprint = serializers.ReadOnlyField(label=_('SSH key fingerprint')) @@ -51,15 +56,9 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): fields_m2m = ['cmd_filters', 'assets_amount', 'applications_amount', 'nodes'] fields = fields_small + fields_m2m extra_kwargs = { - 'password': { - "write_only": True, - 'trim_whitespace': False, - "validators": [validate_password_contains_left_double_curly_bracket] - }, 'cmd_filters': {"required": False, 'label': _('Command filter')}, 'public_key': {"write_only": True}, 'private_key': {"write_only": True}, - 'token': {"write_only": True}, 'nodes_amount': {'label': _('Nodes amount')}, 'assets_amount': {'label': _('Assets amount')}, 'login_mode_display': {'label': _('Login mode display')}, From 556ce0a1463b9c5f5fe6779821aa2ff5c0e63eb1 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 13 Jun 2022 16:26:36 +0800 Subject: [PATCH 5/5] =?UTF-8?q?perf:=20=E7=BB=A7=E7=BB=AD=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E4=B8=80=E6=B3=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0004_auto_20211201_1901.py | 40 +++++++++++++------ utils/test_run_migrations.py | 24 +++++++---- 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/apps/rbac/migrations/0004_auto_20211201_1901.py b/apps/rbac/migrations/0004_auto_20211201_1901.py index eafe37539..9d59d99fc 100644 --- a/apps/rbac/migrations/0004_auto_20211201_1901.py +++ b/apps/rbac/migrations/0004_auto_20211201_1901.py @@ -1,3 +1,5 @@ +# Generated by Django 3.1.13 on 2021-12-01 11:01 + import time from django.db import migrations @@ -8,35 +10,48 @@ def migrate_system_role_binding(apps, schema_editor): db_alias = schema_editor.connection.alias user_model = apps.get_model('users', 'User') role_binding_model = apps.get_model('rbac', 'SystemRoleBinding') - users = user_model.objects.using(db_alias).all() - grouped_users = [users[i:i+1000] for i in range(0, len(users), 1000)] - for i, group in enumerate(grouped_users): + count = 0 + bulk_size = 1000 + while True: + users = user_model.objects.using(db_alias) \ + .only('role', 'id') \ + .all()[count:count+bulk_size] + if not users: + break + role_bindings = [] start = time.time() - for user in group: + for user in users: role = BuiltinRole.get_system_role_by_old_name(user.role) role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id) role_bindings.append(role_binding) + role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) print("Create role binding: {}-{} using: {:.2f}s".format( - i*1000, i*1000 + len(group), time.time()-start + count, count + len(users), time.time()-start )) + count += len(users) def migrate_org_role_binding(apps, schema_editor): db_alias = schema_editor.connection.alias org_member_model = apps.get_model('orgs', 'OrganizationMember') role_binding_model = apps.get_model('rbac', 'RoleBinding') - members = org_member_model.objects.using(db_alias)\ - .only('role', 'user_id', 'org_id')\ - .all() - grouped_members = [members[i:i+1000] for i in range(0, len(members), 1000)] - for i, group in enumerate(grouped_members): + count = 0 + bulk_size = 1000 + + while True: + members = org_member_model.objects.using(db_alias)\ + .only('role', 'user_id', 'org_id')\ + .all()[count:count+bulk_size] + if not members: + break role_bindings = [] start = time.time() - for member in group: + + for member in members: role = BuiltinRole.get_org_role_by_old_name(member.role) role_binding = role_binding_model( scope='org', @@ -47,8 +62,9 @@ def migrate_org_role_binding(apps, schema_editor): role_bindings.append(role_binding) role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) print("Create role binding: {}-{} using: {:.2f}s".format( - i*1000, i*1000 + len(group), time.time()-start + count, count + len(members), time.time()-start )) + count += len(members) class Migration(migrations.Migration): diff --git a/utils/test_run_migrations.py b/utils/test_run_migrations.py index c45c46ec1..33c7c3c5c 100644 --- a/utils/test_run_migrations.py +++ b/utils/test_run_migrations.py @@ -20,24 +20,32 @@ from django.db import migrations from rbac.builtin import BuiltinRole -def migrate_some_binding(apps, schema_editor): +def migrate_system_role_binding(apps, schema_editor): db_alias = schema_editor.connection.alias user_model = apps.get_model('users', 'User') role_binding_model = apps.get_model('rbac', 'SystemRoleBinding') - users = user_model.objects.using(db_alias).all() - grouped_users = [users[i:i+1000] for i in range(0, len(users), 1000)] - for i, group in enumerate(grouped_users): + count = 0 + bulk_size = 1000 + while True: + users = user_model.objects.using(db_alias) \ + .only('role', 'id') \ + .all()[count:count+bulk_size] + if not users: + break + role_bindings = [] start = time.time() - for user in group: + for user in users: role = BuiltinRole.get_system_role_by_old_name(user.role) role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id) role_bindings.append(role_binding) + role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True) print("Create role binding: {}-{} using: {:.2f}s".format( - i*1000, i*1000 + len(group), time.time()-start + count, count + len(users), time.time()-start )) + count += len(users) class Migration(migrations.Migration): @@ -47,14 +55,14 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(migrate_some_binding), + migrations.RunPython(migrate_system_role_binding), ] # ================== 添加到下方 ====================== def main(): schema_editor = connection.schema_editor() - migrate_some_binding(apps, schema_editor) + migrate_system_role_binding(apps, schema_editor) # main()