Merge pull request #8379 from jumpserver/dev

v2.23.0-rc3

apps/assets/serializers/account.py

@@ -5,7 +5,6 @@ from assets.models import AuthBook
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from .base import AuthSerializerMixin
-from .utils import validate_password_contains_left_double_curly_bracket
 from common.utils.encode import ssh_pubkey_gen
 from common.drf.serializers import SecretReadableMixin
 
@@ -32,10 +31,6 @@ class AccountSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         fields = fields_small + fields_fk
         extra_kwargs = {
             'username': {'required': True},
-            'password': {
-                'write_only': True,
-                "validators": [validate_password_contains_left_double_curly_bracket]
-            },
             'private_key': {'write_only': True},
             'public_key': {'write_only': True},
             'systemuser_display': {'label': _('System user display')}

apps/assets/serializers/base.py

@@ -8,6 +8,7 @@ from rest_framework import serializers
 from common.utils import ssh_pubkey_gen, ssh_private_key_gen, validate_ssh_private_key
 from common.drf.fields import EncryptedField
 from assets.models import Type
+from .utils import validate_password_contains_left_double_curly_bracket
 
 
 class AuthSerializer(serializers.ModelSerializer):
@@ -33,7 +34,8 @@ class AuthSerializer(serializers.ModelSerializer):
 
 class AuthSerializerMixin(serializers.ModelSerializer):
     password = EncryptedField(
-        label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024
+        label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024,
+        validators=[validate_password_contains_left_double_curly_bracket]
     )
     private_key = EncryptedField(
         label=_('SSH private key'), required=False, allow_blank=True, allow_null=True, max_length=4096

apps/assets/serializers/system_user.py

@@ -25,6 +25,11 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     """
     系统用户
     """
+    password = EncryptedField(
+        label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024,
+        trim_whitespace=False, validators=[validate_password_contains_left_double_curly_bracket],
+        write_only=True
+    )
     auto_generate_key = serializers.BooleanField(initial=True, required=False, write_only=True)
     type_display = serializers.ReadOnlyField(source='get_type_display', label=_('Type display'))
     ssh_key_fingerprint = serializers.ReadOnlyField(label=_('SSH key fingerprint'))
@@ -51,15 +56,9 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
         fields_m2m = ['cmd_filters', 'assets_amount', 'applications_amount', 'nodes']
         fields = fields_small + fields_m2m
         extra_kwargs = {
-            'password': {
-                "write_only": True,
-                'trim_whitespace': False,
-                "validators": [validate_password_contains_left_double_curly_bracket]
-            },
             'cmd_filters': {"required": False, 'label': _('Command filter')},
             'public_key': {"write_only": True},
             'private_key': {"write_only": True},
-            'token': {"write_only": True},
             'nodes_amount': {'label': _('Nodes amount')},
             'assets_amount': {'label': _('Assets amount')},
             'login_mode_display': {'label': _('Login mode display')},

apps/assets/tasks/push_system_user.py

@@ -33,16 +33,17 @@ def _dump_args(args: dict):
 
 
 def get_push_unixlike_system_user_tasks(system_user, username=None, **kwargs):
-    comment = system_user.name
     algorithm = kwargs.get('algorithm')
     if username is None:
         username = system_user.username
 
+    comment = system_user.name
     if system_user.username_same_with_user:
         from users.models import User
         user = User.objects.filter(username=username).only('name', 'username').first()
         if user:
             comment = f'{system_user.name}[{str(user)}]'
+    comment = comment.replace(' ', '')
 
     password = system_user.password
     public_key = system_user.public_key

apps/rbac/builtin.py

@@ -126,6 +126,8 @@ class BuiltinRole:
     org_user = PredefineRole(
         '7', ugettext_noop('OrgUser'), Scope.org, user_perms
     )
+    system_role_mapper = None
+    org_role_mapper = None
 
     @classmethod
     def get_roles(cls):
@@ -138,22 +140,24 @@ class BuiltinRole:
 
     @classmethod
     def get_system_role_by_old_name(cls, name):
-        mapper = {
-            'App': cls.system_component,
-            'Admin': cls.system_admin,
-            'User': cls.system_user,
-            'Auditor': cls.system_auditor
-        }
-        return mapper[name].get_role()
+        if not cls.system_role_mapper:
+            cls.system_role_mapper = {
+                'App': cls.system_component.get_role(),
+                'Admin': cls.system_admin.get_role(),
+                'User': cls.system_user.get_role(),
+                'Auditor': cls.system_auditor.get_role()
+            }
+        return cls.system_role_mapper[name]
 
     @classmethod
     def get_org_role_by_old_name(cls, name):
-        mapper = {
-            'Admin': cls.org_admin,
-            'User': cls.org_user,
-            'Auditor': cls.org_auditor,
-        }
-        return mapper[name].get_role()
+        if not cls.org_role_mapper:
+            cls.org_role_mapper = {
+                'Admin': cls.org_admin.get_role(),
+                'User': cls.org_user.get_role(),
+                'Auditor': cls.org_auditor.get_role(),
+            }
+        return cls.org_role_mapper[name]
 
     @classmethod
     def sync_to_db(cls, show_msg=False):

apps/rbac/migrations/0004_auto_20211201_1901.py

@@ -1,5 +1,6 @@
 # Generated by Django 3.1.13 on 2021-12-01 11:01
 
+import time
 from django.db import migrations
 
 from rbac.builtin import BuiltinRole
@@ -9,33 +10,61 @@ def migrate_system_role_binding(apps, schema_editor):
     db_alias = schema_editor.connection.alias
     user_model = apps.get_model('users', 'User')
     role_binding_model = apps.get_model('rbac', 'SystemRoleBinding')
-    users = user_model.objects.using(db_alias).all()
 
-    role_bindings = []
-    for user in users:
-        role = BuiltinRole.get_system_role_by_old_name(user.role)
-        role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id)
-        role_bindings.append(role_binding)
-    role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True)
+    count = 0
+    bulk_size = 1000
+    while True:
+        users = user_model.objects.using(db_alias) \
+            .only('role', 'id') \
+            .all()[count:count+bulk_size]
+        if not users:
+            break
+
+        role_bindings = []
+        start = time.time()
+        for user in users:
+            role = BuiltinRole.get_system_role_by_old_name(user.role)
+            role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id)
+            role_bindings.append(role_binding)
+
+        role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True)
+        print("Create role binding: {}-{} using: {:.2f}s".format(
+            count, count + len(users), time.time()-start
+        ))
+        count += len(users)
 
 
 def migrate_org_role_binding(apps, schema_editor):
     db_alias = schema_editor.connection.alias
     org_member_model = apps.get_model('orgs', 'OrganizationMember')
     role_binding_model = apps.get_model('rbac', 'RoleBinding')
-    members = org_member_model.objects.using(db_alias).all()
 
-    role_bindings = []
-    for member in members:
-        role = BuiltinRole.get_org_role_by_old_name(member.role)
-        role_binding = role_binding_model(
-            scope='org',
-            user_id=member.user.id,
-            role_id=role.id,
-            org_id=member.org.id
-        )
-        role_bindings.append(role_binding)
-    role_binding_model.objects.bulk_create(role_bindings)
+    count = 0
+    bulk_size = 1000
+
+    while True:
+        members = org_member_model.objects.using(db_alias)\
+            .only('role', 'user_id', 'org_id')\
+            .all()[count:count+bulk_size]
+        if not members:
+            break
+        role_bindings = []
+        start = time.time()
+
+        for member in members:
+            role = BuiltinRole.get_org_role_by_old_name(member.role)
+            role_binding = role_binding_model(
+                scope='org',
+                user_id=member.user_id,
+                role_id=role.id,
+                org_id=member.org_id
+            )
+            role_bindings.append(role_binding)
+        role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True)
+        print("Create role binding: {}-{} using: {:.2f}s".format(
+            count, count + len(members), time.time()-start
+        ))
+        count += len(members)
 
 
 class Migration(migrations.Migration):

apps/templates/resource_download.html

@@ -15,7 +15,7 @@ p {
 </style>
 <div style="margin: 0 200px">
     <div class="group">
-        <h2>JumpServer {% trans 'Client' %} v1.1.5</h2>
+        <h2>JumpServer {% trans 'Client' %} v1.1.6</h2>
         <p>
             {% trans 'JumpServer Client, currently used to launch the client, now only support launch RDP SSH client, The Telnet client will next' %}
         </p>

utils/test_run_migrations.py

@@ -0,0 +1,68 @@
+# Generated by Django 3.1.13 on 2021-12-01 11:01
+import os
+import sys
+import django
+import time
+
+app_path = '***** Change me *******'
+sys.path.insert(0, app_path)
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jumpserver.settings")
+django.setup()
+
+from django.apps import apps
+from django.db import connection
+
+# ========================== 添加到需要测试的 migrations 上方 ==========================
+
+
+from django.db import migrations
+
+from rbac.builtin import BuiltinRole
+
+
+def migrate_system_role_binding(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    user_model = apps.get_model('users', 'User')
+    role_binding_model = apps.get_model('rbac', 'SystemRoleBinding')
+
+    count = 0
+    bulk_size = 1000
+    while True:
+        users = user_model.objects.using(db_alias) \
+                    .only('role', 'id') \
+                    .all()[count:count+bulk_size]
+        if not users:
+            break
+
+        role_bindings = []
+        start = time.time()
+        for user in users:
+            role = BuiltinRole.get_system_role_by_old_name(user.role)
+            role_binding = role_binding_model(scope='system', user_id=user.id, role_id=role.id)
+            role_bindings.append(role_binding)
+
+        role_binding_model.objects.bulk_create(role_bindings, ignore_conflicts=True)
+        print("Create role binding: {}-{} using: {:.2f}s".format(
+            count, count + len(users), time.time()-start
+        ))
+        count += len(users)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rbac', '0003_auto_20211130_1037'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_system_role_binding),
+    ]
+
+
+# ================== 添加到下方 ======================
+def main():
+    schema_editor = connection.schema_editor()
+    migrate_system_role_binding(apps, schema_editor)
+
+
+# main()