perf: 优化 account 结构

apps/assets/migrations/0099_auto_20220711_1409.py

@@ -23,10 +23,8 @@ class Migration(migrations.Migration):
                 ('id', models.UUIDField(db_index=True, default=uuid.uuid4)),
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
                 ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
-                ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
+                ('secret_type', models.CharField(choices=[('password', 'Password'), ('ssh_key', 'SSH key'), ('access_key', 'Access key'), ('token', 'Token')], default='password', max_length=16, verbose_name='Secret type')),
-                ('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
+                ('secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')),
-                ('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
-                ('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
                 ('comment', models.TextField(blank=True, verbose_name='Comment')),
                 ('date_created', models.DateTimeField(blank=True, editable=False, verbose_name='Date created')),
                 ('date_updated', models.DateTimeField(blank=True, editable=False, verbose_name='Date updated')),
@@ -55,10 +53,8 @@ class Migration(migrations.Migration):
                 ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
                 ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
-                ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
+                ('secret_type', models.CharField(choices=[('password', 'Password'), ('ssh_key', 'SSH key'), ('access_key', 'Access key'), ('token', 'Token')], default='password', max_length=16, verbose_name='Secret type')),
-                ('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
+                ('secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')),
-                ('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
-                ('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
                 ('comment', models.TextField(blank=True, verbose_name='Comment')),
                 ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
@@ -70,7 +66,7 @@ class Migration(migrations.Migration):
             options={
                 'verbose_name': 'Account',
                 'permissions': [('view_accountsecret', 'Can view asset account secret'), ('change_accountsecret', 'Can change asset account secret'), ('view_historyaccount', 'Can view asset history account'), ('view_historyaccountsecret', 'Can view asset history account secret')],
-                'unique_together': {('username', 'asset'), ('name', 'asset')},
+                'unique_together': {('name', 'asset'), ('username', 'asset', 'secret_type')},
             },
         ),
         migrations.AddField(

apps/assets/migrations/0100_auto_20220711_1413.py

@@ -29,28 +29,50 @@ def migrate_accounts(apps, schema_editor):
         accounts = []
         # auth book 和 account 相同的属性
         same_attrs = [
-            'id', 'comment', 'date_created', 'date_updated',
+            'id', 'username', 'comment', 'date_created', 'date_updated',
             'created_by', 'asset_id', 'org_id',
         ]
         # 认证的属性，可能是 authbook 的，可能是 systemuser 的
-        auth_attrs = ['username', 'password', 'private_key', 'public_key']
+        auth_attrs = ['password', 'private_key', 'token']
+        all_attrs = same_attrs + auth_attrs
 
         for auth_book in auth_books:
-            values = {attr: getattr(auth_book, attr) for attr in same_attrs}
+            values = {'version': 1}
-            values['version'] = 1
 
             system_user = auth_book.systemuser
             if system_user:
-                values.update({attr: getattr(system_user, attr) for attr in auth_attrs})
+                # 更新一次系统用户的认证属性
+                values.update({attr: getattr(system_user, attr, '') for attr in all_attrs})
                 values['created_by'] = str(system_user.id)
                 values['privileged'] = system_user.type == 'admin'
 
-            auth_book_auth = {attr: getattr(auth_book, attr) for attr in auth_attrs}
+            auth_book_auth = {attr: getattr(auth_book, attr, '') for attr in all_attrs if getattr(auth_book, attr, '')}
-            auth_book_auth = {attr: value for attr, value in auth_book_auth.items() if value}
+            # 最终使用 authbook 的认证属性
             values.update(auth_book_auth)
-            values['name'] = values['username']
 
-            account = account_model(**values)
+            auth_infos = []
+            username = values['username']
+            for attr in auth_attrs:
+                secret = values.pop(attr, None)
+                if not secret:
+                    continue
+
+                if attr == 'private_key':
+                    secret_type = 'ssh_key'
+                    name = f'{username}(ssh key)'
+                elif attr == 'token':
+                    secret_type = 'token'
+                    name = f'{username}(token)'
+                else:
+                    secret_type = attr
+                    name = username
+                auth_infos.append((name, secret_type, secret))
+
+            if not auth_infos:
+                auth_infos.append((username, 'password', ''))
+
+            for name, secret_type, secret in auth_infos:
+                account = account_model(**values, name=name, secret=secret, secret_type=secret_type)
                 accounts.append(account)
 
         account_model.objects.bulk_create(accounts, ignore_conflicts=True)

apps/assets/migrations/0101_auto_20220803_1448.py

@@ -10,24 +10,11 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RemoveField(
-            model_name='asset',
-            name='port',
-        ),
-        migrations.RemoveField(
-            model_name='asset',
-            name='protocol',
-        ),
         migrations.RenameField(
             model_name='asset',
             old_name='protocols',
             new_name='_protocols',
         ),
-        migrations.AlterField(
-            model_name='systemuser',
-            name='protocol',
-            field=models.CharField(default='ssh', max_length=16, verbose_name='Protocol'),
-        ),
         migrations.CreateModel(
             name='Protocol',
             fields=[

apps/assets/migrations/0102_auto_20220803_1859.py

@@ -6,12 +6,10 @@ from django.db import migrations
 def migrate_asset_protocols(apps, schema_editor):
     asset_model = apps.get_model('assets', 'Asset')
     protocol_model = apps.get_model('assets', 'Protocol')
-    asset_protocol_through = asset_model.protocols.through
 
     count = 0
     bulk_size = 1000
     print("\nStart migrate asset protocols")
-    protocol_map = {}
     while True:
         start = time.time()
         assets = asset_model.objects.all()[count:count+bulk_size]
@@ -19,23 +17,25 @@ def migrate_asset_protocols(apps, schema_editor):
             break
         count += len(assets)
         assets_protocols = []
-        for asset in assets:
-            old_protocols = asset._protocols
 
-            for name_port in old_protocols.split(','):
+        for asset in assets:
+            old_protocols = asset._protocols or '{}/{}'.format(asset.protocol, asset.port) or 'ssh/22'
+
+            if ',' in old_protocols:
+                _protocols = old_protocols.split(',')
+            else:
+                _protocols = old_protocols.split()
+
+            for name_port in _protocols:
                 name_port_list = name_port.split('/')
                 if len(name_port_list) != 2:
                     continue
 
                 name, port = name_port_list
-                protocol = protocol_map.get(name_port)
+                protocol = protocol_model(**{'name': name, 'port': port, 'asset': asset})
-                if not protocol:
+                assets_protocols.append(protocol)
-                    protocol = protocol_model.objects.get_or_create(
+
-                        defaults={'name': name, 'port': port, 'asset': asset},
+        protocol_model.objects.bulk_create(assets_protocols, ignore_conflicts=True)
-                        name=name, port=port
-                    )[0]
-                assets_protocols.append(asset_protocol_through(asset_id=asset.id, protocol_id=protocol.id))
-        asset_model.protocols.through.objects.bulk_create(assets_protocols, ignore_conflicts=True)
         print("Create asset protocols: {}-{} using: {:.2f}s".format(
             count - len(assets), count, time.time()-start
         ))

apps/assets/migrations/0103_auto_20220811_1511.py

@@ -18,11 +18,24 @@ class Migration(migrations.Migration):
         ),
         migrations.RemoveField(
             model_name='asset',
-            name='_protocols',
+            name='admin_user',
         ),
         migrations.RemoveField(
             model_name='asset',
-            name='admin_user',
+            name='port',
+        ),
+        migrations.RemoveField(
+            model_name='asset',
+            name='protocol',
+        ),
+        migrations.RemoveField(
+            model_name='asset',
+            name='_protocols',
+        ),
+        migrations.AlterField(
+            model_name='systemuser',
+            name='protocol',
+            field=models.CharField(default='ssh', max_length=16, verbose_name='Protocol'),
         ),
         migrations.RemoveField(
             model_name='asset',

apps/assets/migrations/0106_auto_20220819_1523.py

@@ -20,14 +20,12 @@ class Migration(migrations.Migration):
                 ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
                 ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
-                ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
+                ('secret_type', models.CharField(choices=[('password', 'Password'), ('ssh_key', 'SSH key'), ('access_key', 'Access key'), ('token', 'Token')], default='password', max_length=16, verbose_name='Secret type'),),
-                ('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
+                ('secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')),
-                ('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
                 ('comment', models.TextField(blank=True, verbose_name='Comment')),
                 ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
                 ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
-                ('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
                 ('privileged', models.BooleanField(default=False, verbose_name='Privileged')),
             ],
             options={

apps/assets/migrations/0108_auto_20220915_1032.py

@@ -11,11 +11,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AddField(
-            model_name='platform',
-            name='brand',
-            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Brand'),
-        ),
         migrations.CreateModel(
             name='PlatformAutomation',
             fields=[

apps/assets/models/account.py

@@ -23,7 +23,7 @@ class Account(BaseAccount):
     class Meta:
         verbose_name = _('Account')
         unique_together = [
-            ('username', 'asset'),
+            ('username', 'asset', 'secret_type'),
             ('name', 'asset'),
         ]
         permissions = [

apps/assets/models/asset/common.py

@@ -77,6 +77,9 @@ class Protocol(models.Model):
     port = models.IntegerField(verbose_name=_("Port"))
     asset = models.ForeignKey('Asset', on_delete=models.CASCADE, related_name='protocols', verbose_name=_("Asset"))
 
+    def __str__(self):
+        return '{}/{}'.format(self.name, self.port)
+
 
 class Asset(AbsConnectivity, NodesRelationMixin, JMSOrgBaseModel):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)

apps/assets/models/base.py

@@ -55,15 +55,17 @@ class AbsConnectivity(models.Model):
 
 
 class BaseAccount(OrgModelMixin):
+    class SecretType(models.TextChoices):
+        password = 'password', _('Password')
+        ssh_key = 'ssh_key', _('SSH key')
+        access_key = 'access_key', _('Access key')
+        token = 'token', _('Token')
+
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_("Name"))
     username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True)
-    secret_type = models.CharField(max_length=16, default='password', verbose_name=_('Secret type'))
+    secret_type = models.CharField(max_length=16, choices=SecretType.choices, default='password', verbose_name=_('Secret type'))
     secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
-    password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
-    private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
-    public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))
-    token = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Token'))
     privileged = models.BooleanField(verbose_name=_("Privileged"), default=False)
     comment = models.TextField(blank=True, verbose_name=_('Comment'))
     date_created = models.DateTimeField(auto_now_add=True, verbose_name=_("Date created"))
@@ -76,6 +78,28 @@ class BaseAccount(OrgModelMixin):
     APPS_AMOUNT_CACHE_KEY = "APP_USER_{}_APPS_AMOUNT"
     APP_USER_CACHE_TIME = 600
 
+    @property
+    def public_key(self):
+        return ''
+
+    @property
+    def private_key(self):
+        return ''
+
+    @private_key.setter
+    def private_key(self, value):
+        self.secret = value
+        self.secret_type = 'private_key'
+
+    @property
+    def password(self):
+        return self.secret
+
+    @password.setter
+    def password(self, value):
+        self.secret = value
+        self.secret_type = 'password'
+
     def expire_assets_amount(self):
         cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id)
         cache.delete(cache_key)

apps/assets/models/domain.py

@@ -10,6 +10,7 @@ from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
 from common.utils import get_logger, lazyproperty
+from common.db import fields
 from orgs.mixins.models import OrgModelMixin
 from .base import BaseAccount
 
@@ -64,7 +65,12 @@ class Gateway(BaseAccount):
     domain = models.ForeignKey(Domain, on_delete=models.CASCADE, verbose_name=_("Domain"))
     comment = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Comment"))
     is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
-    token = None
+    password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
+    private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
+    public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))
+
+    secret = None
+    secret_type = None
     privileged = None
 
     def __str__(self):

apps/assets/models/platform.py

@@ -52,7 +52,6 @@ class Platform(models.Model):
     comment = models.TextField(blank=True, null=True, verbose_name=_("Comment"))
     # 资产有关的
     charset = models.CharField(default='utf8', choices=CHARSET_CHOICES, max_length=8, verbose_name=_("Charset"))
-    brand = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Brand"))  # 厂商主要是给网络设备
     domain_enabled = models.BooleanField(default=True, verbose_name=_("Domain enabled"))
     protocols_enabled = models.BooleanField(default=True, verbose_name=_("Protocols enabled"))
     protocols = models.ManyToManyField(PlatformProtocol, blank=True, verbose_name=_("Protocols"))

apps/assets/serializers/account/common.py

@@ -9,16 +9,16 @@ class AccountFieldsSerializerMixin(serializers.ModelSerializer):
     class Meta:
         fields_mini = [
             'id', 'name', 'username', 'privileged',
-            'platform', 'version'
+            'platform', 'version', 'secret_type',
         ]
-        fields_write_only = ['password', 'private_key', 'public_key', 'passphrase']
+        fields_write_only = ['secret', 'passphrase']
         fields_other = ['date_created', 'date_updated', 'comment']
         fields_small = fields_mini + fields_write_only + fields_other
         fields_fk = ['asset']
         fields = fields_small + fields_fk
         extra_kwargs = {
-            'private_key': {'write_only': True},
+            'secret': {'write_only': True},
-            'public_key': {'write_only': True},
+            'passphrase': {'write_only': True},
             'token': {'write_only': True},
             'password': {'write_only': True},
         }

apps/assets/serializers/asset/common.py

@@ -22,13 +22,6 @@ class AssetProtocolsSerializer(serializers.ModelSerializer):
         model = Protocol
         fields = ['id', 'name', 'port']
 
-    def create(self, validated_data):
-        instance = Protocol.objects.filter(**validated_data).first()
-        if instance:
-            return instance
-        instance = Protocol.objects.create(**validated_data)
-        return instance
-
 
 class AssetLabelSerializer(serializers.ModelSerializer):
     class Meta:
@@ -55,10 +48,10 @@ class AssetAccountSerializer(AccountSerializer):
     class Meta(AccountSerializer.Meta):
         fields_mini = [
             'id', 'name', 'username', 'privileged', 'version',
+            'secret_type',
         ]
         fields_write_only = [
-            'password', 'private_key', 'public_key',
+            'secret', 'passphrase', 'push_now'
-            'passphrase', 'token', 'push_now'
         ]
         fields = fields_mini + fields_write_only
 

apps/assets/serializers/platform.py

@@ -70,7 +70,6 @@ class PlatformSerializer(JMSWritableNestedModelSerializer):
         choices=[('sudo', 'sudo su -'), ('su', 'su - ')],
         label='切换方式', required=False, default='sudo'
     )
-    brand = LabeledChoiceField(choices=[], label='厂商', required=False, allow_null=True)
 
     class Meta:
         model = Platform
@@ -80,7 +79,7 @@ class PlatformSerializer(JMSWritableNestedModelSerializer):
         ]
         fields = fields_small + [
             'protocols_enabled', 'protocols', 'domain_enabled',
-            'su_enabled', 'su_method', 'brand', 'automation', 'comment',
+            'su_enabled', 'su_method', 'automation', 'comment',
         ]
         extra_kwargs = {
             'su_enabled': {'label': '启用切换账号'},

apps/rbac/migrations/0004_auto_20211201_1901.py

@@ -13,6 +13,7 @@ def migrate_system_role_binding(apps, schema_editor):
 
     count = 0
     bulk_size = 1000
+    print('')
     while True:
         users = user_model.objects.using(db_alias) \
             .only('role', 'id') \

apps/tickets/migrations/0020_auto_20220817_1346.py

@@ -16,7 +16,7 @@ def migrate_system_to_account(apps, schema_editor):
     )
 
     for model, old_field, new_field, m2m in model_system_user_account:
-        print("Start migrate '{}' system user to account".format(model))
+        print("Start migrate '{}' system user to account".format(model.__name__))
         count = 0
         bulk_size = 1000