perf: 修改数据库 migrations

apps/applications/migrations/0022_auto_20220816_1015.py

@@ -1,24 +0,0 @@
-# Generated by Django 3.2.14 on 2022-08-16 02:15
-
-import common.db.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('applications', '0021_auto_20220629_1826'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='account',
-            name='token',
-            field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
-        ),
-        migrations.AddField(
-            model_name='historicalaccount',
-            name='token',
-            field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
-        ),
-    ]

apps/applications/migrations/0022_auto_20220817_1346.py

@@ -6,7 +6,7 @@ from django.db import migrations
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('applications', '0023_auto_20220816_1021'),
+        ('applications', '0021_auto_20220629_1826'),
     ]
 
     operations = [
@@ -22,10 +22,6 @@ class Migration(migrations.Migration):
             model_name='account',
             name='systemuser',
         ),
-        migrations.AlterUniqueTogether(
-            name='application',
-            unique_together=None,
-        ),
         migrations.RemoveField(
             model_name='application',
             name='domain',
@@ -42,16 +38,4 @@ class Migration(migrations.Migration):
             model_name='historicalaccount',
             name='systemuser',
         ),
-        migrations.DeleteModel(
-            name='ApplicationUser',
-        ),
-        migrations.DeleteModel(
-            name='Account',
-        ),
-        migrations.DeleteModel(
-            name='Application',
-        ),
-        migrations.DeleteModel(
-            name='HistoricalAccount',
-        ),
     ]

apps/applications/migrations/0023_auto_20220816_1021.py

@@ -1,21 +0,0 @@
-# Generated by Django 3.2.14 on 2022-08-16 02:21
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('applications', '0022_auto_20220816_1015'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='account',
-            name='token',
-        ),
-        migrations.RemoveField(
-            model_name='historicalaccount',
-            name='token',
-        ),
-    ]

apps/applications/migrations/0023_auto_20220817_1716.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.14 on 2022-08-17 09:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0022_auto_20220817_1346'),
+        ('perms', '0031_auto_20220816_1600'),
+        ('ops', '0022_auto_20220817_1346'),
+        ('assets', '0109_auto_20220817_1544'),
+        ('tickets', '0020_auto_20220817_1346'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='Account',
+        ),
+        migrations.DeleteModel(
+            name='HistoricalAccount',
+        ),
+        migrations.DeleteModel(
+            name='ApplicationUser',
+        ),
+    ]

apps/applications/migrations/0024_auto_20220818_1057.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.14 on 2022-08-18 02:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0023_auto_20220817_1716'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='application',
+            name='category',
+            field=models.CharField(max_length=16, verbose_name='Category'),
+        ),
+        migrations.AlterField(
+            model_name='application',
+            name='type',
+            field=models.CharField(max_length=16, verbose_name='Type'),
+        ),
+    ]

apps/applications/models.py

@@ -0,0 +1,28 @@
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from orgs.mixins.models import OrgModelMixin
+from common.mixins import CommonModelMixin
+
+
+class Application(CommonModelMixin, OrgModelMixin):
+    name = models.CharField(max_length=128, verbose_name=_('Name'))
+    category = models.CharField(
+        max_length=16, verbose_name=_('Category')
+    )
+    type = models.CharField(
+        max_length=16, verbose_name=_('Type')
+    )
+    attrs = models.JSONField(default=dict, verbose_name=_('Attrs'))
+    comment = models.TextField(
+        max_length=128, default='', blank=True, verbose_name=_('Comment')
+    )
+
+    class Meta:
+        verbose_name = _('Application')
+        unique_together = [('org_id', 'name')]
+        ordering = ('name',)
+        permissions = [
+            ('match_application', _('Can match application')),
+        ]

apps/assets/migrations/0101_auto_20220711_1409.py

@@ -29,12 +29,12 @@ class Migration(migrations.Migration):
                 ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
                 ('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
                 ('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
+                ('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
                 ('comment', models.TextField(blank=True, verbose_name='Comment')),
                 ('date_created', models.DateTimeField(blank=True, editable=False, verbose_name='Date created')),
                 ('date_updated', models.DateTimeField(blank=True, editable=False, verbose_name='Date updated')),
                 ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
-                ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
+                ('privileged', models.BooleanField(default=False, verbose_name='Privileged account')),
-                ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
                 ('version', models.IntegerField(default=0, verbose_name='Version')),
                 ('history_id', models.AutoField(primary_key=True, serialize=False)),
                 ('history_date', models.DateTimeField(db_index=True)),
@@ -63,12 +63,12 @@ class Migration(migrations.Migration):
                 ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
                 ('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
                 ('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
+                ('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
                 ('comment', models.TextField(blank=True, verbose_name='Comment')),
                 ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
                 ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
-                ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
+                ('privileged', models.BooleanField(default=False, verbose_name='Privileged account')),
-                ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
                 ('version', models.IntegerField(default=0, verbose_name='Version')),
                 ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.asset', verbose_name='Asset')),
             ],

apps/assets/migrations/0102_auto_20220711_1413.py

@@ -16,10 +16,10 @@ def migrate_accounts(apps, schema_editor):
         auth_books = auth_book_model.objects \
             .prefetch_related('systemuser') \
             .all()[count:count+bulk_size]
-        count += len(auth_books)
         if not auth_books:
             break
 
+        count += len(auth_books)
         accounts = []
         # auth book 和 account 相同的属性
         same_attrs = [
@@ -50,7 +50,7 @@ def migrate_accounts(apps, schema_editor):
 
         account_model.objects.bulk_create(accounts, ignore_conflicts=True)
         print("Create accounts: {}-{} using: {:.2f}s".format(
-            count - bulk_size, count, time.time()-start
+            count - len(auth_books), count, time.time()-start
         ))
 
 

apps/assets/migrations/0104_auto_20220803_1859.py

@@ -15,10 +15,9 @@ def migrate_asset_protocols(apps, schema_editor):
     while True:
         start = time.time()
         assets = asset_model.objects.all()[count:count+bulk_size]
-        count += len(assets)
         if not assets:
             break
-
+        count += len(assets)
         assets_protocols = []
         for asset in assets:
             old_protocols = asset._protocols
@@ -38,7 +37,7 @@ def migrate_asset_protocols(apps, schema_editor):
                 assets_protocols.append(asset_protocol_through(asset_id=asset.id, protocol_id=protocol.id))
         asset_model.protocols.through.objects.bulk_create(assets_protocols, ignore_conflicts=True)
         print("Create asset protocols: {}-{} using: {:.2f}s".format(
-            count - bulk_size, count, time.time()-start
+            count - len(assets), count, time.time()-start
         ))
 
 

apps/assets/migrations/0105_auto_20220810_1449.py

@@ -38,4 +38,9 @@ class Migration(migrations.Migration):
             name='type',
             field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('general', 'General'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'),
         ),
+        migrations.AlterField(
+            model_name='platform',
+            name='su_enabled',
+            field=models.BooleanField(default=False, verbose_name='Su enabled'),
+        ),
     ]

apps/assets/migrations/0106_auto_20220811_1358.py

@@ -1,22 +0,0 @@
-# Generated by Django 3.2.14 on 2022-08-11 05:58
-
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0105_auto_20220810_1449'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='account',
-            name='protocol',
-        ),
-        migrations.RemoveField(
-            model_name='historicalaccount',
-            name='protocol',
-        ),
-    ]

apps/assets/migrations/0106_auto_20220811_1449.py

@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('assets', '0106_auto_20220811_1358'),
+        ('assets', '0105_auto_20220810_1449'),
     ]
 
     operations = [

apps/assets/migrations/0107_auto_20220811_1511.py

@@ -6,7 +6,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('assets', '0107_auto_20220811_1449'),
+        ('assets', '0106_auto_20220811_1449'),
     ]
 
     operations = [
@@ -34,8 +34,4 @@ class Migration(migrations.Migration):
             name='created_by',
             field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by'),
         ),
-        migrations.AlterUniqueTogether(
-            name='asset',
-            unique_together={('org_id', 'name')},
-        ),
     ]

apps/assets/migrations/0108_auto_20220816_1022.py

@@ -13,11 +13,10 @@ def migrate_command_filter_to_assets(apps, schema_editor):
     while True:
         start = time.time()
         command_filters = command_filter_model.objects.all() \
-                              .prefetch_related('system_users')[count:count + bulk_size]
+            .prefetch_related('system_users')[count:count + bulk_size]
-        count += len(command_filters)
         if not command_filters:
             break
-
+        count += len(command_filters)
         updated = []
         for command_filter in command_filters:
             command_filter.accounts = [s.username for s in command_filter.system_users.all()]
@@ -25,7 +24,7 @@ def migrate_command_filter_to_assets(apps, schema_editor):
         command_filter_model.objects.bulk_update(updated, ['accounts'])
 
         print("Create assets: {}-{} using: {:.2f}s".format(
-            count - bulk_size, count, time.time() - start
+            count - len(command_filters), count, time.time() - start
         ))
 
 
@@ -47,7 +46,7 @@ def migrate_command_filter_apps(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('assets', '0110_auto_20220815_1831'),
+        ('assets', '0107_auto_20220811_1511'),
     ]
 
     operations = [

apps/assets/migrations/0109_auto_20220815_1811.py

@@ -1,36 +0,0 @@
-# Generated by Django 3.2.14 on 2022-08-15 10:11
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0108_auto_20220811_1511'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='account',
-            name='type',
-        ),
-        migrations.RemoveField(
-            model_name='historicalaccount',
-            name='type',
-        ),
-        migrations.AddField(
-            model_name='account',
-            name='privileged',
-            field=models.BooleanField(default=False, verbose_name='Privileged account'),
-        ),
-        migrations.AddField(
-            model_name='historicalaccount',
-            name='privileged',
-            field=models.BooleanField(default=False, verbose_name='Privileged account'),
-        ),
-        migrations.AlterField(
-            model_name='platform',
-            name='su_enabled',
-            field=models.BooleanField(default=False, verbose_name='Su enabled'),
-        ),
-    ]

apps/assets/migrations/0109_auto_20220817_1544.py

@@ -6,7 +6,7 @@ from django.db import migrations
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('assets', '0111_auto_20220816_1022'),
+        ('assets', '0108_auto_20220816_1022'),
     ]
 
     operations = [
@@ -25,10 +25,6 @@ class Migration(migrations.Migration):
             model_name='historicalauthbook',
             name='systemuser',
         ),
-        migrations.AlterUniqueTogether(
-            name='systemuser',
-            unique_together=None,
-        ),
         migrations.RemoveField(
             model_name='systemuser',
             name='assets',
@@ -41,21 +37,8 @@ class Migration(migrations.Migration):
             model_name='systemuser',
             name='nodes',
         ),
-        migrations.RemoveField(
-            model_name='systemuser',
-            name='su_from',
-        ),
         migrations.RemoveField(
             model_name='systemuser',
             name='users',
         ),
-        migrations.DeleteModel(
-            name='AuthBook',
-        ),
-        migrations.DeleteModel(
-            name='HistoricalAuthBook',
-        ),
-        migrations.DeleteModel(
-            name='SystemUser',
-        ),
     ]

apps/assets/migrations/0110_auto_20220815_1831.py

@@ -1,24 +0,0 @@
-# Generated by Django 3.2.14 on 2022-08-15 10:31
-
-import common.db.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0109_auto_20220815_1811'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='account',
-            name='token',
-            field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
-        ),
-        migrations.AddField(
-            model_name='historicalaccount',
-            name='token',
-            field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
-        ),
-    ]

apps/assets/migrations/0110_auto_20220817_1716.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.14 on 2022-08-17 09:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0109_auto_20220817_1544'),
+        ('applications', '0022_auto_20220817_1346'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='authbook',
+            unique_together=None,
+        ),
+        migrations.RemoveField(
+            model_name='authbook',
+            name='asset',
+        ),
+        migrations.RemoveField(
+            model_name='authbook',
+            name='systemuser',
+        ),
+        migrations.DeleteModel(
+            name='HistoricalAuthBook',
+        ),
+        migrations.DeleteModel(
+            name='AuthBook',
+        ),
+    ]

apps/assets/models/__init__.py

@@ -1,6 +1,5 @@
 from .base import *
 from .platform import *
-# from ._user import *
 from .asset import *
 from .label import Label
 from .group import *
@@ -11,6 +10,7 @@ from .gathered_user import *
 from .favorite_asset import *
 from .account import *
 from .backup import *
+from ._user import *
 # 废弃以下
 # from ._authbook import *
 from .protocol import *

apps/assets/models/_user.py

@@ -8,8 +8,6 @@ from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from django.core.validators import MinValueValidator, MaxValueValidator
 
-from assets.const import Protocol
-from common.utils import signer
 from .base import BaseAccount
 from .protocol import ProtocolMixin
 
@@ -31,14 +29,6 @@ class SystemUser(ProtocolMixin, BaseAccount):
         admin = 'admin', _('Admin user')
 
     username_same_with_user = models.BooleanField(default=False, verbose_name=_("Username same with user"))
-    nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes"))
-    assets = models.ManyToManyField(
-        'assets.Asset', blank=True, verbose_name=_("Assets"),
-        through='assets.AuthBook', through_fields=['systemuser', 'asset'],
-        related_name='system_users'
-    )
-    users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users"))
-    groups = models.ManyToManyField('users.UserGroup', blank=True, verbose_name=_("User groups"))
     type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_('Type'))
     priority = models.IntegerField(default=81, verbose_name=_("Priority"), help_text=_("1-100, the lower the value will be match first"), validators=[MinValueValidator(1), MaxValueValidator(100)])
     protocol = models.CharField(max_length=16, default='ssh', verbose_name=_('Protocol'))
@@ -55,84 +45,6 @@ class SystemUser(ProtocolMixin, BaseAccount):
     su_enabled = models.BooleanField(default=False, verbose_name=_('User switch'))
     su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from"))
 
-    def __str__(self):
-        username = self.username
-        if self.username_same_with_user:
-            username = '*'
-        return '{0.name}({1})'.format(self, username)
-
-    @property
-    def nodes_amount(self):
-        return self.nodes.all().count()
-
-    @property
-    def login_mode_display(self):
-        return self.get_login_mode_display()
-
-    def is_need_push(self):
-        if self.auto_push and self.is_protocol_support_push:
-            return True
-        else:
-            return False
-
-    @property
-    def is_admin_user(self):
-        return self.type == self.Type.admin
-
-    @property
-    def is_need_cmd_filter(self):
-        return self.protocol not in [self.Protocol.rdp, self.Protocol.vnc]
-
-    @property
-    def is_need_test_asset_connective(self):
-        return self.protocol in self.ASSET_CATEGORY_PROTOCOLS
-
-    @property
-    def cmd_filter_rules(self):
-        from .cmd_filter import CommandFilterRule
-        rules = CommandFilterRule.objects.filter(
-            filter__in=self.cmd_filters.all()
-        ).distinct()
-        return rules
-
-    def is_command_can_run(self, command):
-        for rule in self.cmd_filter_rules:
-            action, matched_cmd = rule.match(command)
-            if action == rule.ActionChoices.allow:
-                return True, None
-            elif action == rule.ActionChoices.deny:
-                return False, matched_cmd
-        return True, None
-
-    def get_all_assets(self):
-        from assets.models import Node, Asset
-        nodes_keys = self.nodes.all().values_list('key', flat=True)
-        asset_ids = set(self.assets.all().values_list('id', flat=True))
-        nodes_asset_ids = Node.get_nodes_all_asset_ids_by_keys(nodes_keys)
-        asset_ids.update(nodes_asset_ids)
-        assets = Asset.objects.filter(id__in=asset_ids)
-        return assets
-
-    def add_related_assets(self, assets_or_ids):
-        self.assets.add(*tuple(assets_or_ids))
-        self.add_related_assets_to_su_from_if_need(assets_or_ids)
-
-    def add_related_assets_to_su_from_if_need(self, assets_or_ids):
-        if self.protocol not in [self.Protocol.ssh.value]:
-            return
-        if not self.su_enabled:
-            return
-        if not self.su_from:
-            return
-        if self.su_from.protocol != self.protocol:
-            return
-        self.su_from.assets.add(*tuple(assets_or_ids))
-
-    # TODO 暂时为了接口文档添加
-    @property
-    def auto_push_account(self):
-        return
-
     class Meta:
         ordering = ['name']
         unique_together = [('name', 'org_id')]
@@ -140,52 +52,3 @@ class SystemUser(ProtocolMixin, BaseAccount):
         permissions = [
             ('match_systemuser', _('Can match system user')),
         ]
-
-
-# Deprecated: 准备废弃
-class AdminUser(BaseAccount):
-    """
-    A privileged user that ansible can use it to push system user and so on
-    """
-    BECOME_METHOD_CHOICES = (
-        ('sudo', 'sudo'),
-        ('su', 'su'),
-    )
-    become = models.BooleanField(default=True)
-    become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4)
-    become_user = models.CharField(default='root', max_length=64)
-    _become_pass = models.CharField(default='', blank=True, max_length=128)
-    CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}'
-    _prefer = "admin_user"
-
-    def __str__(self):
-        return self.name
-
-    @property
-    def become_pass(self):
-        password = signer.unsign(self._become_pass)
-        if password:
-            return password
-        else:
-            return ""
-
-    @become_pass.setter
-    def become_pass(self, password):
-        self._become_pass = signer.sign(password)
-
-    @property
-    def become_info(self):
-        if self.become:
-            info = {
-                "method": self.become_method,
-                "user": self.become_user,
-                "pass": self.become_pass,
-            }
-        else:
-            info = None
-        return info
-
-    class Meta:
-        ordering = ['name']
-        unique_together = [('name', 'org_id')]
-        verbose_name = _("Admin user")

apps/tickets/migrations/0020_auto_20220817_1346.py

@@ -34,12 +34,12 @@ def migrate_system_to_account(apps, schema_editor):
                     new_value = [s.username for s in old_value]
                 else:
                     old_value = getattr(obj, old_field)
-                    new_value = old_value.username
+                    new_value = old_value.username if old_value else ''
                 setattr(obj, new_field, new_value)
                 updated.append(obj)
             model.objects.bulk_update(updated, [new_field])
             print("Migrate account: {}-{} using: {:.2f}s".format(
-                count - bulk_size, count, time.time()-start
+                count - len(objects), count, time.time()-start
             ))