perf: 修改 command filter

apps/applications/migrations/0022_auto_20220816_1015.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.14 on 2022-08-16 02:15
+
+import common.db.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0021_auto_20220629_1826'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='account',
+            name='token',
+            field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
+        ),
+        migrations.AddField(
+            model_name='historicalaccount',
+            name='token',
+            field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
+        ),
+    ]

apps/applications/migrations/0023_auto_20220816_1021.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.14 on 2022-08-16 02:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0022_auto_20220816_1015'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='account',
+            name='token',
+        ),
+        migrations.RemoveField(
+            model_name='historicalaccount',
+            name='token',
+        ),
+    ]

apps/applications/models/account.py

@@ -4,11 +4,11 @@ from django.db.models import F
 from django.utils.translation import ugettext_lazy as _
 
 from common.utils import lazyproperty
-from assets.models.base import BaseUser
+from assets.models.base import BaseAccount
 from assets.models import SystemUser
 
 
-class Account(BaseUser):
+class Account(BaseAccount):
     app = models.ForeignKey(
         'applications.Application', on_delete=models.CASCADE, null=True, verbose_name=_('Application')
     )

apps/assets/migrations/0107_delete_accounttemplate.py

@@ -1,16 +0,0 @@
-# Generated by Django 3.2.13 on 2022-08-11 09:34
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0106_auto_20220811_1358'),
-    ]
-
-    operations = [
-        migrations.DeleteModel(
-            name='AccountTemplate',
-        ),
-    ]

apps/assets/migrations/0108_accounttemplate.py

@@ -1,39 +0,0 @@
-# Generated by Django 3.2.13 on 2022-08-11 09:34
-
-import assets.models.base
-import common.db.fields
-from django.db import migrations, models
-import uuid
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('assets', '0107_delete_accounttemplate'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='AccountTemplate',
-            fields=[
-                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
-                ('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')),
-                ('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')),
-                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('name', models.CharField(max_length=128, verbose_name='Name')),
-                ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
-                ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
-                ('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
-                ('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
-                ('comment', models.TextField(blank=True, verbose_name='Comment')),
-                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
-                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
-                ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
-                ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
-            ],
-            options={
-                'verbose_name': 'Account Template',
-            },
-            bases=(models.Model, assets.models.base.AuthMixin),
-        ),
-    ]

apps/assets/migrations/0111_auto_20220816_1022.py

@@ -0,0 +1,71 @@
+# Generated by Django 3.2.14 on 2022-08-16 02:22
+import time
+from django.db import migrations, models
+
+
+def migrate_command_filter_to_assets(apps, schema_editor):
+    command_filter_model = apps.get_model('assets', 'CommandFilter')
+
+    count = 0
+    bulk_size = 1000
+    print("\nStart migrate command filters to assets")
+    while True:
+        start = time.time()
+        command_filters = command_filter_model.objects.all() \
+                              .prefetch_related('system_users')[count:count + bulk_size]
+        count += len(command_filters)
+        if not command_filters:
+            break
+
+        updated = []
+        for command_filter in command_filters:
+            command_filter.accounts = [s.username for s in command_filter.system_users.all()]
+            updated.append(command_filter)
+        command_filter_model.objects.bulk_update(updated, ['accounts'])
+
+        print("Create assets: {}-{} using: {:.2f}s".format(
+            count - bulk_size, count, time.time() - start
+        ))
+
+
+def migrate_command_filter_apps(apps, schema_editor):
+    command_filter_model = apps.get_model('assets', 'CommandFilter')
+    command_filters = command_filter_model.objects \
+        .annotate(app_count=Count('applications')) \
+        .filter(app_count__gt=0)
+
+    for command_filter in command_filters:
+        app_ids = command_filter.applications.all().values_list('id', flat=True)
+
+        try:
+            command_filter.assets.add(*app_ids)
+        except:
+            print("Migrate command filter apps failed: {}, skip".format(command_filter.id))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0110_auto_20220815_1831'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='AccountTemplate',
+        ),
+        migrations.AddField(
+            model_name='commandfilter',
+            name='accounts',
+            field=models.JSONField(default=list, verbose_name='Accounts'),
+        ),
+        migrations.RunPython(migrate_command_filter_to_assets),
+        migrations.RemoveField(
+            model_name='commandfilter',
+            name='system_users',
+        ),
+        migrations.RunPython(migrate_command_filter_apps),
+        migrations.RemoveField(
+            model_name='commandfilter',
+            name='applications',
+        ),
+    ]

apps/assets/models/_authbook.py

@@ -7,7 +7,7 @@ from django.utils.translation import ugettext_lazy as _
 from simple_history.models import HistoricalRecords
 
 from common.utils import lazyproperty, get_logger
-from .base import BaseUser, AbsConnectivity
+from .base import BaseAccount, AbsConnectivity
 
 logger = get_logger(__name__)
 
@@ -15,7 +15,7 @@ logger = get_logger(__name__)
 __all__ = ['AuthBook']
 
 
-class AuthBook(BaseUser, AbsConnectivity):
+class AuthBook(BaseAccount, AbsConnectivity):
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
     systemuser = models.ForeignKey('assets.SystemUser', on_delete=models.CASCADE, null=True, verbose_name=_("System user"))
     version = models.IntegerField(default=1, verbose_name=_('Version'))

apps/assets/models/_user.py

@@ -10,7 +10,7 @@ from django.core.validators import MinValueValidator, MaxValueValidator
 
 from assets.const import Protocol
 from common.utils import signer
-from .base import BaseUser
+from .base import BaseAccount
 from .protocol import ProtocolMixin
 
 
@@ -18,7 +18,7 @@ __all__ = ['SystemUser']
 logger = logging.getLogger(__name__)
 
 
-class SystemUser(ProtocolMixin, BaseUser):
+class SystemUser(ProtocolMixin, BaseAccount):
     LOGIN_AUTO = 'auto'
     LOGIN_MANUAL = 'manual'
     LOGIN_MODE_CHOICES = (
@@ -143,7 +143,7 @@ class SystemUser(ProtocolMixin, BaseUser):
 
 
 # Deprecated: 准备废弃
-class AdminUser(BaseUser):
+class AdminUser(BaseAccount):
     """
     A privileged user that ansible can use it to push system user and so on
     """

apps/assets/models/account.py

@@ -2,14 +2,13 @@ from django.db import models
 from django.utils.translation import gettext_lazy as _
 from simple_history.models import HistoricalRecords
 
-from common.db.models import JMSBaseModel
 from common.db import fields
-from .base import BaseUser, AbsConnectivity
+from .base import BaseAccount, AbsConnectivity
 
 __all__ = ['Account']
 
 
-class Account(BaseUser, AbsConnectivity):
+class Account(BaseAccount, AbsConnectivity):
     token = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Token'))
     privileged = models.BooleanField(verbose_name=_("Privileged account"), default=False)
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
@@ -28,15 +27,3 @@ class Account(BaseUser, AbsConnectivity):
 
     def __str__(self):
         return '{}@{}'.format(self.username, self.asset.name)
-
-
-class AccountTemplate(BaseUser, AbsConnectivity):
-    type = models.CharField(
-        max_length=16, choices=Account.Type.choices, default=Account.Type.common, verbose_name=_("Type")
-    )
-
-    class Meta:
-        verbose_name = _('Account Template')
-
-    def __str__(self):
-        return '{}'.format(self.username)

apps/assets/models/asset/common.py

@@ -84,7 +84,7 @@ class Asset(AbsConnectivity, NodesRelationMixin, JMSOrgBaseModel):
     objects = AssetManager.from_queryset(AssetQuerySet)()
 
     def __str__(self):
-        return '{0.hostname}({0.ip})'.format(self)
+        return '{0.name}({0.ip})'.format(self)
 
     def get_target_ip(self):
         return self.ip

apps/assets/models/base.py

@@ -169,13 +169,14 @@ class AuthMixin:
         )
 
 
-class BaseUser(OrgModelMixin, AuthMixin):
+class BaseAccount(OrgModelMixin, AuthMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'))
     username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True)
     password = fields.EncryptCharField(max_length=256, blank=True, null=True, verbose_name=_('Password'))
     private_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH private key'))
     public_key = fields.EncryptTextField(blank=True, null=True, verbose_name=_('SSH public key'))
+    # token = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Token'))
     comment = models.TextField(blank=True, verbose_name=_('Comment'))
     date_created = models.DateTimeField(auto_now_add=True, verbose_name=_("Date created"))
     date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))

apps/assets/models/cmd_filter.py

@@ -9,7 +9,7 @@ from django.core.validators import MinValueValidator, MaxValueValidator
 from django.utils.translation import ugettext_lazy as _
 
 from users.models import User, UserGroup
-from ..models import SystemUser, Asset
+from ..models import Asset
 
 from common.utils import lazyproperty, get_logger, get_object_or_none
 from orgs.mixins.models import OrgModelMixin
@@ -36,13 +36,7 @@ class CommandFilter(OrgModelMixin):
         'assets.Asset', related_name='cmd_filters', blank=True,
         verbose_name=_("Asset")
     )
-    system_users = models.ManyToManyField(
+    accounts = models.JSONField(default=list, verbose_name=_("Accounts"))
-        'assets.SystemUser', related_name='cmd_filters', blank=True,
-        verbose_name=_("System user"))
-    applications = models.ManyToManyField(
-        'applications.Application', related_name='cmd_filters', blank=True,
-        verbose_name=_("Application")
-    )
     is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
     comment = models.TextField(blank=True, default='', verbose_name=_("Comment"))
     date_created = models.DateTimeField(auto_now_add=True)

apps/assets/models/domain.py

@@ -11,7 +11,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from common.utils import get_logger, lazyproperty
 from orgs.mixins.models import OrgModelMixin
-from .base import BaseUser
+from .base import BaseAccount
 
 logger = get_logger(__file__)
 
@@ -50,7 +50,7 @@ class Domain(OrgModelMixin):
             return random.choice(self.gateways)
 
 
-class Gateway(BaseUser):
+class Gateway(BaseAccount):
     UNCONNECTIVE_KEY_TMPL = 'asset_unconnective_gateway_{}'
     UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL = 'asset_unconnective_gateway_silence_period_{}'
     UNCONNECTIVE_SILENCE_PERIOD_BEGIN_VALUE = 60 * 5