perf: 暂存一下

apps/assets/api/account_history.py

@@ -1,23 +1,21 @@
-from django.db.models import F
-
 from assets.api.accounts import (
     AccountFilterSet, AccountViewSet, AccountSecretsViewSet
 )
 from common.mixins import RecordViewLogMixin
 from .. import serializers
-from ..models import AuthBook
+from ..models import Account
 
 __all__ = ['AccountHistoryViewSet', 'AccountHistorySecretsViewSet']
 
 
 class AccountHistoryFilterSet(AccountFilterSet):
     class Meta:
-        model = AuthBook.history.model
+        model = Account.history.model
         fields = AccountFilterSet.Meta.fields
 
 
 class AccountHistoryViewSet(AccountViewSet):
-    model = AuthBook.history.model
+    model = Account.history.model
     filterset_class = AccountHistoryFilterSet
     serializer_classes = {
         'default': serializers.AccountHistorySerializer,
@@ -26,13 +24,8 @@ class AccountHistoryViewSet(AccountViewSet):
         'list': 'assets.view_assethistoryaccount',
         'retrieve': 'assets.view_assethistoryaccount',
     }
-
     http_method_names = ['get', 'options']
 
-    def get_queryset(self):
-        queryset = AuthBook.get_queryset(is_history_model=True)
-        return queryset
-
 
 class AccountHistorySecretsViewSet(RecordViewLogMixin, AccountHistoryViewSet):
     serializer_classes = {

apps/assets/api/accounts.py

@@ -12,7 +12,7 @@ from common.mixins import RecordViewLogMixin
 from common.permissions import UserConfirmation
 from authentication.const import ConfirmType
 from ..tasks.account_connectivity import test_accounts_connectivity_manual
-from ..models import AuthBook, Node
+from ..models import Node, Account
 from .. import serializers
 
 __all__ = ['AccountFilterSet', 'AccountViewSet', 'AccountSecretsViewSet', 'AccountTaskCreateAPI']
@@ -50,16 +50,16 @@ class AccountFilterSet(BaseFilterSet):
         return qs
 
     class Meta:
-        model = AuthBook
+        model = Account
         fields = [
-            'asset', 'systemuser', 'id',
+            'asset', 'id',
         ]
 
 
 class AccountViewSet(OrgBulkModelViewSet):
-    model = AuthBook
+    model = Account
-    filterset_fields = ("username", "asset", "systemuser", 'ip', 'hostname')
+    filterset_fields = ("username", "asset", 'ip', 'hostname')
-    search_fields = ('username', 'ip', 'hostname', 'systemuser__username')
+    search_fields = ('username', 'ip', 'hostname')
     filterset_class = AccountFilterSet
     serializer_classes = {
         'default': serializers.AccountSerializer,
@@ -70,10 +70,6 @@ class AccountViewSet(OrgBulkModelViewSet):
         'partial_update': 'assets.change_assetaccountsecret',
     }
 
-    def get_queryset(self):
-        queryset = AuthBook.get_queryset()
-        return queryset
-
     @action(methods=['post'], detail=True, url_path='verify')
     def verify_account(self, request, *args, **kwargs):
         account = super().get_object()
@@ -106,7 +102,7 @@ class AccountTaskCreateAPI(CreateAPIView):
         return request.user.has_perm('assets.test_assetconnectivity')
 
     def get_accounts(self):
-        queryset = AuthBook.objects.all()
+        queryset = Account.objects.all()
         queryset = self.filter_queryset(queryset)
         return queryset
 

apps/assets/api/system_user_relation.py

@@ -68,7 +68,6 @@ class BaseRelationViewSet(RelationMixin, OrgBulkModelViewSet):
 
 
 class SystemUserAssetRelationViewSet(BaseRelationViewSet):
-    perm_model = models.AuthBook
     serializer_class = serializers.SystemUserAssetRelationSerializer
     model = models.SystemUser.assets.through
     filterset_fields = [

apps/assets/migrations/0092_auto_20220711_1409.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.2.12 on 2022-07-11 06:08
+# Generated by Django 3.2.12 on 2022-07-11 08:59
 
 import assets.models.base
 import assets.models.user
@@ -22,6 +22,8 @@ class Migration(migrations.Migration):
             name='HistoricalAccount',
             fields=[
                 ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')),
+                ('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')),
                 ('id', models.UUIDField(db_index=True, default=uuid.uuid4)),
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
                 ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
@@ -33,6 +35,7 @@ class Migration(migrations.Migration):
                 ('date_updated', models.DateTimeField(blank=True, editable=False, verbose_name='Date updated')),
                 ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
                 ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
+                ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
                 ('version', models.IntegerField(default=1, verbose_name='Version')),
                 ('history_id', models.AutoField(primary_key=True, serialize=False)),
                 ('history_date', models.DateTimeField(db_index=True)),
@@ -53,6 +56,8 @@ class Migration(migrations.Migration):
             name='Account',
             fields=[
                 ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')),
+                ('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')),
                 ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
                 ('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
@@ -64,6 +69,7 @@ class Migration(migrations.Migration):
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
                 ('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
                 ('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
+                ('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
                 ('version', models.IntegerField(default=1, verbose_name='Version')),
                 ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.asset', verbose_name='Asset')),
             ],

apps/assets/migrations/0093_auto_20220711_1413.py

@@ -10,16 +10,18 @@ def migrate_accounts(apps, schema_editor):
 
     count = 0
     bulk_size = 1000
+    print("\nStart migrate accounts")
     while True:
+        start = time.time()
         auth_books = auth_book_model.objects \
-                    .prefetch_related('systemuser') \
+            .prefetch_related('systemuser') \
-                    .all()[count:count+bulk_size]
+            .all()[count:count+bulk_size]
+        count += len(auth_books)
         if not auth_books:
             break
 
         accounts = []
-        start = time.time()
+        # auth book 和 account 相同的属性
-        # authbook 和 account 相同的属性
         same_attrs = [
             'id', 'comment', 'date_created', 'date_updated',
             'created_by', 'asset_id', 'org_id',
@@ -47,9 +49,8 @@ def migrate_accounts(apps, schema_editor):
 
         account_model.objects.bulk_create(accounts, ignore_conflicts=True)
         print("Create accounts: {}-{} using: {:.2f}s".format(
-            count, count + len(accounts), time.time()-start
+            count - bulk_size, count, time.time()-start
         ))
-        count += len(auth_books)
 
 
 class Migration(migrations.Migration):

apps/assets/migrations/0094_alter_systemuser_assets.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.12 on 2022-07-13 09:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0093_auto_20220711_1413'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='systemuser',
+            name='assets',
+        ),
+        migrations.AddField(
+            model_name='systemuser',
+            name='assets',
+            field=models.ManyToManyField(blank=True, related_name='system_users', to='assets.Asset', verbose_name='Assets'),
+        ),
+    ]

apps/assets/migrations/0095_auto_20220713_1746.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.2.12 on 2022-07-13 09:46
+
+import time
+from django.db import migrations
+
+
+def migrate_asset_system_user_relations(apps, schema_editor):
+    system_user_model = apps.get_model('assets', 'SystemUser')
+    old_model = apps.get_model('assets', 'AuthBook')
+    new_model = system_user_model.assets.through
+
+    count = 0
+    bulk_size = 1000
+    print("\nStart migrate asset system user relations")
+    while True:
+        start = time.time()
+        auth_books = old_model.objects.only('asset_id', 'systemuser_id')[count:count+bulk_size]
+        auth_books = list(auth_books)
+        count += len(auth_books)
+        if not auth_books:
+            break
+        asset_system_users = []
+        for auth_book in auth_books:
+            if not auth_book.asset_id or not auth_book.systemuser_id:
+                continue
+            asset_system_user = new_model(
+                asset_id=auth_book.asset_id,
+                systemuser_id=auth_book.systemuser_id
+            )
+            asset_system_users.append(asset_system_user)
+        new_model.objects.bulk_create(asset_system_users, ignore_conflicts=True)
+        print("Create asset system user relations: {}-{} using: {:.2f}s".format(
+            count - bulk_size, count, time.time()-start
+        ))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0094_alter_systemuser_assets'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_asset_system_user_relations)
+    ]

apps/assets/models/account.py

@@ -3,15 +3,20 @@ from django.utils.translation import gettext_lazy as _
 from simple_history.models import HistoricalRecords
 
 from .user import ProtocolMixin
-from .base import BaseUser
+from .base import BaseUser, AbsConnectivity
 
 
 __all__ = ['Account']
 
 
-class Account(BaseUser, ProtocolMixin):
+class Account(BaseUser, AbsConnectivity, ProtocolMixin):
+    class Type(models.TextChoices):
+        common = 'common', _('Common user')
+        admin = 'admin', _('Admin user')
+
     protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices,
                                 default='ssh', verbose_name=_('Protocol'))
+    type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_("Type"))
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
     version = models.IntegerField(default=1, verbose_name=_('Version'))
     history = HistoricalRecords()

apps/assets/models/user.py

@@ -242,7 +242,6 @@ class SystemUser(ProtocolMixin, AuthMixin, BaseUser):
     nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes"))
     assets = models.ManyToManyField(
         'assets.Asset', blank=True, verbose_name=_("Assets"),
-        through='assets.AuthBook', through_fields=['systemuser', 'asset'],
         related_name='system_users'
     )
     users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users"))

apps/assets/serializers/system_user.py

@@ -298,7 +298,7 @@ class SystemUserAssetRelationSerializer(RelationMixin, serializers.ModelSerializ
     asset_display = serializers.ReadOnlyField(label=_('Asset hostname'))
 
     class Meta:
-        model = SystemUser.assets.through
+        model = SystemUser
         fields = [
             "id", "asset", "asset_display", 'systemuser', 'systemuser_display',
             "connectivity", 'date_verified', 'org_id'

apps/assets/tasks/asset_connectivity.py

@@ -6,7 +6,7 @@ from django.utils.translation import gettext_noop
 
 from common.utils import get_logger
 from orgs.utils import org_aware_func
-from ..models import Asset, Connectivity, AuthBook
+from ..models import Asset, Connectivity, Account
 from . import const
 from .utils import clean_ansible_task_hosts, group_asset_by_platform
 
@@ -18,6 +18,7 @@ __all__ = [
 ]
 
 
+# Todo: 这里可能有问题了
 def set_assets_accounts_connectivity(assets, results_summary):
     asset_ids_ok = set()
     asset_ids_failed = set()
@@ -33,11 +34,11 @@ def set_assets_accounts_connectivity(assets, results_summary):
     Asset.bulk_set_connectivity(asset_ids_ok, Connectivity.ok)
     Asset.bulk_set_connectivity(asset_ids_failed, Connectivity.failed)
 
-    accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser__type='admin')
+    accounts_ok = Account.objects.filter(asset_id__in=asset_ids_ok,)
-    accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser__type='admin')
+    accounts_failed = Account.objects.filter(asset_id__in=asset_ids_faile)
 
-    AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok)
+    Account.bulk_set_connectivity(accounts_ok, Connectivity.ok)
-    AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed)
+    Account.bulk_set_connectivity(accounts_failed, Connectivity.failed)
 
 
 @shared_task(queue="ansible")

apps/assets/tasks/common.py

@@ -17,6 +17,7 @@ def add_nodes_assets_to_system_users(nodes_keys, system_users):
 
     nodes = Node.objects.filter(key__in=nodes_keys)
     assets = Node.get_nodes_all_assets(*nodes)
+
     for system_user in system_users:
         """ 解决资产和节点进行关联时，已经关联过的节点不会触发 authbook post_save 信号， 
         无法更新节点下所有资产的管理用户的问题 """
@@ -28,7 +29,7 @@ def add_nodes_assets_to_system_users(nodes_keys, system_users):
             )
             if created:
                 need_push_asset_ids.append(asset.id)
-            # # 不再自动更新资产管理用户，只允许用户手动指定。
+            # 不再自动更新资产管理用户，只允许用户手动指定。
             # 只要关联都需要更新资产的管理用户
             # instance.update_asset_admin_user_if_need()
 

apps/assets/tasks/system_user_connectivity.py

@@ -8,7 +8,7 @@ from django.utils.translation import ugettext as _, gettext_noop
 from assets.models import Asset
 from common.utils import get_logger
 from orgs.utils import tmp_to_org, org_aware_func
-from ..models import SystemUser, Connectivity, AuthBook
+from ..models import SystemUser, Connectivity, Account
 from . import const
 from .utils import (
     clean_ansible_task_hosts, group_asset_by_platform
@@ -34,11 +34,11 @@ def set_assets_accounts_connectivity(system_user, assets, results_summary):
         else:
             asset_ids_failed.add(asset.id)
 
-    accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user)
+    accounts_ok = Account.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user)
-    accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user)
+    accounts_failed = Account.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user)
 
-    AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok)
+    Account.bulk_set_connectivity(accounts_ok, Connectivity.ok)
-    AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed)
+    Account.bulk_set_connectivity(accounts_failed, Connectivity.failed)
 
 
 @org_aware_func("system_user")