perf: 修改迁移

apps/assets/api/asset/common.py

@@ -49,7 +49,8 @@ class AssetViewSet(SuggestionMixin, FilterAssetByNodeMixin, OrgBulkModelViewSet)
         'gateways': 'assets.view_gateway'
     }
     extra_filter_backends = [
-        FilterAssetByNodeFilterBackend, LabelFilterBackend,
+        FilterAssetByNodeFilterBackend,
+        LabelFilterBackend,
         IpInFilterBackend,
     ]
 

apps/assets/const.py

@@ -46,9 +46,9 @@ class DatabaseTypes(models.TextChoices):
 
 class RemoteAppTypes(models.TextChoices):
     CHROME = 'chrome', 'Chrome'
-    VSPHERE = 'vsphere', 'vSphere client'
+    VSPHERE = 'vmware_client', 'vSphere client'
     MYSQL_WORKBENCH = 'mysql_workbench', 'MySQL workbench'
-    CUSTOM_REMOTE_APP = 'custom_remote_app', _("Custom")
+    GENERAL_REMOTE_APP = 'general_remote_app', _("Custom")
 
 
 class CloudTypes(models.TextChoices):

apps/assets/filters.py

@@ -68,6 +68,7 @@ class FilterAssetByNodeFilterBackend(filters.BaseFilterBackend):
                 Q(nodes__key=node.key)
             ).distinct()
         else:
+            print("Query query origin: ", queryset.count())
             return queryset.filter(nodes__key=node.key).distinct()
 
 

apps/assets/migrations/0095_auto_20220406_1541.py

@@ -20,7 +20,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='asset',
             name='type',
-            field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vsphere', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('custom_remote_app', 'Custom'), ('k8s', 'Kubernetes')], max_length=128, verbose_name='Type'),
+            field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vmware_client', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('general_remote_app', 'Custom'), ('k8s', 'Kubernetes')], max_length=128, verbose_name='Type'),
             preserve_default=False,
         ),
     ]

apps/assets/migrations/0097_auto_20220407_1726.py

@@ -24,7 +24,7 @@ class Migration(migrations.Migration):
         migrations.AlterField(
             model_name='platform',
             name='type',
-            field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vsphere', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('custom_remote_app', 'Custom'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'),
+            field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vmware_client', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('general_remote_app', 'Custom'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'),
         ),
         migrations.AlterField(
             model_name='systemuser',

apps/assets/migrations/0098_auto_20220426_1550.py

@@ -36,6 +36,7 @@ class Migration(migrations.Migration):
             name='RemoteApp',
             fields=[
                 ('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
+                ('connect_host', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='assets.host')),
                 ('app_path', models.CharField(max_length=1024, verbose_name='App path')),
                 ('attrs', models.JSONField(default=dict, verbose_name='Attrs')),
             ],
@@ -44,4 +45,15 @@ class Migration(migrations.Migration):
             },
             bases=('assets.asset',),
         ),
+        migrations.CreateModel(
+            name='Cloud',
+            fields=[
+                ('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
+                ('cluster', models.CharField(max_length=4096, verbose_name='Cluster')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=('assets.asset',),
+        ),
     ]

apps/assets/migrations/0099_auto_20220426_1558.py

@@ -1,12 +1,14 @@
 # Generated by Django 3.1.14 on 2022-04-26 07:58
+import uuid
 
 from django.db import migrations
-from django.db.utils import IntegrityError
+
+failed_apps = []
 
 
 def create_app_platform(apps, *args):
     platform_model = apps.get_model('assets', 'Platform')
-    apps = [
+    platforms = [
         # DB
         {'name': 'MySQL', 'category': 'database', 'type': 'mysql'},
         {'name': 'MariaDB', 'category': 'database', 'type': 'mariadb'},
@@ -15,13 +17,30 @@ def create_app_platform(apps, *args):
         {'name': 'SQLServer', 'category': 'database', 'type': 'sqlserver'},
         {'name': 'MongoDB', 'category': 'database', 'type': 'mongodb'},
         {'name': 'Redis', 'category': 'database', 'type': 'redis'},
-        {'name': 'RemoteApp', 'category': 'remote_app', 'type': 'remote_app'},
+        {'name': 'Chrome', 'category': 'remote_app', 'type': 'chrome'},
-        {'name': 'Kubernetes', 'category': 'cloud', 'type': 'kubernetes'},
+        {'name': 'vSphereClient', 'category': 'remote_app', 'type': 'vmware_client'},
+        {'name': 'MySQLWorkbench', 'category': 'remote_app', 'type': 'mysql_workbench'},
+        {'name': 'GeneralRemoteApp', 'category': 'remote_app', 'type': 'general_remote_app'},
+        {'name': 'Kubernetes', 'category': 'cloud', 'type': 'k8s'},
     ]
 
-    for app in apps:
+    for platform in platforms:
-        app['internal'] = True
+        platform['internal'] = True
-        platform_model.objects.update_or_create(defaults=app, name=app['name'])
+        print("Create platform: {}".format(platform['name']))
+        platform_model.objects.update_or_create(defaults=platform, name=platform['name'])
+
+
+def get_prop_name_id(apps, app, category):
+    asset_model = apps.get_model('assets', 'Asset')
+    _id = app.id
+    id_exists = asset_model.objects.filter(id=_id).exists()
+    if (id_exists):
+        _id = uuid.uuid4()
+    name = app.name
+    name_exists = asset_model.objects.filter(hostname=name).exists()
+    if name_exists:
+        name = category + '-' + app.name
+    return _id, name
 
 
 def migrate_database_to_asset(apps, *args):
@@ -33,12 +52,11 @@ def migrate_database_to_asset(apps, *args):
     platforms = platform_model.objects.all()
     platforms_map = {p.type: p for p in platforms}
 
-    dbs = []
     for app in applications:
         attrs = {'host': '', 'port': 0, 'database': ''}
         _attrs = app.attrs or {}
         attrs.update(_attrs)
-        print("Create: ", app.name)
+
         db = db_model(
             id=app.id, hostname=app.name, ip=attrs['host'],
             protocols='{}/{}'.format(app.type, attrs['port']),
@@ -47,20 +65,129 @@ def migrate_database_to_asset(apps, *args):
             platform=platforms_map[app.type],
             org_id=app.org_id
         )
-        for i in range(3):
+        try:
-            try:
+            print("Create database: ", app.name)
-                db.save()
+            db.save()
-                break
+        except:
-            except IntegrityError:
+            failed_apps.append(app)
-                db.name = 'DB-' + db.name
+            pass
+                # db.hostname = 'DB-' + db.hostname
 
 
 def migrate_remote_app_to_asset(apps, *args):
-    pass
+    app_model = apps.get_model('applications', 'Application')
+    remote_app_model = apps.get_model('assets', 'RemoteApp')
+    host_model = apps.get_model('assets', 'Host')
+    platform_model = apps.get_model('assets', 'Platform')
+    applications = app_model.objects.filter(category='remote_app')
+    platforms = platform_model.objects.filter(category='remote_app')
+    platforms_map = {p.type: p for p in platforms}
+
+    connect_host_map = {}
+
+    for app in applications:
+        attrs = app.attrs
+        connect_host = attrs.pop('asset')
+        if connect_host:
+            connect_host = host_model.objects.filter(asset_ptr_id=connect_host).first()
+            connect_host_map[app.id] = connect_host
+
+    for app in applications:
+        tp = app.type
+        app_path = attrs.pop('path', '')
+        if tp == 'custom':
+            tp = 'general_remote_app'
+
+        print("Create remote app: {}".format(app.name))
+        remote_app = remote_app_model(
+            id=app.id, hostname=app.name, ip='',
+            category='remote_app', type=tp,
+            platform=platforms_map[tp],
+            org_id=app.org_id,
+
+            app_path=app_path,
+            connect_host=connect_host_map.get(app.id),
+            attrs=attrs,
+        )
+        try:
+            remote_app.save()
+        except Exception as e:
+            print("Error: ", e)
+            # remote_app.hostname = 'RemoteApp-' + remote_app.hostname
 
 
 def migrate_cloud_to_asset(apps, *args):
-    pass
+    app_model = apps.get_model('applications', 'Application')
+    cloud_model = apps.get_model('assets', 'Cloud')
+    platform_model = apps.get_model('assets', 'Platform')
+
+    applications = app_model.objects.filter(category='cloud')
+    platform = platform_model.objects.filter(type='k8s').first()
+
+    for app in applications:
+        attrs = app.attrs
+        print("Create cloud: {}".format(app.name))
+        cloud = cloud_model(
+            id=app.id, hostname=app.name, ip='',
+            category='remote_app', type='k8s',
+            platform=platform,
+            org_id=app.org_id,
+            cluster=attrs.get('cluster', '')
+        )
+
+        try:
+            cloud.save()
+        except Exception as e:
+            failed_apps.append(cloud)
+            print("Error: ", e)
+
+
+def create_app_nodes(apps, org_id):
+    node_model = apps.get_model('assets', 'Node')
+
+    child_pattern = r'^[0-9]+:[0-9]+$'
+    node_keys = node_model.objects.filter(org_id=org_id) \
+        .filter(key__regex=child_pattern) \
+        .values_list('key', flat=True)
+    if not node_keys:
+        return
+    node_key_split = [key.split(':') for key in node_keys]
+    next_value = int(max([k[1] for k in node_key_split])) + 1
+    parent_key = node_key_split[0][0]
+    parent = node_model.objects.get(key=parent_key)
+    next_key = '{}:{}'.format(node_key_split[0][0], next_value)
+    name = 'Apps'
+    full_value = parent.full_value + '/' + name
+    defaults = {
+        'key': next_key, 'value': name, 'parent_key': parent_key,
+        'full_value': full_value, 'org_id': org_id
+    }
+    node, created = node_model.objects.get_or_create(
+        defaults=defaults, value=name, org_id=org_id,
+    )
+    node.parent = parent
+    return node
+
+
+def migrate_to_nodes(apps, *args):
+    org_model = apps.get_model('orgs', 'Organization')
+    asset_model = apps.get_model('assets', 'Asset')
+    orgs = org_model.objects.all()
+
+    for org in orgs:
+        node = create_app_nodes(apps, org.id)
+        assets = asset_model.objects.filter(
+            category__in=['remote_app', 'database', 'cloud'],
+            org_id=org.id
+        )
+        print("Set node asset: ", node)
+        if node:
+            node.assets_amount = len(assets)
+            node.save()
+            node.assets.set(assets)
+            parent = node.parent
+            parent.assets_amount += len(assets)
+            parent.save()
 
 
 class Migration(migrations.Migration):
@@ -70,6 +197,9 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(create_app_platform),
+        # migrations.RunPython(create_app_platform),
-        migrations.RunPython(migrate_database_to_asset),
+        # migrations.RunPython(migrate_database_to_asset),
+        # migrations.RunPython(migrate_remote_app_to_asset),
+        # migrations.RunPython(migrate_cloud_to_asset),
+        migrations.RunPython(migrate_to_nodes)
     ]

apps/assets/models/asset/cloud.py

@@ -0,0 +1,11 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from .common import Asset
+
+
+class Cloud(Asset):
+    cluster = models.CharField(max_length=4096, verbose_name=_("Cluster"))
+
+    def __str__(self):
+        return self.hostname

apps/assets/models/asset/database.py

@@ -7,5 +7,8 @@ from .common import Asset
 class Database(Asset):
     db_name = models.CharField(max_length=1024, verbose_name=_("Database"), blank=True)
 
+    def __str__(self):
+        return '{}({}://{}/{})'.format(self.hostname, self.type, self.ip, self.db_name)
+
     class Meta:
         verbose_name = _("Database")

apps/assets/models/asset/remote_app.py

@@ -1,16 +1,10 @@
 from django.utils.translation import gettext_lazy as _
 from django.db import models
 
-from orgs.mixins.models import OrgModelMixin
-from common.mixins.models import CommonModelMixin
 from .common import Asset
 
 
-class RemoteAppHost(CommonModelMixin, OrgModelMixin):
-    host = models.ForeignKey('assets.Host', verbose_name=_("Host"))
-    system_user = models.ForeignKey('assets.SystemUser', verbose_name=_("System user"))
-
-
 class RemoteApp(Asset):
     app_path = models.CharField(max_length=1024, verbose_name=_("App path"))
+    connect_host = models.ForeignKey('assets.Host', null=True, on_delete=models.SET_NULL)
     attrs = models.JSONField(default=dict, verbose_name=_('Attrs'))

apps/assets/pagination.py

@@ -42,11 +42,11 @@ class AssetPaginationBase(LimitOffsetPagination):
 class NodeAssetTreePagination(AssetPaginationBase):
     def get_count_from_nodes(self, queryset):
         is_query_all = self._view.is_query_node_all_assets
-        if is_query_all:
+        if not is_query_all:
-            node = self._view.node
+            return None
-            if not node:
+        node = self._view.node
-                node = Node.org_root()
+        if not node:
-            if node:
+            node = Node.org_root()
-                logger.debug(f'Hit node.assets_amount[{node.assets_amount}] -> {self._request.get_full_path()}')
+        if node:
-                return node.assets_amount
+            logger.debug(f'Hit node assets_amount cache: [{node.assets_amount}]')
-        return None
+            return node.assets_amount