diff --git a/apps/assets/api/asset/common.py b/apps/assets/api/asset/common.py index 4cec7495a..007d7fbd0 100644 --- a/apps/assets/api/asset/common.py +++ b/apps/assets/api/asset/common.py @@ -49,7 +49,8 @@ class AssetViewSet(SuggestionMixin, FilterAssetByNodeMixin, OrgBulkModelViewSet) 'gateways': 'assets.view_gateway' } extra_filter_backends = [ - FilterAssetByNodeFilterBackend, LabelFilterBackend, + FilterAssetByNodeFilterBackend, + LabelFilterBackend, IpInFilterBackend, ] diff --git a/apps/assets/const.py b/apps/assets/const.py index ccd48205b..0697c2112 100644 --- a/apps/assets/const.py +++ b/apps/assets/const.py @@ -46,9 +46,9 @@ class DatabaseTypes(models.TextChoices): class RemoteAppTypes(models.TextChoices): CHROME = 'chrome', 'Chrome' - VSPHERE = 'vsphere', 'vSphere client' + VSPHERE = 'vmware_client', 'vSphere client' MYSQL_WORKBENCH = 'mysql_workbench', 'MySQL workbench' - CUSTOM_REMOTE_APP = 'custom_remote_app', _("Custom") + GENERAL_REMOTE_APP = 'general_remote_app', _("Custom") class CloudTypes(models.TextChoices): diff --git a/apps/assets/filters.py b/apps/assets/filters.py index b807396e0..346c507b1 100644 --- a/apps/assets/filters.py +++ b/apps/assets/filters.py @@ -68,6 +68,7 @@ class FilterAssetByNodeFilterBackend(filters.BaseFilterBackend): Q(nodes__key=node.key) ).distinct() else: + print("Query query origin: ", queryset.count()) return queryset.filter(nodes__key=node.key).distinct() diff --git a/apps/assets/migrations/0095_auto_20220406_1541.py b/apps/assets/migrations/0095_auto_20220406_1541.py index 9647aed54..7b43c92a1 100644 --- a/apps/assets/migrations/0095_auto_20220406_1541.py +++ b/apps/assets/migrations/0095_auto_20220406_1541.py @@ -20,7 +20,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='asset', name='type', - field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vsphere', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('custom_remote_app', 'Custom'), ('k8s', 'Kubernetes')], max_length=128, verbose_name='Type'), + field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vmware_client', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('general_remote_app', 'Custom'), ('k8s', 'Kubernetes')], max_length=128, verbose_name='Type'), preserve_default=False, ), ] diff --git a/apps/assets/migrations/0097_auto_20220407_1726.py b/apps/assets/migrations/0097_auto_20220407_1726.py index e16a6ed92..61d260458 100644 --- a/apps/assets/migrations/0097_auto_20220407_1726.py +++ b/apps/assets/migrations/0097_auto_20220407_1726.py @@ -24,7 +24,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='platform', name='type', - field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vsphere', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('custom_remote_app', 'Custom'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'), + field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vmware_client', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('general_remote_app', 'Custom'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'), ), migrations.AlterField( model_name='systemuser', diff --git a/apps/assets/migrations/0098_auto_20220426_1550.py b/apps/assets/migrations/0098_auto_20220426_1550.py index 54aeb7970..bbce73dec 100644 --- a/apps/assets/migrations/0098_auto_20220426_1550.py +++ b/apps/assets/migrations/0098_auto_20220426_1550.py @@ -36,6 +36,7 @@ class Migration(migrations.Migration): name='RemoteApp', fields=[ ('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')), + ('connect_host', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='assets.host')), ('app_path', models.CharField(max_length=1024, verbose_name='App path')), ('attrs', models.JSONField(default=dict, verbose_name='Attrs')), ], @@ -44,4 +45,15 @@ class Migration(migrations.Migration): }, bases=('assets.asset',), ), + migrations.CreateModel( + name='Cloud', + fields=[ + ('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')), + ('cluster', models.CharField(max_length=4096, verbose_name='Cluster')), + ], + options={ + 'abstract': False, + }, + bases=('assets.asset',), + ), ] diff --git a/apps/assets/migrations/0099_auto_20220426_1558.py b/apps/assets/migrations/0099_auto_20220426_1558.py index b3d2c0c4e..2a7c34e3b 100644 --- a/apps/assets/migrations/0099_auto_20220426_1558.py +++ b/apps/assets/migrations/0099_auto_20220426_1558.py @@ -1,12 +1,14 @@ # Generated by Django 3.1.14 on 2022-04-26 07:58 +import uuid from django.db import migrations -from django.db.utils import IntegrityError + +failed_apps = [] def create_app_platform(apps, *args): platform_model = apps.get_model('assets', 'Platform') - apps = [ + platforms = [ # DB {'name': 'MySQL', 'category': 'database', 'type': 'mysql'}, {'name': 'MariaDB', 'category': 'database', 'type': 'mariadb'}, @@ -15,13 +17,30 @@ def create_app_platform(apps, *args): {'name': 'SQLServer', 'category': 'database', 'type': 'sqlserver'}, {'name': 'MongoDB', 'category': 'database', 'type': 'mongodb'}, {'name': 'Redis', 'category': 'database', 'type': 'redis'}, - {'name': 'RemoteApp', 'category': 'remote_app', 'type': 'remote_app'}, - {'name': 'Kubernetes', 'category': 'cloud', 'type': 'kubernetes'}, + {'name': 'Chrome', 'category': 'remote_app', 'type': 'chrome'}, + {'name': 'vSphereClient', 'category': 'remote_app', 'type': 'vmware_client'}, + {'name': 'MySQLWorkbench', 'category': 'remote_app', 'type': 'mysql_workbench'}, + {'name': 'GeneralRemoteApp', 'category': 'remote_app', 'type': 'general_remote_app'}, + {'name': 'Kubernetes', 'category': 'cloud', 'type': 'k8s'}, ] - for app in apps: - app['internal'] = True - platform_model.objects.update_or_create(defaults=app, name=app['name']) + for platform in platforms: + platform['internal'] = True + print("Create platform: {}".format(platform['name'])) + platform_model.objects.update_or_create(defaults=platform, name=platform['name']) + + +def get_prop_name_id(apps, app, category): + asset_model = apps.get_model('assets', 'Asset') + _id = app.id + id_exists = asset_model.objects.filter(id=_id).exists() + if (id_exists): + _id = uuid.uuid4() + name = app.name + name_exists = asset_model.objects.filter(hostname=name).exists() + if name_exists: + name = category + '-' + app.name + return _id, name def migrate_database_to_asset(apps, *args): @@ -33,12 +52,11 @@ def migrate_database_to_asset(apps, *args): platforms = platform_model.objects.all() platforms_map = {p.type: p for p in platforms} - dbs = [] for app in applications: attrs = {'host': '', 'port': 0, 'database': ''} _attrs = app.attrs or {} attrs.update(_attrs) - print("Create: ", app.name) + db = db_model( id=app.id, hostname=app.name, ip=attrs['host'], protocols='{}/{}'.format(app.type, attrs['port']), @@ -47,20 +65,129 @@ def migrate_database_to_asset(apps, *args): platform=platforms_map[app.type], org_id=app.org_id ) - for i in range(3): - try: - db.save() - break - except IntegrityError: - db.name = 'DB-' + db.name + try: + print("Create database: ", app.name) + db.save() + except: + failed_apps.append(app) + pass + # db.hostname = 'DB-' + db.hostname def migrate_remote_app_to_asset(apps, *args): - pass + app_model = apps.get_model('applications', 'Application') + remote_app_model = apps.get_model('assets', 'RemoteApp') + host_model = apps.get_model('assets', 'Host') + platform_model = apps.get_model('assets', 'Platform') + applications = app_model.objects.filter(category='remote_app') + platforms = platform_model.objects.filter(category='remote_app') + platforms_map = {p.type: p for p in platforms} + + connect_host_map = {} + + for app in applications: + attrs = app.attrs + connect_host = attrs.pop('asset') + if connect_host: + connect_host = host_model.objects.filter(asset_ptr_id=connect_host).first() + connect_host_map[app.id] = connect_host + + for app in applications: + tp = app.type + app_path = attrs.pop('path', '') + if tp == 'custom': + tp = 'general_remote_app' + + print("Create remote app: {}".format(app.name)) + remote_app = remote_app_model( + id=app.id, hostname=app.name, ip='', + category='remote_app', type=tp, + platform=platforms_map[tp], + org_id=app.org_id, + + app_path=app_path, + connect_host=connect_host_map.get(app.id), + attrs=attrs, + ) + try: + remote_app.save() + except Exception as e: + print("Error: ", e) + # remote_app.hostname = 'RemoteApp-' + remote_app.hostname def migrate_cloud_to_asset(apps, *args): - pass + app_model = apps.get_model('applications', 'Application') + cloud_model = apps.get_model('assets', 'Cloud') + platform_model = apps.get_model('assets', 'Platform') + + applications = app_model.objects.filter(category='cloud') + platform = platform_model.objects.filter(type='k8s').first() + + for app in applications: + attrs = app.attrs + print("Create cloud: {}".format(app.name)) + cloud = cloud_model( + id=app.id, hostname=app.name, ip='', + category='remote_app', type='k8s', + platform=platform, + org_id=app.org_id, + cluster=attrs.get('cluster', '') + ) + + try: + cloud.save() + except Exception as e: + failed_apps.append(cloud) + print("Error: ", e) + + +def create_app_nodes(apps, org_id): + node_model = apps.get_model('assets', 'Node') + + child_pattern = r'^[0-9]+:[0-9]+$' + node_keys = node_model.objects.filter(org_id=org_id) \ + .filter(key__regex=child_pattern) \ + .values_list('key', flat=True) + if not node_keys: + return + node_key_split = [key.split(':') for key in node_keys] + next_value = int(max([k[1] for k in node_key_split])) + 1 + parent_key = node_key_split[0][0] + parent = node_model.objects.get(key=parent_key) + next_key = '{}:{}'.format(node_key_split[0][0], next_value) + name = 'Apps' + full_value = parent.full_value + '/' + name + defaults = { + 'key': next_key, 'value': name, 'parent_key': parent_key, + 'full_value': full_value, 'org_id': org_id + } + node, created = node_model.objects.get_or_create( + defaults=defaults, value=name, org_id=org_id, + ) + node.parent = parent + return node + + +def migrate_to_nodes(apps, *args): + org_model = apps.get_model('orgs', 'Organization') + asset_model = apps.get_model('assets', 'Asset') + orgs = org_model.objects.all() + + for org in orgs: + node = create_app_nodes(apps, org.id) + assets = asset_model.objects.filter( + category__in=['remote_app', 'database', 'cloud'], + org_id=org.id + ) + print("Set node asset: ", node) + if node: + node.assets_amount = len(assets) + node.save() + node.assets.set(assets) + parent = node.parent + parent.assets_amount += len(assets) + parent.save() class Migration(migrations.Migration): @@ -70,6 +197,9 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(create_app_platform), - migrations.RunPython(migrate_database_to_asset), + # migrations.RunPython(create_app_platform), + # migrations.RunPython(migrate_database_to_asset), + # migrations.RunPython(migrate_remote_app_to_asset), + # migrations.RunPython(migrate_cloud_to_asset), + migrations.RunPython(migrate_to_nodes) ] diff --git a/apps/assets/models/asset/cloud.py b/apps/assets/models/asset/cloud.py index e69de29bb..b8eed03b0 100644 --- a/apps/assets/models/asset/cloud.py +++ b/apps/assets/models/asset/cloud.py @@ -0,0 +1,11 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from .common import Asset + + +class Cloud(Asset): + cluster = models.CharField(max_length=4096, verbose_name=_("Cluster")) + + def __str__(self): + return self.hostname diff --git a/apps/assets/models/asset/database.py b/apps/assets/models/asset/database.py index 7c688e6d6..4947b79ca 100644 --- a/apps/assets/models/asset/database.py +++ b/apps/assets/models/asset/database.py @@ -7,5 +7,8 @@ from .common import Asset class Database(Asset): db_name = models.CharField(max_length=1024, verbose_name=_("Database"), blank=True) + def __str__(self): + return '{}({}://{}/{})'.format(self.hostname, self.type, self.ip, self.db_name) + class Meta: verbose_name = _("Database") diff --git a/apps/assets/models/asset/remote_app.py b/apps/assets/models/asset/remote_app.py index c7528baae..caa4c89a0 100644 --- a/apps/assets/models/asset/remote_app.py +++ b/apps/assets/models/asset/remote_app.py @@ -1,16 +1,10 @@ from django.utils.translation import gettext_lazy as _ from django.db import models -from orgs.mixins.models import OrgModelMixin -from common.mixins.models import CommonModelMixin from .common import Asset -class RemoteAppHost(CommonModelMixin, OrgModelMixin): - host = models.ForeignKey('assets.Host', verbose_name=_("Host")) - system_user = models.ForeignKey('assets.SystemUser', verbose_name=_("System user")) - - class RemoteApp(Asset): app_path = models.CharField(max_length=1024, verbose_name=_("App path")) + connect_host = models.ForeignKey('assets.Host', null=True, on_delete=models.SET_NULL) attrs = models.JSONField(default=dict, verbose_name=_('Attrs')) diff --git a/apps/assets/pagination.py b/apps/assets/pagination.py index f75ff023c..f913e9eed 100644 --- a/apps/assets/pagination.py +++ b/apps/assets/pagination.py @@ -42,11 +42,11 @@ class AssetPaginationBase(LimitOffsetPagination): class NodeAssetTreePagination(AssetPaginationBase): def get_count_from_nodes(self, queryset): is_query_all = self._view.is_query_node_all_assets - if is_query_all: - node = self._view.node - if not node: - node = Node.org_root() - if node: - logger.debug(f'Hit node.assets_amount[{node.assets_amount}] -> {self._request.get_full_path()}') - return node.assets_amount - return None + if not is_query_all: + return None + node = self._view.node + if not node: + node = Node.org_root() + if node: + logger.debug(f'Hit node assets_amount cache: [{node.assets_amount}]') + return node.assets_amount