github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: 修改迁移

pull/8605/head
ibuler 2022-04-28 12:47:39 +08:00
parent 770d2508d7
commit ba0a017aa4
11 changed files with 191 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/assets/api/asset/common.py
View File

@ -49,7 +49,8 @@ class AssetViewSet(SuggestionMixin, FilterAssetByNodeMixin, OrgBulkModelViewSet)
'gateways': 'assets.view_gateway'
}
extra_filter_backends = [
FilterAssetByNodeFilterBackend, LabelFilterBackend,
FilterAssetByNodeFilterBackend,
LabelFilterBackend,
IpInFilterBackend,
]

4
apps/assets/const.py
View File

@ -46,9 +46,9 @@ class DatabaseTypes(models.TextChoices):
class RemoteAppTypes(models.TextChoices):
CHROME = 'chrome', 'Chrome'
VSPHERE = 'vsphere', 'vSphere client'
VSPHERE = 'vmware_client', 'vSphere client'
MYSQL_WORKBENCH = 'mysql_workbench', 'MySQL workbench'
CUSTOM_REMOTE_APP = 'custom_remote_app', _("Custom")
GENERAL_REMOTE_APP = 'general_remote_app', _("Custom")
class CloudTypes(models.TextChoices):

1
apps/assets/filters.py
View File

@ -68,6 +68,7 @@ class FilterAssetByNodeFilterBackend(filters.BaseFilterBackend):
Q(nodes__key=node.key)
).distinct()
else:
print("Query query origin: ", queryset.count())
return queryset.filter(nodes__key=node.key).distinct()

2
apps/assets/migrations/0095_auto_20220406_1541.py
View File

@ -20,7 +20,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='asset',
name='type',
field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vsphere', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('custom_remote_app', 'Custom'), ('k8s', 'Kubernetes')], max_length=128, verbose_name='Type'),
field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vmware_client', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('general_remote_app', 'Custom'), ('k8s', 'Kubernetes')], max_length=128, verbose_name='Type'),
preserve_default=False,
),
]

2
apps/assets/migrations/0097_auto_20220407_1726.py
View File

@ -24,7 +24,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='platform',
name='type',
field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vsphere', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('custom_remote_app', 'Custom'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'),
field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('chrome', 'Chrome'), ('vmware_client', 'vSphere client'), ('mysql_workbench', 'MySQL workbench'), ('general_remote_app', 'Custom'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'),
),
migrations.AlterField(
model_name='systemuser',

12
apps/assets/migrations/0098_auto_20220426_1550.py
View File

@ -36,6 +36,7 @@ class Migration(migrations.Migration):
name='RemoteApp',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
('connect_host', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='assets.host')),
('app_path', models.CharField(max_length=1024, verbose_name='App path')),
('attrs', models.JSONField(default=dict, verbose_name='Attrs')),
],
@ -44,4 +45,15 @@ class Migration(migrations.Migration):
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Cloud',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
('cluster', models.CharField(max_length=4096, verbose_name='Cluster')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
]

168
apps/assets/migrations/0099_auto_20220426_1558.py
View File

@ -1,12 +1,14 @@
# Generated by Django 3.1.14 on 2022-04-26 07:58
import uuid
from django.db import migrations
from django.db.utils import IntegrityError
failed_apps = []
def create_app_platform(apps, *args):
platform_model = apps.get_model('assets', 'Platform')
apps = [
platforms = [
# DB
{'name': 'MySQL', 'category': 'database', 'type': 'mysql'},
{'name': 'MariaDB', 'category': 'database', 'type': 'mariadb'},
@ -15,13 +17,30 @@ def create_app_platform(apps, *args):
{'name': 'SQLServer', 'category': 'database', 'type': 'sqlserver'},
{'name': 'MongoDB', 'category': 'database', 'type': 'mongodb'},
{'name': 'Redis', 'category': 'database', 'type': 'redis'},
{'name': 'RemoteApp', 'category': 'remote_app', 'type': 'remote_app'},
{'name': 'Kubernetes', 'category': 'cloud', 'type': 'kubernetes'},
{'name': 'Chrome', 'category': 'remote_app', 'type': 'chrome'},
{'name': 'vSphereClient', 'category': 'remote_app', 'type': 'vmware_client'},
{'name': 'MySQLWorkbench', 'category': 'remote_app', 'type': 'mysql_workbench'},
{'name': 'GeneralRemoteApp', 'category': 'remote_app', 'type': 'general_remote_app'},
{'name': 'Kubernetes', 'category': 'cloud', 'type': 'k8s'},
]
for app in apps:
app['internal'] = True
platform_model.objects.update_or_create(defaults=app, name=app['name'])
for platform in platforms:
platform['internal'] = True
print("Create platform: {}".format(platform['name']))
platform_model.objects.update_or_create(defaults=platform, name=platform['name'])
def get_prop_name_id(apps, app, category):
asset_model = apps.get_model('assets', 'Asset')
_id = app.id
id_exists = asset_model.objects.filter(id=_id).exists()
if (id_exists):
_id = uuid.uuid4()
name = app.name
name_exists = asset_model.objects.filter(hostname=name).exists()
if name_exists:
name = category + '-' + app.name
return _id, name
def migrate_database_to_asset(apps, *args):
@ -33,12 +52,11 @@ def migrate_database_to_asset(apps, *args):
platforms = platform_model.objects.all()
platforms_map = {p.type: p for p in platforms}
dbs = []
for app in applications:
attrs = {'host': '', 'port': 0, 'database': ''}
_attrs = app.attrs or {}
attrs.update(_attrs)
print("Create: ", app.name)
db = db_model(
id=app.id, hostname=app.name, ip=attrs['host'],
protocols='{}/{}'.format(app.type, attrs['port']),
@ -47,20 +65,129 @@ def migrate_database_to_asset(apps, *args):
platform=platforms_map[app.type],
org_id=app.org_id
)
for i in range(3):
try:
db.save()
break
except IntegrityError:
db.name = 'DB-' + db.name
try:
print("Create database: ", app.name)
db.save()
except:
failed_apps.append(app)
pass
# db.hostname = 'DB-' + db.hostname
def migrate_remote_app_to_asset(apps, *args):
pass
app_model = apps.get_model('applications', 'Application')
remote_app_model = apps.get_model('assets', 'RemoteApp')
host_model = apps.get_model('assets', 'Host')
platform_model = apps.get_model('assets', 'Platform')
applications = app_model.objects.filter(category='remote_app')
platforms = platform_model.objects.filter(category='remote_app')
platforms_map = {p.type: p for p in platforms}
connect_host_map = {}
for app in applications:
attrs = app.attrs
connect_host = attrs.pop('asset')
if connect_host:
connect_host = host_model.objects.filter(asset_ptr_id=connect_host).first()
connect_host_map[app.id] = connect_host
for app in applications:
tp = app.type
app_path = attrs.pop('path', '')
if tp == 'custom':
tp = 'general_remote_app'
print("Create remote app: {}".format(app.name))
remote_app = remote_app_model(
id=app.id, hostname=app.name, ip='',
category='remote_app', type=tp,
platform=platforms_map[tp],
org_id=app.org_id,
app_path=app_path,
connect_host=connect_host_map.get(app.id),
attrs=attrs,
)
try:
remote_app.save()
except Exception as e:
print("Error: ", e)
# remote_app.hostname = 'RemoteApp-' + remote_app.hostname
def migrate_cloud_to_asset(apps, *args):
pass
app_model = apps.get_model('applications', 'Application')
cloud_model = apps.get_model('assets', 'Cloud')
platform_model = apps.get_model('assets', 'Platform')
applications = app_model.objects.filter(category='cloud')
platform = platform_model.objects.filter(type='k8s').first()
for app in applications:
attrs = app.attrs
print("Create cloud: {}".format(app.name))
cloud = cloud_model(
id=app.id, hostname=app.name, ip='',
category='remote_app', type='k8s',
platform=platform,
org_id=app.org_id,
cluster=attrs.get('cluster', '')
)
try:
cloud.save()
except Exception as e:
failed_apps.append(cloud)
print("Error: ", e)
def create_app_nodes(apps, org_id):
node_model = apps.get_model('assets', 'Node')
child_pattern = r'^[0-9]+:[0-9]+$'
node_keys = node_model.objects.filter(org_id=org_id) \
.filter(key__regex=child_pattern) \
.values_list('key', flat=True)
if not node_keys:
return
node_key_split = [key.split(':') for key in node_keys]
next_value = int(max([k[1] for k in node_key_split])) + 1
parent_key = node_key_split[0][0]
parent = node_model.objects.get(key=parent_key)
next_key = '{}:{}'.format(node_key_split[0][0], next_value)
name = 'Apps'
full_value = parent.full_value + '/' + name
defaults = {
'key': next_key, 'value': name, 'parent_key': parent_key,
'full_value': full_value, 'org_id': org_id
}
node, created = node_model.objects.get_or_create(
defaults=defaults, value=name, org_id=org_id,
)
node.parent = parent
return node
def migrate_to_nodes(apps, *args):
org_model = apps.get_model('orgs', 'Organization')
asset_model = apps.get_model('assets', 'Asset')
orgs = org_model.objects.all()
for org in orgs:
node = create_app_nodes(apps, org.id)
assets = asset_model.objects.filter(
category__in=['remote_app', 'database', 'cloud'],
org_id=org.id
)
print("Set node asset: ", node)
if node:
node.assets_amount = len(assets)
node.save()
node.assets.set(assets)
parent = node.parent
parent.assets_amount += len(assets)
parent.save()
class Migration(migrations.Migration):
@ -70,6 +197,9 @@ class Migration(migrations.Migration):
]
operations = [
migrations.RunPython(create_app_platform),
migrations.RunPython(migrate_database_to_asset),
# migrations.RunPython(create_app_platform),
# migrations.RunPython(migrate_database_to_asset),
# migrations.RunPython(migrate_remote_app_to_asset),
# migrations.RunPython(migrate_cloud_to_asset),
migrations.RunPython(migrate_to_nodes)
]

11
apps/assets/models/asset/cloud.py
View File

@ -0,0 +1,11 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
from .common import Asset
class Cloud(Asset):
cluster = models.CharField(max_length=4096, verbose_name=_("Cluster"))
def __str__(self):
return self.hostname

3
apps/assets/models/asset/database.py
View File

@ -7,5 +7,8 @@ from .common import Asset
class Database(Asset):
db_name = models.CharField(max_length=1024, verbose_name=_("Database"), blank=True)
def __str__(self):
return '{}({}://{}/{})'.format(self.hostname, self.type, self.ip, self.db_name)
class Meta:
verbose_name = _("Database")

8
apps/assets/models/asset/remote_app.py
View File

@ -1,16 +1,10 @@
from django.utils.translation import gettext_lazy as _
from django.db import models
from orgs.mixins.models import OrgModelMixin
from common.mixins.models import CommonModelMixin
from .common import Asset
class RemoteAppHost(CommonModelMixin, OrgModelMixin):
host = models.ForeignKey('assets.Host', verbose_name=_("Host"))
system_user = models.ForeignKey('assets.SystemUser', verbose_name=_("System user"))
class RemoteApp(Asset):
app_path = models.CharField(max_length=1024, verbose_name=_("App path"))
connect_host = models.ForeignKey('assets.Host', null=True, on_delete=models.SET_NULL)
attrs = models.JSONField(default=dict, verbose_name=_('Attrs'))

16
apps/assets/pagination.py
View File

@ -42,11 +42,11 @@ class AssetPaginationBase(LimitOffsetPagination):
class NodeAssetTreePagination(AssetPaginationBase):
def get_count_from_nodes(self, queryset):
is_query_all = self._view.is_query_node_all_assets
if is_query_all:
node = self._view.node
if not node:
node = Node.org_root()
if node:
logger.debug(f'Hit node.assets_amount[{node.assets_amount}] -> {self._request.get_full_path()}')
return node.assets_amount
return None
if not is_query_all:
return None
node = self._view.node
if not node:
node = Node.org_root()
if node:
logger.debug(f'Hit node assets_amount cache: [{node.assets_amount}]')
return node.assets_amount