From 8282a6869a2facda24f623805d0551731c0c2828 Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 17 Aug 2022 15:44:59 +0800 Subject: [PATCH] perf: remove system users --- .../migrations/0024_auto_20220817_1346.py | 57 +++++++++++++ .../migrations/0112_auto_20220817_1544.py | 61 ++++++++++++++ apps/ops/api/command.py | 1 - .../ops/migrations/0022_auto_20220817_1346.py | 47 +++++++++++ apps/ops/models/command.py | 3 +- .../migrations/0011_auto_20200721_1739.py | 2 +- apps/tickets/api/ticket.py | 4 +- .../migrations/0020_auto_20220817_1346.py | 81 +++++++++++++++++++ apps/tickets/models/ticket/apply_asset.py | 3 - apps/tickets/models/ticket/command_confirm.py | 6 +- .../models/ticket/login_asset_confirm.py | 6 +- 11 files changed, 253 insertions(+), 18 deletions(-) create mode 100644 apps/applications/migrations/0024_auto_20220817_1346.py create mode 100644 apps/assets/migrations/0112_auto_20220817_1544.py create mode 100644 apps/ops/migrations/0022_auto_20220817_1346.py create mode 100644 apps/tickets/migrations/0020_auto_20220817_1346.py diff --git a/apps/applications/migrations/0024_auto_20220817_1346.py b/apps/applications/migrations/0024_auto_20220817_1346.py new file mode 100644 index 000000000..a6d03a5ca --- /dev/null +++ b/apps/applications/migrations/0024_auto_20220817_1346.py @@ -0,0 +1,57 @@ +# Generated by Django 3.2.14 on 2022-08-17 05:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('applications', '0023_auto_20220816_1021'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='account', + unique_together=None, + ), + migrations.RemoveField( + model_name='account', + name='app', + ), + migrations.RemoveField( + model_name='account', + name='systemuser', + ), + migrations.AlterUniqueTogether( + name='application', + unique_together=None, + ), + migrations.RemoveField( + model_name='application', + name='domain', + ), + migrations.RemoveField( + model_name='historicalaccount', + name='app', + ), + migrations.RemoveField( + model_name='historicalaccount', + name='history_user', + ), + migrations.RemoveField( + model_name='historicalaccount', + name='systemuser', + ), + migrations.DeleteModel( + name='ApplicationUser', + ), + migrations.DeleteModel( + name='Account', + ), + migrations.DeleteModel( + name='Application', + ), + migrations.DeleteModel( + name='HistoricalAccount', + ), + ] diff --git a/apps/assets/migrations/0112_auto_20220817_1544.py b/apps/assets/migrations/0112_auto_20220817_1544.py new file mode 100644 index 000000000..d1ba8875b --- /dev/null +++ b/apps/assets/migrations/0112_auto_20220817_1544.py @@ -0,0 +1,61 @@ +# Generated by Django 3.2.14 on 2022-08-17 07:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0111_auto_20220816_1022'), + ] + + operations = [ + migrations.DeleteModel( + name='AdminUser', + ), + migrations.RemoveField( + model_name='historicalauthbook', + name='asset', + ), + migrations.RemoveField( + model_name='historicalauthbook', + name='history_user', + ), + migrations.RemoveField( + model_name='historicalauthbook', + name='systemuser', + ), + migrations.AlterUniqueTogether( + name='systemuser', + unique_together=None, + ), + migrations.RemoveField( + model_name='systemuser', + name='assets', + ), + migrations.RemoveField( + model_name='systemuser', + name='groups', + ), + migrations.RemoveField( + model_name='systemuser', + name='nodes', + ), + migrations.RemoveField( + model_name='systemuser', + name='su_from', + ), + migrations.RemoveField( + model_name='systemuser', + name='users', + ), + migrations.DeleteModel( + name='AuthBook', + ), + migrations.DeleteModel( + name='HistoricalAuthBook', + ), + migrations.DeleteModel( + name='SystemUser', + ), + ] diff --git a/apps/ops/api/command.py b/apps/ops/api/command.py index 0d513cd9e..a129ec416 100644 --- a/apps/ops/api/command.py +++ b/apps/ops/api/command.py @@ -9,7 +9,6 @@ from django.conf import settings from assets.models import Asset, Node from orgs.mixins.api import RootOrgViewMixin -from common.permissions import IsValidUser from rbac.permissions import RBACPermission from ..models import CommandExecution from ..serializers import CommandExecutionSerializer diff --git a/apps/ops/migrations/0022_auto_20220817_1346.py b/apps/ops/migrations/0022_auto_20220817_1346.py new file mode 100644 index 000000000..b06c85a18 --- /dev/null +++ b/apps/ops/migrations/0022_auto_20220817_1346.py @@ -0,0 +1,47 @@ +# Generated by Django 3.2.14 on 2022-08-17 05:46 + +from django.db import migrations, models + + +def migrate_run_system_user_to_account(apps, schema_editor): + execution_model = apps.get_model('ops', 'CommandExecution') + count = 0 + bulk_size = 1000 + + while True: + executions = execution_model.objects.all().prefetch_related('run_as')[count:bulk_size] + if not executions: + break + count += len(executions) + updated = [] + for obj in executions: + run_as = obj.run_as + if not run_as: + continue + obj.account = run_as.username + updated.append(obj) + execution_model.objects.bulk_update(updated, ['account']) + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0021_auto_20211130_1037'), + ] + + operations = [ + migrations.RemoveField( + model_name='adhoc', + name='run_system_user', + ), + migrations.AddField( + model_name='commandexecution', + name='account', + field=models.CharField(default='', max_length=128, verbose_name='account'), + ), + migrations.RunPython(migrate_run_system_user_to_account), + migrations.RemoveField( + model_name='commandexecution', + name='run_as', + ), + ] diff --git a/apps/ops/models/command.py b/apps/ops/models/command.py index 0c1038f89..14cb99081 100644 --- a/apps/ops/models/command.py +++ b/apps/ops/models/command.py @@ -22,8 +22,7 @@ from ..inventory import JMSInventory class CommandExecution(OrgModelMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) hosts = models.ManyToManyField('assets.Asset') - run_as = models.ForeignKey('assets.SystemUser', on_delete=models.CASCADE) - account = models.CharField(max_length=128, verbose_name=_('account')) + account = models.CharField(max_length=128, default='', verbose_name=_('account')) command = models.TextField(verbose_name=_("Command")) _result = models.TextField(blank=True, null=True, verbose_name=_('Result')) user = models.ForeignKey('users.User', on_delete=models.CASCADE, null=True) diff --git a/apps/perms/migrations/0011_auto_20200721_1739.py b/apps/perms/migrations/0011_auto_20200721_1739.py index 352bd6b79..df8b46cde 100644 --- a/apps/perms/migrations/0011_auto_20200721_1739.py +++ b/apps/perms/migrations/0011_auto_20200721_1739.py @@ -3,7 +3,7 @@ from django.db import migrations, models from django.db.models import F -from ..models.base import Action +from perms.models import Action def migrate_asset_permission(apps, schema_editor): diff --git a/apps/tickets/api/ticket.py b/apps/tickets/api/ticket.py index 9933a5401..369bc155d 100644 --- a/apps/tickets/api/ticket.py +++ b/apps/tickets/api/ticket.py @@ -15,8 +15,8 @@ from tickets import serializers from tickets import filters from tickets.permissions.ticket import IsAssignee, IsApplicant from tickets.models import ( - Ticket, ApplyAssetTicket, - ApplyLoginTicket, ApplyLoginAssetTicket, ApplyCommandTicket + Ticket, ApplyAssetTicket, ApplyLoginTicket, + ApplyLoginAssetTicket, ApplyCommandTicket ) __all__ = [ diff --git a/apps/tickets/migrations/0020_auto_20220817_1346.py b/apps/tickets/migrations/0020_auto_20220817_1346.py new file mode 100644 index 000000000..76437f936 --- /dev/null +++ b/apps/tickets/migrations/0020_auto_20220817_1346.py @@ -0,0 +1,81 @@ +# Generated by Django 3.2.14 on 2022-08-17 05:46 + +import time +from django.db import migrations, models + + +def migrate_system_to_account(apps, schema_editor): + apply_asset_ticket_model = apps.get_model('tickets', 'ApplyAssetTicket') + apply_command_ticket_model = apps.get_model('tickets', 'ApplyCommandTicket') + apply_login_asset_ticket_model = apps.get_model('tickets', 'ApplyLoginAssetTicket') + + model_system_user_account = ( + (apply_asset_ticket_model, 'apply_system_users', 'apply_accounts', True), + (apply_command_ticket_model, 'apply_run_system_user', 'apply_run_account', False), + (apply_login_asset_ticket_model, 'apply_login_system_user', 'apply_login_account', False), + ) + + for model, old_field, new_field, m2m in model_system_user_account: + print("Start migrate '{}' system user to account".format(model)) + count = 0 + bulk_size = 1000 + + while True: + start = time.time() + objects = model.objects.all().prefetch_related(old_field)[count:bulk_size] + if not objects: + break + count += len(objects) + + updated = [] + for obj in objects: + if m2m: + old_value = getattr(obj, old_field).all() + new_value = [s.username for s in old_value] + else: + old_value = getattr(obj, old_field) + new_value = old_value.username + setattr(obj, new_field, new_value) + updated.append(obj) + model.objects.bulk_update(updated, [new_field]) + print("Migrate account: {}-{} using: {:.2f}s".format( + count - bulk_size, count, time.time()-start + )) + + +class Migration(migrations.Migration): + + dependencies = [ + ('tickets', '0019_delete_applyapplicationticket'), + ] + + operations = [ + migrations.AddField( + model_name='applyassetticket', + name='apply_accounts', + field=models.JSONField(default=list, verbose_name='Apply accounts'), + ), + migrations.AddField( + model_name='applycommandticket', + name='apply_run_account', + field=models.CharField(default='', max_length=128, verbose_name='Run account'), + ), + migrations.AddField( + model_name='applyloginassetticket', + name='apply_login_account', + field=models.CharField(default='', max_length=128, verbose_name='Login account'), + ), + migrations.RunPython(migrate_system_to_account), + migrations.RemoveField( + model_name='applyassetticket', + name='apply_system_users', + ), + migrations.RemoveField( + model_name='applycommandticket', + name='apply_run_system_user', + ), + migrations.RemoveField( + model_name='applyloginassetticket', + name='apply_login_system_user', + ), + ] diff --git a/apps/tickets/models/ticket/apply_asset.py b/apps/tickets/models/ticket/apply_asset.py index 1b358f5c1..c0ec46cc0 100644 --- a/apps/tickets/models/ticket/apply_asset.py +++ b/apps/tickets/models/ticket/apply_asset.py @@ -14,9 +14,6 @@ class ApplyAssetTicket(Ticket): apply_nodes = models.ManyToManyField('assets.Node', verbose_name=_('Apply nodes')) # 申请信息 apply_assets = models.ManyToManyField('assets.Asset', verbose_name=_('Apply assets')) - apply_system_users = models.ManyToManyField( - 'assets.SystemUser', verbose_name=_('Apply system users') - ) apply_accounts = models.JSONField(default=list, verbose_name=_('Apply accounts')) apply_actions = models.IntegerField( choices=Action.DB_CHOICES, default=Action.ALL, verbose_name=_('Actions') diff --git a/apps/tickets/models/ticket/command_confirm.py b/apps/tickets/models/ticket/command_confirm.py index b8c37ec54..601102ba3 100644 --- a/apps/tickets/models/ticket/command_confirm.py +++ b/apps/tickets/models/ticket/command_confirm.py @@ -10,11 +10,7 @@ class ApplyCommandTicket(Ticket): null=True, verbose_name=_('Run user') ) apply_run_asset = models.CharField(max_length=128, verbose_name=_('Run asset')) - apply_run_system_user = models.ForeignKey( - 'assets.SystemUser', on_delete=models.SET_NULL, - null=True, verbose_name=_('Run system user') - ) - apply_run_account = models.CharField(max_length=128, verbose_name=_('Run account')) + apply_run_account = models.CharField(max_length=128, default='', verbose_name=_('Run account')) apply_run_command = models.CharField(max_length=4096, verbose_name=_('Run command')) apply_from_session = models.ForeignKey( 'terminal.Session', on_delete=models.SET_NULL, diff --git a/apps/tickets/models/ticket/login_asset_confirm.py b/apps/tickets/models/ticket/login_asset_confirm.py index 43130e052..2b97cd7a2 100644 --- a/apps/tickets/models/ticket/login_asset_confirm.py +++ b/apps/tickets/models/ticket/login_asset_confirm.py @@ -15,8 +15,6 @@ class ApplyLoginAssetTicket(Ticket): 'assets.Asset', on_delete=models.SET_NULL, null=True, verbose_name=_('Login asset'), ) - apply_login_system_user = models.ForeignKey( - 'assets.SystemUser', on_delete=models.SET_NULL, null=True, - verbose_name=_('Login system user'), + apply_login_account = models.CharField( + max_length=128, default='', verbose_name=_('Login account') ) - apply_login_account = models.CharField(max_length=128, verbose_name=_('Login account'))