github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: 修改数据库 migrations

pull/8873/head
ibuler 2022-08-18 11:15:17 +08:00
parent 8282a6869a
commit 2c3239e238
21 changed files with 133 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
apps/applications/migrations/0022_auto_20220816_1015.py
View File

@ -1,24 +0,0 @@
# Generated by Django 3.2.14 on 2022-08-16 02:15
import common.db.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('applications', '0021_auto_20220629_1826'),
]
operations = [
migrations.AddField(
model_name='account',
name='token',
field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
),
migrations.AddField(
model_name='historicalaccount',
name='token',
field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
),
]

18
apps/applications/migrations/0024_auto_20220817_1346.py → apps/applications/migrations/0022_auto_20220817_1346.py
View File

@ -6,7 +6,7 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('applications', '0023_auto_20220816_1021'),
('applications', '0021_auto_20220629_1826'),
]
operations = [
@ -22,10 +22,6 @@ class Migration(migrations.Migration):
model_name='account',
name='systemuser',
),
migrations.AlterUniqueTogether(
name='application',
unique_together=None,
),
migrations.RemoveField(
model_name='application',
name='domain',
@ -42,16 +38,4 @@ class Migration(migrations.Migration):
model_name='historicalaccount',
name='systemuser',
),
migrations.DeleteModel(
name='ApplicationUser',
),
migrations.DeleteModel(
name='Account',
),
migrations.DeleteModel(
name='Application',
),
migrations.DeleteModel(
name='HistoricalAccount',
),
]

21
apps/applications/migrations/0023_auto_20220816_1021.py
View File

@ -1,21 +0,0 @@
# Generated by Django 3.2.14 on 2022-08-16 02:21
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('applications', '0022_auto_20220816_1015'),
]
operations = [
migrations.RemoveField(
model_name='account',
name='token',
),
migrations.RemoveField(
model_name='historicalaccount',
name='token',
),
]

26
apps/applications/migrations/0023_auto_20220817_1716.py Normal file
View File

@ -0,0 +1,26 @@
# Generated by Django 3.2.14 on 2022-08-17 09:16
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('applications', '0022_auto_20220817_1346'),
('perms', '0031_auto_20220816_1600'),
('ops', '0022_auto_20220817_1346'),
('assets', '0109_auto_20220817_1544'),
('tickets', '0020_auto_20220817_1346'),
]
operations = [
migrations.DeleteModel(
name='Account',
),
migrations.DeleteModel(
name='HistoricalAccount',
),
migrations.DeleteModel(
name='ApplicationUser',
),
]

23
apps/applications/migrations/0024_auto_20220818_1057.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 3.2.14 on 2022-08-18 02:57
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('applications', '0023_auto_20220817_1716'),
]
operations = [
migrations.AlterField(
model_name='application',
name='category',
field=models.CharField(max_length=16, verbose_name='Category'),
),
migrations.AlterField(
model_name='application',
name='type',
field=models.CharField(max_length=16, verbose_name='Type'),
),
]

28
apps/applications/models.py Normal file
View File

@ -0,0 +1,28 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from orgs.mixins.models import OrgModelMixin
from common.mixins import CommonModelMixin
class Application(CommonModelMixin, OrgModelMixin):
name = models.CharField(max_length=128, verbose_name=_('Name'))
category = models.CharField(
max_length=16, verbose_name=_('Category')
)
type = models.CharField(
max_length=16, verbose_name=_('Type')
)
attrs = models.JSONField(default=dict, verbose_name=_('Attrs'))
comment = models.TextField(
max_length=128, default='', blank=True, verbose_name=_('Comment')
)
class Meta:
verbose_name = _('Application')
unique_together = [('org_id', 'name')]
ordering = ('name',)
permissions = [
('match_application', _('Can match application')),
]

8
apps/assets/migrations/0101_auto_20220711_1409.py
View File

@ -29,12 +29,12 @@ class Migration(migrations.Migration):
('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
('comment', models.TextField(blank=True, verbose_name='Comment')),
('date_created', models.DateTimeField(blank=True, editable=False, verbose_name='Date created')),
('date_updated', models.DateTimeField(blank=True, editable=False, verbose_name='Date updated')),
('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
('privileged', models.BooleanField(default=False, verbose_name='Privileged account')),
('version', models.IntegerField(default=0, verbose_name='Version')),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField(db_index=True)),
@ -63,12 +63,12 @@ class Migration(migrations.Migration):
('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')),
('private_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH private key')),
('public_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH public key')),
('token', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token')),
('comment', models.TextField(blank=True, verbose_name='Comment')),
('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
('privileged', models.BooleanField(default=False, verbose_name='Privileged account')),
('version', models.IntegerField(default=0, verbose_name='Version')),
('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.asset', verbose_name='Asset')),
],

4
apps/assets/migrations/0102_auto_20220711_1413.py
View File

@ -16,10 +16,10 @@ def migrate_accounts(apps, schema_editor):
auth_books = auth_book_model.objects \
.prefetch_related('systemuser') \
.all()[count:count+bulk_size]
count += len(auth_books)
if not auth_books:
break
count += len(auth_books)
accounts = []
# auth book 和 account 相同的属性
same_attrs = [
@ -50,7 +50,7 @@ def migrate_accounts(apps, schema_editor):
account_model.objects.bulk_create(accounts, ignore_conflicts=True)
print("Create accounts: {}-{} using: {:.2f}s".format(
count - bulk_size, count, time.time()-start
count - len(auth_books), count, time.time()-start
))

5
apps/assets/migrations/0104_auto_20220803_1859.py
View File

@ -15,10 +15,9 @@ def migrate_asset_protocols(apps, schema_editor):
while True:
start = time.time()
assets = asset_model.objects.all()[count:count+bulk_size]
count += len(assets)
if not assets:
break
count += len(assets)
assets_protocols = []
for asset in assets:
old_protocols = asset._protocols
@ -38,7 +37,7 @@ def migrate_asset_protocols(apps, schema_editor):
assets_protocols.append(asset_protocol_through(asset_id=asset.id, protocol_id=protocol.id))
asset_model.protocols.through.objects.bulk_create(assets_protocols, ignore_conflicts=True)
print("Create asset protocols: {}-{} using: {:.2f}s".format(
count - bulk_size, count, time.time()-start
count - len(assets), count, time.time()-start
))

5
apps/assets/migrations/0105_auto_20220810_1449.py
View File

@ -38,4 +38,9 @@ class Migration(migrations.Migration):
name='type',
field=models.CharField(choices=[('linux', 'Linux'), ('windows', 'Windows'), ('unix', 'Unix'), ('bsd', 'BSD'), ('macos', 'MacOS'), ('mainframe', 'Mainframe'), ('other_host', 'Other host'), ('switch', 'Switch'), ('router', 'Router'), ('firewall', 'Firewall'), ('other_network', 'Other device'), ('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('oracle', 'Oracle'), ('sqlserver', 'SQLServer'), ('mongodb', 'MongoDB'), ('redis', 'Redis'), ('general', 'General'), ('k8s', 'Kubernetes')], default='Linux', max_length=32, verbose_name='Type'),
),
migrations.AlterField(
model_name='platform',
name='su_enabled',
field=models.BooleanField(default=False, verbose_name='Su enabled'),
),
]

22
apps/assets/migrations/0106_auto_20220811_1358.py
View File

@ -1,22 +0,0 @@
# Generated by Django 3.2.14 on 2022-08-11 05:58
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
('assets', '0105_auto_20220810_1449'),
]
operations = [
migrations.RemoveField(
model_name='account',
name='protocol',
),
migrations.RemoveField(
model_name='historicalaccount',
name='protocol',
),
]

2
apps/assets/migrations/0107_auto_20220811_1449.py → apps/assets/migrations/0106_auto_20220811_1449.py
View File

@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0106_auto_20220811_1358'),
('assets', '0105_auto_20220810_1449'),
]
operations = [

6
apps/assets/migrations/0108_auto_20220811_1511.py → apps/assets/migrations/0107_auto_20220811_1511.py
View File

@ -6,7 +6,7 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0107_auto_20220811_1449'),
('assets', '0106_auto_20220811_1449'),
]
operations = [
@ -34,8 +34,4 @@ class Migration(migrations.Migration):
name='created_by',
field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by'),
),
migrations.AlterUniqueTogether(
name='asset',
unique_together={('org_id', 'name')},
),
]

9
apps/assets/migrations/0111_auto_20220816_1022.py → apps/assets/migrations/0108_auto_20220816_1022.py
View File

@ -13,11 +13,10 @@ def migrate_command_filter_to_assets(apps, schema_editor):
while True:
start = time.time()
command_filters = command_filter_model.objects.all() \
.prefetch_related('system_users')[count:count + bulk_size]
count += len(command_filters)
.prefetch_related('system_users')[count:count + bulk_size]
if not command_filters:
break
count += len(command_filters)
updated = []
for command_filter in command_filters:
command_filter.accounts = [s.username for s in command_filter.system_users.all()]
@ -25,7 +24,7 @@ def migrate_command_filter_to_assets(apps, schema_editor):
command_filter_model.objects.bulk_update(updated, ['accounts'])
print("Create assets: {}-{} using: {:.2f}s".format(
count - bulk_size, count, time.time() - start
count - len(command_filters), count, time.time() - start
))
@ -47,7 +46,7 @@ def migrate_command_filter_apps(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('assets', '0110_auto_20220815_1831'),
('assets', '0107_auto_20220811_1511'),
]
operations = [

36
apps/assets/migrations/0109_auto_20220815_1811.py
View File

@ -1,36 +0,0 @@
# Generated by Django 3.2.14 on 2022-08-15 10:11
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0108_auto_20220811_1511'),
]
operations = [
migrations.RemoveField(
model_name='account',
name='type',
),
migrations.RemoveField(
model_name='historicalaccount',
name='type',
),
migrations.AddField(
model_name='account',
name='privileged',
field=models.BooleanField(default=False, verbose_name='Privileged account'),
),
migrations.AddField(
model_name='historicalaccount',
name='privileged',
field=models.BooleanField(default=False, verbose_name='Privileged account'),
),
migrations.AlterField(
model_name='platform',
name='su_enabled',
field=models.BooleanField(default=False, verbose_name='Su enabled'),
),
]

19
apps/assets/migrations/0112_auto_20220817_1544.py → apps/assets/migrations/0109_auto_20220817_1544.py
View File

@ -6,7 +6,7 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('assets', '0111_auto_20220816_1022'),
('assets', '0108_auto_20220816_1022'),
]
operations = [
@ -25,10 +25,6 @@ class Migration(migrations.Migration):
model_name='historicalauthbook',
name='systemuser',
),
migrations.AlterUniqueTogether(
name='systemuser',
unique_together=None,
),
migrations.RemoveField(
model_name='systemuser',
name='assets',
@ -41,21 +37,8 @@ class Migration(migrations.Migration):
model_name='systemuser',
name='nodes',
),
migrations.RemoveField(
model_name='systemuser',
name='su_from',
),
migrations.RemoveField(
model_name='systemuser',
name='users',
),
migrations.DeleteModel(
name='AuthBook',
),
migrations.DeleteModel(
name='HistoricalAuthBook',
),
migrations.DeleteModel(
name='SystemUser',
),
]

24
apps/assets/migrations/0110_auto_20220815_1831.py
View File

@ -1,24 +0,0 @@
# Generated by Django 3.2.14 on 2022-08-15 10:31
import common.db.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('assets', '0109_auto_20220815_1811'),
]
operations = [
migrations.AddField(
model_name='account',
name='token',
field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
),
migrations.AddField(
model_name='historicalaccount',
name='token',
field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Token'),
),
]

32
apps/assets/migrations/0110_auto_20220817_1716.py Normal file
View File

@ -0,0 +1,32 @@
# Generated by Django 3.2.14 on 2022-08-17 09:16
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('assets', '0109_auto_20220817_1544'),
('applications', '0022_auto_20220817_1346'),
]
operations = [
migrations.AlterUniqueTogether(
name='authbook',
unique_together=None,
),
migrations.RemoveField(
model_name='authbook',
name='asset',
),
migrations.RemoveField(
model_name='authbook',
name='systemuser',
),
migrations.DeleteModel(
name='HistoricalAuthBook',
),
migrations.DeleteModel(
name='AuthBook',
),
]

2
apps/assets/models/__init__.py
View File

@ -1,6 +1,5 @@
from .base import *
from .platform import *
# from ._user import *
from .asset import *
from .label import Label
from .group import *
@ -11,6 +10,7 @@ from .gathered_user import *
from .favorite_asset import *
from .account import *
from .backup import *
from ._user import *
# 废弃以下
# from ._authbook import *
from .protocol import *

137
apps/assets/models/_user.py
View File

@ -8,8 +8,6 @@ from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.validators import MinValueValidator, MaxValueValidator
from assets.const import Protocol
from common.utils import signer
from .base import BaseAccount
from .protocol import ProtocolMixin
@ -31,14 +29,6 @@ class SystemUser(ProtocolMixin, BaseAccount):
admin = 'admin', _('Admin user')
username_same_with_user = models.BooleanField(default=False, verbose_name=_("Username same with user"))
nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes"))
assets = models.ManyToManyField(
'assets.Asset', blank=True, verbose_name=_("Assets"),
through='assets.AuthBook', through_fields=['systemuser', 'asset'],
related_name='system_users'
)
users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users"))
groups = models.ManyToManyField('users.UserGroup', blank=True, verbose_name=_("User groups"))
type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_('Type'))
priority = models.IntegerField(default=81, verbose_name=_("Priority"), help_text=_("1-100, the lower the value will be match first"), validators=[MinValueValidator(1), MaxValueValidator(100)])
protocol = models.CharField(max_length=16, default='ssh', verbose_name=_('Protocol'))
@ -55,84 +45,6 @@ class SystemUser(ProtocolMixin, BaseAccount):
su_enabled = models.BooleanField(default=False, verbose_name=_('User switch'))
su_from = models.ForeignKey('self', on_delete=models.SET_NULL, related_name='su_to', null=True, verbose_name=_("Switch from"))
def __str__(self):
username = self.username
if self.username_same_with_user:
username = '*'
return '{0.name}({1})'.format(self, username)
@property
def nodes_amount(self):
return self.nodes.all().count()
@property
def login_mode_display(self):
return self.get_login_mode_display()
def is_need_push(self):
if self.auto_push and self.is_protocol_support_push:
return True
else:
return False
@property
def is_admin_user(self):
return self.type == self.Type.admin
@property
def is_need_cmd_filter(self):
return self.protocol not in [self.Protocol.rdp, self.Protocol.vnc]
@property
def is_need_test_asset_connective(self):
return self.protocol in self.ASSET_CATEGORY_PROTOCOLS
@property
def cmd_filter_rules(self):
from .cmd_filter import CommandFilterRule
rules = CommandFilterRule.objects.filter(
filter__in=self.cmd_filters.all()
).distinct()
return rules
def is_command_can_run(self, command):
for rule in self.cmd_filter_rules:
action, matched_cmd = rule.match(command)
if action == rule.ActionChoices.allow:
return True, None
elif action == rule.ActionChoices.deny:
return False, matched_cmd
return True, None
def get_all_assets(self):
from assets.models import Node, Asset
nodes_keys = self.nodes.all().values_list('key', flat=True)
asset_ids = set(self.assets.all().values_list('id', flat=True))
nodes_asset_ids = Node.get_nodes_all_asset_ids_by_keys(nodes_keys)
asset_ids.update(nodes_asset_ids)
assets = Asset.objects.filter(id__in=asset_ids)
return assets
def add_related_assets(self, assets_or_ids):
self.assets.add(*tuple(assets_or_ids))
self.add_related_assets_to_su_from_if_need(assets_or_ids)
def add_related_assets_to_su_from_if_need(self, assets_or_ids):
if self.protocol not in [self.Protocol.ssh.value]:
return
if not self.su_enabled:
return
if not self.su_from:
return
if self.su_from.protocol != self.protocol:
return
self.su_from.assets.add(*tuple(assets_or_ids))
# TODO 暂时为了接口文档添加
@property
def auto_push_account(self):
return
class Meta:
ordering = ['name']
unique_together = [('name', 'org_id')]
@ -140,52 +52,3 @@ class SystemUser(ProtocolMixin, BaseAccount):
permissions = [
('match_systemuser', _('Can match system user')),
]
# Deprecated: 准备废弃
class AdminUser(BaseAccount):
"""
A privileged user that ansible can use it to push system user and so on
"""
BECOME_METHOD_CHOICES = (
('sudo', 'sudo'),
('su', 'su'),
)
become = models.BooleanField(default=True)
become_method = models.CharField(choices=BECOME_METHOD_CHOICES, default='sudo', max_length=4)
become_user = models.CharField(default='root', max_length=64)
_become_pass = models.CharField(default='', blank=True, max_length=128)
CONNECTIVITY_CACHE_KEY = '_ADMIN_USER_CONNECTIVE_{}'
_prefer = "admin_user"
def __str__(self):
return self.name
@property
def become_pass(self):
password = signer.unsign(self._become_pass)
if password:
return password
else:
return ""
@become_pass.setter
def become_pass(self, password):
self._become_pass = signer.sign(password)
@property
def become_info(self):
if self.become:
info = {
"method": self.become_method,
"user": self.become_user,
"pass": self.become_pass,
}
else:
info = None
return info
class Meta:
ordering = ['name']
unique_together = [('name', 'org_id')]
verbose_name = _("Admin user")

4
apps/tickets/migrations/0020_auto_20220817_1346.py
View File

@ -34,12 +34,12 @@ def migrate_system_to_account(apps, schema_editor):
new_value = [s.username for s in old_value]
else:
old_value = getattr(obj, old_field)
new_value = old_value.username
new_value = old_value.username if old_value else ''
setattr(obj, new_field, new_value)
updated.append(obj)
model.objects.bulk_update(updated, [new_field])
print("Migrate account: {}-{} using: {:.2f}s".format(
count - bulk_size, count, time.time()-start
count - len(objects), count, time.time()-start
))