github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: 修改合并引起的 migrations 依赖问题

pull/9158/head
ibuler 2022-12-05 16:07:14 +08:00
parent e91cbb9c97
commit 2168610ffe
24 changed files with 347 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/acls/serializers/login_asset_check.py
View File

@ -1,8 +1,8 @@
from rest_framework import serializers
from common.utils import get_object_or_none
from orgs.utils import tmp_to_root_org
from common.utils import get_object_or_none, lazyproperty
from users.models import User
from assets.models import Asset, Account
__all__ = ['LoginAssetCheckSerializer']
@ -22,6 +22,7 @@ class LoginAssetCheckSerializer(serializers.Serializer):
return user_id
def validate_asset_id(self, asset_id):
from assets.models import Asset
self.asset = self.get_object(Asset, asset_id)
return asset_id

22
apps/applications/migrations/0022_auto_20220714_1046.py Normal file
View File

@ -0,0 +1,22 @@
# Generated by Django 3.2.12 on 2022-07-14 02:46
from django.db import migrations
def migrate_db_oracle_version_to_attrs(apps, schema_editor):
db_alias = schema_editor.connection.alias
model = apps.get_model("applications", "Application")
oracles = list(model.objects.using(db_alias).filter(type='oracle'))
for o in oracles:
o.attrs['version'] = '12c'
model.objects.using(db_alias).bulk_update(oracles, ['attrs'])
class Migration(migrations.Migration):
dependencies = [
('applications', '0021_auto_20220629_1826'),
]
operations = [
migrations.RunPython(migrate_db_oracle_version_to_attrs)
]

47
apps/applications/migrations/0023_auto_20220715_1556.py Normal file
View File

@ -0,0 +1,47 @@
# Generated by Django 3.1.14 on 2022-07-15 07:56
import time
from django.db import migrations
def migrate_account_dirty_data(apps, schema_editor):
db_alias = schema_editor.connection.alias
account_model = apps.get_model('applications', 'Account')
count = 0
bulk_size = 1000
while True:
accounts = account_model.objects.using(db_alias) \
.filter(org_id='')[count:count + bulk_size]
if not accounts:
break
accounts = list(accounts)
start = time.time()
for i in accounts:
if i.app:
org_id = i.app.org_id
elif i.systemuser:
org_id = i.systemuser.org_id
else:
org_id = ''
if org_id:
i.org_id = org_id
account_model.objects.bulk_update(accounts, ['org_id', ])
print("Update account org is empty: {}-{} using: {:.2f}s".format(
count, count + len(accounts), time.time() - start
))
count += len(accounts)
class Migration(migrations.Migration):
dependencies = [
('applications', '0022_auto_20220714_1046'),
]
operations = [
migrations.RunPython(migrate_account_dirty_data),
]

23
apps/applications/migrations/0024_auto_20220818_1057.py
View File

@ -1,23 +0,0 @@
# Generated by Django 3.2.14 on 2022-08-18 02:57
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('applications', '0023_auto_20220817_1716'),
]
operations = [
migrations.AlterField(
model_name='application',
name='category',
field=models.CharField(max_length=16, verbose_name='Category'),
),
migrations.AlterField(
model_name='application',
name='type',
field=models.CharField(max_length=16, verbose_name='Type'),
),
]

14
apps/applications/migrations/0022_auto_20220817_1346.py → apps/applications/migrations/0025_auto_20220817_1346.py
View File

@ -1,6 +1,6 @@
# Generated by Django 3.2.14 on 2022-08-17 05:46
from django.db import migrations
from django.db import migrations, models
def migrate_db_oracle_version_to_attrs(apps, schema_editor):
@ -14,7 +14,7 @@ def migrate_db_oracle_version_to_attrs(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('applications', '0021_auto_20220629_1826'),
('applications', '0024_alter_application_type'),
]
operations = [
@ -47,4 +47,14 @@ class Migration(migrations.Migration):
model_name='historicalaccount',
name='systemuser',
),
migrations.AlterField(
model_name='application',
name='category',
field=models.CharField(max_length=16, verbose_name='Category'),
),
migrations.AlterField(
model_name='application',
name='type',
field=models.CharField(max_length=16, verbose_name='Type'),
),
]

2
apps/applications/migrations/0023_auto_20220817_1716.py → apps/applications/migrations/0026_auto_20220817_1716.py
View File

@ -39,7 +39,7 @@ def migrate_account_dirty_data(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('applications', '0022_auto_20220817_1346'),
('applications', '0025_auto_20220817_1346'),
('perms', '0031_auto_20220816_1600'),
('ops', '0022_auto_20220817_1346'),
('assets', '0105_auto_20220817_1544'),

106
apps/assets/migrations/0092_add_host.py
View File

@ -1,106 +0,0 @@
# Generated by Django 3.1.14 on 2022-03-30 10:35
import uuid
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('assets', '0091_auto_20220629_1826'),
]
operations = [
migrations.AddField(
model_name='asset',
name='info',
field=models.JSONField(blank=True, default=dict, verbose_name='Info'),
),
migrations.RenameField(
model_name='asset',
old_name='hostname',
new_name='name',
),
migrations.AlterField(
model_name='asset',
name='name',
field=models.CharField(max_length=128, verbose_name='Name'),
),
migrations.AlterModelOptions(
name='asset',
options={'ordering': ['name'], 'permissions': [('refresh_assethardwareinfo', 'Can refresh asset hardware info'), ('test_assetconnectivity', 'Can test asset connectivity'), ('push_assetsystemuser', 'Can push system user to asset'), ('match_asset', 'Can match asset'), ('add_assettonode', 'Add asset to node'), ('move_assettonode', 'Move asset to node')], 'verbose_name': 'Asset'},
),
migrations.RenameField(
model_name='asset',
old_name='ip',
new_name='address',
),
migrations.AddField(
model_name='asset',
name='date_updated',
field=models.DateTimeField(auto_now=True, verbose_name='Date updated'),
),
migrations.AddField(
model_name='asset',
name='updated_by',
field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by'),
),
migrations.AlterField(
model_name='asset',
name='created_by',
field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by'),
),
migrations.CreateModel(
name='Host',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
],
),
migrations.CreateModel(
name='Database',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
('db_name', models.CharField(blank=True, max_length=1024, verbose_name='Database')),
],
options={
'verbose_name': 'Database',
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Device',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Cloud',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Web',
fields=[
('asset_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.asset')),
('autofill', models.CharField(choices=[('no', 'Disabled'), ('basic', 'Basic'), ('script', 'Script')], default='basic', max_length=16, verbose_name='Autofill')),
('password_selector', models.CharField(blank=True, default='', max_length=128, verbose_name='Password selector')),
('submit_selector', models.CharField(blank=True, default='', max_length=128, verbose_name='Submit selector')),
('username_selector', models.CharField(blank=True, default='', max_length=128, verbose_name='Username selector')),
('script', models.JSONField(blank=True, default=list, verbose_name='Script')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
]

18
apps/assets/migrations/0093_alter_systemuser_protocol.py
View File

@ -1,18 +0,0 @@
# Generated by Django 3.2.14 on 2022-11-04 07:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0092_commandfilter_nodes'),
]
operations = [
migrations.AlterField(
model_name='systemuser',
name='protocol',
field=models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('clickhouse', 'ClickHouse'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol'),
),
]

119
apps/assets/migrations/0093_auto_20220403_1627.py
View File

@ -1,6 +1,6 @@
# Generated by Django 3.1.14 on 2022-04-02 08:27
from django.utils import timezone
import django.db
from django.db import migrations, models
@ -13,7 +13,7 @@ def migrate_to_host(apps, schema_editor):
batch_size = 1000
while True:
assets = asset_model.objects.using(db_alias).all()[count:count+batch_size]
assets = asset_model.objects.using(db_alias).all()[count:count + batch_size]
if not assets:
break
count += len(assets)
@ -33,7 +33,7 @@ def migrate_hardware_info(apps, *args):
]
while True:
assets = asset_model.objects.all()[count:count+batch_size]
assets = asset_model.objects.all()[count:count + batch_size]
if not assets:
break
count += len(assets)
@ -49,12 +49,121 @@ def migrate_hardware_info(apps, *args):
class Migration(migrations.Migration):
dependencies = [
('assets', '0092_add_host'),
('assets', '0092_commandfilter_nodes'),
]
operations = [
migrations.AddField(
model_name='asset',
name='info',
field=models.JSONField(blank=True, default=dict, verbose_name='Info'),
),
migrations.RenameField(
model_name='asset',
old_name='hostname',
new_name='name',
),
migrations.AlterField(
model_name='asset',
name='name',
field=models.CharField(max_length=128, verbose_name='Name'),
),
migrations.AlterModelOptions(
name='asset',
options={'ordering': ['name'],
'permissions': [('refresh_assethardwareinfo', 'Can refresh asset hardware info'),
('test_assetconnectivity', 'Can test asset connectivity'),
('push_assetsystemuser', 'Can push system user to asset'),
('match_asset', 'Can match asset'), ('add_assettonode', 'Add asset to node'),
('move_assettonode', 'Move asset to node')], 'verbose_name': 'Asset'},
),
migrations.RenameField(
model_name='asset',
old_name='ip',
new_name='address',
),
migrations.AddField(
model_name='asset',
name='date_updated',
field=models.DateTimeField(auto_now=True, verbose_name='Date updated'),
),
migrations.AddField(
model_name='asset',
name='updated_by',
field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by'),
),
migrations.AlterField(
model_name='asset',
name='created_by',
field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by'),
),
migrations.CreateModel(
name='Host',
fields=[
('asset_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.asset')),
],
),
migrations.CreateModel(
name='Database',
fields=[
('asset_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.asset')),
('db_name', models.CharField(blank=True, max_length=1024, verbose_name='Database')),
],
options={
'verbose_name': 'Database',
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Device',
fields=[
('asset_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.asset')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Cloud',
fields=[
('asset_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.asset')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
migrations.CreateModel(
name='Web',
fields=[
('asset_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.asset')),
('autofill', models.CharField(choices=[('no', 'Disabled'), ('basic', 'Basic'), ('script', 'Script')],
default='basic', max_length=16, verbose_name='Autofill')),
('password_selector',
models.CharField(blank=True, default='', max_length=128, verbose_name='Password selector')),
('submit_selector',
models.CharField(blank=True, default='', max_length=128, verbose_name='Submit selector')),
('username_selector',
models.CharField(blank=True, default='', max_length=128, verbose_name='Username selector')),
('script', models.JSONField(blank=True, default=list, verbose_name='Script')),
],
options={
'abstract': False,
},
bases=('assets.asset',),
),
migrations.RunPython(migrate_hardware_info),
migrations.RunPython(migrate_to_host),
]

69
apps/assets/migrations/0107_auto_20221019_1115.py
View File

@ -1,14 +1,15 @@
# Generated by Django 3.2.14 on 2022-10-19 03:15
import common.db.fields
import uuid
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import uuid
import common.db.fields
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('assets', '0106_auto_20220916_1556'),
@ -18,14 +19,17 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='AutomationExecution',
fields=[
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('org_id',
models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('status', models.CharField(default='pending', max_length=16)),
('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')),
('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')),
('snapshot', common.db.fields.EncryptJsonDictTextField(blank=True, default=dict, null=True, verbose_name='Automation snapshot')),
('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')], default='manual', max_length=128, verbose_name='Trigger mode')),
('snapshot', common.db.fields.EncryptJsonDictTextField(blank=True, default=dict, null=True,
verbose_name='Automation snapshot')),
('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')],
default='manual', max_length=128, verbose_name='Trigger mode')),
],
options={
'verbose_name': 'Automation task execution',
@ -38,9 +42,10 @@ class Migration(migrations.Migration):
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('org_id',
models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('name', models.CharField(max_length=128, verbose_name='Name')),
('is_periodic', models.BooleanField(default=False)),
('is_periodic', models.BooleanField(default=False, verbose_name='Periodic perform')),
('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')),
('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')),
('accounts', models.JSONField(default=list, verbose_name='Accounts')),
@ -88,7 +93,9 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='DiscoveryAccountAutomation',
fields=[
('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')),
('baseautomation_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.baseautomation')),
],
options={
'verbose_name': 'Discovery account automation',
@ -98,7 +105,9 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='GatherFactsAutomation',
fields=[
('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')),
('baseautomation_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.baseautomation')),
],
options={
'verbose_name': 'Gather asset facts',
@ -108,7 +117,9 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='PushAccountAutomation',
fields=[
('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')),
('baseautomation_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.baseautomation')),
],
options={
'verbose_name': 'Push asset account',
@ -118,7 +129,9 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='VerifyAccountAutomation',
fields=[
('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')),
('baseautomation_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.baseautomation')),
],
options={
'verbose_name': 'Verify asset account',
@ -139,8 +152,10 @@ class Migration(migrations.Migration):
('date_finished', models.DateTimeField(blank=True, null=True, verbose_name='Date finished')),
('status', models.CharField(default='pending', max_length=16)),
('error', models.TextField(blank=True, null=True, verbose_name='Error')),
('account', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='assets.account')),
('execution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.automationexecution')),
('account',
models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='assets.account')),
('execution',
models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.automationexecution')),
],
options={
'verbose_name': 'Change secret record',
@ -149,18 +164,30 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='automationexecution',
name='automation',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='assets.baseautomation', verbose_name='Automation task'),
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions',
to='assets.baseautomation', verbose_name='Automation task'),
),
migrations.CreateModel(
name='ChangeSecretAutomation',
fields=[
('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')),
('secret_type', models.CharField(choices=[('password', 'Password'), ('ssh_key', 'SSH key'), ('access_key', 'Access key'), ('token', 'Token')], default='password', max_length=16, verbose_name='Secret type')),
('secret_strategy', models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='specific', max_length=16, verbose_name='Secret strategy')),
('baseautomation_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.baseautomation')),
('secret_type', models.CharField(
choices=[('password', 'Password'), ('ssh_key', 'SSH key'), ('access_key', 'Access key'),
('token', 'Token')], default='password', max_length=16, verbose_name='Secret type')),
('secret_strategy', models.CharField(
choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'),
('random_all', 'All assets use different random password')], default='specific',
max_length=16, verbose_name='Secret strategy')),
('secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')),
('password_rules', models.JSONField(default=dict, verbose_name='Password rules')),
('ssh_key_change_strategy', models.CharField(choices=[('add', 'Append SSH KEY'), ('set', 'Empty and append SSH KEY'), ('set_jms', 'Replace (The key generated by JumpServer) ')], default='add', max_length=16, verbose_name='SSH key change strategy')),
('recipients', models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL, verbose_name='Recipient')),
('ssh_key_change_strategy', models.CharField(
choices=[('add', 'Append SSH KEY'), ('set', 'Empty and append SSH KEY'),
('set_jms', 'Replace (The key generated by JumpServer) ')], default='add', max_length=16,
verbose_name='SSH key change strategy')),
('recipients',
models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL, verbose_name='Recipient')),
],
options={
'verbose_name': 'Change secret automation',

12
apps/audits/migrations/0015_auto_20221111_1919.py → apps/audits/migrations/0016_auto_20221111_1919.py
View File

@ -4,21 +4,25 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('audits', '0014_auto_20220505_1902'),
('audits', '0015_auto_20221011_1745'),
]
operations = [
migrations.AlterField(
model_name='ftplog',
name='operate',
field=models.CharField(choices=[('mkdir', 'Mkdir'), ('rmdir', 'Rmdir'), ('delete', 'Delete'), ('upload', 'Upload'), ('rename', 'Rename'), ('symlink', 'Symlink'), ('download', 'Download')], max_length=16, verbose_name='Operate'),
field=models.CharField(
choices=[('mkdir', 'Mkdir'), ('rmdir', 'Rmdir'), ('delete', 'Delete'), ('upload', 'Upload'),
('rename', 'Rename'), ('symlink', 'Symlink'), ('download', 'Download')], max_length=16,
verbose_name='Operate'),
),
migrations.AlterField(
model_name='operatelog',
name='action',
field=models.CharField(choices=[('view', 'View'), ('update', 'Update'), ('delete', 'Delete'), ('create', 'Create')], max_length=16, verbose_name='Action'),
field=models.CharField(
choices=[('view', 'View'), ('update', 'Update'), ('delete', 'Delete'), ('create', 'Create')],
max_length=16, verbose_name='Action'),
),
migrations.AlterField(
model_name='userloginlog',

46
apps/audits/signal_handlers.py
View File

@ -2,41 +2,37 @@
#
import uuid
from django.dispatch import receiver
from django.conf import settings
from django.contrib.auth import BACKEND_SESSION_KEY
from django.db import transaction
from django.db.models.signals import post_save, pre_save, m2m_changed, pre_delete
from django.dispatch import receiver
from django.utils import timezone, translation
from django.utils.functional import LazyObject
from django.contrib.auth import BACKEND_SESSION_KEY
from django.utils.translation import ugettext_lazy as _
from django.db.models.signals import post_save, pre_save, m2m_changed, pre_delete
from rest_framework.request import Request
from rest_framework.renderers import JSONRenderer
from rest_framework.request import Request
from users.models import User
from audits.utils import model_to_dict_for_operate_log as model_to_dict
from audits.handler import (
get_instance_current_with_cache_diff, cache_instance_before_data,
create_or_update_operate_log, get_instance_dict_from_cache
)
from audits.utils import model_to_dict_for_operate_log as model_to_dict
from authentication.signals import post_auth_failed, post_auth_success
from authentication.utils import check_different_city_login_if_need
from terminal.models import Session, Command
from jumpserver.utils import current_request
from users.signals import post_user_change_password
from .models import OperateLog
from .const import MODELS_NEED_RECORD
from terminal.serializers import SessionSerializer
from terminal.backends.command.serializers import SessionCommandSerializer
from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR, SKIP_SIGNAL
from common.utils import get_request_ip, get_logger, get_syslogger
from common.utils.encode import data_to_json
from jumpserver.utils import current_request
from terminal.backends.command.serializers import SessionCommandSerializer
from terminal.models import Session, Command
from terminal.serializers import SessionSerializer
from users.models import User
from users.signals import post_user_change_password
from . import models, serializers
from .const import MODELS_NEED_RECORD, ActionChoices
from .utils import write_login_log
logger = get_logger(__name__)
sys_logger = get_syslogger(__name__)
json_render = JSONRenderer()
@ -66,9 +62,9 @@ class AuthBackendLabelMapping(LazyObject):
AUTH_BACKEND_LABEL_MAPPING = AuthBackendLabelMapping()
M2M_ACTION = {
POST_ADD: OperateLog.ACTION_CREATE,
POST_REMOVE: OperateLog.ACTION_DELETE,
POST_CLEAR: OperateLog.ACTION_DELETE,
POST_ADD: ActionChoices.create,
POST_REMOVE: ActionChoices.delete,
POST_CLEAR: ActionChoices.delete,
}
@ -92,9 +88,9 @@ def on_m2m_changed(sender, action, instance, reverse, model, pk_set, **kwargs):
changed_field = current_instance.get(field_name, [])
after, before, before_value = None, None, None
if action == OperateLog.ACTION_CREATE:
if action == ActionChoices.create:
before_value = list(set(changed_field) - set(objs_display))
elif action == OperateLog.ACTION_DELETE:
elif action == ActionChoices.delete:
before_value = list(
set(changed_field).symmetric_difference(set(objs_display))
)
@ -108,7 +104,7 @@ def on_m2m_changed(sender, action, instance, reverse, model, pk_set, **kwargs):
return
create_or_update_operate_log(
OperateLog.ACTION_UPDATE, resource_type,
ActionChoices.update, resource_type,
resource=instance, log_id=log_id, before=before, after=after
)
@ -159,11 +155,11 @@ def on_object_created_or_update(sender, instance=None, created=False, update_fie
log_id, before, after = None, None, None
if created:
action = models.OperateLog.ACTION_CREATE
action = models.ActionChoices.create
after = model_to_dict(instance)
log_id = getattr(instance, 'operate_log_id', None)
else:
action = models.OperateLog.ACTION_UPDATE
action = ActionChoices.update
current_instance = model_to_dict(instance)
log_id, before, after = get_instance_current_with_cache_diff(current_instance)
@ -182,7 +178,7 @@ def on_object_delete(sender, instance=None, **kwargs):
resource_type = sender._meta.verbose_name
create_or_update_operate_log(
models.OperateLog.ACTION_DELETE, resource_type,
ActionChoices.delete, resource_type,
resource=instance, before=model_to_dict(instance)
)

5
apps/common/db/models.py
View File

@ -9,15 +9,14 @@
- 此文件中添加代码的时候，注意不要跟 `django.db.models` 中的命名冲突
"""
import inspect
import uuid
from functools import reduce, partial
import inspect
from django.db import models
from django.db.models import F, Value, ExpressionWrapper
from django.db import transaction
from django.db.models import F, ExpressionWrapper, CASCADE
from django.db.models import QuerySet
from django.db.models.functions import Concat
from django.utils.translation import ugettext_lazy as _
from ..const.signals import SKIP_SIGNAL

18
apps/perms/migrations/0029_alter_applicationpermission_type.py
View File

@ -1,18 +0,0 @@
# Generated by Django 3.2.14 on 2022-11-04 07:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('perms', '0028_auto_20220316_2028'),
]
operations = [
migrations.AlterField(
model_name='applicationpermission',
name='type',
field=models.CharField(choices=[('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('oracle', 'Oracle'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('clickhouse', 'ClickHouse'), ('chrome', 'Chrome'), ('mysql_workbench', 'MySQL Workbench'), ('vmware_client', 'vSphere Client'), ('custom', 'Custom'), ('k8s', 'Kubernetes')], max_length=16, verbose_name='Type'),
),
]

32
apps/terminal/migrations/0053_auto_20220830_1244.py
View File

@ -1,32 +0,0 @@
# Generated by Django 3.2.13 on 2022-08-30 04:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('terminal', '0052_auto_20220713_1417'),
]
operations = [
migrations.AlterField(
model_name='session',
name='protocol',
field=models.CharField(db_index=True, default='ssh', max_length=16),
),
migrations.RenameField(
model_name='session',
old_name='system_user',
new_name='account',
),
migrations.RemoveField(
model_name='session',
name='system_user_id',
),
migrations.AlterField(
model_name='session',
name='account',
field=models.CharField(db_index=True, max_length=128, verbose_name='Account'),
),
]

18
apps/terminal/migrations/0054_alter_session_protocol.py
View File

@ -1,18 +0,0 @@
# Generated by Django 3.2.14 on 2022-11-04 07:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('terminal', '0053_auto_20221009_1755'),
]
operations = [
migrations.AlterField(
model_name='session',
name='protocol',
field=models.CharField(choices=[('ssh', 'ssh'), ('rdp', 'rdp'), ('vnc', 'vnc'), ('telnet', 'telnet'), ('mysql', 'mysql'), ('oracle', 'oracle'), ('mariadb', 'mariadb'), ('sqlserver', 'sqlserver'), ('postgresql', 'postgresql'), ('redis', 'redis'), ('mongodb', 'MongoDB'), ('clickhouse', 'ClickHouse'), ('k8s', 'kubernetes')], db_index=True, default='ssh', max_length=16),
),
]

47
apps/terminal/migrations/0054_auto_20221027_1125.py
View File

@ -1,18 +1,37 @@
# Generated by Django 3.2.14 on 2022-10-27 03:25
from django.db import migrations, models
import django.db.models.deletion
import uuid
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0107_auto_20221019_1115'),
('terminal', '0053_auto_20220830_1244'),
('terminal', '0053_auto_20221009_1755'),
]
operations = [
migrations.AlterField(
model_name='session',
name='protocol',
field=models.CharField(db_index=True, default='ssh', max_length=16),
),
migrations.RenameField(
model_name='session',
old_name='system_user',
new_name='account',
),
migrations.RemoveField(
model_name='session',
name='system_user_id',
),
migrations.AlterField(
model_name='session',
name='account',
field=models.CharField(db_index=True, max_length=128, verbose_name='Account'),
),
migrations.CreateModel(
name='Applet',
fields=[
@ -25,7 +44,9 @@ class Migration(migrations.Migration):
('display_name', models.CharField(max_length=128, verbose_name='Display name')),
('version', models.CharField(max_length=16, verbose_name='Version')),
('author', models.CharField(max_length=128, verbose_name='Author')),
('type', models.CharField(choices=[('general', 'General'), ('web', 'Web')], default='general', max_length=16, verbose_name='Type')),
('type',
models.CharField(choices=[('general', 'General'), ('web', 'Web')], default='general', max_length=16,
verbose_name='Type')),
('is_active', models.BooleanField(default=True, verbose_name='Is active')),
('protocols', models.JSONField(default=list, verbose_name='Protocol')),
('tags', models.JSONField(default=list, verbose_name='Tags')),
@ -38,7 +59,9 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='AppletHost',
fields=[
('host_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.host')),
('host_ptr',
models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True,
primary_key=True, serialize=False, to='assets.host')),
('date_synced', models.DateTimeField(blank=True, null=True, verbose_name='Date synced')),
('status', models.CharField(max_length=16, verbose_name='Status')),
],
@ -57,8 +80,10 @@ class Migration(migrations.Migration):
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('status', models.CharField(default='', max_length=16, verbose_name='Status')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('applet', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.applet', verbose_name='Applet')),
('host', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.applethost', verbose_name='Host')),
('applet', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.applet',
verbose_name='Applet')),
('host', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.applethost',
verbose_name='Host')),
],
options={
'unique_together': {('applet', 'host')},
@ -74,7 +99,8 @@ class Migration(migrations.Migration):
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('status', models.CharField(max_length=16, default='', verbose_name='Status')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('host', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='terminal.applethost', verbose_name='Hosting')),
('host', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='terminal.applethost',
verbose_name='Hosting')),
],
options={
'abstract': False,
@ -83,6 +109,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='applethost',
name='applets',
field=models.ManyToManyField(through='terminal.AppletPublication', to='terminal.Applet', verbose_name='Applet'),
field=models.ManyToManyField(through='terminal.AppletPublication', to='terminal.Applet',
verbose_name='Applet'),
),
]

11
apps/terminal/models/component/endpoint.py
View File

@ -1,13 +1,11 @@
from django.core.validators import MinValueValidator, MaxValueValidator
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.core.validators import MinValueValidator, MaxValueValidator
from common.db.models import JMSBaseModel
from assets.models import Asset
from common.db.fields import PortField
from common.db.models import JMSBaseModel
from common.utils.ip import contains_ip
from ..utils import db_port_manager, DBPortManager
db_port_manager: DBPortManager
class Endpoint(JMSBaseModel):
@ -31,9 +29,10 @@ class Endpoint(JMSBaseModel):
return self.name
def get_port(self, target_instance, protocol):
from terminal.utils import db_port_manager
if protocol in ['https', 'http', 'ssh', 'rdp']:
port = getattr(self, f'{protocol}_port', 0)
elif isinstance(target_instance, Application) and target_instance.category_db:
elif isinstance(target_instance, Asset) and target_instance.category == 'dabase':
port = db_port_manager.get_port_by_db(target_instance)
else:
port = 0

7
apps/terminal/models/component/terminal.py
View File

@ -5,8 +5,8 @@ from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger, lazyproperty
from common.const.signals import SKIP_SIGNAL
from common.utils import get_logger, lazyproperty
from orgs.utils import tmp_to_root_org
from terminal.const import TerminalType as TypeChoices
from users.models import User
@ -85,8 +85,9 @@ class Terminal(StorageMixin, TerminalStatusMixin, models.Model):
remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address'))
command_storage = models.CharField(max_length=128, verbose_name=_("Command storage"), default='default')
replay_storage = models.CharField(max_length=128, verbose_name=_("Replay storage"), default='default')
user = models.OneToOneField(User, related_name='terminal', verbose_name=_('Application User'), null=True, on_delete=models.CASCADE)
is_accepted = models.BooleanField(default=False, verbose_name=_('Is Accepted'))
user = models.OneToOneField(User, related_name='terminal', verbose_name=_('Application User'), null=True,
on_delete=models.CASCADE)
is_deleted = models.BooleanField(default=False, verbose_name=_('Is deleted'))
date_created = models.DateTimeField(auto_now_add=True)
comment = models.TextField(blank=True, verbose_name=_('Comment'))

44
apps/terminal/utils/common.py
View File

@ -2,55 +2,11 @@
#
from common.utils import get_logger
from .. import const
from tickets.models import TicketSession
logger = get_logger(__name__)
class ComputeStatUtil:
# system status
@staticmethod
def _common_compute_system_status(value, thresholds):
if thresholds[0] <= value <= thresholds[1]:
return const.ComponentStatusChoices.normal.value
elif thresholds[1] < value <= thresholds[2]:
return const.ComponentStatusChoices.high.value
else:
return const.ComponentStatusChoices.critical.value
@classmethod
def _compute_system_stat_status(cls, stat):
system_stat_thresholds_mapper = {
'cpu_load': [0, 5, 20],
'memory_used': [0, 85, 95],
'disk_used': [0, 80, 99]
}
system_status = {}
for stat_key, thresholds in system_stat_thresholds_mapper.items():
stat_value = getattr(stat, stat_key)
if stat_value is None:
msg = 'stat: {}, stat_key: {}, stat_value: {}'
logger.debug(msg.format(stat, stat_key, stat_value))
stat_value = 0
status = cls._common_compute_system_status(stat_value, thresholds)
system_status[stat_key] = status
return system_status
@classmethod
def compute_component_status(cls, stat):
if not stat:
return const.ComponentStatusChoices.offline
system_status_values = cls._compute_system_stat_status(stat).values()
if const.ComponentStatusChoices.critical in system_status_values:
return const.ComponentStatusChoices.critical
elif const.ComponentStatusChoices.high in system_status_values:
return const.ComponentStatusChoices.high
else:
return const.ComponentStatusChoices.normal
def is_session_approver(session_id, user_id):
ticket = TicketSession.get_ticket_by_session_id(session_id)
if not ticket:

91
apps/terminal/utils/components.py
View File

@ -1,94 +1,24 @@
# -*- coding: utf-8 -*-
#
import os
import time
from itertools import groupby, chain
from collections import defaultdict
from django.utils import timezone
from django.conf import settings
from django.core.files.storage import default_storage
import jms_storage
from itertools import groupby
from common.utils import get_logger
from tickets.models import TicketSession
from . import const
from ..models import ReplayStorage
from terminal.const import ComponentLoad
logger = get_logger(__name__)
def find_session_replay_local(session):
# 存在外部存储上，所有可能的路径名
session_paths = session.get_all_possible_relative_path()
# 存在本地存储上，所有可能的路径名
local_paths = session.get_all_possible_local_path()
for _local_path in chain(session_paths, local_paths):
if default_storage.exists(_local_path):
url = default_storage.url(_local_path)
return _local_path, url
return None, None
def download_session_replay(session):
replay_storages = ReplayStorage.objects.all()
configs = {
storage.name: storage.config
for storage in replay_storages
if not storage.type_null_or_server
}
if settings.SERVER_REPLAY_STORAGE:
configs['SERVER_REPLAY_STORAGE'] = settings.SERVER_REPLAY_STORAGE
if not configs:
msg = "Not found replay file, and not remote storage set"
return None, msg
storage = jms_storage.get_multi_object_storage(configs)
# 获取外部存储路径名
session_path = session.find_ok_relative_path_in_storage(storage)
if not session_path:
msg = "Not found session replay file"
return None, msg
# 通过外部存储路径名后缀，构造真实的本地存储路径
local_path = session.get_local_path_by_relative_path(session_path)
# 保存到storage的路径
target_path = os.path.join(default_storage.base_location, local_path)
target_dir = os.path.dirname(target_path)
if not os.path.isdir(target_dir):
os.makedirs(target_dir, exist_ok=True)
ok, err = storage.download(session_path, target_path)
if not ok:
msg = "Failed download replay file: {}".format(err)
logger.error(msg)
return None, msg
url = default_storage.url(local_path)
return local_path, url
def get_session_replay_url(session):
local_path, url = find_session_replay_local(session)
if local_path is None:
local_path, url = download_session_replay(session)
return local_path, url
class ComputeLoadUtil:
# system status
@staticmethod
def _common_compute_system_status(value, thresholds):
if thresholds[0] <= value <= thresholds[1]:
return const.ComponentLoad.normal.value
return ComponentLoad.normal.value
elif thresholds[1] < value <= thresholds[2]:
return const.ComponentLoad.high.value
return ComponentLoad.high.value
else:
return const.ComponentLoad.critical.value
return ComponentLoad.critical.value
@classmethod
def _compute_system_stat_status(cls, stat):
@ -111,14 +41,14 @@ class ComputeLoadUtil:
@classmethod
def compute_load(cls, stat):
if not stat or time.time() - stat.date_created.timestamp() > 150:
return const.ComponentLoad.offline
return ComponentLoad.offline
system_status_values = cls._compute_system_stat_status(stat).values()
if const.ComponentLoad.critical in system_status_values:
return const.ComponentLoad.critical
elif const.ComponentLoad.high in system_status_values:
return const.ComponentLoad.high
if ComponentLoad.critical in system_status_values:
return ComponentLoad.critical
elif ComponentLoad.high in system_status_values:
return ComponentLoad.high
else:
return const.ComponentLoad.normal
return ComponentLoad.normal
class TypedComponentsStatusMetricsUtil(object):
@ -236,4 +166,3 @@ class ComponentsPrometheusMetricsUtil(TypedComponentsStatusMetricsUtil):
prometheus_metrics.append('\n')
prometheus_metrics_text = '\n'.join(prometheus_metrics)
return prometheus_metrics_text

19
apps/terminal/utils/db_port_mapper.py
View File

@ -1,15 +1,14 @@
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from django.conf import settings
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from assets.const import Category
from assets.models import Asset
from common.decorator import Singleton
from common.utils import get_logger, get_object_or_none
from common.exceptions import JMSException
from applications.const import AppCategory
from applications.models import Application
from common.utils import get_logger, get_object_or_none
from orgs.utils import tmp_to_root_org
logger = get_logger(__file__)
@ -37,19 +36,19 @@ class DBPortManager(object):
def init(self):
with tmp_to_root_org():
db_ids = Application.objects.filter(category=AppCategory.db).values_list('id', flat=True)
db_ids = Asset.objects.filter(platform__category=Category.DATABASE).values_list('id', flat=True)
db_ids = [str(i) for i in db_ids]
mapper = dict(zip(self.all_available_ports, list(db_ids)))
self.set_mapper(mapper)
def add(self, db: Application):
def add(self, db: Asset):
mapper = self.get_mapper()
available_port = self.get_next_available_port()
mapper.update({available_port: str(db.id)})
self.set_mapper(mapper)
return True
def pop(self, db: Application):
def pop(self, db: Asset):
mapper = self.get_mapper()
to_delete_port = self.get_port_by_db(db, raise_exception=False)
mapper.pop(to_delete_port, None)
@ -79,7 +78,7 @@ class DBPortManager(object):
if not db_id:
raise JMSException('Database not in port-db mapper, port: {}'.format(port))
with tmp_to_root_org():
db = get_object_or_none(Application, id=db_id)
db = get_object_or_none(Asset, id=db_id)
if not db:
raise JMSException('Database not exists, db id: {}'.format(db_id))
return db

18
apps/tickets/migrations/0019_alter_applyapplicationticket_apply_type.py
View File

@ -1,18 +0,0 @@
# Generated by Django 3.2.14 on 2022-11-04 07:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('tickets', '0018_applyapplicationticket_apply_actions'),
]
operations = [
migrations.AlterField(
model_name='applyapplicationticket',
name='apply_type',
field=models.CharField(choices=[('mysql', 'MySQL'), ('mariadb', 'MariaDB'), ('oracle', 'Oracle'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('clickhouse', 'ClickHouse'), ('chrome', 'Chrome'), ('mysql_workbench', 'MySQL Workbench'), ('vmware_client', 'vSphere Client'), ('custom', 'Custom'), ('k8s', 'Kubernetes')], max_length=16, verbose_name='Type'),
),
]

15
apps/tickets/serializers/ticket/ticket.py
View File

@ -1,14 +1,14 @@
# -*- coding: utf-8 -*-
#
from rest_framework import serializers
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from common.drf.fields import LabeledChoiceField
from users.models import User
from orgs.models import Organization
from orgs.mixins.serializers import OrgResourceModelSerializerMixin
from tickets.models import Ticket, TicketFlow
from orgs.models import Organization
from tickets.const import TicketType, TicketStatus, TicketState
from tickets.models import Ticket, TicketFlow
from users.models import User
__all__ = [
'TicketApplySerializer', 'TicketApproveSerializer', 'TicketSerializer',
@ -63,13 +63,6 @@ class TicketApplySerializer(TicketSerializer):
)
applicant = serializers.CharField(required=False, allow_blank=True)
class Meta:
model = Ticket
fields = TicketSerializer.Meta.fields
extra_kwargs = {
'type': {'required': True}
}
def get_applicant(self, applicant_id):
current_user = self.context['request'].user
want_applicant = User.objects.filter(id=applicant_id).first()