github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: 暂存一下

pull/8023/head^2
ibuler 2022-07-14 10:56:09 +08:00
parent dac0b44b99
commit d3c67d2f04
13 changed files with 112 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/assets/api/account_history.py
View File

@ -1,23 +1,21 @@
from django.db.models import F
from assets.api.accounts import (
AccountFilterSet, AccountViewSet, AccountSecretsViewSet
)
from common.mixins import RecordViewLogMixin
from .. import serializers
from ..models import AuthBook
from ..models import Account
__all__ = ['AccountHistoryViewSet', 'AccountHistorySecretsViewSet']
class AccountHistoryFilterSet(AccountFilterSet):
class Meta:
model = AuthBook.history.model
model = Account.history.model
fields = AccountFilterSet.Meta.fields
class AccountHistoryViewSet(AccountViewSet):
model = AuthBook.history.model
model = Account.history.model
filterset_class = AccountHistoryFilterSet
serializer_classes = {
'default': serializers.AccountHistorySerializer,
@ -26,13 +24,8 @@ class AccountHistoryViewSet(AccountViewSet):
'list': 'assets.view_assethistoryaccount',
'retrieve': 'assets.view_assethistoryaccount',
}
http_method_names = ['get', 'options']
def get_queryset(self):
queryset = AuthBook.get_queryset(is_history_model=True)
return queryset
class AccountHistorySecretsViewSet(RecordViewLogMixin, AccountHistoryViewSet):
serializer_classes = {

18
apps/assets/api/accounts.py
View File

@ -12,7 +12,7 @@ from common.mixins import RecordViewLogMixin
from common.permissions import UserConfirmation
from authentication.const import ConfirmType
from ..tasks.account_connectivity import test_accounts_connectivity_manual
from ..models import AuthBook, Node
from ..models import Node, Account
from .. import serializers
__all__ = ['AccountFilterSet', 'AccountViewSet', 'AccountSecretsViewSet', 'AccountTaskCreateAPI']
@ -50,16 +50,16 @@ class AccountFilterSet(BaseFilterSet):
return qs
class Meta:
model = AuthBook
model = Account
fields = [
'asset', 'systemuser', 'id',
'asset', 'id',
]
class AccountViewSet(OrgBulkModelViewSet):
model = AuthBook
filterset_fields = ("username", "asset", "systemuser", 'ip', 'hostname')
search_fields = ('username', 'ip', 'hostname', 'systemuser__username')
model = Account
filterset_fields = ("username", "asset", 'ip', 'hostname')
search_fields = ('username', 'ip', 'hostname')
filterset_class = AccountFilterSet
serializer_classes = {
'default': serializers.AccountSerializer,
@ -70,10 +70,6 @@ class AccountViewSet(OrgBulkModelViewSet):
'partial_update': 'assets.change_assetaccountsecret',
}
def get_queryset(self):
queryset = AuthBook.get_queryset()
return queryset
@action(methods=['post'], detail=True, url_path='verify')
def verify_account(self, request, *args, **kwargs):
account = super().get_object()
@ -106,7 +102,7 @@ class AccountTaskCreateAPI(CreateAPIView):
return request.user.has_perm('assets.test_assetconnectivity')
def get_accounts(self):
queryset = AuthBook.objects.all()
queryset = Account.objects.all()
queryset = self.filter_queryset(queryset)
return queryset

1
apps/assets/api/system_user_relation.py
View File

@ -68,7 +68,6 @@ class BaseRelationViewSet(RelationMixin, OrgBulkModelViewSet):
class SystemUserAssetRelationViewSet(BaseRelationViewSet):
perm_model = models.AuthBook
serializer_class = serializers.SystemUserAssetRelationSerializer
model = models.SystemUser.assets.through
filterset_fields = [

8
apps/assets/migrations/0092_auto_20220711_1409.py
View File

@ -1,4 +1,4 @@
# Generated by Django 3.2.12 on 2022-07-11 06:08
# Generated by Django 3.2.12 on 2022-07-11 08:59
import assets.models.base
import assets.models.user
@ -22,6 +22,8 @@ class Migration(migrations.Migration):
name='HistoricalAccount',
fields=[
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')),
('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')),
('id', models.UUIDField(db_index=True, default=uuid.uuid4)),
('name', models.CharField(max_length=128, verbose_name='Name')),
('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
@ -33,6 +35,7 @@ class Migration(migrations.Migration):
('date_updated', models.DateTimeField(blank=True, editable=False, verbose_name='Date updated')),
('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
('version', models.IntegerField(default=1, verbose_name='Version')),
('history_id', models.AutoField(primary_key=True, serialize=False)),
('history_date', models.DateTimeField(db_index=True)),
@ -53,6 +56,8 @@ class Migration(migrations.Migration):
name='Account',
fields=[
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('connectivity', models.CharField(choices=[('unknown', 'Unknown'), ('ok', 'Ok'), ('failed', 'Failed')], default='unknown', max_length=16, verbose_name='Connectivity')),
('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('name', models.CharField(max_length=128, verbose_name='Name')),
('username', models.CharField(blank=True, db_index=True, max_length=128, verbose_name='Username')),
@ -64,6 +69,7 @@ class Migration(migrations.Migration):
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('created_by', models.CharField(max_length=128, null=True, verbose_name='Created by')),
('protocol', models.CharField(choices=[('ssh', 'SSH'), ('rdp', 'RDP'), ('telnet', 'Telnet'), ('vnc', 'VNC'), ('mysql', 'MySQL'), ('oracle', 'Oracle'), ('mariadb', 'MariaDB'), ('postgresql', 'PostgreSQL'), ('sqlserver', 'SQLServer'), ('redis', 'Redis'), ('mongodb', 'MongoDB'), ('k8s', 'K8S')], default='ssh', max_length=16, verbose_name='Protocol')),
('type', models.CharField(choices=[('common', 'Common user'), ('admin', 'Admin user')], default='common', max_length=16, verbose_name='Type')),
('version', models.IntegerField(default=1, verbose_name='Version')),
('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.asset', verbose_name='Asset')),
],

13
apps/assets/migrations/0093_auto_20220711_1413.py
View File

@ -10,16 +10,18 @@ def migrate_accounts(apps, schema_editor):
count = 0
bulk_size = 1000
print("\nStart migrate accounts")
while True:
start = time.time()
auth_books = auth_book_model.objects \
.prefetch_related('systemuser') \
.all()[count:count+bulk_size]
.prefetch_related('systemuser') \
.all()[count:count+bulk_size]
count += len(auth_books)
if not auth_books:
break
accounts = []
start = time.time()
# authbook 和 account 相同的属性
# auth book 和 account 相同的属性
same_attrs = [
'id', 'comment', 'date_created', 'date_updated',
'created_by', 'asset_id', 'org_id',
@ -47,9 +49,8 @@ def migrate_accounts(apps, schema_editor):
account_model.objects.bulk_create(accounts, ignore_conflicts=True)
print("Create accounts: {}-{} using: {:.2f}s".format(
count, count + len(accounts), time.time()-start
count - bulk_size, count, time.time()-start
))
count += len(auth_books)
class Migration(migrations.Migration):

22
apps/assets/migrations/0094_alter_systemuser_assets.py Normal file
View File

@ -0,0 +1,22 @@
# Generated by Django 3.2.12 on 2022-07-13 09:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0093_auto_20220711_1413'),
]
operations = [
migrations.RemoveField(
model_name='systemuser',
name='assets',
),
migrations.AddField(
model_name='systemuser',
name='assets',
field=models.ManyToManyField(blank=True, related_name='system_users', to='assets.Asset', verbose_name='Assets'),
),
]

45
apps/assets/migrations/0095_auto_20220713_1746.py Normal file
View File

@ -0,0 +1,45 @@
# Generated by Django 3.2.12 on 2022-07-13 09:46
import time
from django.db import migrations
def migrate_asset_system_user_relations(apps, schema_editor):
system_user_model = apps.get_model('assets', 'SystemUser')
old_model = apps.get_model('assets', 'AuthBook')
new_model = system_user_model.assets.through
count = 0
bulk_size = 1000
print("\nStart migrate asset system user relations")
while True:
start = time.time()
auth_books = old_model.objects.only('asset_id', 'systemuser_id')[count:count+bulk_size]
auth_books = list(auth_books)
count += len(auth_books)
if not auth_books:
break
asset_system_users = []
for auth_book in auth_books:
if not auth_book.asset_id or not auth_book.systemuser_id:
continue
asset_system_user = new_model(
asset_id=auth_book.asset_id,
systemuser_id=auth_book.systemuser_id
)
asset_system_users.append(asset_system_user)
new_model.objects.bulk_create(asset_system_users, ignore_conflicts=True)
print("Create asset system user relations: {}-{} using: {:.2f}s".format(
count - bulk_size, count, time.time()-start
))
class Migration(migrations.Migration):
dependencies = [
('assets', '0094_alter_systemuser_assets'),
]
operations = [
migrations.RunPython(migrate_asset_system_user_relations)
]

9
apps/assets/models/account.py
View File

@ -3,15 +3,20 @@ from django.utils.translation import gettext_lazy as _
from simple_history.models import HistoricalRecords
from .user import ProtocolMixin
from .base import BaseUser
from .base import BaseUser, AbsConnectivity
__all__ = ['Account']
class Account(BaseUser, ProtocolMixin):
class Account(BaseUser, AbsConnectivity, ProtocolMixin):
class Type(models.TextChoices):
common = 'common', _('Common user')
admin = 'admin', _('Admin user')
protocol = models.CharField(max_length=16, choices=ProtocolMixin.Protocol.choices,
default='ssh', verbose_name=_('Protocol'))
type = models.CharField(max_length=16, choices=Type.choices, default=Type.common, verbose_name=_("Type"))
asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset'))
version = models.IntegerField(default=1, verbose_name=_('Version'))
history = HistoricalRecords()

1
apps/assets/models/user.py
View File

@ -242,7 +242,6 @@ class SystemUser(ProtocolMixin, AuthMixin, BaseUser):
nodes = models.ManyToManyField('assets.Node', blank=True, verbose_name=_("Nodes"))
assets = models.ManyToManyField(
'assets.Asset', blank=True, verbose_name=_("Assets"),
through='assets.AuthBook', through_fields=['systemuser', 'asset'],
related_name='system_users'
)
users = models.ManyToManyField('users.User', blank=True, verbose_name=_("Users"))

2
apps/assets/serializers/system_user.py
View File

@ -298,7 +298,7 @@ class SystemUserAssetRelationSerializer(RelationMixin, serializers.ModelSerializ
asset_display = serializers.ReadOnlyField(label=_('Asset hostname'))
class Meta:
model = SystemUser.assets.through
model = SystemUser
fields = [
"id", "asset", "asset_display", 'systemuser', 'systemuser_display',
"connectivity", 'date_verified', 'org_id'

11
apps/assets/tasks/asset_connectivity.py
View File

@ -6,7 +6,7 @@ from django.utils.translation import gettext_noop
from common.utils import get_logger
from orgs.utils import org_aware_func
from ..models import Asset, Connectivity, AuthBook
from ..models import Asset, Connectivity, Account
from . import const
from .utils import clean_ansible_task_hosts, group_asset_by_platform
@ -18,6 +18,7 @@ __all__ = [
]
# Todo: 这里可能有问题了
def set_assets_accounts_connectivity(assets, results_summary):
asset_ids_ok = set()
asset_ids_failed = set()
@ -33,11 +34,11 @@ def set_assets_accounts_connectivity(assets, results_summary):
Asset.bulk_set_connectivity(asset_ids_ok, Connectivity.ok)
Asset.bulk_set_connectivity(asset_ids_failed, Connectivity.failed)
accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser__type='admin')
accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser__type='admin')
accounts_ok = Account.objects.filter(asset_id__in=asset_ids_ok,)
accounts_failed = Account.objects.filter(asset_id__in=asset_ids_faile)
AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok)
AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed)
Account.bulk_set_connectivity(accounts_ok, Connectivity.ok)
Account.bulk_set_connectivity(accounts_failed, Connectivity.failed)
@shared_task(queue="ansible")

3
apps/assets/tasks/common.py
View File

@ -17,6 +17,7 @@ def add_nodes_assets_to_system_users(nodes_keys, system_users):
nodes = Node.objects.filter(key__in=nodes_keys)
assets = Node.get_nodes_all_assets(*nodes)
for system_user in system_users:
""" 解决资产和节点进行关联时，已经关联过的节点不会触发 authbook post_save 信号，
无法更新节点下所有资产的管理用户的问题 """
@ -28,7 +29,7 @@ def add_nodes_assets_to_system_users(nodes_keys, system_users):
)
if created:
need_push_asset_ids.append(asset.id)
# # 不再自动更新资产管理用户，只允许用户手动指定。
# 不再自动更新资产管理用户，只允许用户手动指定。
# 只要关联都需要更新资产的管理用户
# instance.update_asset_admin_user_if_need()

10
apps/assets/tasks/system_user_connectivity.py
View File

@ -8,7 +8,7 @@ from django.utils.translation import ugettext as _, gettext_noop
from assets.models import Asset
from common.utils import get_logger
from orgs.utils import tmp_to_org, org_aware_func
from ..models import SystemUser, Connectivity, AuthBook
from ..models import SystemUser, Connectivity, Account
from . import const
from .utils import (
clean_ansible_task_hosts, group_asset_by_platform
@ -34,11 +34,11 @@ def set_assets_accounts_connectivity(system_user, assets, results_summary):
else:
asset_ids_failed.add(asset.id)
accounts_ok = AuthBook.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user)
accounts_failed = AuthBook.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user)
accounts_ok = Account.objects.filter(asset_id__in=asset_ids_ok, systemuser=system_user)
accounts_failed = Account.objects.filter(asset_id__in=asset_ids_failed, systemuser=system_user)
AuthBook.bulk_set_connectivity(accounts_ok, Connectivity.ok)
AuthBook.bulk_set_connectivity(accounts_failed, Connectivity.failed)
Account.bulk_set_connectivity(accounts_ok, Connectivity.ok)
Account.bulk_set_connectivity(accounts_failed, Connectivity.failed)
@org_aware_func("system_user")