github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: remove gather model (#9246) 

Co-authored-by: feng <1304903146@qq.com>
pull/9248/head
fit2bot 2022-12-27 17:45:41 +08:00 committed by GitHub
parent 0328fd1bb0
commit c81f36cc27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 77 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/assets/api/__init__.py
View File

@ -4,7 +4,6 @@ from .automations import *
from .category import *
from .domain import *
from .favorite_asset import *
from .gathered_user import *
from .label import *
from .mixin import *
from .node import *

20
apps/assets/api/gathered_user.py
View File

@ -1,20 +0,0 @@
# -*- coding: utf-8 -*-
#
from orgs.mixins.api import OrgModelViewSet
from assets.models import GatheredUser
from ..serializers import GatheredUserSerializer
from ..filters import AssetRelatedByNodeFilterBackend
__all__ = ['GatheredUserViewSet']
class GatheredUserViewSet(OrgModelViewSet):
model = GatheredUser
serializer_class = GatheredUserSerializer
extra_filter_backends = [AssetRelatedByNodeFilterBackend]
filterset_fields = ['asset', 'username', 'present', 'asset__address', 'asset__name', 'asset_id']
search_fields = ['username', 'asset__address', 'asset__name']

38
apps/assets/automations/gather_accounts/manager.py
View File

@ -1,8 +1,9 @@
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger
from assets.const import AutomationTypes
from assets.const import AutomationTypes, Source
from orgs.utils import tmp_to_org
from .filter import GatherAccountsFilter
from ...models import GatheredUser
from ..base.manager import BasePlaybookManager
logger = get_logger(__name__)
@ -26,20 +27,33 @@ class GatherAccountsManager(BasePlaybookManager):
result = GatherAccountsFilter(host).run(self.method_id_meta_mapper, result)
return result
@staticmethod
def bulk_create_accounts(asset, result):
account_objs = []
account_model = asset.accounts.model
account_usernames = set(asset.accounts.values_list('username', flat=True))
with tmp_to_org(asset.org_id):
accounts_dict = {}
for username, data in result.items():
comment = ''
d = {'asset': asset, 'username': username, 'name': username, 'source': Source.COLLECTED}
if data.get('date'):
comment += f"{_('Date last login')}: {data['date']}\n "
if data.get('address'):
comment += f"{_('IP last login')}: {data['address'][:32]}"
d['comment'] = comment
accounts_dict[username] = d
for username, data in accounts_dict.items():
if username in account_usernames:
continue
account_objs.append(account_model(**data))
account_model.objects.bulk_create(account_objs)
def on_host_success(self, host, result):
info = result.get('debug', {}).get('res', {}).get('info', {})
asset = self.host_asset_mapper.get(host)
org_id = asset.org_id
if asset and info:
result = self.filter_success_result(host, info)
with tmp_to_org(org_id):
GatheredUser.objects.filter(asset=asset, present=True).update(present=False)
for username, data in result.items():
defaults = {'asset': asset, 'present': True, 'username': username}
if data.get('date'):
defaults['date_last_login'] = data['date']
if data.get('address'):
defaults['ip_last_login'] = data['address'][:32]
GatheredUser.objects.update_or_create(defaults=defaults, asset=asset, username=username)
self.bulk_create_accounts(asset, result)
else:
logger.error("Not found info".format(host))

11
apps/assets/const/account.py
View File

@ -13,3 +13,14 @@ class SecretType(TextChoices):
SSH_KEY = 'ssh_key', _('SSH key')
ACCESS_KEY = 'access_key', _('Access key')
TOKEN = 'token', _('Token')
class AliasAccount(TextChoices):
ALL = '@ALL', _('All')
INPUT = '@INPUT', _('Manual input')
USER = '@USER', _('Dynamic user')
class Source(TextChoices):
LOCAL = 'local', _('Local')
COLLECTED = 'collected', _('Collected')

11
apps/assets/filters.py
View File

@ -126,17 +126,6 @@ class LabelFilterBackend(filters.BaseFilterBackend):
return queryset
class AssetRelatedByNodeFilterBackend(AssetByNodeFilterBackend):
def filter_node_related_all(self, queryset, node):
return queryset.filter(
Q(asset__nodes__key__istartswith=f'{node.key}:') |
Q(asset__nodes__key=node.key)
).distinct()
def filter_node_related_direct(self, queryset, node):
return queryset.filter(asset__nodes__key=node.key).distinct()
class IpInFilterBackend(filters.BaseFilterBackend):
def filter_queryset(self, request, queryset, view):
ips = request.query_params.get('ips')

21
apps/assets/migrations/0119_auto_20221227_1740.py Normal file
View File

@ -0,0 +1,21 @@
# Generated by Django 3.2.16 on 2022-12-27 09:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('assets', '0118_auto_20221227_1504'),
]
operations = [
migrations.AddField(
model_name='account',
name='source',
field=models.CharField(default='local', max_length=30, verbose_name='Source'),
),
migrations.DeleteModel(
name='GatheredUser',
),
]

1
apps/assets/models/__init__.py
View File

@ -7,7 +7,6 @@ from .gateway import *
from .domain import *
from .node import *
from .utils import *
from .gathered_user import *
from .favorite_asset import *
from .account import *
from .backup import *

11
apps/assets/models/account.py
View File

@ -3,6 +3,7 @@ from django.utils.translation import gettext_lazy as _
from simple_history.models import HistoricalRecords
from common.utils import lazyproperty
from ..const import AliasAccount, Source
from .base import AbsConnectivity, BaseAccount
__all__ = ['Account', 'AccountTemplate']
@ -40,11 +41,6 @@ class AccountHistoricalRecords(HistoricalRecords):
class Account(AbsConnectivity, BaseAccount):
class AliasAccount(models.TextChoices):
ALL = '@ALL', _('All')
INPUT = '@INPUT', _('Manual input')
USER = '@USER', _('Dynamic user')
asset = models.ForeignKey(
'assets.Asset', related_name='accounts',
on_delete=models.CASCADE, verbose_name=_('Asset')
@ -55,6 +51,7 @@ class Account(AbsConnectivity, BaseAccount):
)
version = models.IntegerField(default=0, verbose_name=_('Version'))
history = AccountHistoricalRecords(included_fields=['id', 'secret', 'secret_type', 'version'])
source = models.CharField(max_length=30, default=Source.LOCAL, verbose_name=_('Source'))
class Meta:
verbose_name = _('Account')
@ -89,12 +86,12 @@ class Account(AbsConnectivity, BaseAccount):
@classmethod
def get_manual_account(cls):
""" @INPUT 手动登录的账号(any) """
return cls(name=cls.AliasAccount.INPUT.label, username=cls.AliasAccount.INPUT.value, secret=None)
return cls(name=AliasAccount.INPUT.label, username=AliasAccount.INPUT.value, secret=None)
@classmethod
def get_user_account(cls, username):
""" @USER 动态用户的账号(self) """
return cls(name=cls.AliasAccount.USER.label, username=cls.AliasAccount.USER.value)
return cls(name=AliasAccount.USER.label, username=AliasAccount.USER.value)
def get_su_from_accounts(self):
""" 排除自己和以自己为 su-from 的账号 """

31
apps/assets/models/gathered_user.py
View File

@ -1,31 +0,0 @@
# -*- coding: utf-8 -*-
#
from django.db import models
from django.utils.translation import ugettext_lazy as _
from orgs.mixins.models import JMSOrgBaseModel
__all__ = ['GatheredUser']
class GatheredUser(JMSOrgBaseModel):
asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_("Asset"))
username = models.CharField(max_length=32, blank=True, db_index=True, verbose_name=_('Username'))
present = models.BooleanField(default=True, verbose_name=_("Present"))
date_last_login = models.DateTimeField(null=True, verbose_name=_("Date last login"))
ip_last_login = models.CharField(max_length=39, default='', verbose_name=_("IP last login"))
@property
def name(self):
return self.asset.name
@property
def ip(self):
return self.asset.address
class Meta:
verbose_name = _('GatherUser')
ordering = ['asset']
def __str__(self):
return '{}: {}'.format(self.asset.name, self.username)

1
apps/assets/serializers/__init__.py
View File

@ -6,7 +6,6 @@ from .label import *
from .node import *
from .gateway import *
from .domain import *
from .gathered_user import *
from .favorite_asset import *
from .account import *
from .platform import *

26
apps/assets/serializers/gathered_user.py
View File

@ -1,26 +0,0 @@
# -*- coding: utf-8 -*-
#
from django.utils.translation import ugettext_lazy as _
from orgs.mixins.serializers import OrgResourceModelSerializerMixin
from common.drf.fields import ObjectRelatedField
from ..models import GatheredUser, Asset
class GatheredUserSerializer(OrgResourceModelSerializerMixin):
asset = ObjectRelatedField(queryset=Asset.objects, label=_('Asset'))
class Meta:
model = GatheredUser
fields_mini = ['id']
fields_small = fields_mini + [
'username', 'ip_last_login', 'present', 'name',
'date_last_login', 'date_created', 'date_updated'
]
fields_fk = ['asset', 'ip']
fields = fields_small + fields_fk
read_only_fields = fields
extra_kwargs = {
'name': {'label': _("Hostname")},
'ip': {'label': 'IP'},
}

2
apps/assets/urls/api_urls.py
View File

@ -23,7 +23,6 @@ router.register(r'labels', api.LabelViewSet, 'label')
router.register(r'nodes', api.NodeViewSet, 'node')
router.register(r'domains', api.DomainViewSet, 'domain')
router.register(r'gateways', api.GatewayViewSet, 'gateway')
router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset')
router.register(r'account-backup-plans', api.AccountBackupPlanViewSet, 'account-backup')
router.register(r'account-backup-plan-executions', api.AccountBackupPlanExecutionViewSet, 'account-backup-execution')
@ -51,7 +50,6 @@ urlpatterns = [
name='account-secret-history'),
path('nodes/category/tree/', api.CategoryTreeApi.as_view(), name='asset-category-tree'),
# path('nodes/tree/', api.NodeListAsTreeApi.as_view(), name='node-tree'),
path('nodes/children/tree/', api.NodeChildrenAsTreeApi.as_view(), name='node-children-tree'),
path('nodes/<uuid:pk>/children/', api.NodeChildrenApi.as_view(), name='node-children'),
path('nodes/children/', api.NodeChildrenApi.as_view(), name='node-children-2'),

9
apps/audits/const.py
View File

@ -17,12 +17,10 @@ MODELS_NEED_RECORD = (
"LoginAssetACL",
"LoginConfirmSetting",
# assets
'Asset', 'Node', 'AdminUser', 'SystemUser', 'Domain', 'Gateway', 'CommandFilterRule',
'Asset', 'Node', 'Domain', 'Gateway', 'CommandFilterRule',
'CommandFilter', 'Platform', 'Label',
# applications
'Application',
# account
'AuthBook',
'Account',
# orgs
"Organization",
# settings
@ -36,8 +34,7 @@ MODELS_NEED_RECORD = (
# rbac
'Role', 'SystemRole', 'OrgRole', 'RoleBinding', 'OrgRoleBinding', 'SystemRoleBinding',
# xpack
'License', 'Account', 'SyncInstanceTask', 'ChangeAuthPlan',
'GatherUserTask', 'Interface',
'License', 'Account', 'SyncInstanceTask', 'Interface',
)

5
apps/orgs/api.py
View File

@ -15,7 +15,7 @@ from .serializers import (
from users.models import User, UserGroup
from assets.models import (
Asset, Domain, Label, Node,
CommandFilter, CommandFilterRule, GatheredUser
CommandFilter, CommandFilterRule
)
from perms.models import AssetPermission
from orgs.utils import current_org, tmp_to_root_org
@ -28,8 +28,7 @@ logger = get_logger(__file__)
# 部分 org 相关的 model，需要清空这些数据之后才能删除该组织
org_related_models = [
User, UserGroup, Asset, Label, Domain, Node, Label,
CommandFilter, CommandFilterRule, GatheredUser,
AssetPermission,
CommandFilter, CommandFilterRule, AssetPermission,
]

3
apps/perms/api/user_permission/tree/node_with_asset.py
View File

@ -11,6 +11,7 @@ from rest_framework.exceptions import PermissionDenied, NotFound
from assets.utils import KubernetesTree
from assets.models import Asset, Account
from assets.const import AliasAccount
from assets.api import SerializeToTreeNodeMixin
from authentication.models import ConnectionToken
from common.utils import get_object_or_none, lazyproperty
@ -157,7 +158,7 @@ class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView):
raise NotFound('Account is not found')
account = accounts[0]
if account.username in [
Account.AliasAccount.INPUT, Account.AliasAccount.USER
AliasAccount.INPUT, AliasAccount.USER
]:
return token.input_secret
else:

5
apps/perms/models/asset_permission.py
View File

@ -13,6 +13,7 @@ from common.utils import date_expired_default
from common.utils.timezone import local_now
from perms.const import ActionChoices
from assets.const import AliasAccount
__all__ = ['AssetPermission', 'ActionChoices']
@ -38,7 +39,7 @@ class AssetPermissionQuerySet(models.QuerySet):
def filter_by_accounts(self, accounts):
q = Q(accounts__contains=list(accounts)) | \
Q(accounts__contains=Account.AliasAccount.ALL.value)
Q(accounts__contains=AliasAccount.ALL.value)
return self.filter(q)
@ -127,7 +128,7 @@ class AssetPermission(JMSOrgBaseModel):
"""
asset_ids = self.get_all_assets(flat=True)
q = Q(asset_id__in=asset_ids)
if Account.AliasAccount.ALL not in self.accounts:
if AliasAccount.ALL not in self.accounts:
q &= Q(username__in=self.accounts)
accounts = Account.objects.filter(q).order_by('asset__name', 'name', 'username')
if not flat:

9
apps/perms/utils/account.py
View File

@ -1,6 +1,7 @@
from collections import defaultdict
from assets.models import Account
from assets.const import AliasAccount
from .permission import AssetPermissionUtil
__all__ = ['PermAccountUtil']
@ -44,21 +45,21 @@ class PermAccountUtil(AssetPermissionUtil):
cleaned_accounts_expired = defaultdict(list)
# @ALL 账号先处理，后面的每个最多映射一个账号
all_action_bit = alias_action_bit_mapper.pop(Account.AliasAccount.ALL, None)
all_action_bit = alias_action_bit_mapper.pop(AliasAccount.ALL, None)
if all_action_bit:
for account in asset_accounts:
cleaned_accounts_action_bit[account] |= all_action_bit
cleaned_accounts_expired[account].extend(
alias_expired_mapper[Account.AliasAccount.ALL]
alias_expired_mapper[AliasAccount.ALL]
)
for alias, action_bit in alias_action_bit_mapper.items():
if alias == Account.AliasAccount.USER:
if alias == AliasAccount.USER:
if user.username in username_account_mapper:
account = username_account_mapper[user.username]
else:
account = Account.get_user_account(user.username)
elif alias == Account.AliasAccount.INPUT:
elif alias == AliasAccount.INPUT:
account = Account.get_manual_account()
elif alias in username_account_mapper:
account = username_account_mapper[alias]