perf: remove gather model (#9246)

Co-authored-by: feng <1304903146@qq.com>

apps/assets/api/__init__.py

@@ -4,7 +4,6 @@ from .automations import *
 from .category import *
 from .domain import *
 from .favorite_asset import *
-from .gathered_user import *
 from .label import *
 from .mixin import *
 from .node import *

apps/assets/api/gathered_user.py

@@ -1,20 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-
-from orgs.mixins.api import OrgModelViewSet
-from assets.models import GatheredUser
-
-from ..serializers import GatheredUserSerializer
-from ..filters import AssetRelatedByNodeFilterBackend
-
-
-__all__ = ['GatheredUserViewSet']
-
-
-class GatheredUserViewSet(OrgModelViewSet):
-    model = GatheredUser
-    serializer_class = GatheredUserSerializer
-    extra_filter_backends = [AssetRelatedByNodeFilterBackend]
-
-    filterset_fields = ['asset', 'username', 'present', 'asset__address', 'asset__name', 'asset_id']
-    search_fields = ['username', 'asset__address', 'asset__name']

apps/assets/automations/gather_accounts/manager.py

@@ -1,8 +1,9 @@
+from django.utils.translation import ugettext_lazy as _
+
 from common.utils import get_logger
-from assets.const import AutomationTypes
+from assets.const import AutomationTypes, Source
 from orgs.utils import tmp_to_org
 from .filter import GatherAccountsFilter
-from ...models import GatheredUser
 from ..base.manager import BasePlaybookManager
 
 logger = get_logger(__name__)
@@ -26,20 +27,33 @@ class GatherAccountsManager(BasePlaybookManager):
         result = GatherAccountsFilter(host).run(self.method_id_meta_mapper, result)
         return result
 
+    @staticmethod
+    def bulk_create_accounts(asset, result):
+        account_objs = []
+        account_model = asset.accounts.model
+        account_usernames = set(asset.accounts.values_list('username', flat=True))
+        with tmp_to_org(asset.org_id):
+            accounts_dict = {}
+            for username, data in result.items():
+                comment = ''
+                d = {'asset': asset, 'username': username, 'name': username, 'source': Source.COLLECTED}
+                if data.get('date'):
+                    comment += f"{_('Date last login')}: {data['date']}\n "
+                if data.get('address'):
+                    comment += f"{_('IP last login')}: {data['address'][:32]}"
+                d['comment'] = comment
+                accounts_dict[username] = d
+            for username, data in accounts_dict.items():
+                if username in account_usernames:
+                    continue
+                account_objs.append(account_model(**data))
+            account_model.objects.bulk_create(account_objs)
+
     def on_host_success(self, host, result):
         info = result.get('debug', {}).get('res', {}).get('info', {})
         asset = self.host_asset_mapper.get(host)
-        org_id = asset.org_id
         if asset and info:
             result = self.filter_success_result(host, info)
-            with tmp_to_org(org_id):
+            self.bulk_create_accounts(asset, result)
-                GatheredUser.objects.filter(asset=asset, present=True).update(present=False)
-                for username, data in result.items():
-                    defaults = {'asset': asset, 'present': True, 'username': username}
-                    if data.get('date'):
-                        defaults['date_last_login'] = data['date']
-                    if data.get('address'):
-                        defaults['ip_last_login'] = data['address'][:32]
-                    GatheredUser.objects.update_or_create(defaults=defaults, asset=asset, username=username)
         else:
             logger.error("Not found info".format(host))

apps/assets/const/account.py

@@ -13,3 +13,14 @@ class SecretType(TextChoices):
     SSH_KEY = 'ssh_key', _('SSH key')
     ACCESS_KEY = 'access_key', _('Access key')
     TOKEN = 'token', _('Token')
+
+
+class AliasAccount(TextChoices):
+    ALL = '@ALL', _('All')
+    INPUT = '@INPUT', _('Manual input')
+    USER = '@USER', _('Dynamic user')
+
+
+class Source(TextChoices):
+    LOCAL = 'local', _('Local')
+    COLLECTED = 'collected', _('Collected')

apps/assets/filters.py

@@ -126,17 +126,6 @@ class LabelFilterBackend(filters.BaseFilterBackend):
         return queryset
 
 
-class AssetRelatedByNodeFilterBackend(AssetByNodeFilterBackend):
-    def filter_node_related_all(self, queryset, node):
-        return queryset.filter(
-            Q(asset__nodes__key__istartswith=f'{node.key}:') |
-            Q(asset__nodes__key=node.key)
-        ).distinct()
-
-    def filter_node_related_direct(self, queryset, node):
-        return queryset.filter(asset__nodes__key=node.key).distinct()
-
-
 class IpInFilterBackend(filters.BaseFilterBackend):
     def filter_queryset(self, request, queryset, view):
         ips = request.query_params.get('ips')

apps/assets/migrations/0119_auto_20221227_1740.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.2.16 on 2022-12-27 09:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0118_auto_20221227_1504'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='account',
+            name='source',
+            field=models.CharField(default='local', max_length=30, verbose_name='Source'),
+        ),
+        migrations.DeleteModel(
+            name='GatheredUser',
+        ),
+    ]

apps/assets/models/__init__.py

@@ -7,7 +7,6 @@ from .gateway import *
 from .domain import *
 from .node import *
 from .utils import *
-from .gathered_user import *
 from .favorite_asset import *
 from .account import *
 from .backup import *

apps/assets/models/account.py

@@ -3,6 +3,7 @@ from django.utils.translation import gettext_lazy as _
 from simple_history.models import HistoricalRecords
 
 from common.utils import lazyproperty
+from ..const import AliasAccount, Source
 from .base import AbsConnectivity, BaseAccount
 
 __all__ = ['Account', 'AccountTemplate']
@@ -40,11 +41,6 @@ class AccountHistoricalRecords(HistoricalRecords):
 
 
 class Account(AbsConnectivity, BaseAccount):
-    class AliasAccount(models.TextChoices):
-        ALL = '@ALL', _('All')
-        INPUT = '@INPUT', _('Manual input')
-        USER = '@USER', _('Dynamic user')
-
     asset = models.ForeignKey(
         'assets.Asset', related_name='accounts',
         on_delete=models.CASCADE, verbose_name=_('Asset')
@@ -55,6 +51,7 @@ class Account(AbsConnectivity, BaseAccount):
     )
     version = models.IntegerField(default=0, verbose_name=_('Version'))
     history = AccountHistoricalRecords(included_fields=['id', 'secret', 'secret_type', 'version'])
+    source = models.CharField(max_length=30, default=Source.LOCAL, verbose_name=_('Source'))
 
     class Meta:
         verbose_name = _('Account')
@@ -89,12 +86,12 @@ class Account(AbsConnectivity, BaseAccount):
     @classmethod
     def get_manual_account(cls):
         """ @INPUT 手动登录的账号(any) """
-        return cls(name=cls.AliasAccount.INPUT.label, username=cls.AliasAccount.INPUT.value, secret=None)
+        return cls(name=AliasAccount.INPUT.label, username=AliasAccount.INPUT.value, secret=None)
 
     @classmethod
     def get_user_account(cls, username):
         """ @USER 动态用户的账号(self) """
-        return cls(name=cls.AliasAccount.USER.label, username=cls.AliasAccount.USER.value)
+        return cls(name=AliasAccount.USER.label, username=AliasAccount.USER.value)
 
     def get_su_from_accounts(self):
         """ 排除自己和以自己为 su-from 的账号 """

apps/assets/models/gathered_user.py

@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-from django.db import models
-from django.utils.translation import ugettext_lazy as _
-
-from orgs.mixins.models import JMSOrgBaseModel
-
-__all__ = ['GatheredUser']
-
-
-class GatheredUser(JMSOrgBaseModel):
-    asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_("Asset"))
-    username = models.CharField(max_length=32, blank=True, db_index=True, verbose_name=_('Username'))
-    present = models.BooleanField(default=True, verbose_name=_("Present"))
-    date_last_login = models.DateTimeField(null=True, verbose_name=_("Date last login"))
-    ip_last_login = models.CharField(max_length=39, default='', verbose_name=_("IP last login"))
-
-    @property
-    def name(self):
-        return self.asset.name
-
-    @property
-    def ip(self):
-        return self.asset.address
-
-    class Meta:
-        verbose_name = _('GatherUser')
-        ordering = ['asset']
-
-    def __str__(self):
-        return '{}: {}'.format(self.asset.name, self.username)

apps/assets/serializers/__init__.py

@@ -6,7 +6,6 @@ from .label import *
 from .node import *
 from .gateway import *
 from .domain import *
-from .gathered_user import *
 from .favorite_asset import *
 from .account import *
 from .platform import *

apps/assets/serializers/gathered_user.py

@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-from django.utils.translation import ugettext_lazy as _
-
-from orgs.mixins.serializers import OrgResourceModelSerializerMixin
-from common.drf.fields import ObjectRelatedField
-from ..models import GatheredUser, Asset
-
-
-class GatheredUserSerializer(OrgResourceModelSerializerMixin):
-    asset = ObjectRelatedField(queryset=Asset.objects, label=_('Asset'))
-
-    class Meta:
-        model = GatheredUser
-        fields_mini = ['id']
-        fields_small = fields_mini + [
-            'username', 'ip_last_login', 'present', 'name',
-            'date_last_login', 'date_created', 'date_updated'
-        ]
-        fields_fk = ['asset', 'ip']
-        fields = fields_small + fields_fk
-        read_only_fields = fields
-        extra_kwargs = {
-            'name': {'label': _("Hostname")},
-            'ip': {'label': 'IP'},
-        }

apps/assets/urls/api_urls.py

@@ -23,7 +23,6 @@ router.register(r'labels', api.LabelViewSet, 'label')
 router.register(r'nodes', api.NodeViewSet, 'node')
 router.register(r'domains', api.DomainViewSet, 'domain')
 router.register(r'gateways', api.GatewayViewSet, 'gateway')
-router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset')
 router.register(r'account-backup-plans', api.AccountBackupPlanViewSet, 'account-backup')
 router.register(r'account-backup-plan-executions', api.AccountBackupPlanExecutionViewSet, 'account-backup-execution')
@@ -51,7 +50,6 @@ urlpatterns = [
          name='account-secret-history'),
 
     path('nodes/category/tree/', api.CategoryTreeApi.as_view(), name='asset-category-tree'),
-    # path('nodes/tree/', api.NodeListAsTreeApi.as_view(), name='node-tree'),
     path('nodes/children/tree/', api.NodeChildrenAsTreeApi.as_view(), name='node-children-tree'),
     path('nodes/<uuid:pk>/children/', api.NodeChildrenApi.as_view(), name='node-children'),
     path('nodes/children/', api.NodeChildrenApi.as_view(), name='node-children-2'),

apps/audits/const.py

@@ -17,12 +17,10 @@ MODELS_NEED_RECORD = (
     "LoginAssetACL",
     "LoginConfirmSetting",
     # assets
-    'Asset', 'Node', 'AdminUser', 'SystemUser', 'Domain', 'Gateway', 'CommandFilterRule',
+    'Asset', 'Node', 'Domain', 'Gateway', 'CommandFilterRule',
     'CommandFilter', 'Platform', 'Label',
-    # applications
-    'Application',
     # account
-    'AuthBook',
+    'Account',
     # orgs
     "Organization",
     # settings
@@ -36,8 +34,7 @@ MODELS_NEED_RECORD = (
     # rbac
     'Role', 'SystemRole', 'OrgRole', 'RoleBinding', 'OrgRoleBinding', 'SystemRoleBinding',
     # xpack
-    'License', 'Account', 'SyncInstanceTask', 'ChangeAuthPlan',
+    'License', 'Account', 'SyncInstanceTask', 'Interface',
-    'GatherUserTask', 'Interface',
 )
 
 

apps/orgs/api.py

@@ -15,7 +15,7 @@ from .serializers import (
 from users.models import User, UserGroup
 from assets.models import (
     Asset, Domain, Label, Node,
-    CommandFilter, CommandFilterRule, GatheredUser
+    CommandFilter, CommandFilterRule
 )
 from perms.models import AssetPermission
 from orgs.utils import current_org, tmp_to_root_org
@@ -28,8 +28,7 @@ logger = get_logger(__file__)
 # 部分 org 相关的 model，需要清空这些数据之后才能删除该组织
 org_related_models = [
     User, UserGroup, Asset, Label, Domain, Node, Label,
-    CommandFilter, CommandFilterRule, GatheredUser,
+    CommandFilter, CommandFilterRule, AssetPermission,
-    AssetPermission,
 ]
 
 

apps/perms/api/user_permission/tree/node_with_asset.py

@@ -11,6 +11,7 @@ from rest_framework.exceptions import PermissionDenied, NotFound
 
 from assets.utils import KubernetesTree
 from assets.models import Asset, Account
+from assets.const import AliasAccount
 from assets.api import SerializeToTreeNodeMixin
 from authentication.models import ConnectionToken
 from common.utils import get_object_or_none, lazyproperty
@@ -157,7 +158,7 @@ class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView):
             raise NotFound('Account is not found')
         account = accounts[0]
         if account.username in [
-            Account.AliasAccount.INPUT, Account.AliasAccount.USER
+            AliasAccount.INPUT, AliasAccount.USER
         ]:
             return token.input_secret
         else:

apps/perms/models/asset_permission.py

@@ -13,6 +13,7 @@ from common.utils import date_expired_default
 from common.utils.timezone import local_now
 
 from perms.const import ActionChoices
+from assets.const import AliasAccount
 
 __all__ = ['AssetPermission', 'ActionChoices']
 
@@ -38,7 +39,7 @@ class AssetPermissionQuerySet(models.QuerySet):
 
     def filter_by_accounts(self, accounts):
         q = Q(accounts__contains=list(accounts)) | \
-            Q(accounts__contains=Account.AliasAccount.ALL.value)
+            Q(accounts__contains=AliasAccount.ALL.value)
         return self.filter(q)
 
 
@@ -127,7 +128,7 @@ class AssetPermission(JMSOrgBaseModel):
         """
         asset_ids = self.get_all_assets(flat=True)
         q = Q(asset_id__in=asset_ids)
-        if Account.AliasAccount.ALL not in self.accounts:
+        if AliasAccount.ALL not in self.accounts:
             q &= Q(username__in=self.accounts)
         accounts = Account.objects.filter(q).order_by('asset__name', 'name', 'username')
         if not flat:

apps/perms/utils/account.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 
 from assets.models import Account
+from assets.const import AliasAccount
 from .permission import AssetPermissionUtil
 
 __all__ = ['PermAccountUtil']
@@ -44,21 +45,21 @@ class PermAccountUtil(AssetPermissionUtil):
         cleaned_accounts_expired = defaultdict(list)
 
         # @ALL 账号先处理，后面的每个最多映射一个账号
-        all_action_bit = alias_action_bit_mapper.pop(Account.AliasAccount.ALL, None)
+        all_action_bit = alias_action_bit_mapper.pop(AliasAccount.ALL, None)
         if all_action_bit:
             for account in asset_accounts:
                 cleaned_accounts_action_bit[account] |= all_action_bit
                 cleaned_accounts_expired[account].extend(
-                    alias_expired_mapper[Account.AliasAccount.ALL]
+                    alias_expired_mapper[AliasAccount.ALL]
                 )
 
         for alias, action_bit in alias_action_bit_mapper.items():
-            if alias == Account.AliasAccount.USER:
+            if alias == AliasAccount.USER:
                 if user.username in username_account_mapper:
                     account = username_account_mapper[user.username]
                 else:
                     account = Account.get_user_account(user.username)
-            elif alias == Account.AliasAccount.INPUT:
+            elif alias == AliasAccount.INPUT:
                 account = Account.get_manual_account()
             elif alias in username_account_mapper:
                 account = username_account_mapper[alias]