From 39f87c1c648967312f008d991b4a1367af2ba389 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 12 Nov 2024 16:00:41 +0800 Subject: [PATCH] perf: update pam --- .../automations/gather_accounts/filter.py | 1 - .../gather_accounts/host/posix/main.yml | 16 ++--- .../automations/gather_accounts/manager.py | 60 ++++++++++++------- ...rd_gatheredaccount_date_password_change.py | 18 ++++++ .../models/automations/gather_account.py | 2 +- apps/accounts/serializers/account/__init__.py | 1 - .../serializers/account/gathered_account.py | 42 ------------- .../serializers/automations/check_accounts.py | 2 +- .../automations/gather_accounts.py | 44 ++++++++++++-- 9 files changed, 103 insertions(+), 83 deletions(-) create mode 100644 apps/accounts/migrations/0011_rename_date_change_password_gatheredaccount_date_password_change.py delete mode 100644 apps/accounts/serializers/account/gathered_account.py diff --git a/apps/accounts/automations/gather_accounts/filter.py b/apps/accounts/automations/gather_accounts/filter.py index 0d6742ba2..209616dee 100644 --- a/apps/accounts/automations/gather_accounts/filter.py +++ b/apps/accounts/automations/gather_accounts/filter.py @@ -96,7 +96,6 @@ class GatherAccountsFilter: user['groups'] = username_groups.get(username) or '' user['sudoers'] = username_sudo.get(username) or '' user['authorized_keys'] = username_authorized.get(username) or '' - result[username] = user return result diff --git a/apps/accounts/automations/gather_accounts/host/posix/main.yml b/apps/accounts/automations/gather_accounts/host/posix/main.yml index cee37213b..59f09d948 100644 --- a/apps/accounts/automations/gather_accounts/host/posix/main.yml +++ b/apps/accounts/automations/gather_accounts/host/posix/main.yml @@ -49,13 +49,13 @@ register: user_authorized - set_fact: - info: - users: "{{ users.stdout_lines }}" - last_login: "{{ last_login.stdout_lines }}" - user_groups: "{{ user_groups.stdout_lines }}" - user_sudo: "{{ user_sudo.stdout_lines }}" - user_authorized: "{{ user_authorized.stdout_lines }}" - passwd_date: "{{ passwd_date.stdout_lines }}" + info: + users: "{{ users.stdout_lines }}" + last_login: "{{ last_login.stdout_lines }}" + user_groups: "{{ user_groups.stdout_lines }}" + user_sudo: "{{ user_sudo.stdout_lines }}" + user_authorized: "{{ user_authorized.stdout_lines }}" + passwd_date: "{{ passwd_date.stdout_lines }}" - debug: - var: info + var: info diff --git a/apps/accounts/automations/gather_accounts/manager.py b/apps/accounts/automations/gather_accounts/manager.py index 172dbc534..d0529e536 100644 --- a/apps/accounts/automations/gather_accounts/manager.py +++ b/apps/accounts/automations/gather_accounts/manager.py @@ -20,9 +20,13 @@ logger = get_logger(__name__) class GatherAccountsManager(AccountBasePlaybookManager): diff_items = [ 'authorized_keys', 'sudoers', 'groups', - 'date_password_change', 'date_password_expired', ] long_time = timezone.timedelta(days=90) + datetime_check_items = [ + {'field': 'date_last_login', 'risk': 'zombie', 'delta': long_time}, + {'field': 'date_password_change', 'risk': 'long_time_password', 'delta': long_time}, + {'field': 'date_password_expired', 'risk': 'password_expired', 'delta': timezone.timedelta(seconds=1)} + ] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -70,6 +74,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.asset_account_info[asset] = accounts def on_runner_failed(self, runner, e): + print("Runner failed: ", e) raise e def on_host_success(self, host, result): @@ -167,6 +172,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): now = timezone.now().isoformat() diff = self.get_items_diff(ori_account, d) + print("Diff items: ", diff) if not diff: return @@ -175,20 +181,43 @@ class GatherAccountsManager(AccountBasePlaybookManager): asset=ori_account.asset, username=ori_account.username, risk=k+'_changed', details=[{'datetime': now, 'diff': v}] )) + print("Pending add risks: ", self.pending_add_risks) @staticmethod def perform_save_risks(risks): + # 提前取出来，避免每次都查数据库 assets = {r.asset for r in risks} assets_risks = AccountRisk.objects.filter(asset__in=assets) - assets_risks = {f"{r.asset_id}_{r.username}": r for r in assets_risks} + assets_risks = {f"{r.asset_id}_{r.username}_{r.risk}": r for r in assets_risks} for r in risks: - found = assets_risks.get(f"{r.asset_id}_{r.username}") + found = assets_risks.get(f"{r.asset_id}_{r.username}_{r.risk}") if not found: r.save() - else: - found.details.extend(r.details) + continue + + found.details.extend(r.details) + found.save(update_fields=['details']) + + def _analyse_datetime_changed(self, ori_account, d, asset, username): + for item in self.datetime_check_items: + field = item['field'] + risk = item['risk'] + delta = item['delta'] + + date = d.get(field) + if not date: + continue + + pre_date = ori_account and getattr(ori_account, field) + if pre_date == date: + continue + + if date and date < timezone.now() - delta: + self.pending_add_risks.append( + AccountRisk(asset=asset, username=username, risk=risk) + ) def batch_analyse_risk(self, asset, ori_account, d, batch_size=20): if asset is None: @@ -204,26 +233,10 @@ class GatherAccountsManager(AccountBasePlaybookManager): self._analyse_item_changed(ori_account, d) else: self.pending_add_risks.append( - AccountRisk(**basic, risk='ghost', ) + AccountRisk(**basic, risk='ghost') ) - last_login = d.get('date_last_login') - if last_login and last_login < timezone.now() - self.long_time: - self.pending_add_risks.append( - AccountRisk(**basic, risk='zombie') - ) - - date_password_change = d.get('date_password_change') - if date_password_change and date_password_change < timezone.now() - self.long_time: - self.pending_add_risks.append( - AccountRisk(**basic, risk='long_time_password') - ) - - date_password_expired = d.get('date_password_expired') - if date_password_expired and date_password_expired < timezone.now(): - self.pending_add_risks.append( - AccountRisk(**basic, risk='password_expired') - ) + self._analyse_datetime_changed(ori_account, d, asset, d['username']) if len(self.pending_add_risks) > batch_size: self.batch_analyse_risk(None, None, {}) @@ -279,6 +292,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): else: self.batch_update_gathered_account(ori_account, d) + print("Batch analyse risk") self.batch_analyse_risk(asset, ori_account, d) self.update_gather_accounts_status(asset) diff --git a/apps/accounts/migrations/0011_rename_date_change_password_gatheredaccount_date_password_change.py b/apps/accounts/migrations/0011_rename_date_change_password_gatheredaccount_date_password_change.py new file mode 100644 index 000000000..c55844b33 --- /dev/null +++ b/apps/accounts/migrations/0011_rename_date_change_password_gatheredaccount_date_password_change.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.13 on 2024-11-12 06:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("accounts", "0010_accountrisk_details_alter_accountrisk_comment"), + ] + + operations = [ + migrations.RenameField( + model_name="gatheredaccount", + old_name="date_change_password", + new_name="date_password_change", + ), + ] diff --git a/apps/accounts/models/automations/gather_account.py b/apps/accounts/models/automations/gather_account.py index b78ee84a5..b575d4f42 100644 --- a/apps/accounts/models/automations/gather_account.py +++ b/apps/accounts/models/automations/gather_account.py @@ -22,7 +22,7 @@ class GatheredAccount(JMSOrgBaseModel): groups = models.TextField(default='', blank=True, verbose_name=_("Groups")) remote_present = models.BooleanField(default=True, verbose_name=_("Remote present")) # 远端资产上是否还存在 present = models.BooleanField(default=False, verbose_name=_("Present")) # 系统资产上是否还存在 - date_change_password = models.DateTimeField(null=True, verbose_name=_("Date change password")) + date_password_change = models.DateTimeField(null=True, verbose_name=_("Date change password")) date_password_expired = models.DateTimeField(null=True, verbose_name=_("Date password expired")) status = models.CharField(max_length=32, default='', blank=True, choices=ConfirmOrIgnore.choices, verbose_name=_("Status")) diff --git a/apps/accounts/serializers/account/__init__.py b/apps/accounts/serializers/account/__init__.py index 207029047..556f346df 100644 --- a/apps/accounts/serializers/account/__init__.py +++ b/apps/accounts/serializers/account/__init__.py @@ -1,6 +1,5 @@ from .account import * from .backup import * from .base import * -from .gathered_account import * from .template import * from .virtual import * diff --git a/apps/accounts/serializers/account/gathered_account.py b/apps/accounts/serializers/account/gathered_account.py deleted file mode 100644 index f3d83a9e9..000000000 --- a/apps/accounts/serializers/account/gathered_account.py +++ /dev/null @@ -1,42 +0,0 @@ -from django.utils.translation import gettext_lazy as _ - -from accounts.models import GatheredAccount -from orgs.mixins.serializers import BulkOrgResourceModelSerializer -from .account import AccountAssetSerializer as _AccountAssetSerializer -from .base import BaseAccountSerializer - -__all__ = [ - 'GatheredAccountSerializer', - 'GatheredAccountActionSerializer', -] - - -class AccountAssetSerializer(_AccountAssetSerializer): - class Meta(_AccountAssetSerializer.Meta): - fields = [f for f in _AccountAssetSerializer.Meta.fields if f != 'auto_config'] - - -class GatheredAccountSerializer(BulkOrgResourceModelSerializer): - asset = AccountAssetSerializer(label=_('Asset')) - - class Meta(BaseAccountSerializer.Meta): - model = GatheredAccount - fields = [ - 'id', 'asset', 'username', - 'date_updated', 'address_last_login', - 'groups', 'sudoers', 'authorized_keys', - 'remote_present', 'present', - 'date_last_login', 'status' - ] - read_only_fields = fields - - @classmethod - def setup_eager_loading(cls, queryset): - """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('asset', 'asset__platform') - return queryset - - -class GatheredAccountActionSerializer(GatheredAccountSerializer): - class Meta(GatheredAccountSerializer.Meta): - read_only_fields = list(set(GatheredAccountSerializer.Meta.read_only_fields) - {'status'}) diff --git a/apps/accounts/serializers/automations/check_accounts.py b/apps/accounts/serializers/automations/check_accounts.py index d0c446949..809c01737 100644 --- a/apps/accounts/serializers/automations/check_accounts.py +++ b/apps/accounts/serializers/automations/check_accounts.py @@ -28,7 +28,7 @@ class AccountRiskSerializer(serializers.ModelSerializer): model = AccountRisk fields = [ 'id', 'asset', 'username', 'risk', 'status', - 'date_created', 'comment' + 'date_created', 'details', ] diff --git a/apps/accounts/serializers/automations/gather_accounts.py b/apps/accounts/serializers/automations/gather_accounts.py index cbef21307..2f4d05ce1 100644 --- a/apps/accounts/serializers/automations/gather_accounts.py +++ b/apps/accounts/serializers/automations/gather_accounts.py @@ -1,14 +1,16 @@ -# -*- coding: utf-8 -*- -# +from django.utils.translation import gettext_lazy as _ + from accounts.const import AutomationTypes from accounts.models import GatherAccountsAutomation -from common.utils import get_logger - +from accounts.models import GatheredAccount +from accounts.serializers.account.account import AccountAssetSerializer as _AccountAssetSerializer +from accounts.serializers.account.base import BaseAccountSerializer +from orgs.mixins.serializers import BulkOrgResourceModelSerializer from .base import BaseAutomationSerializer -logger = get_logger(__file__) - __all__ = [ + 'GatheredAccountSerializer', + 'GatheredAccountActionSerializer', 'GatherAccountAutomationSerializer', ] @@ -25,3 +27,33 @@ class GatherAccountAutomationSerializer(BaseAutomationSerializer): @property def model_type(self): return AutomationTypes.gather_accounts + + +class AccountAssetSerializer(_AccountAssetSerializer): + class Meta(_AccountAssetSerializer.Meta): + fields = [f for f in _AccountAssetSerializer.Meta.fields if f != 'auto_config'] + + +class GatheredAccountSerializer(BulkOrgResourceModelSerializer): + asset = AccountAssetSerializer(label=_('Asset')) + + class Meta(BaseAccountSerializer.Meta): + model = GatheredAccount + fields = [ + 'id', 'asset', 'username', + 'date_last_login', 'address_last_login', + 'remote_present', 'present', + 'date_updated', 'status', + ] + read_only_fields = fields + + @classmethod + def setup_eager_loading(cls, queryset): + """ Perform necessary eager loading of data. """ + queryset = queryset.prefetch_related('asset', 'asset__platform') + return queryset + + +class GatheredAccountActionSerializer(GatheredAccountSerializer): + class Meta(GatheredAccountSerializer.Meta): + read_only_fields = list(set(GatheredAccountSerializer.Meta.read_only_fields) - {'status'})