From f5d611acce8e9e27b0b3d7a8de064df21e0475aa Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 11 Nov 2024 11:12:10 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=9B=B4=E6=96=B0=E9=A3=8E=E9=99=A9?= =?UTF-8?q?=E5=8F=91=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accounts/api/automations/check_account.py | 7 +- .../automations/gather_accounts/filter.py | 46 +++++-- .../gather_accounts/host/posix/main.yml | 35 +++--- .../automations/gather_accounts/manager.py | 116 +++++++++++++++--- .../0009_alter_accountrisk_comment.py | 58 +++++++++ .../models/automations/check_account.py | 28 ++--- .../models/automations/gather_account.py | 23 ++-- 7 files changed, 232 insertions(+), 81 deletions(-) create mode 100644 apps/accounts/migrations/0009_alter_accountrisk_comment.py diff --git a/apps/accounts/api/automations/check_account.py b/apps/accounts/api/automations/check_account.py index 0f00217e4..363591486 100644 --- a/apps/accounts/api/automations/check_account.py +++ b/apps/accounts/api/automations/check_account.py @@ -39,11 +39,16 @@ class CheckAccountExecutionViewSet(AutomationExecutionViewSet): class AccountRiskViewSet(OrgBulkModelViewSet): model = AccountRisk - search_fields = ('username',) + search_fields = ('username', 'asset') + filterset_fields = ('risk', 'status', 'asset') serializer_classes = { 'default': serializers.AccountRiskSerializer, 'assets': serializers.AssetRiskSerializer, } + ordering_fields = ( + 'asset', 'risk', 'status', 'username', 'date_created' + ) + ordering = ('-asset', 'date_created') rbac_perms = { 'sync_accounts': 'assets.add_accountrisk', 'assets': 'accounts.view_accountrisk' diff --git a/apps/accounts/automations/gather_accounts/filter.py b/apps/accounts/automations/gather_accounts/filter.py index 34e582e43..0d6742ba2 100644 --- a/apps/accounts/automations/gather_accounts/filter.py +++ b/apps/accounts/automations/gather_accounts/filter.py @@ -44,6 +44,14 @@ class GatherAccountsFilter: continue username_sudo[username.strip()] = sudo.strip() + last_login = info.pop('last_login', '') + user_last_login = {} + for line in last_login: + if not line.strip() or ' ' not in line: + continue + username, login = line.split(' ', 1) + user_last_login[username] = login + user_authorized = info.pop('user_authorized', []) username_authorized = {} for line in user_authorized: @@ -52,26 +60,38 @@ class GatherAccountsFilter: username, authorized = line.split(':', 1) username_authorized[username.strip()] = authorized.strip() + passwd_date = info.pop('passwd_date', []) + username_password_date = {} + for line in passwd_date: + if ':' not in line: + continue + username, password_date = line.split(':', 1) + username_password_date[username.strip()] = password_date.strip().split() + result = {} users = info.pop('users', '') - for line in users: - parts = line.split() - if len(parts) < 4: - continue - username = parts[0] + for username in users: if not username: continue user = dict() - address = parts[2] - user['address_last_login'] = address - login_time = parts[3] - try: - login_date = timezone.datetime.fromisoformat(login_time) - user['date_last_login'] = login_date - except ValueError: - pass + login = user_last_login.get(username) or '' + if login and len(login) == 3: + user['address_last_login'] = login[1][:32] + try: + login_date = timezone.datetime.fromisoformat(login[2]) + user['date_last_login'] = login_date + except ValueError: + pass + + start_date = timezone.make_aware(timezone.datetime(1970, 1, 1)) + _password_date = username_password_date.get(username) or '' + if _password_date and len(_password_date) == 2: + if _password_date[0] and _password_date[0] != '0': + user['date_password_change'] = start_date + timezone.timedelta(days=int(_password_date[0])) + if _password_date[1] and _password_date[1] != '0': + user['date_password_expired'] = start_date + timezone.timedelta(days=int(_password_date[1])) user['groups'] = username_groups.get(username) or '' user['sudoers'] = username_sudo.get(username) or '' diff --git a/apps/accounts/automations/gather_accounts/host/posix/main.yml b/apps/accounts/automations/gather_accounts/host/posix/main.yml index a229869d4..cee37213b 100644 --- a/apps/accounts/automations/gather_accounts/host/posix/main.yml +++ b/apps/accounts/automations/gather_accounts/host/posix/main.yml @@ -4,23 +4,28 @@ - name: Get users ansible.builtin.shell: cmd: > - getent passwd | awk -F: '$7 !~ /(false|nologin)$/' | grep -v '^$' | awk -F":" '{ print $1 }' + getent passwd | awk -F: '$7 !~ /(false|nologin|true|sync)$/' | grep -v '^$' | awk -F":" '{ print $1 }' register: users - - name: Gather posix account + - name: Gather posix account last login ansible.builtin.shell: | for user in {{ users.stdout_lines | join(" ") }}; do - k=$(last -i --time-format iso -1 ${user} | head -1 | grep -v ^$ ) - if [ -n "$k" ]; then - echo $k - fi + last -i --time-format iso -n 1 ${user} | awk '{ print $1,$3,$4, $NF }' | head -1 | grep -v ^$ done register: last_login + - name: Get user password change date and expiry + ansible.builtin.shell: | + for user in {{ users.stdout_lines | join(" ") }}; do + k=$(getent shadow $user | awk -F: '{ print $3, $5 }') + echo "$user:$k" + done + register: passwd_date + - name: Get user groups ansible.builtin.shell: | for user in {{ users.stdout_lines | join(" ") }}; do - echo "$(groups $user)" + echo "$(groups $user)" | sed 's@ : @:@g' done register: user_groups @@ -38,17 +43,19 @@ echo -n "$user:" if [[ -f ${home}/.ssh/authorized_keys ]]; then cat ${home}/.ssh/authorized_keys | tr '\n' ';' - echo fi + echo done register: user_authorized - set_fact: - info: - users: "{{ last_login.stdout_lines }}" - user_groups: "{{ user_groups.stdout_lines }}" - user_sudo: "{{ user_sudo.stdout_lines }}" - user_authorized: "{{ user_authorized.stdout_lines }}" + info: + users: "{{ users.stdout_lines }}" + last_login: "{{ last_login.stdout_lines }}" + user_groups: "{{ user_groups.stdout_lines }}" + user_sudo: "{{ user_sudo.stdout_lines }}" + user_authorized: "{{ user_authorized.stdout_lines }}" + passwd_date: "{{ passwd_date.stdout_lines }}" - debug: - var: info + var: info diff --git a/apps/accounts/automations/gather_accounts/manager.py b/apps/accounts/automations/gather_accounts/manager.py index ec65c0c03..3413c0c48 100644 --- a/apps/accounts/automations/gather_accounts/manager.py +++ b/apps/accounts/automations/gather_accounts/manager.py @@ -1,7 +1,9 @@ from collections import defaultdict +from django.utils import timezone + from accounts.const import AutomationTypes -from accounts.models import GatheredAccount, Account, GatheredAccountDiff +from accounts.models import GatheredAccount, Account, AccountRisk from assets.models import Asset from common.const import ConfirmOrIgnore from common.utils import get_logger @@ -16,7 +18,11 @@ logger = get_logger(__name__) class GatherAccountsManager(AccountBasePlaybookManager): - diff_items = ['authorized_keys', 'sudoers', 'groups'] + diff_items = [ + 'authorized_keys', 'sudoers', 'groups', + 'date_password_change', 'date_password_expired', + ] + long_time = timezone.timedelta(days=90) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -30,7 +36,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.is_sync_account = self.execution.snapshot.get('is_sync_account') self.pending_add_accounts = [] self.pending_update_accounts = [] - self.pending_add_diffs = [] + self.pending_add_risks = [] @classmethod def method_type(cls): @@ -60,8 +66,6 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.asset_usernames_mapper[asset].add(username) d = {'asset': asset, 'username': username, 'remote_present': True, **info} - if len(d['address_last_login']) > 32: - d['address_last_login'] = d['address_last_login'][:32] accounts.append(d) self.asset_account_info[asset] = accounts @@ -147,7 +151,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): def batch_create_gathered_account(self, d, batch_size=20): if d is None: if self.pending_add_accounts: - GatheredAccount.objects.bulk_create(self.pending_add_accounts) + GatheredAccount.objects.bulk_create(self.pending_add_accounts, ignore_conflicts=True) self.pending_add_accounts = [] return @@ -159,32 +163,103 @@ class GatherAccountsManager(AccountBasePlaybookManager): if len(self.pending_add_accounts) > batch_size: self.batch_create_gathered_account(None) - def batch_update_gathered_account(self, ori_account, d, batch_size=20): - if not ori_account or d is None: - if self.pending_update_accounts: - GatheredAccount.objects.bulk_update(self.pending_update_accounts, [*self.diff_items]) - self.pending_update_accounts = [] + def _analyse_item_changed(self, ori_account, d): + diff = self.get_items_diff(ori_account, d) - if self.pending_add_diffs: - GatheredAccountDiff.objects.bulk_create(self.pending_add_diffs) - self.pending_add_diffs = [] + if not diff: return + for k, v in diff.items(): + self.pending_add_risks.append(AccountRisk( + asset=ori_account.asset, username=ori_account.username, + risk=k+'_changed', comment=v + )) + + @staticmethod + def perform_save_risks(risks): + assets = {r.asset for r in risks} + assets_risks = AccountRisk.objects.filter(asset__in=assets) + assets_risks = {f"{r.asset_id}_{r.username}": r for r in assets_risks} + + for r in risks: + found = assets_risks.get(f"{r.asset_id}_{r.username}") + + if not found: + r.save() + else: + found.comment = r.comment + '\n------\n' + found.comment + + def batch_analyse_risk(self, asset, ori_account, d, batch_size=20): + if asset is None: + if self.pending_add_risks: + self.perform_save_risks(self.pending_add_risks) + self.pending_add_risks = [] + return + + now = timezone.now().strftime('%Y-%m-%d %H:%M:%S') + basic = {'asset': asset, 'username': d['username']} + if ori_account: + self._analyse_item_changed(ori_account, d) + else: + self.pending_add_risks.append( + AccountRisk(**basic, risk='ghost', comment='{}'.format(now)) + ) + + last_login = d.get('date_last_login') + if last_login and last_login < timezone.now() - self.long_time: + self.pending_add_risks.append( + AccountRisk(**basic, risk='zombie', comment='{}'.format(last_login)) + ) + + date_password_change = d.get('date_password_change') + if date_password_change and date_password_change < timezone.now() - self.long_time: + self.pending_add_risks.append( + AccountRisk(**basic, risk='long_time_password', comment='{}'.format(date_password_change)) + ) + + date_password_expired = d.get('date_password_expired') + if date_password_expired and date_password_expired < timezone.now(): + self.pending_add_risks.append( + AccountRisk(**basic, risk='password_expired', comment='{}'.format(date_password_expired)) + ) + + if len(self.pending_add_risks) > batch_size: + self.batch_analyse_risk(None, None, {}) + + def get_items_diff(self, ori_account, d): + if hasattr(ori_account, '_diff'): + return ori_account._diff + diff = {} for item in self.diff_items: ori = getattr(ori_account, item) new = d.get(item, '') + if not ori: + continue + + if isinstance(new, timezone.datetime): + new = ori.strftime('%Y-%m-%d %H:%M:%S') + ori = ori.strftime('%Y-%m-%d %H:%M:%S') + if new != ori: - setattr(ori_account, item, new) diff[item] = get_text_diff(ori, new) + ori_account._diff = diff + return diff + + def batch_update_gathered_account(self, ori_account, d, batch_size=20): + if not ori_account or d is None: + if self.pending_update_accounts: + GatheredAccount.objects.bulk_update(self.pending_update_accounts, [*self.diff_items]) + self.pending_update_accounts = [] + return + + diff = self.get_items_diff(ori_account, d) if diff: + for k in diff: + setattr(ori_account, k, d[k]) self.pending_update_accounts.append(ori_account) - for k, v in diff.items(): - self.pending_add_diffs.append( - GatheredAccountDiff(account=ori_account, item=k, diff=v) - ) if len(self.pending_update_accounts) > batch_size: self.batch_update_gathered_account(None, None) @@ -202,11 +277,14 @@ class GatherAccountsManager(AccountBasePlaybookManager): else: self.batch_update_gathered_account(ori_account, d) + self.batch_analyse_risk(asset, ori_account, d) + self.update_gather_accounts_status(asset) GatheredAccount.sync_accounts(gathered_accounts, self.is_sync_account) self.batch_create_gathered_account(None) self.batch_update_gathered_account(None, None) + self.batch_analyse_risk(None, None, {}) def run(self, *args, **kwargs): super().run(*args, **kwargs) diff --git a/apps/accounts/migrations/0009_alter_accountrisk_comment.py b/apps/accounts/migrations/0009_alter_accountrisk_comment.py new file mode 100644 index 000000000..f2c2fb456 --- /dev/null +++ b/apps/accounts/migrations/0009_alter_accountrisk_comment.py @@ -0,0 +1,58 @@ +# Generated by Django 4.1.13 on 2024-11-08 09:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("assets", "0006_baseautomation_start_time"), + ("accounts", "0008_remove_accountrisk_confirmed_accountrisk_status_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="gatheredaccount", + name="date_change_password", + field=models.DateTimeField(null=True, verbose_name="Date change password"), + ), + migrations.AddField( + model_name="gatheredaccount", + name="date_password_expired", + field=models.DateTimeField(null=True, verbose_name="Date password expired"), + ), + migrations.AlterField( + model_name="accountrisk", + name="comment", + field=models.TextField(default="", verbose_name="Comment"), + ), + migrations.AlterField( + model_name="accountrisk", + name="risk", + field=models.CharField( + choices=[ + ("zombie", "Long time no login"), + ("ghost", "Not managed"), + ("groups_changed", "Groups change"), + ("sudoers_changed", "Sudo changed"), + ("authorized_keys_changed", "Authorized keys changed"), + ("account_deleted", "Account delete"), + ("password_expired", "Password expired"), + ("long_time_password", "Long time no change"), + ("weak_password", "Weak password"), + ("password_error", "Password error"), + ("no_admin_account", "No admin account"), + ("others", "Others"), + ], + max_length=128, + verbose_name="Risk", + ), + ), + migrations.AlterUniqueTogether( + name="accountrisk", + unique_together={("asset", "username", "risk")}, + ), + migrations.DeleteModel( + name="GatheredAccountDiff", + ), + ] diff --git a/apps/accounts/models/automations/check_account.py b/apps/accounts/models/automations/check_account.py index de592a2ca..9680744fc 100644 --- a/apps/accounts/models/automations/check_account.py +++ b/apps/accounts/models/automations/check_account.py @@ -37,16 +37,18 @@ class AccountCheckAutomation(AccountBaseAutomation): class RiskChoice(TextChoices): + # 依赖自动发现的 zombie = 'zombie', _('Long time no login') # 好久没登录的账号, 禁用、删除 ghost = 'ghost', _('Not managed') # 未被纳管的账号, 纳管, 删除, 禁用 - long_time_password = 'long_time_password', _('Long time no change') # 好久没改密码的账号, 改密码 - weak_password = 'weak_password', _('Weak password') # 弱密码, 改密 - password_error = 'password_error', _('Password error') # 密码错误, 修改账号 - password_expired = 'password_expired', _('Password expired') # 密码过期, 修改密码 - group_changed = 'group_changed', _('Group change') # 组变更, 确认 - sudo_changed = 'sudo_changed', _('Sudo changed') # sudo 变更, 确认 + group_changed = 'groups_changed', _('Groups change') # 组变更, 确认 + sudo_changed = 'sudoers_changed', _('Sudo changed') # sudo 变更, 确认 authorized_keys_changed = 'authorized_keys_changed', _('Authorized keys changed') # authorized_keys 变更, 确认 account_deleted = 'account_deleted', _('Account delete') # 账号被删除, 确认 + password_expired = 'password_expired', _('Password expired') # 密码过期, 修改密码 + long_time_password = 'long_time_password', _('Long time no change') # 好久没改密码的账号, 改密码 + + weak_password = 'weak_password', _('Weak password') # 弱密码, 改密 + password_error = 'password_error', _('Password error') # 密码错误, 修改账号 no_admin_account = 'no_admin_account', _('No admin account') # 无管理员账号, 设置账号 others = 'others', _('Others') # 其他风险, 确认 @@ -56,25 +58,15 @@ class AccountRisk(JMSOrgBaseModel): username = models.CharField(max_length=32, verbose_name=_('Username')) risk = models.CharField(max_length=128, verbose_name=_('Risk'), choices=RiskChoice.choices) status = models.CharField(max_length=32, choices=ConfirmOrIgnore.choices, default='', blank=True, verbose_name=_('Status')) + comment = models.TextField(default='', verbose_name=_('Comment')) class Meta: verbose_name = _('Account risk') + unique_together = ('asset', 'username', 'risk') def __str__(self): return f"{self.username}@{self.asset} - {self.risk}" - def disable_account(self): - pass - - def remove_account(self): - pass - - def change_password(self): - pass - - def handle_risk(self): - pass - @classmethod def gen_fake_data(cls, count=1000, batch_size=50): from assets.models import Asset diff --git a/apps/accounts/models/automations/gather_account.py b/apps/accounts/models/automations/gather_account.py index d5c78742d..b78ee84a5 100644 --- a/apps/accounts/models/automations/gather_account.py +++ b/apps/accounts/models/automations/gather_account.py @@ -9,36 +9,27 @@ from common.utils.timezone import is_date_more_than from orgs.mixins.models import JMSOrgBaseModel from .base import AccountBaseAutomation -__all__ = ['GatherAccountsAutomation', 'GatheredAccount', 'GatheredAccountDiff'] - - -class GatheredAccountDiff(models.Model): - account = models.ForeignKey('GatheredAccount', on_delete=models.CASCADE, verbose_name=_("Gathered account")) - diff = models.TextField(default='', verbose_name=_("Diff")) - item = models.CharField(max_length=32, default='', verbose_name=_("Item")) - date_created = models.DateTimeField(auto_now_add=True, verbose_name=_("Date created")) +__all__ = ['GatherAccountsAutomation', 'GatheredAccount', ] class GatheredAccount(JMSOrgBaseModel): - remote_present = models.BooleanField(default=True, verbose_name=_("Remote present")) # 远端资产上是否还存在 - present = models.BooleanField(default=False, verbose_name=_("Present")) # 系统资产上是否还存在 - date_last_login = models.DateTimeField(null=True, verbose_name=_("Date login")) asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, verbose_name=_("Asset")) username = models.CharField(max_length=32, blank=True, db_index=True, verbose_name=_('Username')) address_last_login = models.CharField(max_length=39, default='', verbose_name=_("Address login")) - status = models.CharField(max_length=32, default='', blank=True, choices=ConfirmOrIgnore.choices, verbose_name=_("Status")) + date_last_login = models.DateTimeField(null=True, verbose_name=_("Date login")) authorized_keys = models.TextField(default='', blank=True, verbose_name=_("Authorized keys")) sudoers = models.TextField(default='', verbose_name=_("Sudoers"), blank=True) groups = models.TextField(default='', blank=True, verbose_name=_("Groups")) + remote_present = models.BooleanField(default=True, verbose_name=_("Remote present")) # 远端资产上是否还存在 + present = models.BooleanField(default=False, verbose_name=_("Present")) # 系统资产上是否还存在 + date_change_password = models.DateTimeField(null=True, verbose_name=_("Date change password")) + date_password_expired = models.DateTimeField(null=True, verbose_name=_("Date password expired")) + status = models.CharField(max_length=32, default='', blank=True, choices=ConfirmOrIgnore.choices, verbose_name=_("Status")) @property def address(self): return self.asset.address - @classmethod - def find_account_risk(cls, gathered_account, accounts): - pass - @classmethod def update_exists_accounts(cls, gathered_account, accounts): if not gathered_account.date_last_login: