From b87fb8bba08c3045c2bea8e6893129ea9c3eea68 Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 22 Nov 2024 14:30:45 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E6=94=B6=E9=9B=86?= =?UTF-8?q?=E8=B4=A6=E5=8F=B7=E7=9A=84=E6=8A=A5=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/automations/gather_account.py | 5 +- .../automations/gather_account/manager.py | 216 +++++++++--------- .../accounts/gather_account_report.html | 57 +++-- apps/assets/automations/base/manager.py | 205 ++++++++++------- apps/assets/automations/methods.py | 2 - 5 files changed, 275 insertions(+), 210 deletions(-) diff --git a/apps/accounts/api/automations/gather_account.py b/apps/accounts/api/automations/gather_account.py index 6a9026bb8..e691d3aa3 100644 --- a/apps/accounts/api/automations/gather_account.py +++ b/apps/accounts/api/automations/gather_account.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # from django.db import transaction +from django.http import HttpResponse from django.shortcuts import get_object_or_404 from rest_framework import status from rest_framework.decorators import action @@ -86,8 +87,8 @@ class GatheredAccountViewSet(OrgBulkModelViewSet): } execution.save() execution.start() - accounts = self.model.objects.filter(asset=asset).prefetch_related('asset', 'asset__platform') - return self.get_paginated_response_from_queryset(accounts) + report = execution.manager.gen_report() + return HttpResponse(report) @action(methods=['post'], detail=False, url_path='sync-accounts') def sync_accounts(self, request, *args, **kwargs): diff --git a/apps/accounts/automations/gather_account/manager.py b/apps/accounts/automations/gather_account/manager.py index cc78d3bc1..04588920c 100644 --- a/apps/accounts/automations/gather_account/manager.py +++ b/apps/accounts/automations/gather_account/manager.py @@ -19,25 +19,27 @@ logger = get_logger(__name__) diff_items = [ - 'authorized_keys', 'sudoers', 'groups', + "authorized_keys", + "sudoers", + "groups", ] def get_items_diff(ori_account, d): - if hasattr(ori_account, '_diff'): + if hasattr(ori_account, "_diff"): return ori_account._diff diff = {} for item in diff_items: ori = getattr(ori_account, item) - new = d.get(item, '') + new = d.get(item, "") if not ori: continue if isinstance(new, timezone.datetime): - new = ori.strftime('%Y-%m-%d %H:%M:%S') - ori = ori.strftime('%Y-%m-%d %H:%M:%S') + new = ori.strftime("%Y-%m-%d %H:%M:%S") + ori = ori.strftime("%Y-%m-%d %H:%M:%S") if new != ori: diff[item] = get_text_diff(ori, new) @@ -48,9 +50,17 @@ def get_items_diff(ori_account, d): class AnalyseAccountRisk: long_time = timezone.timedelta(days=90) datetime_check_items = [ - {'field': 'date_last_login', 'risk': 'zombie', 'delta': long_time}, - {'field': 'date_password_change', 'risk': 'long_time_password', 'delta': long_time}, - {'field': 'date_password_expired', 'risk': 'password_expired', 'delta': timezone.timedelta(seconds=1)} + {"field": "date_last_login", "risk": "zombie", "delta": long_time}, + { + "field": "date_password_change", + "risk": "long_time_password", + "delta": long_time, + }, + { + "field": "date_password_expired", + "risk": "password_expired", + "delta": timezone.timedelta(seconds=1), + }, ] def __init__(self, check_risk=True): @@ -66,20 +76,24 @@ class AnalyseAccountRisk: risks = [] for k, v in diff.items(): - risks.append(dict( - asset=ori_account.asset, username=ori_account.username, - risk=k+'_changed', detail={'diff': v} - )) + risks.append( + dict( + asset=ori_account.asset, + username=ori_account.username, + risk=k + "_changed", + detail={"diff": v}, + ) + ) self.save_or_update_risks(risks) def _analyse_datetime_changed(self, ori_account, d, asset, username): - basic = {'asset': asset, 'username': username} + basic = {"asset": asset, "username": username} risks = [] for item in self.datetime_check_items: - field = item['field'] - risk = item['risk'] - delta = item['delta'] + field = item["field"] + risk = item["risk"] + delta = item["delta"] date = d.get(field) if not date: @@ -91,20 +105,20 @@ class AnalyseAccountRisk: if date and date < timezone.now() - delta: risks.append( - dict(**basic, risk=risk, detail={'date': date.isoformat()}) + dict(**basic, risk=risk, detail={"date": date.isoformat()}) ) self.save_or_update_risks(risks) def save_or_update_risks(self, risks): # 提前取出来，避免每次都查数据库 - assets = {r['asset'] for r in risks} + assets = {r["asset"] for r in risks} assets_risks = AccountRisk.objects.filter(asset__in=assets) assets_risks = {f"{r.asset_id}_{r.username}_{r.risk}": r for r in assets_risks} for d in risks: - detail = d.pop('detail', {}) - detail['datetime'] = self.now.isoformat() + detail = d.pop("detail", {}) + detail["datetime"] = self.now.isoformat() key = f"{d['asset'].id}_{d['username']}_{d['risk']}" found = assets_risks.get(key) @@ -119,7 +133,7 @@ class AnalyseAccountRisk: def _create_risk(self, data): return AccountRisk(**data) - @bulk_update_decorator(AccountRisk, update_fields=['details']) + @bulk_update_decorator(AccountRisk, update_fields=["details"]) def _update_risk(self, account): return account @@ -127,13 +141,17 @@ class AnalyseAccountRisk: if not self.check_risk: return - basic = {'asset': asset, 'username': d['username']} + basic = {"asset": asset, "username": d["username"]} if ori_account: self._analyse_item_changed(ori_account, d) else: - self._create_risk(dict(**basic, risk='ghost', details=[{'datetime': self.now.isoformat()}])) + self._create_risk( + dict( + **basic, risk="ghost", details=[{"datetime": self.now.isoformat()}] + ) + ) - self._analyse_datetime_changed(ori_account, d, asset, d['username']) + self._analyse_datetime_changed(ori_account, d, asset, d["username"]) class GatherAccountsManager(AccountBasePlaybookManager): @@ -145,8 +163,8 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.ori_asset_usernames = defaultdict(set) self.ori_gathered_usernames = defaultdict(set) self.ori_gathered_accounts_mapper = dict() - self.is_sync_account = self.execution.snapshot.get('is_sync_account') - self.check_risk = self.execution.snapshot.get('check_risk', False) + self.is_sync_account = self.execution.snapshot.get("is_sync_account") + self.check_risk = self.execution.snapshot.get("check_risk", False) @classmethod def method_type(cls): @@ -154,7 +172,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): def host_callback(self, host, asset=None, **kwargs): super().host_callback(host, asset=asset, **kwargs) - self.host_asset_mapper[host['name']] = asset + self.host_asset_mapper[host["name"]] = asset return host def _filter_success_result(self, tp, result): @@ -175,43 +193,39 @@ class GatherAccountsManager(AccountBasePlaybookManager): for username, info in result.items(): self.asset_usernames_mapper[asset].add(username) - d = {'asset': asset, 'username': username, 'remote_present': True, **info} + d = {"asset": asset, "username": username, "remote_present": True, **info} accounts.append(d) self.asset_account_info[asset] = accounts - def on_runner_failed(self, runner, e): - print("Runner failed: ", e) - raise e - - def on_host_error(self, host, error, result): - print(f'\033[31m {host} error: {error} \033[0m\n') - self.summary['error_assets'] += 1 - def on_host_success(self, host, result): - info = self._get_nested_info(result, 'debug', 'res', 'info') + super().on_host_success(host, result) + info = self._get_nested_info(result, "debug", "res", "info") asset = self.host_asset_mapper.get(host) - self.summary['success_assets'] += 1 if asset and info: self._collect_asset_account_info(asset, info) else: - print(f'\033[31m Not found {host} info \033[0m\n') - + print(f"\033[31m Not found {host} info \033[0m\n") + def prefetch_origin_account_usernames(self): """ 提起查出来，避免每次 sql 查询 :return: """ assets = self.asset_usernames_mapper.keys() - accounts = Account.objects.filter(asset__in=assets).values_list('asset', 'username') + accounts = Account.objects.filter(asset__in=assets).values_list( + "asset", "username" + ) for asset, username in accounts: self.ori_asset_usernames[asset].add(username) - ga_accounts = GatheredAccount.objects.filter(asset__in=assets).prefetch_related('asset') + ga_accounts = GatheredAccount.objects.filter(asset__in=assets).prefetch_related( + "asset" + ) for account in ga_accounts: self.ori_gathered_usernames[account.asset].add(account.username) - key = '{}_{}'.format(account.asset_id, account.username) + key = "{}_{}".format(account.asset_id, account.username) self.ori_gathered_accounts_mapper[key] = account def update_gather_accounts_status(self, asset): @@ -225,25 +239,45 @@ class GatherAccountsManager(AccountBasePlaybookManager): ori_users = self.ori_asset_usernames[asset] ori_ga_users = self.ori_gathered_usernames[asset] - queryset = (GatheredAccount.objects - .filter(asset=asset) - .exclude(status=ConfirmOrIgnore.ignored)) + queryset = GatheredAccount.objects.filter(asset=asset).exclude( + status=ConfirmOrIgnore.ignored + ) # 远端账号 比 收集账号多的 # 新增创建，不用处理状态 + new_found_users = remote_users - ori_ga_users + if new_found_users: + self.summary["new_accounts"] += len(new_found_users) + for username in new_found_users: + self.result["new_accounts"].append( + { + "asset": str(asset), + "username": username, + } + ) # 远端上 比 收集账号少的 # 标识 remote_present=False, 标记为待处理 # 远端资产上不存在的，标识为待处理，需要管理员介入 lost_users = ori_ga_users - remote_users if lost_users: - queryset.filter(username__in=lost_users).update(status='', remote_present=False) + queryset.filter(username__in=lost_users).update( + status="", remote_present=False + ) + self.summary["lost_accounts"] += len(lost_users) + for username in lost_users: + self.result["lost_accounts"].append( + { + "asset": str(asset), + "username": username, + } + ) # 收集的账号 比 账号列表多的, 有可能是账号中删掉了, 但这时候状态已经是 confirm 了 # 标识状态为 待处理, 让管理员去确认 ga_added_users = ori_ga_users - ori_users if ga_added_users: - queryset.filter(username__in=ga_added_users).update(status='') + queryset.filter(username__in=ga_added_users).update(status="") # 收集的账号 比 账号列表少的 # 这个好像不不用对比，原始情况就这样 @@ -255,22 +289,33 @@ class GatherAccountsManager(AccountBasePlaybookManager): # 正常情况, 不用处理，因为远端账号会创建到收集账号，收集账号再去对比 # 不过这个好像也处理一下 status，因为已存在，这是状态应该是确认 - (queryset.filter(username__in=ori_users) - .exclude(status=ConfirmOrIgnore.confirmed) - .update(status=ConfirmOrIgnore.confirmed)) - + ( + queryset.filter(username__in=ori_users) + .exclude(status=ConfirmOrIgnore.confirmed) + .update(status=ConfirmOrIgnore.confirmed) + ) + # 远端存在的账号，标识为已存在 - queryset.filter(username__in=remote_users, remote_present=False).update(remote_present=True) + ( + queryset.filter(username__in=remote_users, remote_present=False).update( + remote_present=True + ) + ) # 资产上没有的，标识为为存在 - queryset.exclude(username__in=ori_users).filter(present=False).update(present=True) + ( + queryset.exclude(username__in=ori_users) + .filter(present=False) + .update(present=True) + ) @bulk_create_decorator(GatheredAccount) def create_gathered_account(self, d): - gathered_account = GatheredAccount() + ga = GatheredAccount() for k, v in d.items(): - setattr(gathered_account, k, v) - return gathered_account + setattr(ga, k, v) + + return ga @bulk_update_decorator(GatheredAccount, update_fields=diff_items) def update_gathered_account(self, ori_account, d): @@ -287,11 +332,13 @@ class GatherAccountsManager(AccountBasePlaybookManager): risk_analyser = AnalyseAccountRisk(self.check_risk) for asset, accounts_data in self.asset_account_info.items(): - with (tmp_to_org(asset.org_id)): + with tmp_to_org(asset.org_id): gathered_accounts = [] for d in accounts_data: - username = d['username'] - ori_account = self.ori_gathered_accounts_mapper.get('{}_{}'.format(asset.id, username)) + username = d["username"] + ori_account = self.ori_gathered_accounts_mapper.get( + "{}_{}".format(asset.id, username) + ) if not ori_account: self.create_gathered_account(d) @@ -304,50 +351,5 @@ class GatherAccountsManager(AccountBasePlaybookManager): # 因为有 bulk create, bulk update, 所以这里需要 sleep 一下，等待数据同步 time.sleep(0.5) - def send_report_if_need(self): - pass - - def generate_send_users_and_change_info(self): - recipients = self.execution.recipients - if not self.asset_usernames_mapper or not recipients: - return None, None - - users = recipients - asset_ids = self.asset_usernames_mapper.keys() - assets = Asset.objects.filter(id__in=asset_ids).prefetch_related('accounts') - gather_accounts = GatheredAccount.objects.filter(asset_id__in=asset_ids, remote_present=True) - - asset_id_map = {str(asset.id): asset for asset in assets} - asset_id_username = list(assets.values_list('id', 'accounts__username')) - asset_id_username.extend(list(gather_accounts.values_list('asset_id', 'username'))) - - system_asset_usernames_mapper = defaultdict(set) - for asset_id, username in asset_id_username: - system_asset_usernames_mapper[str(asset_id)].add(username) - - change_info = defaultdict(dict) - for asset_id, usernames in self.asset_usernames_mapper.items(): - system_usernames = system_asset_usernames_mapper.get(asset_id) - if not system_usernames: - continue - - add_usernames = usernames - system_usernames - remove_usernames = system_usernames - usernames - - if not add_usernames and not remove_usernames: - continue - - change_info[str(asset_id_map[asset_id])] = { - 'add_usernames': add_usernames, - 'remove_usernames': remove_usernames - } - - return users, dict(change_info) - - def send_email_if_need(self): - users, change_info = self.generate_send_users_and_change_info() - if not users or not change_info: - return - - for user in users: - GatherAccountChangeMsg(user, change_info).publish_async() + def get_report_template(self): + return "accounts/gather_account_report.html" diff --git a/apps/accounts/templates/accounts/gather_account_report.html b/apps/accounts/templates/accounts/gather_account_report.html index cba9c5bda..691f97979 100644 --- a/apps/accounts/templates/accounts/gather_account_report.html +++ b/apps/accounts/templates/accounts/gather_account_report.html @@ -28,11 +28,11 @@ {% trans 'Assets count' %}: - {{ summary.assets }} + {{ summary.total_assets }} {% trans 'Asset success count' %}: - {{ summary.success_assets }} + {{ summary.ok_assets }} {% trans 'Asset failed count' %}: @@ -40,32 +40,24 @@ {% trans 'Asset not support count' %}: - {{ summary.na_assets }} + {{ summary.error_assets }} {% trans 'Account new found count' %}: {{ summary.new_accounts }} - - - {% trans 'Sudo changed count' %}: - {{ summary.sudo_changed }} - - - {% trans 'Groups changed count' %}: - {{ summary.groups_changed }} - - {% trans 'Authorized key changed count' %}: - {{ summary.authorized_key_changed }} + {% trans 'Account lost count' %}: + {{ summary.lost_accounts }}
-

{% trans 'New found accounts' %}:

+

{% trans 'New found accounts' %}: {{ summary.new_accounts }}

+ {% if summary.new_accounts %} @@ -84,10 +76,36 @@ {% endfor %}
+ {% endif %} +
+
+

{% trans 'Lost accounts' %}: {{ summary.lost_accounts }}

+ {% if summary.lost_accounts %} + + + + + + + + + + {% for account in result.lost_accounts %} + + + + + + {% endfor %} + +
{% trans 'No.' %}{% trans 'Asset' %}{% trans 'Username' %}
{{ forloop.counter }}{{ account.asset }}{{ account.username }}
+ {% endif %}
+
-

{% trans 'New found risk' %}:

+

{% trans 'New found risks' %}: {{ summary.new_risks }}

+ {% if summary.new_risks %} @@ -108,6 +126,7 @@ {% endfor %}
+ {% endif %}