diff --git a/apps/accounts/api/automations/check_account.py b/apps/accounts/api/automations/check_account.py index 363591486..ca536efbc 100644 --- a/apps/accounts/api/automations/check_account.py +++ b/apps/accounts/api/automations/check_account.py @@ -10,12 +10,12 @@ from orgs.mixins.api import OrgBulkModelViewSet from .base import AutomationExecutionViewSet __all__ = [ - 'CheckAccountsAutomationViewSet', 'CheckAccountExecutionViewSet', + 'CheckAccountAutomationViewSet', 'CheckAccountExecutionViewSet', 'AccountRiskViewSet', 'AccountCheckEngineViewSet', ] -class CheckAccountsAutomationViewSet(OrgBulkModelViewSet): +class CheckAccountAutomationViewSet(OrgBulkModelViewSet): model = AccountCheckAutomation filterset_fields = ('name',) search_fields = filterset_fields @@ -29,7 +29,7 @@ class CheckAccountExecutionViewSet(AutomationExecutionViewSet): ("create", "accounts.add_gatheraccountsexecution"), ) - tp = AutomationTypes.gather_accounts + tp = AutomationTypes.check_account def get_queryset(self): queryset = super().get_queryset() diff --git a/apps/accounts/automations/gather_accounts/__init__.py b/apps/accounts/automations/check_account/__init__.py similarity index 100% rename from apps/accounts/automations/gather_accounts/__init__.py rename to apps/accounts/automations/check_account/__init__.py diff --git a/apps/accounts/automations/gather_accounts/filter.py b/apps/accounts/automations/check_account/filter.py similarity index 100% rename from apps/accounts/automations/gather_accounts/filter.py rename to apps/accounts/automations/check_account/filter.py diff --git a/apps/accounts/automations/gather_accounts/host/posix/main.yml b/apps/accounts/automations/check_account/host/posix/main.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/host/posix/main.yml rename to apps/accounts/automations/check_account/host/posix/main.yml diff --git a/apps/accounts/automations/gather_accounts/host/posix/manifest.yml b/apps/accounts/automations/check_account/host/posix/manifest.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/host/posix/manifest.yml rename to apps/accounts/automations/check_account/host/posix/manifest.yml diff --git a/apps/accounts/automations/gather_accounts/host/windows/main.yml b/apps/accounts/automations/check_account/host/windows/main.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/host/windows/main.yml rename to apps/accounts/automations/check_account/host/windows/main.yml diff --git a/apps/accounts/automations/gather_accounts/host/windows/manifest.yml b/apps/accounts/automations/check_account/host/windows/manifest.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/host/windows/manifest.yml rename to apps/accounts/automations/check_account/host/windows/manifest.yml diff --git a/apps/accounts/automations/gather_accounts/manager.py b/apps/accounts/automations/check_account/manager.py similarity index 94% rename from apps/accounts/automations/gather_accounts/manager.py rename to apps/accounts/automations/check_account/manager.py index d0529e536..956a3f5c8 100644 --- a/apps/accounts/automations/gather_accounts/manager.py +++ b/apps/accounts/automations/check_account/manager.py @@ -41,6 +41,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.pending_add_accounts = [] self.pending_update_accounts = [] self.pending_add_risks = [] + self.now = timezone.now() @classmethod def method_type(cls): @@ -169,38 +170,40 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.batch_create_gathered_account(None) def _analyse_item_changed(self, ori_account, d): - now = timezone.now().isoformat() diff = self.get_items_diff(ori_account, d) - print("Diff items: ", diff) if not diff: return for k, v in diff.items(): - self.pending_add_risks.append(AccountRisk( + self.pending_add_risks.append(dict( asset=ori_account.asset, username=ori_account.username, - risk=k+'_changed', details=[{'datetime': now, 'diff': v}] + risk=k+'_changed', detail={'diff': v} )) - print("Pending add risks: ", self.pending_add_risks) - @staticmethod - def perform_save_risks(risks): + def perform_save_risks(self, risks): # 提前取出来，避免每次都查数据库 - assets = {r.asset for r in risks} + assets = {r['asset'] for r in risks} assets_risks = AccountRisk.objects.filter(asset__in=assets) assets_risks = {f"{r.asset_id}_{r.username}_{r.risk}": r for r in assets_risks} - for r in risks: - found = assets_risks.get(f"{r.asset_id}_{r.username}_{r.risk}") + for d in risks: + detail = d.pop('detail', {}) + detail['datetime'] = self.now.isoformat() + key = f"{d['asset'].id}_{d['username']}_{d['risk']}" + found = assets_risks.get(key) if not found: + r = AccountRisk(**d, details=[detail]) r.save() continue - found.details.extend(r.details) + found.details.append(detail) found.save(update_fields=['details']) def _analyse_datetime_changed(self, ori_account, d, asset, username): + basic = {'asset': asset, 'username': username} + for item in self.datetime_check_items: field = item['field'] risk = item['risk'] @@ -216,7 +219,7 @@ class GatherAccountsManager(AccountBasePlaybookManager): if date and date < timezone.now() - delta: self.pending_add_risks.append( - AccountRisk(asset=asset, username=username, risk=risk) + dict(**basic, risk=risk, detail={'date': date.isoformat()}) ) def batch_analyse_risk(self, asset, ori_account, d, batch_size=20): @@ -226,14 +229,12 @@ class GatherAccountsManager(AccountBasePlaybookManager): self.pending_add_risks = [] return - now = timezone.now().isoformat() - basic = {'asset': asset, 'username': d['username'], 'details': [{'datetime': now}]} - + basic = {'asset': asset, 'username': d['username']} if ori_account: self._analyse_item_changed(ori_account, d) else: self.pending_add_risks.append( - AccountRisk(**basic, risk='ghost') + dict(**basic, risk='ghost') ) self._analyse_datetime_changed(ori_account, d, asset, d['username']) @@ -292,7 +293,6 @@ class GatherAccountsManager(AccountBasePlaybookManager): else: self.batch_update_gathered_account(ori_account, d) - print("Batch analyse risk") self.batch_analyse_risk(asset, ori_account, d) self.update_gather_accounts_status(asset) diff --git a/apps/accounts/automations/endpoint.py b/apps/accounts/automations/endpoint.py index f045858a7..124cf60b4 100644 --- a/apps/accounts/automations/endpoint.py +++ b/apps/accounts/automations/endpoint.py @@ -1,6 +1,6 @@ from .backup_account.manager import AccountBackupManager from .change_secret.manager import ChangeSecretManager -from .gather_accounts.manager import GatherAccountsManager +from .gather_account.manager import GatherAccountsManager from .push_account.manager import PushAccountManager from .remove_account.manager import RemoveAccountManager from .verify_account.manager import VerifyAccountManager diff --git a/apps/accounts/automations/gather_account/__init__.py b/apps/accounts/automations/gather_account/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/accounts/automations/gather_accounts/database/mongodb/main.yml b/apps/accounts/automations/gather_account/database/mongodb/main.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/mongodb/main.yml rename to apps/accounts/automations/gather_account/database/mongodb/main.yml diff --git a/apps/accounts/automations/gather_accounts/database/mongodb/manifest.yml b/apps/accounts/automations/gather_account/database/mongodb/manifest.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/mongodb/manifest.yml rename to apps/accounts/automations/gather_account/database/mongodb/manifest.yml diff --git a/apps/accounts/automations/gather_accounts/database/mysql/main.yml b/apps/accounts/automations/gather_account/database/mysql/main.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/mysql/main.yml rename to apps/accounts/automations/gather_account/database/mysql/main.yml diff --git a/apps/accounts/automations/gather_accounts/database/mysql/manifest.yml b/apps/accounts/automations/gather_account/database/mysql/manifest.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/mysql/manifest.yml rename to apps/accounts/automations/gather_account/database/mysql/manifest.yml diff --git a/apps/accounts/automations/gather_accounts/database/oracle/main.yml b/apps/accounts/automations/gather_account/database/oracle/main.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/oracle/main.yml rename to apps/accounts/automations/gather_account/database/oracle/main.yml diff --git a/apps/accounts/automations/gather_accounts/database/oracle/manifest.yml b/apps/accounts/automations/gather_account/database/oracle/manifest.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/oracle/manifest.yml rename to apps/accounts/automations/gather_account/database/oracle/manifest.yml diff --git a/apps/accounts/automations/gather_accounts/database/postgresql/main.yml b/apps/accounts/automations/gather_account/database/postgresql/main.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/postgresql/main.yml rename to apps/accounts/automations/gather_account/database/postgresql/main.yml diff --git a/apps/accounts/automations/gather_accounts/database/postgresql/manifest.yml b/apps/accounts/automations/gather_account/database/postgresql/manifest.yml similarity index 100% rename from apps/accounts/automations/gather_accounts/database/postgresql/manifest.yml rename to apps/accounts/automations/gather_account/database/postgresql/manifest.yml diff --git a/apps/accounts/automations/gather_account/filter.py b/apps/accounts/automations/gather_account/filter.py new file mode 100644 index 000000000..209616dee --- /dev/null +++ b/apps/accounts/automations/gather_account/filter.py @@ -0,0 +1,123 @@ +from django.utils import timezone + +__all__ = ['GatherAccountsFilter'] + + +# TODO 后期会挪到 playbook 中 +class GatherAccountsFilter: + def __init__(self, tp): + self.tp = tp + + @staticmethod + def mysql_filter(info): + result = {} + for _, user_dict in info.items(): + for username, _ in user_dict.items(): + if len(username.split('.')) == 1: + result[username] = {} + return result + + @staticmethod + def postgresql_filter(info): + result = {} + for username in info: + result[username] = {} + return result + + @staticmethod + def posix_filter(info): + user_groups = info.pop('user_groups', []) + username_groups = {} + for line in user_groups: + if ':' not in line: + continue + username, groups = line.split(':', 1) + username_groups[username.strip()] = groups.strip() + + user_sudo = info.pop('user_sudo', []) + username_sudo = {} + for line in user_sudo: + if ':' not in line: + continue + username, sudo = line.split(':', 1) + if not sudo.strip(): + continue + username_sudo[username.strip()] = sudo.strip() + + last_login = info.pop('last_login', '') + user_last_login = {} + for line in last_login: + if not line.strip() or ' ' not in line: + continue + username, login = line.split(' ', 1) + user_last_login[username] = login + + user_authorized = info.pop('user_authorized', []) + username_authorized = {} + for line in user_authorized: + if ':' not in line: + continue + username, authorized = line.split(':', 1) + username_authorized[username.strip()] = authorized.strip() + + passwd_date = info.pop('passwd_date', []) + username_password_date = {} + for line in passwd_date: + if ':' not in line: + continue + username, password_date = line.split(':', 1) + username_password_date[username.strip()] = password_date.strip().split() + + result = {} + users = info.pop('users', '') + + for username in users: + if not username: + continue + user = dict() + + login = user_last_login.get(username) or '' + if login and len(login) == 3: + user['address_last_login'] = login[1][:32] + try: + login_date = timezone.datetime.fromisoformat(login[2]) + user['date_last_login'] = login_date + except ValueError: + pass + + start_date = timezone.make_aware(timezone.datetime(1970, 1, 1)) + _password_date = username_password_date.get(username) or '' + if _password_date and len(_password_date) == 2: + if _password_date[0] and _password_date[0] != '0': + user['date_password_change'] = start_date + timezone.timedelta(days=int(_password_date[0])) + if _password_date[1] and _password_date[1] != '0': + user['date_password_expired'] = start_date + timezone.timedelta(days=int(_password_date[1])) + + user['groups'] = username_groups.get(username) or '' + user['sudoers'] = username_sudo.get(username) or '' + user['authorized_keys'] = username_authorized.get(username) or '' + result[username] = user + return result + + @staticmethod + def windows_filter(info): + info = info[4:-2] + result = {} + for i in info: + for username in i.split(): + result[username] = {} + return result + + def run(self, method_id_meta_mapper, info): + run_method_name = None + for k, v in method_id_meta_mapper.items(): + if self.tp not in v['type']: + continue + run_method_name = k.replace(f'{v["method"]}_', '') + + if not run_method_name: + return info + + if hasattr(self, f'{run_method_name}_filter'): + return getattr(self, f'{run_method_name}_filter')(info) + return info diff --git a/apps/accounts/automations/gather_account/host/posix/main.yml b/apps/accounts/automations/gather_account/host/posix/main.yml new file mode 100644 index 000000000..59f09d948 --- /dev/null +++ b/apps/accounts/automations/gather_account/host/posix/main.yml @@ -0,0 +1,61 @@ +- hosts: demo + gather_facts: no + tasks: + - name: Get users + ansible.builtin.shell: + cmd: > + getent passwd | awk -F: '$7 !~ /(false|nologin|true|sync)$/' | grep -v '^$' | awk -F":" '{ print $1 }' + register: users + + - name: Gather posix account last login + ansible.builtin.shell: | + for user in {{ users.stdout_lines | join(" ") }}; do + last -i --time-format iso -n 1 ${user} | awk '{ print $1,$3,$4, $NF }' | head -1 | grep -v ^$ + done + register: last_login + + - name: Get user password change date and expiry + ansible.builtin.shell: | + for user in {{ users.stdout_lines | join(" ") }}; do + k=$(getent shadow $user | awk -F: '{ print $3, $5 }') + echo "$user:$k" + done + register: passwd_date + + - name: Get user groups + ansible.builtin.shell: | + for user in {{ users.stdout_lines | join(" ") }}; do + echo "$(groups $user)" | sed 's@ : @:@g' + done + register: user_groups + + - name: Get sudoers + ansible.builtin.shell: | + for user in {{ users.stdout_lines | join(" ") }}; do + echo "$user: $(grep "^$user " /etc/sudoers | tr '\n' ';' || echo '')" + done + register: user_sudo + + - name: Get authorized keys + ansible.builtin.shell: | + for user in {{ users.stdout_lines | join(" ") }}; do + home=$(getent passwd $user | cut -d: -f6) + echo -n "$user:" + if [[ -f ${home}/.ssh/authorized_keys ]]; then + cat ${home}/.ssh/authorized_keys | tr '\n' ';' + fi + echo + done + register: user_authorized + + - set_fact: + info: + users: "{{ users.stdout_lines }}" + last_login: "{{ last_login.stdout_lines }}" + user_groups: "{{ user_groups.stdout_lines }}" + user_sudo: "{{ user_sudo.stdout_lines }}" + user_authorized: "{{ user_authorized.stdout_lines }}" + passwd_date: "{{ passwd_date.stdout_lines }}" + + - debug: + var: info diff --git a/apps/accounts/automations/gather_account/host/posix/manifest.yml b/apps/accounts/automations/gather_account/host/posix/manifest.yml new file mode 100644 index 000000000..09240351b --- /dev/null +++ b/apps/accounts/automations/gather_account/host/posix/manifest.yml @@ -0,0 +1,13 @@ +id: gather_accounts_posix +name: "{{ 'Posix account gather' | trans }}" +category: host +type: + - linux + - unix +method: gather_accounts + +i18n: + Posix account gather: + zh: 使用命令 getent passwd 收集 Posix 资产账号 + ja: コマンド getent を使用してアセットアカウントを収集する + en: Using command getent to gather accounts diff --git a/apps/accounts/automations/gather_account/host/windows/main.yml b/apps/accounts/automations/gather_account/host/windows/main.yml new file mode 100644 index 000000000..944ae142f --- /dev/null +++ b/apps/accounts/automations/gather_account/host/windows/main.yml @@ -0,0 +1,14 @@ +- hosts: demo + gather_facts: no + tasks: + - name: Gather windows account + ansible.builtin.win_shell: net user + register: result + ignore_errors: true + + - name: Define info by set_fact + ansible.builtin.set_fact: + info: "{{ result.stdout_lines }}" + + - debug: + var: info diff --git a/apps/accounts/automations/gather_account/host/windows/manifest.yml b/apps/accounts/automations/gather_account/host/windows/manifest.yml new file mode 100644 index 000000000..c98fb0b2b --- /dev/null +++ b/apps/accounts/automations/gather_account/host/windows/manifest.yml @@ -0,0 +1,13 @@ +id: gather_accounts_windows +name: "{{ 'Windows account gather' | trans }}" +version: 1 +method: gather_accounts +category: host +type: + - windows + +i18n: + Windows account gather: + zh: 使用命令 net user 收集 Windows 账号 + ja: コマンド net user を使用して Windows アカウントを収集する + en: Using command net user to gather accounts diff --git a/apps/accounts/automations/gather_account/manager.py b/apps/accounts/automations/gather_account/manager.py new file mode 100644 index 000000000..956a3f5c8 --- /dev/null +++ b/apps/accounts/automations/gather_account/manager.py @@ -0,0 +1,357 @@ +from collections import defaultdict + +from django.utils import timezone + +from accounts.const import AutomationTypes +from accounts.models import GatheredAccount, Account, AccountRisk +from assets.models import Asset +from common.const import ConfirmOrIgnore +from common.utils import get_logger +from common.utils.strings import get_text_diff +from orgs.utils import tmp_to_org +from users.models import User +from .filter import GatherAccountsFilter +from ..base.manager import AccountBasePlaybookManager +from ...notifications import GatherAccountChangeMsg + +logger = get_logger(__name__) + + +class GatherAccountsManager(AccountBasePlaybookManager): + diff_items = [ + 'authorized_keys', 'sudoers', 'groups', + ] + long_time = timezone.timedelta(days=90) + datetime_check_items = [ + {'field': 'date_last_login', 'risk': 'zombie', 'delta': long_time}, + {'field': 'date_password_change', 'risk': 'long_time_password', 'delta': long_time}, + {'field': 'date_password_expired', 'risk': 'password_expired', 'delta': timezone.timedelta(seconds=1)} + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.host_asset_mapper = {} + self.asset_account_info = {} + + self.asset_usernames_mapper = defaultdict(set) + self.ori_asset_usernames = defaultdict(set) + self.ori_gathered_usernames = defaultdict(set) + self.ori_gathered_accounts_mapper = dict() + self.is_sync_account = self.execution.snapshot.get('is_sync_account') + self.pending_add_accounts = [] + self.pending_update_accounts = [] + self.pending_add_risks = [] + self.now = timezone.now() + + @classmethod + def method_type(cls): + return AutomationTypes.gather_accounts + + def host_callback(self, host, asset=None, **kwargs): + super().host_callback(host, asset=asset, **kwargs) + self.host_asset_mapper[host['name']] = asset + return host + + def _filter_success_result(self, tp, result): + result = GatherAccountsFilter(tp).run(self.method_id_meta_mapper, result) + return result + + @staticmethod + def _get_nested_info(data, *keys): + for key in keys: + data = data.get(key, {}) + if not data: + break + return data + + def _collect_asset_account_info(self, asset, info): + result = self._filter_success_result(asset.type, info) + accounts = [] + for username, info in result.items(): + self.asset_usernames_mapper[asset].add(username) + + d = {'asset': asset, 'username': username, 'remote_present': True, **info} + accounts.append(d) + self.asset_account_info[asset] = accounts + + def on_runner_failed(self, runner, e): + print("Runner failed: ", e) + raise e + + def on_host_success(self, host, result): + info = self._get_nested_info(result, 'debug', 'res', 'info') + asset = self.host_asset_mapper.get(host) + if asset and info: + self._collect_asset_account_info(asset, info) + else: + print(f'\033[31m Not found {host} info \033[0m\n') + + def prefetch_origin_account_usernames(self): + """ + 提起查出来，避免每次 sql 查询 + :return: + """ + assets = self.asset_usernames_mapper.keys() + accounts = Account.objects.filter(asset__in=assets).values_list('asset', 'username') + + for asset, username in accounts: + self.ori_asset_usernames[asset].add(username) + + ga_accounts = GatheredAccount.objects.filter(asset__in=assets) + for account in ga_accounts: + self.ori_gathered_usernames[account.asset].add(account.username) + key = '{}_{}'.format(account.asset.id, account.username) + self.ori_gathered_accounts_mapper[key] = account + + def update_gather_accounts_status(self, asset): + """ + 远端账号，收集中的账号，vault 中的账号。 + 要根据账号新增见啥，标识 收集账号的状态, 让管理员关注 + + 远端账号 -> 收集账号 -> 特权账号 + """ + remote_users = self.asset_usernames_mapper[asset] + ori_users = self.ori_asset_usernames[asset] + ori_ga_users = self.ori_gathered_usernames[asset] + + queryset = (GatheredAccount.objects + .filter(asset=asset) + .exclude(status=ConfirmOrIgnore.ignored)) + + # 远端账号 比 收集账号多的 + # 新增创建，不用处理状态 + + # 远端上 比 收集账号少的 + # 标识 remote_present=False, 标记为待处理 + # 远端资产上不存在的，标识为待处理，需要管理员介入 + lost_users = ori_ga_users - remote_users + if lost_users: + queryset.filter(username__in=lost_users).update(status='', remote_present=False) + + # 收集的账号 比 账号列表多的, 有可能是账号中删掉了, 但这时候状态已经是 confirm 了 + # 标识状态为 待处理, 让管理员去确认 + ga_added_users = ori_ga_users - ori_users + if ga_added_users: + queryset.filter(username__in=ga_added_users).update(status='') + + # 收集的账号 比 账号列表少的 + # 这个好像不不用对比，原始情况就这样 + + # 远端账号 比 账号列表少的 + # 创建收集账号，标识 remote_present=False, 状态待处理 + + # 远端账号 比 账号列表多的 + # 正常情况, 不用处理，因为远端账号会创建到收集账号，收集账号再去对比 + + # 不过这个好像也处理一下 status，因为已存在，这是状态应该是确认 + (queryset.filter(username__in=ori_users) + .exclude(status=ConfirmOrIgnore.confirmed) + .update(status=ConfirmOrIgnore.confirmed)) + + # 远端存在的账号，标识为已存在 + queryset.filter(username__in=remote_users, remote_present=False).update(remote_present=True) + + # 资产上没有的，标识为为存在 + queryset.exclude(username__in=ori_users).filter(present=False).update(present=True) + + def batch_create_gathered_account(self, d, batch_size=20): + if d is None: + if self.pending_add_accounts: + GatheredAccount.objects.bulk_create(self.pending_add_accounts, ignore_conflicts=True) + self.pending_add_accounts = [] + return + + gathered_account = GatheredAccount() + for k, v in d.items(): + setattr(gathered_account, k, v) + self.pending_add_accounts.append(gathered_account) + + if len(self.pending_add_accounts) > batch_size: + self.batch_create_gathered_account(None) + + def _analyse_item_changed(self, ori_account, d): + diff = self.get_items_diff(ori_account, d) + + if not diff: + return + + for k, v in diff.items(): + self.pending_add_risks.append(dict( + asset=ori_account.asset, username=ori_account.username, + risk=k+'_changed', detail={'diff': v} + )) + + def perform_save_risks(self, risks): + # 提前取出来，避免每次都查数据库 + assets = {r['asset'] for r in risks} + assets_risks = AccountRisk.objects.filter(asset__in=assets) + assets_risks = {f"{r.asset_id}_{r.username}_{r.risk}": r for r in assets_risks} + + for d in risks: + detail = d.pop('detail', {}) + detail['datetime'] = self.now.isoformat() + key = f"{d['asset'].id}_{d['username']}_{d['risk']}" + found = assets_risks.get(key) + + if not found: + r = AccountRisk(**d, details=[detail]) + r.save() + continue + + found.details.append(detail) + found.save(update_fields=['details']) + + def _analyse_datetime_changed(self, ori_account, d, asset, username): + basic = {'asset': asset, 'username': username} + + for item in self.datetime_check_items: + field = item['field'] + risk = item['risk'] + delta = item['delta'] + + date = d.get(field) + if not date: + continue + + pre_date = ori_account and getattr(ori_account, field) + if pre_date == date: + continue + + if date and date < timezone.now() - delta: + self.pending_add_risks.append( + dict(**basic, risk=risk, detail={'date': date.isoformat()}) + ) + + def batch_analyse_risk(self, asset, ori_account, d, batch_size=20): + if asset is None: + if self.pending_add_risks: + self.perform_save_risks(self.pending_add_risks) + self.pending_add_risks = [] + return + + basic = {'asset': asset, 'username': d['username']} + if ori_account: + self._analyse_item_changed(ori_account, d) + else: + self.pending_add_risks.append( + dict(**basic, risk='ghost') + ) + + self._analyse_datetime_changed(ori_account, d, asset, d['username']) + + if len(self.pending_add_risks) > batch_size: + self.batch_analyse_risk(None, None, {}) + + def get_items_diff(self, ori_account, d): + if hasattr(ori_account, '_diff'): + return ori_account._diff + + diff = {} + for item in self.diff_items: + ori = getattr(ori_account, item) + new = d.get(item, '') + + if not ori: + continue + + if isinstance(new, timezone.datetime): + new = ori.strftime('%Y-%m-%d %H:%M:%S') + ori = ori.strftime('%Y-%m-%d %H:%M:%S') + + if new != ori: + diff[item] = get_text_diff(ori, new) + + ori_account._diff = diff + return diff + + def batch_update_gathered_account(self, ori_account, d, batch_size=20): + if not ori_account or d is None: + if self.pending_update_accounts: + GatheredAccount.objects.bulk_update(self.pending_update_accounts, [*self.diff_items]) + self.pending_update_accounts = [] + return + + diff = self.get_items_diff(ori_account, d) + if diff: + for k in diff: + setattr(ori_account, k, d[k]) + self.pending_update_accounts.append(ori_account) + + if len(self.pending_update_accounts) > batch_size: + self.batch_update_gathered_account(None, None) + + def update_or_create_accounts(self): + for asset, accounts_data in self.asset_account_info.items(): + with (tmp_to_org(asset.org_id)): + gathered_accounts = [] + for d in accounts_data: + username = d['username'] + ori_account = self.ori_gathered_accounts_mapper.get('{}_{}'.format(asset.id, username)) + + if not ori_account: + self.batch_create_gathered_account(d) + else: + self.batch_update_gathered_account(ori_account, d) + + self.batch_analyse_risk(asset, ori_account, d) + + self.update_gather_accounts_status(asset) + GatheredAccount.sync_accounts(gathered_accounts, self.is_sync_account) + + self.batch_create_gathered_account(None) + self.batch_update_gathered_account(None, None) + self.batch_analyse_risk(None, None, {}) + + def run(self, *args, **kwargs): + super().run(*args, **kwargs) + self.prefetch_origin_account_usernames() + self.update_or_create_accounts() + # self.send_email_if_need() + + def generate_send_users_and_change_info(self): + recipients = self.execution.recipients + if not self.asset_usernames_mapper or not recipients: + return None, None + + users = User.objects.filter(id__in=recipients) + if not users.exists(): + return users, None + + asset_ids = self.asset_usernames_mapper.keys() + assets = Asset.objects.filter(id__in=asset_ids).prefetch_related('accounts') + gather_accounts = GatheredAccount.objects.filter(asset_id__in=asset_ids, remote_present=True) + + asset_id_map = {str(asset.id): asset for asset in assets} + asset_id_username = list(assets.values_list('id', 'accounts__username')) + asset_id_username.extend(list(gather_accounts.values_list('asset_id', 'username'))) + + system_asset_usernames_mapper = defaultdict(set) + for asset_id, username in asset_id_username: + system_asset_usernames_mapper[str(asset_id)].add(username) + + change_info = defaultdict(dict) + for asset_id, usernames in self.asset_usernames_mapper.items(): + system_usernames = system_asset_usernames_mapper.get(asset_id) + if not system_usernames: + continue + + add_usernames = usernames - system_usernames + remove_usernames = system_usernames - usernames + + if not add_usernames and not remove_usernames: + continue + + change_info[str(asset_id_map[asset_id])] = { + 'add_usernames': add_usernames, + 'remove_usernames': remove_usernames + } + + return users, dict(change_info) + + def send_email_if_need(self): + users, change_info = self.generate_send_users_and_change_info() + if not users or not change_info: + return + + for user in users: + GatherAccountChangeMsg(user, change_info).publish_async() diff --git a/apps/accounts/urls.py b/apps/accounts/urls.py index 6bfe9056b..839e5a082 100644 --- a/apps/accounts/urls.py +++ b/apps/accounts/urls.py @@ -24,6 +24,8 @@ router.register(r'gather-account-executions', api.GatherAccountsExecutionViewSet router.register(r'push-account-automations', api.PushAccountAutomationViewSet, 'push-account-automation') router.register(r'push-account-executions', api.PushAccountExecutionViewSet, 'push-account-execution') router.register(r'push-account-records', api.PushAccountRecordViewSet, 'push-account-record') +router.register(r'check-account-automations', api.CheckAccountAutomationViewSet, 'check-account-automation') +router.register(r'check-account-executions', api.CheckAccountExecutionViewSet, 'check-account-execution') router.register(r'account-check-engines', api.AccountCheckEngineViewSet, 'account-check-engine') router.register(r'account-risks', api.AccountRiskViewSet, 'account-risks')