perf: remove to check account dir

pull/14387/merge
ibuler 2 weeks ago
parent a750fbb785
commit e3f93a9410

@ -10,12 +10,12 @@ from orgs.mixins.api import OrgBulkModelViewSet
from .base import AutomationExecutionViewSet
__all__ = [
'CheckAccountsAutomationViewSet', 'CheckAccountExecutionViewSet',
'CheckAccountAutomationViewSet', 'CheckAccountExecutionViewSet',
'AccountRiskViewSet', 'AccountCheckEngineViewSet',
]
class CheckAccountsAutomationViewSet(OrgBulkModelViewSet):
class CheckAccountAutomationViewSet(OrgBulkModelViewSet):
model = AccountCheckAutomation
filterset_fields = ('name',)
search_fields = filterset_fields
@ -29,7 +29,7 @@ class CheckAccountExecutionViewSet(AutomationExecutionViewSet):
("create", "accounts.add_gatheraccountsexecution"),
)
tp = AutomationTypes.gather_accounts
tp = AutomationTypes.check_account
def get_queryset(self):
queryset = super().get_queryset()

@ -41,6 +41,7 @@ class GatherAccountsManager(AccountBasePlaybookManager):
self.pending_add_accounts = []
self.pending_update_accounts = []
self.pending_add_risks = []
self.now = timezone.now()
@classmethod
def method_type(cls):
@ -169,38 +170,40 @@ class GatherAccountsManager(AccountBasePlaybookManager):
self.batch_create_gathered_account(None)
def _analyse_item_changed(self, ori_account, d):
now = timezone.now().isoformat()
diff = self.get_items_diff(ori_account, d)
print("Diff items: ", diff)
if not diff:
return
for k, v in diff.items():
self.pending_add_risks.append(AccountRisk(
self.pending_add_risks.append(dict(
asset=ori_account.asset, username=ori_account.username,
risk=k+'_changed', details=[{'datetime': now, 'diff': v}]
risk=k+'_changed', detail={'diff': v}
))
print("Pending add risks: ", self.pending_add_risks)
@staticmethod
def perform_save_risks(risks):
def perform_save_risks(self, risks):
# 提前取出来,避免每次都查数据库
assets = {r.asset for r in risks}
assets = {r['asset'] for r in risks}
assets_risks = AccountRisk.objects.filter(asset__in=assets)
assets_risks = {f"{r.asset_id}_{r.username}_{r.risk}": r for r in assets_risks}
for r in risks:
found = assets_risks.get(f"{r.asset_id}_{r.username}_{r.risk}")
for d in risks:
detail = d.pop('detail', {})
detail['datetime'] = self.now.isoformat()
key = f"{d['asset'].id}_{d['username']}_{d['risk']}"
found = assets_risks.get(key)
if not found:
r = AccountRisk(**d, details=[detail])
r.save()
continue
found.details.extend(r.details)
found.details.append(detail)
found.save(update_fields=['details'])
def _analyse_datetime_changed(self, ori_account, d, asset, username):
basic = {'asset': asset, 'username': username}
for item in self.datetime_check_items:
field = item['field']
risk = item['risk']
@ -216,7 +219,7 @@ class GatherAccountsManager(AccountBasePlaybookManager):
if date and date < timezone.now() - delta:
self.pending_add_risks.append(
AccountRisk(asset=asset, username=username, risk=risk)
dict(**basic, risk=risk, detail={'date': date.isoformat()})
)
def batch_analyse_risk(self, asset, ori_account, d, batch_size=20):
@ -226,14 +229,12 @@ class GatherAccountsManager(AccountBasePlaybookManager):
self.pending_add_risks = []
return
now = timezone.now().isoformat()
basic = {'asset': asset, 'username': d['username'], 'details': [{'datetime': now}]}
basic = {'asset': asset, 'username': d['username']}
if ori_account:
self._analyse_item_changed(ori_account, d)
else:
self.pending_add_risks.append(
AccountRisk(**basic, risk='ghost')
dict(**basic, risk='ghost')
)
self._analyse_datetime_changed(ori_account, d, asset, d['username'])
@ -292,7 +293,6 @@ class GatherAccountsManager(AccountBasePlaybookManager):
else:
self.batch_update_gathered_account(ori_account, d)
print("Batch analyse risk")
self.batch_analyse_risk(asset, ori_account, d)
self.update_gather_accounts_status(asset)

@ -1,6 +1,6 @@
from .backup_account.manager import AccountBackupManager
from .change_secret.manager import ChangeSecretManager
from .gather_accounts.manager import GatherAccountsManager
from .gather_account.manager import GatherAccountsManager
from .push_account.manager import PushAccountManager
from .remove_account.manager import RemoveAccountManager
from .verify_account.manager import VerifyAccountManager

@ -0,0 +1,123 @@
from django.utils import timezone
__all__ = ['GatherAccountsFilter']
# TODO 后期会挪到 playbook 中
class GatherAccountsFilter:
def __init__(self, tp):
self.tp = tp
@staticmethod
def mysql_filter(info):
result = {}
for _, user_dict in info.items():
for username, _ in user_dict.items():
if len(username.split('.')) == 1:
result[username] = {}
return result
@staticmethod
def postgresql_filter(info):
result = {}
for username in info:
result[username] = {}
return result
@staticmethod
def posix_filter(info):
user_groups = info.pop('user_groups', [])
username_groups = {}
for line in user_groups:
if ':' not in line:
continue
username, groups = line.split(':', 1)
username_groups[username.strip()] = groups.strip()
user_sudo = info.pop('user_sudo', [])
username_sudo = {}
for line in user_sudo:
if ':' not in line:
continue
username, sudo = line.split(':', 1)
if not sudo.strip():
continue
username_sudo[username.strip()] = sudo.strip()
last_login = info.pop('last_login', '')
user_last_login = {}
for line in last_login:
if not line.strip() or ' ' not in line:
continue
username, login = line.split(' ', 1)
user_last_login[username] = login
user_authorized = info.pop('user_authorized', [])
username_authorized = {}
for line in user_authorized:
if ':' not in line:
continue
username, authorized = line.split(':', 1)
username_authorized[username.strip()] = authorized.strip()
passwd_date = info.pop('passwd_date', [])
username_password_date = {}
for line in passwd_date:
if ':' not in line:
continue
username, password_date = line.split(':', 1)
username_password_date[username.strip()] = password_date.strip().split()
result = {}
users = info.pop('users', '')
for username in users:
if not username:
continue
user = dict()
login = user_last_login.get(username) or ''
if login and len(login) == 3:
user['address_last_login'] = login[1][:32]
try:
login_date = timezone.datetime.fromisoformat(login[2])
user['date_last_login'] = login_date
except ValueError:
pass
start_date = timezone.make_aware(timezone.datetime(1970, 1, 1))
_password_date = username_password_date.get(username) or ''
if _password_date and len(_password_date) == 2:
if _password_date[0] and _password_date[0] != '0':
user['date_password_change'] = start_date + timezone.timedelta(days=int(_password_date[0]))
if _password_date[1] and _password_date[1] != '0':
user['date_password_expired'] = start_date + timezone.timedelta(days=int(_password_date[1]))
user['groups'] = username_groups.get(username) or ''
user['sudoers'] = username_sudo.get(username) or ''
user['authorized_keys'] = username_authorized.get(username) or ''
result[username] = user
return result
@staticmethod
def windows_filter(info):
info = info[4:-2]
result = {}
for i in info:
for username in i.split():
result[username] = {}
return result
def run(self, method_id_meta_mapper, info):
run_method_name = None
for k, v in method_id_meta_mapper.items():
if self.tp not in v['type']:
continue
run_method_name = k.replace(f'{v["method"]}_', '')
if not run_method_name:
return info
if hasattr(self, f'{run_method_name}_filter'):
return getattr(self, f'{run_method_name}_filter')(info)
return info

@ -0,0 +1,61 @@
- hosts: demo
gather_facts: no
tasks:
- name: Get users
ansible.builtin.shell:
cmd: >
getent passwd | awk -F: '$7 !~ /(false|nologin|true|sync)$/' | grep -v '^$' | awk -F":" '{ print $1 }'
register: users
- name: Gather posix account last login
ansible.builtin.shell: |
for user in {{ users.stdout_lines | join(" ") }}; do
last -i --time-format iso -n 1 ${user} | awk '{ print $1,$3,$4, $NF }' | head -1 | grep -v ^$
done
register: last_login
- name: Get user password change date and expiry
ansible.builtin.shell: |
for user in {{ users.stdout_lines | join(" ") }}; do
k=$(getent shadow $user | awk -F: '{ print $3, $5 }')
echo "$user:$k"
done
register: passwd_date
- name: Get user groups
ansible.builtin.shell: |
for user in {{ users.stdout_lines | join(" ") }}; do
echo "$(groups $user)" | sed 's@ : @:@g'
done
register: user_groups
- name: Get sudoers
ansible.builtin.shell: |
for user in {{ users.stdout_lines | join(" ") }}; do
echo "$user: $(grep "^$user " /etc/sudoers | tr '\n' ';' || echo '')"
done
register: user_sudo
- name: Get authorized keys
ansible.builtin.shell: |
for user in {{ users.stdout_lines | join(" ") }}; do
home=$(getent passwd $user | cut -d: -f6)
echo -n "$user:"
if [[ -f ${home}/.ssh/authorized_keys ]]; then
cat ${home}/.ssh/authorized_keys | tr '\n' ';'
fi
echo
done
register: user_authorized
- set_fact:
info:
users: "{{ users.stdout_lines }}"
last_login: "{{ last_login.stdout_lines }}"
user_groups: "{{ user_groups.stdout_lines }}"
user_sudo: "{{ user_sudo.stdout_lines }}"
user_authorized: "{{ user_authorized.stdout_lines }}"
passwd_date: "{{ passwd_date.stdout_lines }}"
- debug:
var: info

@ -0,0 +1,13 @@
id: gather_accounts_posix
name: "{{ 'Posix account gather' | trans }}"
category: host
type:
- linux
- unix
method: gather_accounts
i18n:
Posix account gather:
zh: 使用命令 getent passwd 收集 Posix 资产账号
ja: コマンド getent を使用してアセットアカウントを収集する
en: Using command getent to gather accounts

@ -0,0 +1,14 @@
- hosts: demo
gather_facts: no
tasks:
- name: Gather windows account
ansible.builtin.win_shell: net user
register: result
ignore_errors: true
- name: Define info by set_fact
ansible.builtin.set_fact:
info: "{{ result.stdout_lines }}"
- debug:
var: info

@ -0,0 +1,13 @@
id: gather_accounts_windows
name: "{{ 'Windows account gather' | trans }}"
version: 1
method: gather_accounts
category: host
type:
- windows
i18n:
Windows account gather:
zh: 使用命令 net user 收集 Windows 账号
ja: コマンド net user を使用して Windows アカウントを収集する
en: Using command net user to gather accounts

@ -0,0 +1,357 @@
from collections import defaultdict
from django.utils import timezone
from accounts.const import AutomationTypes
from accounts.models import GatheredAccount, Account, AccountRisk
from assets.models import Asset
from common.const import ConfirmOrIgnore
from common.utils import get_logger
from common.utils.strings import get_text_diff
from orgs.utils import tmp_to_org
from users.models import User
from .filter import GatherAccountsFilter
from ..base.manager import AccountBasePlaybookManager
from ...notifications import GatherAccountChangeMsg
logger = get_logger(__name__)
class GatherAccountsManager(AccountBasePlaybookManager):
diff_items = [
'authorized_keys', 'sudoers', 'groups',
]
long_time = timezone.timedelta(days=90)
datetime_check_items = [
{'field': 'date_last_login', 'risk': 'zombie', 'delta': long_time},
{'field': 'date_password_change', 'risk': 'long_time_password', 'delta': long_time},
{'field': 'date_password_expired', 'risk': 'password_expired', 'delta': timezone.timedelta(seconds=1)}
]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.host_asset_mapper = {}
self.asset_account_info = {}
self.asset_usernames_mapper = defaultdict(set)
self.ori_asset_usernames = defaultdict(set)
self.ori_gathered_usernames = defaultdict(set)
self.ori_gathered_accounts_mapper = dict()
self.is_sync_account = self.execution.snapshot.get('is_sync_account')
self.pending_add_accounts = []
self.pending_update_accounts = []
self.pending_add_risks = []
self.now = timezone.now()
@classmethod
def method_type(cls):
return AutomationTypes.gather_accounts
def host_callback(self, host, asset=None, **kwargs):
super().host_callback(host, asset=asset, **kwargs)
self.host_asset_mapper[host['name']] = asset
return host
def _filter_success_result(self, tp, result):
result = GatherAccountsFilter(tp).run(self.method_id_meta_mapper, result)
return result
@staticmethod
def _get_nested_info(data, *keys):
for key in keys:
data = data.get(key, {})
if not data:
break
return data
def _collect_asset_account_info(self, asset, info):
result = self._filter_success_result(asset.type, info)
accounts = []
for username, info in result.items():
self.asset_usernames_mapper[asset].add(username)
d = {'asset': asset, 'username': username, 'remote_present': True, **info}
accounts.append(d)
self.asset_account_info[asset] = accounts
def on_runner_failed(self, runner, e):
print("Runner failed: ", e)
raise e
def on_host_success(self, host, result):
info = self._get_nested_info(result, 'debug', 'res', 'info')
asset = self.host_asset_mapper.get(host)
if asset and info:
self._collect_asset_account_info(asset, info)
else:
print(f'\033[31m Not found {host} info \033[0m\n')
def prefetch_origin_account_usernames(self):
"""
提起查出来避免每次 sql 查询
:return:
"""
assets = self.asset_usernames_mapper.keys()
accounts = Account.objects.filter(asset__in=assets).values_list('asset', 'username')
for asset, username in accounts:
self.ori_asset_usernames[asset].add(username)
ga_accounts = GatheredAccount.objects.filter(asset__in=assets)
for account in ga_accounts:
self.ori_gathered_usernames[account.asset].add(account.username)
key = '{}_{}'.format(account.asset.id, account.username)
self.ori_gathered_accounts_mapper[key] = account
def update_gather_accounts_status(self, asset):
"""
远端账号收集中的账号vault 中的账号
要根据账号新增见啥标识 收集账号的状态, 让管理员关注
远端账号 -> 收集账号 -> 特权账号
"""
remote_users = self.asset_usernames_mapper[asset]
ori_users = self.ori_asset_usernames[asset]
ori_ga_users = self.ori_gathered_usernames[asset]
queryset = (GatheredAccount.objects
.filter(asset=asset)
.exclude(status=ConfirmOrIgnore.ignored))
# 远端账号 比 收集账号多的
# 新增创建,不用处理状态
# 远端上 比 收集账号少的
# 标识 remote_present=False, 标记为待处理
# 远端资产上不存在的,标识为待处理,需要管理员介入
lost_users = ori_ga_users - remote_users
if lost_users:
queryset.filter(username__in=lost_users).update(status='', remote_present=False)
# 收集的账号 比 账号列表多的, 有可能是账号中删掉了, 但这时候状态已经是 confirm 了
# 标识状态为 待处理, 让管理员去确认
ga_added_users = ori_ga_users - ori_users
if ga_added_users:
queryset.filter(username__in=ga_added_users).update(status='')
# 收集的账号 比 账号列表少的
# 这个好像不不用对比,原始情况就这样
# 远端账号 比 账号列表少的
# 创建收集账号,标识 remote_present=False, 状态待处理
# 远端账号 比 账号列表多的
# 正常情况, 不用处理,因为远端账号会创建到收集账号,收集账号再去对比
# 不过这个好像也处理一下 status因为已存在这是状态应该是确认
(queryset.filter(username__in=ori_users)
.exclude(status=ConfirmOrIgnore.confirmed)
.update(status=ConfirmOrIgnore.confirmed))
# 远端存在的账号,标识为已存在
queryset.filter(username__in=remote_users, remote_present=False).update(remote_present=True)
# 资产上没有的,标识为为存在
queryset.exclude(username__in=ori_users).filter(present=False).update(present=True)
def batch_create_gathered_account(self, d, batch_size=20):
if d is None:
if self.pending_add_accounts:
GatheredAccount.objects.bulk_create(self.pending_add_accounts, ignore_conflicts=True)
self.pending_add_accounts = []
return
gathered_account = GatheredAccount()
for k, v in d.items():
setattr(gathered_account, k, v)
self.pending_add_accounts.append(gathered_account)
if len(self.pending_add_accounts) > batch_size:
self.batch_create_gathered_account(None)
def _analyse_item_changed(self, ori_account, d):
diff = self.get_items_diff(ori_account, d)
if not diff:
return
for k, v in diff.items():
self.pending_add_risks.append(dict(
asset=ori_account.asset, username=ori_account.username,
risk=k+'_changed', detail={'diff': v}
))
def perform_save_risks(self, risks):
# 提前取出来,避免每次都查数据库
assets = {r['asset'] for r in risks}
assets_risks = AccountRisk.objects.filter(asset__in=assets)
assets_risks = {f"{r.asset_id}_{r.username}_{r.risk}": r for r in assets_risks}
for d in risks:
detail = d.pop('detail', {})
detail['datetime'] = self.now.isoformat()
key = f"{d['asset'].id}_{d['username']}_{d['risk']}"
found = assets_risks.get(key)
if not found:
r = AccountRisk(**d, details=[detail])
r.save()
continue
found.details.append(detail)
found.save(update_fields=['details'])
def _analyse_datetime_changed(self, ori_account, d, asset, username):
basic = {'asset': asset, 'username': username}
for item in self.datetime_check_items:
field = item['field']
risk = item['risk']
delta = item['delta']
date = d.get(field)
if not date:
continue
pre_date = ori_account and getattr(ori_account, field)
if pre_date == date:
continue
if date and date < timezone.now() - delta:
self.pending_add_risks.append(
dict(**basic, risk=risk, detail={'date': date.isoformat()})
)
def batch_analyse_risk(self, asset, ori_account, d, batch_size=20):
if asset is None:
if self.pending_add_risks:
self.perform_save_risks(self.pending_add_risks)
self.pending_add_risks = []
return
basic = {'asset': asset, 'username': d['username']}
if ori_account:
self._analyse_item_changed(ori_account, d)
else:
self.pending_add_risks.append(
dict(**basic, risk='ghost')
)
self._analyse_datetime_changed(ori_account, d, asset, d['username'])
if len(self.pending_add_risks) > batch_size:
self.batch_analyse_risk(None, None, {})
def get_items_diff(self, ori_account, d):
if hasattr(ori_account, '_diff'):
return ori_account._diff
diff = {}
for item in self.diff_items:
ori = getattr(ori_account, item)
new = d.get(item, '')
if not ori:
continue
if isinstance(new, timezone.datetime):
new = ori.strftime('%Y-%m-%d %H:%M:%S')
ori = ori.strftime('%Y-%m-%d %H:%M:%S')
if new != ori:
diff[item] = get_text_diff(ori, new)
ori_account._diff = diff
return diff
def batch_update_gathered_account(self, ori_account, d, batch_size=20):
if not ori_account or d is None:
if self.pending_update_accounts:
GatheredAccount.objects.bulk_update(self.pending_update_accounts, [*self.diff_items])
self.pending_update_accounts = []
return
diff = self.get_items_diff(ori_account, d)
if diff:
for k in diff:
setattr(ori_account, k, d[k])
self.pending_update_accounts.append(ori_account)
if len(self.pending_update_accounts) > batch_size:
self.batch_update_gathered_account(None, None)
def update_or_create_accounts(self):
for asset, accounts_data in self.asset_account_info.items():
with (tmp_to_org(asset.org_id)):
gathered_accounts = []
for d in accounts_data:
username = d['username']
ori_account = self.ori_gathered_accounts_mapper.get('{}_{}'.format(asset.id, username))
if not ori_account:
self.batch_create_gathered_account(d)
else:
self.batch_update_gathered_account(ori_account, d)
self.batch_analyse_risk(asset, ori_account, d)
self.update_gather_accounts_status(asset)
GatheredAccount.sync_accounts(gathered_accounts, self.is_sync_account)
self.batch_create_gathered_account(None)
self.batch_update_gathered_account(None, None)
self.batch_analyse_risk(None, None, {})
def run(self, *args, **kwargs):
super().run(*args, **kwargs)
self.prefetch_origin_account_usernames()
self.update_or_create_accounts()
# self.send_email_if_need()
def generate_send_users_and_change_info(self):
recipients = self.execution.recipients
if not self.asset_usernames_mapper or not recipients:
return None, None
users = User.objects.filter(id__in=recipients)
if not users.exists():
return users, None
asset_ids = self.asset_usernames_mapper.keys()
assets = Asset.objects.filter(id__in=asset_ids).prefetch_related('accounts')
gather_accounts = GatheredAccount.objects.filter(asset_id__in=asset_ids, remote_present=True)
asset_id_map = {str(asset.id): asset for asset in assets}
asset_id_username = list(assets.values_list('id', 'accounts__username'))
asset_id_username.extend(list(gather_accounts.values_list('asset_id', 'username')))
system_asset_usernames_mapper = defaultdict(set)
for asset_id, username in asset_id_username:
system_asset_usernames_mapper[str(asset_id)].add(username)
change_info = defaultdict(dict)
for asset_id, usernames in self.asset_usernames_mapper.items():
system_usernames = system_asset_usernames_mapper.get(asset_id)
if not system_usernames:
continue
add_usernames = usernames - system_usernames
remove_usernames = system_usernames - usernames
if not add_usernames and not remove_usernames:
continue
change_info[str(asset_id_map[asset_id])] = {
'add_usernames': add_usernames,
'remove_usernames': remove_usernames
}
return users, dict(change_info)
def send_email_if_need(self):
users, change_info = self.generate_send_users_and_change_info()
if not users or not change_info:
return
for user in users:
GatherAccountChangeMsg(user, change_info).publish_async()

@ -24,6 +24,8 @@ router.register(r'gather-account-executions', api.GatherAccountsExecutionViewSet
router.register(r'push-account-automations', api.PushAccountAutomationViewSet, 'push-account-automation')
router.register(r'push-account-executions', api.PushAccountExecutionViewSet, 'push-account-execution')
router.register(r'push-account-records', api.PushAccountRecordViewSet, 'push-account-record')
router.register(r'check-account-automations', api.CheckAccountAutomationViewSet, 'check-account-automation')
router.register(r'check-account-executions', api.CheckAccountExecutionViewSet, 'check-account-execution')
router.register(r'account-check-engines', api.AccountCheckEngineViewSet, 'account-check-engine')
router.register(r'account-risks', api.AccountRiskViewSet, 'account-risks')

Loading…
Cancel
Save