perf: automation account username change id (#9867)

* perf: automation account username change id

* perf: 授权账号模版 自推送

---------

Co-authored-by: feng <1304903146@qq.com>
pull/9893/head
fit2bot 2023-03-08 18:52:00 +08:00 committed by GitHub
parent 8a0bd3379c
commit c90a2df28e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 243 additions and 113 deletions

View File

@ -9,12 +9,12 @@
name: "{{ account.username }}"
password: "{{ account.secret | password_hash('des') }}"
update_password: always
when: secret_type == "password"
when: account.secret_type == "password"
- name: create user If it already exists, no operation will be performed
ansible.builtin.user:
name: "{{ account.username }}"
when: secret_type == "ssh_key"
when: account.secret_type == "ssh_key"
- name: remove jumpserver ssh key
ansible.builtin.lineinfile:
@ -22,7 +22,7 @@
regexp: "{{ kwargs.regexp }}"
state: absent
when:
- secret_type == "ssh_key"
- account.secret_type == "ssh_key"
- kwargs.strategy == "set_jms"
- name: Change SSH key
@ -30,7 +30,7 @@
user: "{{ account.username }}"
key: "{{ account.secret }}"
exclusive: "{{ kwargs.exclusive }}"
when: secret_type == "ssh_key"
when: account.secret_type == "ssh_key"
- name: Refresh connection
ansible.builtin.meta: reset_connection
@ -42,7 +42,7 @@
ansible_user: "{{ account.username }}"
ansible_password: "{{ account.secret }}"
ansible_become: no
when: secret_type == "password"
when: account.secret_type == "password"
- name: Verify SSH key
ansible.builtin.ping:
@ -51,4 +51,4 @@
ansible_user: "{{ account.username }}"
ansible_ssh_private_key_file: "{{ account.private_key_path }}"
ansible_become: no
when: secret_type == "ssh_key"
when: account.secret_type == "ssh_key"

View File

@ -9,12 +9,12 @@
name: "{{ account.username }}"
password: "{{ account.secret | password_hash('sha512') }}"
update_password: always
when: secret_type == "password"
when: account.secret_type == "password"
- name: create user If it already exists, no operation will be performed
ansible.builtin.user:
name: "{{ account.username }}"
when: secret_type == "ssh_key"
when: account.secret_type == "ssh_key"
- name: remove jumpserver ssh key
ansible.builtin.lineinfile:
@ -22,7 +22,7 @@
regexp: "{{ kwargs.regexp }}"
state: absent
when:
- secret_type == "ssh_key"
- account.secret_type == "ssh_key"
- kwargs.strategy == "set_jms"
- name: Change SSH key
@ -30,7 +30,7 @@
user: "{{ account.username }}"
key: "{{ account.secret }}"
exclusive: "{{ kwargs.exclusive }}"
when: secret_type == "ssh_key"
when: account.secret_type == "ssh_key"
- name: Refresh connection
ansible.builtin.meta: reset_connection
@ -42,7 +42,7 @@
ansible_user: "{{ account.username }}"
ansible_password: "{{ account.secret }}"
ansible_become: no
when: secret_type == "password"
when: account.secret_type == "password"
- name: Verify SSH key
ansible.builtin.ping:
@ -51,4 +51,4 @@
ansible_user: "{{ account.username }}"
ansible_ssh_private_key_file: "{{ account.private_key_path }}"
ansible_become: no
when: secret_type == "ssh_key"
when: account.secret_type == "ssh_key"

View File

@ -12,7 +12,7 @@ from accounts.models import ChangeSecretRecord
from accounts.notifications import ChangeSecretExecutionTaskMsg
from accounts.serializers import ChangeSecretRecordBackUpSerializer
from assets.const import HostTypes
from common.utils import get_logger, lazyproperty
from common.utils import get_logger
from common.utils.file import encrypt_and_compress_zip_file
from common.utils.timezone import local_now_display
from users.models import User
@ -28,23 +28,23 @@ class ChangeSecretManager(AccountBasePlaybookManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.method_hosts_mapper = defaultdict(list)
self.secret_type = self.execution.snapshot['secret_type']
self.secret_type = self.execution.snapshot.get('secret_type')
self.secret_strategy = self.execution.snapshot.get(
'secret_strategy', SecretStrategy.custom
)
self.ssh_key_change_strategy = self.execution.snapshot.get(
'ssh_key_change_strategy', SSHKeyStrategy.add
)
self.snapshot_account_usernames = self.execution.snapshot['accounts']
self.account_ids = self.execution.snapshot['accounts']
self.name_recorder_mapper = {} # 做个映射,方便后面处理
@classmethod
def method_type(cls):
return AutomationTypes.change_secret
def get_kwargs(self, account, secret):
def get_kwargs(self, account, secret, secret_type):
kwargs = {}
if self.secret_type != SecretType.SSH_KEY:
if secret_type != SecretType.SSH_KEY:
return kwargs
kwargs['strategy'] = self.ssh_key_change_strategy
kwargs['exclusive'] = 'yes' if kwargs['strategy'] == SSHKeyStrategy.set else 'no'
@ -54,18 +54,29 @@ class ChangeSecretManager(AccountBasePlaybookManager):
kwargs['regexp'] = '.*{}$'.format(secret.split()[2].strip())
return kwargs
@lazyproperty
def secret_generator(self):
def secret_generator(self, secret_type):
return SecretGenerator(
self.secret_strategy, self.secret_type,
self.secret_strategy, secret_type,
self.execution.snapshot.get('password_rules')
)
def get_secret(self):
def get_secret(self, secret_type):
if self.secret_strategy == SecretStrategy.custom:
return self.execution.snapshot['secret']
else:
return self.secret_generator.get_secret()
return self.secret_generator(secret_type).get_secret()
def get_accounts(self, privilege_account):
if not privilege_account:
print(f'not privilege account')
return []
asset = privilege_account.asset
accounts = asset.accounts.exclude(username=privilege_account.username)
accounts = accounts.filter(id__in=self.account_ids)
if self.secret_type:
accounts = accounts.filter(secret_type=self.secret_type)
return accounts
def host_callback(
self, host, asset=None, account=None,
@ -78,17 +89,10 @@ class ChangeSecretManager(AccountBasePlaybookManager):
if host.get('error'):
return host
accounts = asset.accounts.all()
if account:
accounts = accounts.exclude(username=account.username)
if '*' not in self.snapshot_account_usernames:
accounts = accounts.filter(username__in=self.snapshot_account_usernames)
accounts = accounts.filter(secret_type=self.secret_type)
accounts = self.get_accounts(account)
if not accounts:
print('没有发现待改密账号: %s 用户: %s 类型: %s' % (
asset.name, self.snapshot_account_usernames, self.secret_type
print('没有发现待改密账号: %s 用户ID: %s 类型: %s' % (
asset.name, self.account_ids, self.secret_type
))
return []
@ -97,16 +101,16 @@ class ChangeSecretManager(AccountBasePlaybookManager):
method_hosts = [h for h in method_hosts if h != host['name']]
inventory_hosts = []
records = []
host['secret_type'] = self.secret_type
if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY:
print(f'Windows {asset} does not support ssh key push \n')
print(f'Windows {asset} does not support ssh key push')
return inventory_hosts
for account in accounts:
h = deepcopy(host)
secret_type = account.secret_type
h['name'] += '(' + account.username + ')'
new_secret = self.get_secret()
new_secret = self.get_secret(secret_type)
recorder = ChangeSecretRecord(
asset=asset, account=account, execution=self.execution,
@ -116,15 +120,15 @@ class ChangeSecretManager(AccountBasePlaybookManager):
self.name_recorder_mapper[h['name']] = recorder
private_key_path = None
if self.secret_type == SecretType.SSH_KEY:
if secret_type == SecretType.SSH_KEY:
private_key_path = self.generate_private_key_path(new_secret, path_dir)
new_secret = self.generate_public_key(new_secret)
h['kwargs'] = self.get_kwargs(account, new_secret)
h['kwargs'] = self.get_kwargs(account, new_secret, secret_type)
h['account'] = {
'name': account.name,
'username': account.username,
'secret_type': account.secret_type,
'secret_type': secret_type,
'secret': new_secret,
'private_key_path': private_key_path
}

View File

@ -1,9 +1,6 @@
from copy import deepcopy
from django.db.models import QuerySet
from accounts.const import AutomationTypes, SecretType
from accounts.models import Account
from assets.const import HostTypes
from common.utils import get_logger
from ..base.manager import AccountBasePlaybookManager
@ -19,36 +16,6 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager):
def method_type(cls):
return AutomationTypes.push_account
def create_nonlocal_accounts(self, accounts, snapshot_account_usernames, asset):
secret_type = self.secret_type
usernames = accounts.filter(secret_type=secret_type).values_list(
'username', flat=True
)
create_usernames = set(snapshot_account_usernames) - set(usernames)
create_account_objs = [
Account(
name=f'{username}-{secret_type}', username=username,
secret_type=secret_type, asset=asset,
)
for username in create_usernames
]
Account.objects.bulk_create(create_account_objs)
def get_accounts(self, privilege_account, accounts: QuerySet):
if not privilege_account:
print(f'not privilege account')
return []
snapshot_account_usernames = self.execution.snapshot['accounts']
if '*' in snapshot_account_usernames:
return accounts.exclude(username=privilege_account.username)
asset = privilege_account.asset
self.create_nonlocal_accounts(accounts, snapshot_account_usernames, asset)
accounts = asset.accounts.exclude(username=privilege_account.username).filter(
username__in=snapshot_account_usernames, secret_type=self.secret_type
)
return accounts
def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs):
host = super(ChangeSecretManager, self).host_callback(
host, asset=asset, account=account, automation=automation,
@ -57,19 +24,21 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager):
if host.get('error'):
return host
accounts = asset.accounts.all()
accounts = self.get_accounts(account, accounts)
accounts = self.get_accounts(account)
inventory_hosts = []
host['secret_type'] = self.secret_type
if asset.type == HostTypes.WINDOWS and self.secret_type == SecretType.SSH_KEY:
msg = f'Windows {asset} does not support ssh key push \n'
msg = f'Windows {asset} does not support ssh key push'
print(msg)
return inventory_hosts
for account in accounts:
h = deepcopy(host)
secret_type = account.secret_type
h['name'] += '(' + account.username + ')'
new_secret = self.get_secret()
if self.secret_type is None:
new_secret = account.secret
else:
new_secret = self.get_secret(secret_type)
self.name_recorder_mapper[h['name']] = {
'account': account, 'new_secret': new_secret,
@ -80,11 +49,11 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager):
private_key_path = self.generate_private_key_path(new_secret, path_dir)
new_secret = self.generate_public_key(new_secret)
h['kwargs'] = self.get_kwargs(account, new_secret)
h['kwargs'] = self.get_kwargs(account, new_secret, secret_type)
h['account'] = {
'name': account.name,
'username': account.username,
'secret_type': account.secret_type,
'secret_type': secret_type,
'secret': new_secret,
'private_key_path': private_key_path
}
@ -112,9 +81,9 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager):
logger.error("Pust account error: ", e)
def run(self, *args, **kwargs):
if not self.check_secret():
if self.secret_type and not self.check_secret():
return
super().run(*args, **kwargs)
super(ChangeSecretManager, self).run(*args, **kwargs)
# @classmethod
# def trigger_by_asset_create(cls, asset):

View File

@ -25,6 +25,15 @@ class VerifyAccountManager(AccountBasePlaybookManager):
f.write('ssh_args = -o ControlMaster=no -o ControlPersist=no\n')
return path
@classmethod
def method_type(cls):
return AutomationTypes.verify_account
def get_accounts(self, privilege_account, accounts: QuerySet):
account_ids = self.execution.snapshot['accounts']
accounts = accounts.filter(id__in=account_ids)
return accounts
def host_callback(self, host, asset=None, account=None, automation=None, path_dir=None, **kwargs):
host = super().host_callback(
host, asset=asset, account=account,
@ -62,16 +71,6 @@ class VerifyAccountManager(AccountBasePlaybookManager):
inventory_hosts.append(h)
return inventory_hosts
@classmethod
def method_type(cls):
return AutomationTypes.verify_account
def get_accounts(self, privilege_account, accounts: QuerySet):
snapshot_account_usernames = self.execution.snapshot['accounts']
if '*' not in snapshot_account_usernames:
accounts = accounts.filter(username__in=snapshot_account_usernames)
return accounts
def on_host_success(self, host, result):
account = self.host_account_mapper.get(host)
account.set_connectivity(Connectivity.OK)

View File

@ -1,6 +1,6 @@
from common.utils import get_logger
from accounts.const import AutomationTypes
from assets.automations.ping_gateway.manager import PingGatewayManager
from common.utils import get_logger
logger = get_logger(__name__)
@ -16,6 +16,6 @@ class VerifyGatewayAccountManager(PingGatewayManager):
logger.info(">>> 开始执行测试网关账号可连接性任务")
def get_accounts(self, gateway):
usernames = self.execution.snapshot['accounts']
accounts = gateway.accounts.filter(username__in=usernames)
account_ids = self.execution.snapshot['accounts']
accounts = gateway.accounts.filter(id__in=account_ids)
return accounts

View File

@ -0,0 +1,69 @@
# Generated by Django 3.2.16 on 2023-03-07 07:36
from django.db import migrations
from django.db.models import Q
def get_nodes_all_assets(apps, *nodes):
node_model = apps.get_model('assets', 'Node')
asset_model = apps.get_model('assets', 'Asset')
node_ids = set()
descendant_node_query = Q()
for n in nodes:
node_ids.add(n.id)
descendant_node_query |= Q(key__istartswith=f'{n.key}:')
if descendant_node_query:
_ids = node_model.objects.order_by().filter(descendant_node_query).values_list('id', flat=True)
node_ids.update(_ids)
return asset_model.objects.order_by().filter(nodes__id__in=node_ids).distinct()
def get_all_assets(apps, snapshot):
node_model = apps.get_model('assets', 'Node')
asset_model = apps.get_model('assets', 'Asset')
asset_ids = snapshot.get('assets', [])
node_ids = snapshot.get('nodes', [])
nodes = node_model.objects.filter(id__in=node_ids)
node_asset_ids = get_nodes_all_assets(apps, *nodes).values_list('id', flat=True)
asset_ids = set(list(asset_ids) + list(node_asset_ids))
return asset_model.objects.filter(id__in=asset_ids)
def migrate_account_usernames_to_ids(apps, schema_editor):
db_alias = schema_editor.connection.alias
execution_model = apps.get_model('accounts', 'AutomationExecution')
account_model = apps.get_model('accounts', 'Account')
executions = execution_model.objects.using(db_alias).all()
executions_update = []
for execution in executions:
snapshot = execution.snapshot
accounts = account_model.objects.none()
account_usernames = snapshot.get('accounts', [])
for asset in get_all_assets(apps, snapshot):
accounts = accounts | asset.accounts.all()
secret_type = snapshot.get('secret_type')
if secret_type:
ids = accounts.filter(
username__in=account_usernames,
secret_type=secret_type
).values_list('id', flat=True)
else:
ids = accounts.filter(
username__in=account_usernames
).values_list('id', flat=True)
snapshot['accounts'] = [str(_id) for _id in ids]
execution.snapshot = snapshot
executions_update.append(execution)
execution_model.objects.bulk_update(executions_update, ['snapshot'])
class Migration(migrations.Migration):
dependencies = [
('accounts', '0008_alter_gatheredaccount_options'),
]
operations = [
migrations.RunPython(migrate_account_usernames_to_ids),
]

View File

@ -1,11 +1,12 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from common.db import fields
from common.db.models import JMSBaseModel
from accounts.const import (
AutomationTypes, SecretType, SecretStrategy, SSHKeyStrategy
)
from accounts.models import Account
from common.db import fields
from common.db.models import JMSBaseModel
from .base import AccountBaseAutomation
__all__ = ['ChangeSecretAutomation', 'ChangeSecretRecord', 'ChangeSecretMixin']
@ -27,18 +28,35 @@ class ChangeSecretMixin(models.Model):
default=SSHKeyStrategy.add, verbose_name=_('SSH key change strategy')
)
accounts: list[str] # account usernames
get_all_assets: callable # get all assets
class Meta:
abstract = True
def create_nonlocal_accounts(self, usernames, asset):
pass
def get_account_ids(self):
usernames = self.accounts
accounts = Account.objects.none()
for asset in self.get_all_assets():
self.create_nonlocal_accounts(usernames, asset)
accounts = accounts | asset.accounts.all()
account_ids = accounts.filter(
username__in=usernames, secret_type=self.secret_type
).values_list('id', flat=True)
return [str(_id) for _id in account_ids]
def to_attr_json(self):
attr_json = super().to_attr_json()
attr_json.update({
'secret': self.secret,
'secret_type': self.secret_type,
'secret_strategy': self.secret_strategy,
'accounts': self.get_account_ids(),
'password_rules': self.password_rules,
'secret_strategy': self.secret_strategy,
'ssh_key_change_strategy': self.ssh_key_change_strategy,
})
return attr_json

View File

@ -2,6 +2,7 @@ from django.db import models
from django.utils.translation import ugettext_lazy as _
from accounts.const import AutomationTypes
from accounts.models import Account
from jumpserver.utils import has_valid_xpack_license
from .base import AccountBaseAutomation
from .change_secret import ChangeSecretMixin
@ -14,6 +15,21 @@ class PushAccountAutomation(ChangeSecretMixin, AccountBaseAutomation):
username = models.CharField(max_length=128, verbose_name=_('Username'))
action = models.CharField(max_length=16, verbose_name=_('Action'))
def create_nonlocal_accounts(self, usernames, asset):
secret_type = self.secret_type
account_usernames = asset.accounts.filter(secret_type=self.secret_type).values_list(
'username', flat=True
)
create_usernames = set(usernames) - set(account_usernames)
create_account_objs = [
Account(
name=f'{username}-{secret_type}', username=username,
secret_type=secret_type, asset=asset,
)
for username in create_usernames
]
Account.objects.bulk_create(create_account_objs)
def set_period_schedule(self):
pass

View File

@ -23,12 +23,10 @@ def push_accounts_to_assets_task(account_ids):
task_name = gettext_noop("Push accounts to assets")
task_name = PushAccountAutomation.generate_unique_name(task_name)
for account in accounts:
task_snapshot = {
'secret': account.secret,
'secret_type': account.secret_type,
'accounts': [account.username],
'assets': [str(account.asset_id)],
}
tp = AutomationTypes.push_account
quickstart_automation_by_snapshot(task_name, tp, task_snapshot)
task_snapshot = {
'accounts': [str(account.id) for account in accounts],
'assets': [str(account.asset_id) for account in accounts],
}
tp = AutomationTypes.push_account
quickstart_automation_by_snapshot(task_name, tp, task_snapshot)

View File

@ -17,9 +17,9 @@ __all__ = [
def verify_connectivity_util(assets, tp, accounts, task_name):
if not assets or not accounts:
return
account_usernames = list(accounts.values_list('username', flat=True))
account_ids = [str(account.id) for account in accounts]
task_snapshot = {
'accounts': account_usernames,
'accounts': account_ids,
'assets': [str(asset.id) for asset in assets],
}
quickstart_automation_by_snapshot(task_name, tp, task_snapshot)

View File

@ -12,8 +12,7 @@ from django.utils.translation import gettext as _
from sshtunnel import SSHTunnelForwarder, BaseSSHTunnelForwarderError
from assets.automations.methods import platform_automation_methods
from common.utils import get_logger, lazyproperty
from common.utils import ssh_pubkey_gen, is_openssh_format_key
from common.utils import get_logger, lazyproperty, is_openssh_format_key, ssh_pubkey_gen
from ops.ansible import JMSInventory, PlaybookRunner, DefaultCallback
logger = get_logger(__name__)

View File

@ -1,10 +1,12 @@
# -*- coding: utf-8 -*-
#
from django.db.models import Q
from django.db.models import Q, QuerySet
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from accounts.models import AccountTemplate, Account
from accounts.tasks import push_accounts_to_assets_task
from assets.models import Asset, Node
from common.serializers.fields import BitChoicesField, ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
@ -31,6 +33,8 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
is_expired = serializers.BooleanField(read_only=True, label=_("Is expired"))
accounts = serializers.ListField(label=_("Account"), required=False)
template_accounts: QuerySet
class Meta:
model = AssetPermission
fields_mini = ["id", "name"]
@ -73,8 +77,55 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
actions.default = list(actions.choices.keys())
@staticmethod
def validate_accounts(accounts):
return list(set(accounts))
def get_all_assets(nodes, assets):
node_asset_ids = Node.get_nodes_all_assets(*nodes).values_list('id', flat=True)
direct_asset_ids = [asset.id for asset in assets]
asset_ids = set(direct_asset_ids + list(node_asset_ids))
return Asset.objects.filter(id__in=asset_ids)
def create_accounts(self, assets):
need_create_accounts = []
account_attribute = [
'name', 'username', 'secret_type', 'secret', 'privileged', 'is_active', 'org_id'
]
for asset in assets:
asset_exist_accounts = Account.objects.none()
for template in self.template_accounts:
asset_exist_accounts |= asset.accounts.filter(
username=template.username,
secret_type=template.secret_type,
)
username_secret_type_dict = asset_exist_accounts.values('username', 'secret_type')
for template in self.template_accounts:
condition = {
'username': template.username,
'secret_type': template.secret_type
}
if condition in username_secret_type_dict:
continue
account_data = {key: getattr(template, key) for key in account_attribute}
account_data['name'] = f"{account_data['name']}-clone"
need_create_accounts.append(Account(**{'asset_id': asset.id, **account_data}))
return Account.objects.bulk_create(need_create_accounts)
def create_and_push_account(self, nodes, assets):
if not self.template_accounts:
return
assets = self.get_all_assets(nodes, assets)
accounts = self.create_accounts(assets)
push_accounts_to_assets_task.delay([str(account.id) for account in accounts])
def validate_accounts(self, usernames: list[str]):
template_ids = []
account_usernames = []
for username in usernames:
if username.startswith('%'):
template_ids.append(username[1:])
else:
account_usernames.append(username)
self.template_accounts = AccountTemplate.objects.filter(id__in=template_ids)
template_usernames = list(self.template_accounts.values_list('username', flat=True))
return list(set(account_usernames + template_usernames))
@classmethod
def setup_eager_loading(cls, queryset):
@ -112,6 +163,13 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
).distinct()
instance.nodes.add(*nodes_to_set)
def validate(self, attrs):
self.create_and_push_account(
attrs.get("nodes", []),
attrs.get("assets", [])
)
return super().validate(attrs)
def create(self, validated_data):
display = {
"users_display": validated_data.pop("users_display", ""),