From 41589c5305d346203ffe7636cbfb6d19a6bf2e3c Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 29 Sep 2022 20:44:45 +0800 Subject: [PATCH 01/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20ansible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/account/account.py | 6 +- apps/assets/api/platform.py | 6 +- apps/assets/models/automation/__init__.py | 4 + .../models/automation/account_discovery.py} | 6 +- .../models/automation/account_reconcile.py} | 6 +- .../models/automation/account_verify.py} | 4 +- .../models/automation/base.py} | 6 +- .../models/automation/change_secret.py} | 35 +--- apps/assets/models/base.py | 22 +-- apps/assets/models/domain.py | 3 + apps/assets/tasks/account_connectivity.py | 5 +- apps/assets/tasks/asset_connectivity.py | 21 ++- .../tasks/gather_asset_hardware_info.py | 27 ++-- apps/assets/tasks/gather_asset_users.py | 3 +- apps/ops/ansible/callback.py | 9 ++ apps/ops/ansible/inventory.py | 150 +++++++++++++++++- apps/ops/ansible/new_callback.py | 65 ++++++++ apps/ops/ansible/new_runner.py | 41 ++++- .../ops/migrations/0023_auto_20220929_2025.py | 44 +++++ .../migrations/0023_automation_strategy.py | 123 -------------- apps/ops/models/__init__.py | 1 - apps/ops/models/automation/__init__.py | 5 - apps/ops/models/automation/base.py | 2 - apps/ops/models/celery.py | 24 +-- apps/ops/models/playbook.py | 16 ++ apps/ops/signal_handlers.py | 55 ++++++- apps/ops/tasks.py | 35 +--- apps/orgs/caches.py | 3 +- apps/orgs/tasks.py | 6 +- 29 files changed, 450 insertions(+), 283 deletions(-) create mode 100644 apps/assets/models/automation/__init__.py rename apps/{ops/models/automation/collect.py => assets/models/automation/account_discovery.py} (69%) rename apps/{ops/models/automation/push.py => assets/models/automation/account_reconcile.py} (70%) rename apps/{ops/models/automation/verify.py => assets/models/automation/account_verify.py} (80%) rename apps/{ops/models/automation/common.py => assets/models/automation/base.py} (93%) rename apps/{ops/models/automation/change_auth.py => assets/models/automation/change_secret.py} (57%) create mode 100644 apps/ops/ansible/new_callback.py create mode 100644 apps/ops/migrations/0023_auto_20220929_2025.py delete mode 100644 apps/ops/migrations/0023_automation_strategy.py delete mode 100644 apps/ops/models/automation/__init__.py delete mode 100644 apps/ops/models/automation/base.py create mode 100644 apps/ops/models/playbook.py diff --git a/apps/assets/api/account/account.py b/apps/assets/api/account/account.py index 4b26c042d..bdf1de116 100644 --- a/apps/assets/api/account/account.py +++ b/apps/assets/api/account/account.py @@ -33,7 +33,7 @@ class AccountViewSet(OrgBulkModelViewSet): @action(methods=['post'], detail=True, url_path='verify') def verify_account(self, request, *args, **kwargs): account = super().get_object() - task = test_accounts_connectivity_manual.delay([account]) + task = test_accounts_connectivity_manual.delay([account.id]) return Response(data={'task': task.id}) @@ -67,8 +67,8 @@ class AccountTaskCreateAPI(CreateAPIView): return queryset def perform_create(self, serializer): - accounts = self.get_accounts() - task = test_accounts_connectivity_manual.delay(accounts) + account_ids = self.get_accounts().values_list('id', flat=True) + task = test_accounts_connectivity_manual.delay(account_ids) data = getattr(serializer, '_data', {}) data["task"] = task.id setattr(serializer, '_data', data) diff --git a/apps/assets/api/platform.py b/apps/assets/api/platform.py index 2c06d504a..31726fc02 100644 --- a/apps/assets/api/platform.py +++ b/apps/assets/api/platform.py @@ -1,12 +1,8 @@ -from rest_framework.decorators import action -from rest_framework.response import Response from common.drf.api import JMSModelViewSet from common.drf.serializers import GroupedChoiceSerializer from assets.models import Platform -from assets.serializers import PlatformSerializer, PlatformOpsMethodSerializer -from assets.const import AllTypes -from assets.playbooks import filter_platform_methods +from assets.serializers import PlatformSerializer __all__ = ['AssetPlatformViewSet'] diff --git a/apps/assets/models/automation/__init__.py b/apps/assets/models/automation/__init__.py new file mode 100644 index 000000000..4e46ff150 --- /dev/null +++ b/apps/assets/models/automation/__init__.py @@ -0,0 +1,4 @@ +from .change_secret import * +from .account_discovery import * +from .account_reconcile import * +from .account_verify import * diff --git a/apps/ops/models/automation/collect.py b/apps/assets/models/automation/account_discovery.py similarity index 69% rename from apps/ops/models/automation/collect.py rename to apps/assets/models/automation/account_discovery.py index 9710e5c52..bfe9d0d80 100644 --- a/apps/ops/models/automation/collect.py +++ b/apps/assets/models/automation/account_discovery.py @@ -1,12 +1,12 @@ from django.utils.translation import ugettext_lazy as _ from ops.const import StrategyChoice -from .common import AutomationStrategy +from .base import BaseAutomation -class CollectStrategy(AutomationStrategy): +class DiscoveryAutomation(BaseAutomation): class Meta: - verbose_name = _("Collect strategy") + verbose_name = _("Discovery strategy") def to_attr_json(self): attr_json = super().to_attr_json() diff --git a/apps/ops/models/automation/push.py b/apps/assets/models/automation/account_reconcile.py similarity index 70% rename from apps/ops/models/automation/push.py rename to apps/assets/models/automation/account_reconcile.py index f7a1bd4be..f69d1c82d 100644 --- a/apps/ops/models/automation/push.py +++ b/apps/assets/models/automation/account_reconcile.py @@ -1,12 +1,12 @@ from django.utils.translation import ugettext_lazy as _ from ops.const import StrategyChoice -from .common import AutomationStrategy +from .base import BaseAutomation -class PushStrategy(AutomationStrategy): +class ReconcileAutomation(BaseAutomation): class Meta: - verbose_name = _("Push strategy") + verbose_name = _("Reconcile strategy") def to_attr_json(self): attr_json = super().to_attr_json() diff --git a/apps/ops/models/automation/verify.py b/apps/assets/models/automation/account_verify.py similarity index 80% rename from apps/ops/models/automation/verify.py rename to apps/assets/models/automation/account_verify.py index 0726704f9..d05cb4a0d 100644 --- a/apps/ops/models/automation/verify.py +++ b/apps/assets/models/automation/account_verify.py @@ -1,10 +1,10 @@ from django.utils.translation import ugettext_lazy as _ from ops.const import StrategyChoice -from .common import AutomationStrategy +from .base import BaseAutomation -class VerifyStrategy(AutomationStrategy): +class VerifyAutomation(BaseAutomation): class Meta: verbose_name = _("Verify strategy") diff --git a/apps/ops/models/automation/common.py b/apps/assets/models/automation/base.py similarity index 93% rename from apps/ops/models/automation/common.py rename to apps/assets/models/automation/base.py index a586c85a9..27c971e0d 100644 --- a/apps/ops/models/automation/common.py +++ b/apps/assets/models/automation/base.py @@ -12,8 +12,7 @@ from ops.tasks import execute_automation_strategy from ops.task_handlers import ExecutionManager -class AutomationStrategy(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) +class BaseAutomation(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): accounts = models.JSONField(default=list, verbose_name=_("Accounts")) nodes = models.ManyToManyField( 'assets.Node', related_name='automation_strategy', blank=True, verbose_name=_("Nodes") @@ -21,6 +20,7 @@ class AutomationStrategy(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): assets = models.ManyToManyField( 'assets.Asset', related_name='automation_strategy', blank=True, verbose_name=_("Assets") ) + type = models.CharField(max_length=16, verbose_name=_('Type')) comment = models.TextField(blank=True, verbose_name=_('Comment')) def __str__(self): @@ -67,7 +67,7 @@ class AutomationStrategyExecution(OrgModelMixin): default=dict, blank=True, null=True, verbose_name=_('Automation snapshot') ) strategy = models.ForeignKey( - 'AutomationStrategy', related_name='execution', on_delete=models.CASCADE, + 'assets.models.automation.base.BaseAutomation', related_name='execution', on_delete=models.CASCADE, verbose_name=_('Automation strategy') ) trigger = models.CharField( diff --git a/apps/ops/models/automation/change_auth.py b/apps/assets/models/automation/change_secret.py similarity index 57% rename from apps/ops/models/automation/change_auth.py rename to apps/assets/models/automation/change_secret.py index 71adb010f..d176f5c6a 100644 --- a/apps/ops/models/automation/change_auth.py +++ b/apps/assets/models/automation/change_secret.py @@ -1,38 +1,19 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ +from common.db import fields from ops.const import SSHKeyStrategy, PasswordStrategy, StrategyChoice from ops.utils import generate_random_password -from common.db.fields import ( - EncryptCharField, EncryptTextField, JsonDictCharField -) -from .common import AutomationStrategy +from .base import BaseAutomation -class ChangeAuthStrategy(AutomationStrategy): - is_password = models.BooleanField(default=True) - password_strategy = models.CharField( - max_length=128, blank=True, null=True, choices=PasswordStrategy.choices, - verbose_name=_('Password strategy') - ) - password_rules = JsonDictCharField( - max_length=2048, blank=True, null=True, verbose_name=_('Password rules') - ) - password = EncryptCharField( - max_length=256, blank=True, null=True, verbose_name=_('Password') - ) +class ChangePasswordAutomation(BaseAutomation): + class PasswordStrategy(models.TextChoices): + custom = 'specific', _('Specific') + random_one = 'random_one', _('All assets use the same random password') + random_all = 'random_all', _('All assets use different random password') - is_ssh_key = models.BooleanField(default=False) - ssh_key_strategy = models.CharField( - max_length=128, blank=True, null=True, choices=SSHKeyStrategy.choices, - verbose_name=_('SSH Key strategy') - ) - private_key = EncryptTextField( - max_length=4096, blank=True, null=True, verbose_name=_('SSH private key') - ) - public_key = EncryptTextField( - max_length=4096, blank=True, null=True, verbose_name=_('SSH public key') - ) + password = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret')) recipients = models.ManyToManyField( 'users.User', related_name='recipients_change_auth_strategy', blank=True, verbose_name=_("Recipient") diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 7f96a07e9..7b5b1d6f8 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -76,6 +76,15 @@ class BaseAccount(OrgModelMixin): def has_secret(self): return bool(self.secret) + @property + def password(self): + return self.secret + + @password.setter + def password(self, value): + self.secret = value + self.secret_type = 'password' + @property def private_key(self): if self.secret_type == self.SecretType.ssh_key: @@ -91,15 +100,6 @@ class BaseAccount(OrgModelMixin): self.secret = value self.secret_type = 'private_key' - @property - def password(self): - return self.secret - - @password.setter - def password(self, value): - self.secret = value - self.secret_type = 'password' - @property def ssh_key_fingerprint(self): if self.public_key: @@ -125,8 +125,8 @@ class BaseAccount(OrgModelMixin): return None @property - def private_key_file(self): - if not self.private_key_obj: + def private_key_path(self): + if not self.secret_type != 'ssh_key' or not self.secret: return None project_dir = settings.PROJECT_DIR tmp_dir = os.path.join(project_dir, 'tmp') diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index 219595d2c..4abe8aa68 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -40,6 +40,9 @@ class Domain(OrgModelMixin): def gateways(self): return self.gateway_set.filter(is_active=True) + def select_gateway(self): + return self.random_gateway() + def random_gateway(self): gateways = [gw for gw in self.gateways if gw.is_connective] if gateways: diff --git a/apps/assets/tasks/account_connectivity.py b/apps/assets/tasks/account_connectivity.py index c28f93110..694a7fe3a 100644 --- a/apps/assets/tasks/account_connectivity.py +++ b/apps/assets/tasks/account_connectivity.py @@ -5,7 +5,7 @@ from django.utils.translation import ugettext as _, gettext_noop from common.utils import get_logger from orgs.utils import org_aware_func -from ..models import Connectivity +from ..models import Connectivity, Account from . import const from .utils import check_asset_can_run_ansible @@ -99,10 +99,11 @@ def test_account_connectivity_util(account, task_name): @shared_task(queue="ansible") -def test_accounts_connectivity_manual(accounts): +def test_accounts_connectivity_manual(account_ids): """ :param accounts: 对象 """ + accounts = Account.objects.filter(id__in=account_ids) for account in accounts: task_name = gettext_noop("Test account connectivity: ") + str(account) test_account_connectivity_util(account, task_name) diff --git a/apps/assets/tasks/asset_connectivity.py b/apps/assets/tasks/asset_connectivity.py index 8c76d8e2b..ce379832a 100644 --- a/apps/assets/tasks/asset_connectivity.py +++ b/apps/assets/tasks/asset_connectivity.py @@ -5,8 +5,8 @@ from celery import shared_task from django.utils.translation import gettext_noop from common.utils import get_logger -from orgs.utils import org_aware_func -from ..models import Asset, Connectivity, Account +from orgs.utils import org_aware_func, tmp_to_root_org +from ..models import Asset, Connectivity, Account, Node from . import const from .utils import clean_ansible_task_hosts, group_asset_by_platform @@ -41,8 +41,7 @@ def set_assets_accounts_connectivity(assets, results_summary): Account.bulk_set_connectivity(accounts_failed, Connectivity.failed) -@shared_task(queue="ansible") -@org_aware_func("assets") +@org_aware_func('assets') def test_asset_connectivity_util(assets, task_name=None): from ops.utils import update_or_create_ansible_task @@ -88,7 +87,10 @@ def test_asset_connectivity_util(assets, task_name=None): @shared_task(queue="ansible") -def test_asset_connectivity_manual(asset): +def test_asset_connectivity_manual(asset_id): + asset = Asset.objects.filter(id=asset_id).first() + if not asset: + return task_name = gettext_noop("Test assets connectivity: ") + str(asset) summary = test_asset_connectivity_util([asset], task_name=task_name) @@ -99,7 +101,9 @@ def test_asset_connectivity_manual(asset): @shared_task(queue="ansible") -def test_assets_connectivity_manual(assets): +def test_assets_connectivity_manual(asset_ids): + with tmp_to_root_org(): + assets = Asset.objects.filter(id__in=asset_ids) task_name = gettext_noop("Test assets connectivity: ") + str([asset.name for asset in assets]) summary = test_asset_connectivity_util(assets, task_name=task_name) @@ -110,7 +114,10 @@ def test_assets_connectivity_manual(assets): @shared_task(queue="ansible") -def test_node_assets_connectivity_manual(node): +def test_node_assets_connectivity_manual(node_id): + with tmp_to_root_org(): + node = Node.objects.get(id=node_id) + task_name = gettext_noop("Test if the assets under the node are connectable: ") + node.name assets = node.get_all_assets() result = test_asset_connectivity_util(assets, task_name=task_name) diff --git a/apps/assets/tasks/gather_asset_hardware_info.py b/apps/assets/tasks/gather_asset_hardware_info.py index 7678f1115..1973dd4ff 100644 --- a/apps/assets/tasks/gather_asset_hardware_info.py +++ b/apps/assets/tasks/gather_asset_hardware_info.py @@ -9,8 +9,9 @@ from django.utils.translation import ugettext as _, gettext_noop from common.utils import ( capacity_convert, sum_capacity, get_logger ) -from orgs.utils import org_aware_func +from orgs.utils import org_aware_func, tmp_to_root_org from . import const +from ..models import Asset, Node from .utils import clean_ansible_task_hosts @@ -27,7 +28,6 @@ def set_assets_hardware_info(assets, result, **kwargs): """ Using ops task run result, to update asset info - @shared_task must be exit, because we using it as a task callback, is must be a celery task also :param assets: :param result: @@ -83,15 +83,15 @@ def set_assets_hardware_info(assets, result, **kwargs): return assets_updated -@shared_task -@org_aware_func("assets") +@org_aware_func('assets') def update_assets_hardware_info_util(assets, task_name=None): """ Using ansible api to update asset hardware info - :param assets: asset seq + :param asset_ids: asset seq :param task_name: task_name running :return: result summary ['contacted': {}, 'dark': {}] """ + from ops.utils import update_or_create_ansible_task if task_name is None: task_name = gettext_noop("Update some assets hardware info. ") @@ -110,15 +110,19 @@ def update_assets_hardware_info_util(assets, task_name=None): @shared_task(queue="ansible") -def update_asset_hardware_info_manual(asset): +def update_asset_hardware_info_manual(asset_id): + with tmp_to_root_org(): + asset = Asset.objects.filter(id=asset_id).first() + if not asset: + return task_name = gettext_noop("Update asset hardware info: ") + str(asset.name) update_assets_hardware_info_util([asset], task_name=task_name) @shared_task(queue="ansible") -def update_assets_hardware_info_manual(assets): +def update_assets_hardware_info_manual(asset_ids): task_name = gettext_noop("Update assets hardware info: ") + str([asset.name for asset in assets]) - update_assets_hardware_info_util(assets, task_name=task_name) + update_assets_hardware_info_util(asset_ids, task_name=task_name) @shared_task(queue="ansible") @@ -133,7 +137,12 @@ def update_assets_hardware_info_period(): @shared_task(queue="ansible") -def update_node_assets_hardware_info_manual(node): +def update_node_assets_hardware_info_manual(node_id): + with tmp_to_root_org(): + node = Node.objects.filter(id=node_id).first() + if not node: + return + task_name = gettext_noop("Update node asset hardware information: ") + str(node.name) assets = node.get_all_assets() result = update_assets_hardware_info_util(assets, task_name=task_name) diff --git a/apps/assets/tasks/gather_asset_users.py b/apps/assets/tasks/gather_asset_users.py index acacbb33d..0ce6c7453 100644 --- a/apps/assets/tasks/gather_asset_users.py +++ b/apps/assets/tasks/gather_asset_users.py @@ -7,7 +7,7 @@ from celery import shared_task from django.utils.translation import gettext_noop from django.utils import timezone -from orgs.utils import tmp_to_org, org_aware_func +from orgs.utils import tmp_to_org, org_aware_func, tmp_to_root_org from common.utils import get_logger from ..models import GatheredUser, Node from .utils import clean_ansible_task_hosts @@ -103,7 +103,6 @@ def add_asset_users(assets, results): ) -@shared_task(queue="ansible") @org_aware_func("assets") def gather_asset_users(assets, task_name=None): from ops.utils import update_or_create_ansible_task diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index 3fe1933ac..cc879cc4c 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -72,6 +72,15 @@ class AdHocResultCallback(CallbackMixin, CallbackModule, CMDCallBackModule): Task result Callback """ context = None + events = [ + 'runner_on_failed', 'runner_on_ok', + 'runner_on_skipped', 'runner_on_unreachable', + ] + + def event_handler(self, data): + event = data.get('event', None) + print("Event: ", event) + print("Event Data: ", json.dumps(data)) def clean_result(self, t, host, task_name, task_result): contacted = self.results_summary["contacted"] diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index e024bc45b..dabda5e4e 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -1,4 +1,7 @@ # ~*~ coding: utf-8 ~*~ +from collections import defaultdict +import json + from ansible.inventory.host import Host from ansible.vars.manager import VariableManager from ansible.inventory.manager import InventoryManager @@ -21,7 +24,7 @@ class BaseHost(Host): # behind is not must be required "username": "", "password": "", - "private_key": "", + "private_key_path": "", "become": { "method": "", "user": "", @@ -49,8 +52,8 @@ class BaseHost(Host): # 添加密码和密钥 if host_data.get('password'): self.set_variable('ansible_ssh_pass', host_data['password']) - if host_data.get('private_key'): - self.set_variable('ansible_ssh_private_key_file', host_data['private_key']) + if host_data.get('private_key_path'): + self.set_variable('ansible_ssh_private_key_file', host_data['private_key_path']) # 添加become支持 become = host_data.get("become", False) @@ -155,13 +158,144 @@ class BaseInventory(InventoryManager): class JMSInventory: - def __init__(self, assets, account=None, ansible_connection='ssh', - account_policy='smart', host_var_callback=None): + def __init__(self, assets, account_username=None, account_policy='smart', host_var_callback=None): """ :param assets: - :param account: account username name if not set use account_policy - :param ansible_connection: ssh, local, + :param account_username: account username name if not set use account_policy :param account_policy: :param host_var_callback: """ - pass + self.assets = self.clean_assets(assets) + self.account_username = account_username + self.account_policy = account_policy + self.host_var_callback = host_var_callback + + @staticmethod + def clean_assets(assets): + from assets.models import Asset + asset_ids = [asset.id for asset in assets] + assets = Asset.objects.filter(id__in=asset_ids)\ + .prefetch_related('platform', 'domain', 'accounts') + return assets + + @staticmethod + def group_by_platform(assets): + groups = defaultdict(list) + for asset in assets: + groups[asset.platform].append(asset) + return groups + + @staticmethod + def make_proxy_command(gateway): + proxy_command_list = [ + "ssh", "-o", "Port={}".format(gateway.port), + "-o", "StrictHostKeyChecking=no", + "{}@{}".format(gateway.username, gateway.address), + "-W", "%h:%p", "-q", + ] + + if gateway.password: + proxy_command_list.insert( + 0, "sshpass -p '{}'".format(gateway.password) + ) + if gateway.private_key: + proxy_command_list.append("-i {}".format(gateway.private_key_file)) + + proxy_command = "'-o ProxyCommand={}'".format( + " ".join(proxy_command_list) + ) + return {"ansible_ssh_common_args": proxy_command} + + def asset_to_host(self, asset, account, automation, protocols): + host = {'name': asset.name, 'vars': { + 'asset_id': str(asset.id), 'asset_name': asset.name, + 'asset_type': asset.type, 'asset_category': asset.category, + }} + ansible_connection = automation.ansible_config.get('ansible_connection', 'ssh') + gateway = None + if asset.domain: + gateway = asset.domain.select_gateway() + + ssh_protocol_matched = list(filter(lambda x: x.name == 'ssh', protocols)) + ssh_protocol = ssh_protocol_matched[0] if ssh_protocol_matched else None + if ansible_connection == 'local': + if gateway: + host['ansible_host'] = gateway.address + host['ansible_port'] = gateway.port + host['ansible_user'] = gateway.username + host['ansible_password'] = gateway.password + host['ansible_connection'] = 'smart' + else: + host['ansible_connection'] = 'local' + else: + host['ansible_host'] = asset.address + host['ansible_port'] = ssh_protocol.port if ssh_protocol else 22 + if account: + host['ansible_user'] = account.username + + if account.secret_type == 'password' and account.secret: + host['ansible_password'] = account.secret + elif account.secret_type == 'private_key' and account.secret: + host['ssh_private_key'] = account.private_key_file + + if gateway: + host['vars'].update(self.make_proxy_command(gateway)) + + if self.host_var_callback: + callback_var = self.host_var_callback(asset) + if isinstance(callback_var, dict): + host['vars'].update(callback_var) + return host + + def select_account(self, asset): + accounts = list(asset.accounts.all()) + if not accounts: + return None + + account_selected = None + account_username = self.account_username + + if isinstance(self.account_username, str): + account_username = [self.account_username] + if account_username: + for username in account_username: + account_matched = list(filter(lambda account: account.username == username, accounts)) + if account_matched: + account_selected = account_matched[0] + return account_selected + + if not account_selected: + if self.account_policy in ['privileged_must', 'privileged_first']: + account_selected = list(filter(lambda account: account.is_privileged, accounts)) + account_selected = account_selected[0] if account_selected else None + + if not account_selected and self.account_policy == 'privileged_first': + account_selected = accounts[0] + return account_selected + + def generate(self): + hosts = [] + platform_assets = self.group_by_platform(self.assets) + for platform, assets in platform_assets.items(): + automation = platform.automation + protocols = platform.protocols.all() + + if not automation.ansible_enabled: + continue + + for asset in self.assets: + account = self.select_account(asset) + host = self.asset_to_host(asset, account, automation, protocols) + hosts.append(host) + return hosts + + def write_to_file(self, path): + hosts = self.generate() + data = {'all': {'hosts': {}}} + for host in hosts: + name = host.pop('name') + var = host.pop('vars', {}) + host.update(var) + data['all']['hosts'][name] = host + with open(path, 'w') as f: + f.write(json.dumps(data, indent=4)) diff --git a/apps/ops/ansible/new_callback.py b/apps/ops/ansible/new_callback.py new file mode 100644 index 000000000..90f945f48 --- /dev/null +++ b/apps/ops/ansible/new_callback.py @@ -0,0 +1,65 @@ + +class JMSCallback: + def event_handler(self, data, runner_config): + event = data.get('event', None) + if not event: + return + event_data = data.get('event_data', {}) + pass + + def runner_on_ok(self, event_data): + pass + + def runer_on_failed(self, event_data): + pass + + def runner_on_skipped(self, event_data): + pass + + def runner_on_unreachable(self, event_data): + pass + + def runner_on_start(self, event_data): + pass + + def runer_retry(self, event_data): + pass + + def runner_on_file_diff(self, event_data): + pass + + def runner_item_on_failed(self, event_data): + pass + + def runner_item_on_skipped(self, event_data): + pass + + def playbook_on_play_start(self, event_data): + pass + + def playbook_on_stats(self, event_data): + pass + + def playbook_on_include(self, event_data): + pass + + def playbook_on_notify(self, event_data): + pass + + def playbook_on_vars_prompt(self, event_data): + pass + + def playbook_on_handler_task_start(self, event_data): + pass + + def playbook_on_no_hosts_matched(self, event_data): + pass + + def playbook_on_no_hosts_remaining(self, event_data): + pass + + def warning(self): + pass + + def status_handler(self): + pass diff --git a/apps/ops/ansible/new_runner.py b/apps/ops/ansible/new_runner.py index 7802c1a82..26cb121a7 100644 --- a/apps/ops/ansible/new_runner.py +++ b/apps/ops/ansible/new_runner.py @@ -1,14 +1,43 @@ +import uuid import ansible_runner - -class AnsibleInventory: - def __init__(self, assets, account=None, ansible_connection='ssh'): - self.assets = assets - self.account = account +from django.conf import settings class AdHocRunner: - pass + cmd_modules_choices = ('shell', 'raw', 'command', 'script', 'win_shell') + cmd_blacklist = [ + "reboot", 'shutdown', 'poweroff', 'halt', 'dd', 'half', 'top' + ] + + def __init__(self, inventory, module, module_args, pattern='*', project_dir='/tmp/'): + self.id = uuid.uuid4() + self.inventory = inventory + self.pattern = pattern + self.module = module + self.module_args = module_args + self.project_dir = project_dir + + def check_module(self): + if self.module not in self.cmd_modules_choices: + return + if self.module_args and self.module_args.split()[0] in self.cmd_blacklist: + raise Exception("command not allowed: {}".format(self.module_args[0])) + + def run(self, verbosity=0, **kwargs): + self.check_module() + if verbosity is None and settings.DEBUG: + verbosity = 1 + + return ansible_runner.run( + host_pattern=self.pattern, + private_data_dir=self.project_dir, + inventory=self.inventory, + module=self.module, + module_args=self.module_args, + verbosity=verbosity, + **kwargs + ) class PlaybookRunner: diff --git a/apps/ops/migrations/0023_auto_20220929_2025.py b/apps/ops/migrations/0023_auto_20220929_2025.py new file mode 100644 index 000000000..b5c7475f4 --- /dev/null +++ b/apps/ops/migrations/0023_auto_20220929_2025.py @@ -0,0 +1,44 @@ +# Generated by Django 3.2.14 on 2022-09-29 12:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0022_auto_20220817_1346'), + ] + + operations = [ + migrations.RemoveField( + model_name='celerytask', + name='log_path', + ), + migrations.RemoveField( + model_name='celerytask', + name='status', + ), + migrations.AddField( + model_name='celerytask', + name='args', + field=models.JSONField(default=[], verbose_name='Args'), + preserve_default=False, + ), + migrations.AddField( + model_name='celerytask', + name='is_finished', + field=models.BooleanField(default=False, verbose_name='Finished'), + ), + migrations.AddField( + model_name='celerytask', + name='kwargs', + field=models.JSONField(default={}, verbose_name='Kwargs'), + preserve_default=False, + ), + migrations.AddField( + model_name='celerytask', + name='state', + field=models.CharField(default='SUCCESS', max_length=16, verbose_name='State'), + preserve_default=False, + ), + ] diff --git a/apps/ops/migrations/0023_automation_strategy.py b/apps/ops/migrations/0023_automation_strategy.py deleted file mode 100644 index d2807023f..000000000 --- a/apps/ops/migrations/0023_automation_strategy.py +++ /dev/null @@ -1,123 +0,0 @@ -# Generated by Django 3.2.14 on 2022-09-08 11:58 - -import common.db.fields -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('assets', '0105_auto_20220817_1544'), - ('ops', '0022_auto_20220817_1346'), - ] - - operations = [ - migrations.CreateModel( - name='AutomationStrategy', - fields=[ - ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), - ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), - ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), - ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), - ('name', models.CharField(max_length=128, verbose_name='Name')), - ('is_periodic', models.BooleanField(default=False)), - ('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')), - ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')), - ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), - ('accounts', models.JSONField(default=list, verbose_name='Accounts')), - ('comment', models.TextField(blank=True, verbose_name='Comment')), - ('assets', models.ManyToManyField(blank=True, related_name='automation_strategy', to='assets.Asset', verbose_name='Assets')), - ('nodes', models.ManyToManyField(blank=True, related_name='automation_strategy', to='assets.Node', verbose_name='Nodes')), - ], - options={ - 'verbose_name': 'Automation plan', - 'unique_together': {('org_id', 'name')}, - }, - ), - migrations.CreateModel( - name='AutomationStrategyExecution', - fields=[ - ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), - ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), - ('date_created', models.DateTimeField(auto_now_add=True)), - ('timedelta', models.FloatField(default=0.0, null=True, verbose_name='Time')), - ('date_start', models.DateTimeField(auto_now_add=True, verbose_name='Date start')), - ('snapshot', common.db.fields.EncryptJsonDictTextField(blank=True, default=dict, null=True, verbose_name='Automation snapshot')), - ('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')], default='manual', max_length=128, verbose_name='Trigger mode')), - ('strategy', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='execution', to='ops.automationstrategy', verbose_name='Automation strategy')), - ], - options={ - 'verbose_name': 'Automation strategy execution', - }, - ), - migrations.CreateModel( - name='CollectStrategy', - fields=[ - ('automationstrategy_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='ops.automationstrategy')), - ], - options={ - 'verbose_name': 'Collect strategy', - }, - bases=('ops.automationstrategy',), - ), - migrations.CreateModel( - name='PushStrategy', - fields=[ - ('automationstrategy_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='ops.automationstrategy')), - ], - options={ - 'verbose_name': 'Push strategy', - }, - bases=('ops.automationstrategy',), - ), - migrations.CreateModel( - name='VerifyStrategy', - fields=[ - ('automationstrategy_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='ops.automationstrategy')), - ], - options={ - 'verbose_name': 'Verify strategy', - }, - bases=('ops.automationstrategy',), - ), - migrations.CreateModel( - name='AutomationStrategyTask', - fields=[ - ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), - ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), - ('is_success', models.BooleanField(default=False, verbose_name='Is success')), - ('timedelta', models.FloatField(default=0.0, null=True, verbose_name='Time')), - ('date_start', models.DateTimeField(auto_now_add=True, verbose_name='Date start')), - ('reason', models.CharField(blank=True, max_length=1024, null=True, verbose_name='Reason')), - ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.account', verbose_name='Account')), - ('asset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.asset', verbose_name='Asset')), - ('execution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='task', to='ops.automationstrategyexecution', verbose_name='Automation strategy execution')), - ], - options={ - 'verbose_name': 'Automation strategy task', - }, - ), - migrations.CreateModel( - name='ChangeAuthStrategy', - fields=[ - ('automationstrategy_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='ops.automationstrategy')), - ('is_password', models.BooleanField(default=True)), - ('password_strategy', models.CharField(blank=True, choices=[('custom', 'Custom password'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], max_length=128, null=True, verbose_name='Password strategy')), - ('password_rules', common.db.fields.JsonDictCharField(blank=True, max_length=2048, null=True, verbose_name='Password rules')), - ('password', common.db.fields.EncryptCharField(blank=True, max_length=256, null=True, verbose_name='Password')), - ('is_ssh_key', models.BooleanField(default=False)), - ('ssh_key_strategy', models.CharField(blank=True, choices=[('add', 'Append SSH KEY'), ('set', 'Empty and append SSH KEY'), ('set_jms', 'Replace (The key generated by JumpServer) ')], max_length=128, null=True, verbose_name='SSH Key strategy')), - ('private_key', common.db.fields.EncryptTextField(blank=True, max_length=4096, null=True, verbose_name='SSH private key')), - ('public_key', common.db.fields.EncryptTextField(blank=True, max_length=4096, null=True, verbose_name='SSH public key')), - ('recipients', models.ManyToManyField(blank=True, related_name='recipients_change_auth_strategy', to=settings.AUTH_USER_MODEL, verbose_name='Recipient')), - ], - options={ - 'verbose_name': 'Change auth strategy', - }, - bases=('ops.automationstrategy',), - ), - ] diff --git a/apps/ops/models/__init__.py b/apps/ops/models/__init__.py index f925b14a5..0a9ed463c 100644 --- a/apps/ops/models/__init__.py +++ b/apps/ops/models/__init__.py @@ -4,4 +4,3 @@ from .adhoc import * from .celery import * from .command import * -from .automation import * diff --git a/apps/ops/models/automation/__init__.py b/apps/ops/models/automation/__init__.py deleted file mode 100644 index 83fe4a6f9..000000000 --- a/apps/ops/models/automation/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .change_auth import * -from .collect import * -from .push import * -from .verify import * -from .common import * diff --git a/apps/ops/models/automation/base.py b/apps/ops/models/automation/base.py deleted file mode 100644 index ec51c5a2b..000000000 --- a/apps/ops/models/automation/base.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- coding: utf-8 -*- -# diff --git a/apps/ops/models/celery.py b/apps/ops/models/celery.py index 9ab5f49e1..2291eb6f1 100644 --- a/apps/ops/models/celery.py +++ b/apps/ops/models/celery.py @@ -9,32 +9,16 @@ from django.db import models class CeleryTask(models.Model): - WAITING = "waiting" - RUNNING = "running" - FINISHED = "finished" LOG_DIR = os.path.join(settings.PROJECT_DIR, 'data', 'celery') - - STATUS_CHOICES = ( - (WAITING, WAITING), - (RUNNING, RUNNING), - (FINISHED, FINISHED), - ) id = models.UUIDField(primary_key=True, default=uuid.uuid4) name = models.CharField(max_length=1024) - status = models.CharField(max_length=128, choices=STATUS_CHOICES, db_index=True) - log_path = models.CharField(max_length=256, blank=True, null=True) + args = models.JSONField(verbose_name=_("Args")) + kwargs = models.JSONField(verbose_name=_("Kwargs")) + state = models.CharField(max_length=16, verbose_name=_("State")) + is_finished = models.BooleanField(default=False, verbose_name=_("Finished")) date_published = models.DateTimeField(auto_now_add=True) date_start = models.DateTimeField(null=True) date_finished = models.DateTimeField(null=True) def __str__(self): return "{}: {}".format(self.name, self.id) - - def is_finished(self): - return self.status == self.FINISHED - - @property - def full_log_path(self): - if not self.log_path: - return None - return os.path.join(self.LOG_DIR, self.log_path) diff --git a/apps/ops/models/playbook.py b/apps/ops/models/playbook.py new file mode 100644 index 000000000..aaec7a4ef --- /dev/null +++ b/apps/ops/models/playbook.py @@ -0,0 +1,16 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from orgs.mixins.models import JMSOrgBaseModel +from ..mixin import PeriodTaskModelMixin + + +class PlaybookTask(PeriodTaskModelMixin, JMSOrgBaseModel): + assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) + account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) + playbook = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) + owner = models.CharField(max_length=1024, verbose_name=_("Owner")) + comment = models.TextField(blank=True, verbose_name=_("Comment")) + + def get_register_task(self): + pass diff --git a/apps/ops/signal_handlers.py b/apps/ops/signal_handlers.py index dfd364845..e48802d84 100644 --- a/apps/ops/signal_handlers.py +++ b/apps/ops/signal_handlers.py @@ -1,15 +1,20 @@ -from django.utils import translation +import ast + +from django.utils import translation, timezone from django.core.cache import cache -from celery.signals import task_prerun, task_postrun, before_task_publish +from celery import signals -from common.db.utils import close_old_connections +from common.db.utils import close_old_connections, get_logger +from .models import CeleryTask +logger = get_logger(__name__) + TASK_LANG_CACHE_KEY = 'TASK_LANG_{}' TASK_LANG_CACHE_TTL = 1800 -@before_task_publish.connect() +@signals.before_task_publish.connect def before_task_publish(headers=None, **kwargs): task_id = headers.get('id') current_lang = translation.get_language() @@ -17,8 +22,10 @@ def before_task_publish(headers=None, **kwargs): cache.set(key, current_lang, 1800) -@task_prerun.connect() +@signals.task_prerun.connect def on_celery_task_pre_run(task_id='', **kwargs): + # 更新状态 + CeleryTask.objects.filter(id=task_id).update(state='RUNNING', date_start=timezone.now()) # 关闭之前的数据库连接 close_old_connections() @@ -29,6 +36,40 @@ def on_celery_task_pre_run(task_id='', **kwargs): translation.activate(task_lang) -@task_postrun.connect() -def on_celery_task_post_run(**kwargs): +@signals.task_postrun.connect +def on_celery_task_post_run(task_id='', state='', **kwargs): close_old_connections() + print("Task post run: ", task_id, state) + + CeleryTask.objects.filter(id=task_id).update( + state=state, date_finished=timezone.now(), is_finished=True + ) + + +@signals.after_task_publish.connect +def task_sent_handler(headers=None, body=None, **kwargs): + info = headers if 'task' in headers else body + task = info.get('task') + i = info.get('id') + if not i or not task: + logger.error("Not found task id or name: {}".format(info)) + return + + args = info.get('argsrepr', '()') + kwargs = info.get('kwargsrepr', '{}') + try: + args = list(ast.literal_eval(args)) + kwargs = ast.literal_eval(kwargs) + except (ValueError, SyntaxError): + args = [] + kwargs = {} + + data = { + 'id': i, + 'name': task, + 'state': 'PENDING', + 'is_finished': False, + 'args': args, + 'kwargs': kwargs + } + CeleryTask.objects.create(**data) diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 2b739e3d6..cb21b5c3d 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -1,10 +1,10 @@ # coding: utf-8 import os import subprocess -import time from django.conf import settings from celery import shared_task, subtask +from celery import signals from celery.exceptions import SoftTimeLimitExceeded from django.utils import timezone @@ -30,7 +30,7 @@ def rerun_task(): pass -@shared_task(queue="ansible") +@shared_task(queue="ansible", verbose_name=_("Run ansible task")) def run_ansible_task(tid, callback=None, **kwargs): """ :param tid: is the tasks serialized data @@ -49,7 +49,7 @@ def run_ansible_task(tid, callback=None, **kwargs): return result -@shared_task(soft_time_limit=60, queue="ansible") +@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible command")) def run_command_execution(cid, **kwargs): with tmp_to_root_org(): execution = get_object_or_none(CommandExecution, id=cid) @@ -136,7 +136,7 @@ def check_server_performance_period(): ServerPerformanceCheckUtil().check_and_publish() -@shared_task(queue="ansible") +@shared_task(queue="ansible", verbose_name=_("Hello")) def hello(name, callback=None): from users.models import User import time @@ -148,38 +148,12 @@ def hello(name, callback=None): return gettext("Hello") -@shared_task -# @after_app_shutdown_clean_periodic -# @register_as_period_task(interval=30) -def hello123(): - return None - - @shared_task def hello_callback(result): print(result) print("Hello callback") -@shared_task -def add(a, b): - time.sleep(5) - return a + b - - -@shared_task -def add_m(x): - from celery import chain - a = range(x) - b = [a[i:i + 10] for i in range(0, len(a), 10)] - s = list() - s.append(add.s(b[0], b[1])) - for i in b[1:]: - s.append(add.s(i)) - res = chain(*tuple(s))() - return res - - @shared_task def execute_automation_strategy(pid, trigger): from .models import AutomationStrategy @@ -191,3 +165,4 @@ def execute_automation_strategy(pid, trigger): with tmp_to_org(instance.org): instance.execute(trigger) + diff --git a/apps/orgs/caches.py b/apps/orgs/caches.py index c3a1cb86d..5df387c91 100644 --- a/apps/orgs/caches.py +++ b/apps/orgs/caches.py @@ -6,7 +6,7 @@ from orgs.utils import current_org, tmp_to_org from common.cache import Cache, IntegerField from common.utils import get_logger from users.models import UserGroup, User -from assets.models import Node, Domain, Gateway, Asset +from assets.models import Node, Domain, Gateway, Asset, Account from terminal.models import Session from perms.models import AssetPermission @@ -52,6 +52,7 @@ class OrgResourceStatisticsCache(OrgRelatedCache): assets_amount = IntegerField() nodes_amount = IntegerField(queryset=Node.objects) + accounts_amount = IntegerField(queryset=Account.objects) domains_amount = IntegerField(queryset=Domain.objects) gateways_amount = IntegerField(queryset=Gateway.objects) asset_perms_amount = IntegerField(queryset=AssetPermission.objects) diff --git a/apps/orgs/tasks.py b/apps/orgs/tasks.py index a33456913..6b6ec9e0d 100644 --- a/apps/orgs/tasks.py +++ b/apps/orgs/tasks.py @@ -6,6 +6,6 @@ logger = get_logger(__file__) @shared_task -def refresh_org_cache_task(cache, *fields): - logger.info(f'CACHE: refresh {cache.key}.{fields}') - cache.refresh(*fields) +def refresh_org_cache_task(*fields): + from .caches import OrgResourceStatisticsCache + OrgResourceStatisticsCache.refresh(*fields) From df5e63b3be6489a3e9e96adcb41c11638206ebd0 Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 30 Sep 2022 18:49:45 +0800 Subject: [PATCH 02/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20ansible=20?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/ansible/callback.py | 398 +++++++---------------------- apps/ops/ansible/display.py | 69 ----- apps/ops/ansible/inventory.py | 151 +---------- apps/ops/ansible/new_callback.py | 65 ----- apps/ops/ansible/new_runner.py | 44 ---- apps/ops/ansible/runner.py | 301 ++++------------------ apps/ops/ansible/test_inventory.py | 63 ----- apps/ops/ansible/test_runner.py | 58 ----- 8 files changed, 151 insertions(+), 998 deletions(-) delete mode 100644 apps/ops/ansible/display.py delete mode 100644 apps/ops/ansible/new_callback.py delete mode 100644 apps/ops/ansible/new_runner.py delete mode 100644 apps/ops/ansible/test_inventory.py delete mode 100644 apps/ops/ansible/test_runner.py diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index cc879cc4c..8b6ad1f8f 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -1,334 +1,126 @@ -# ~*~ coding: utf-8 ~*~ - -import datetime -import json -import os from collections import defaultdict -import ansible.constants as C -from ansible.plugins.callback import CallbackBase -from ansible.plugins.callback.default import CallbackModule -from ansible.plugins.callback.minimal import CallbackModule as CMDCallBackModule -from common.utils.strings import safe_str - - -class CallbackMixin: - def __init__(self, display=None): - # result_raw example: { - # "ok": {"hostname": {"task_name": {}，...},..}, - # "failed": {"hostname": {"task_name": {}..}, ..}, - # "unreachable: {"hostname": {"task_name": {}, ..}}, - # "skipped": {"hostname": {"task_name": {}, ..}, ..}, - # } - # results_summary example: { - # "contacted": {"hostname": {"task_name": {}}, "hostname": {}}, - # "dark": {"hostname": {"task_name": {}, "task_name": {}},...,}, - # "success": True - # } - self.results_raw = dict( +class DefaultCallback: + def __init__(self): + self.result = dict( ok=defaultdict(dict), - failed=defaultdict(dict), - unreachable=defaultdict(dict), - skippe=defaultdict(dict), - ) - self.results_summary = dict( - contacted=defaultdict(dict), + failures=defaultdict(dict), dark=defaultdict(dict), - success=True + skipped=defaultdict(dict), ) - self.results = { - 'raw': self.results_raw, - 'summary': self.results_summary, - } - super().__init__() - if display: - self._display = display + self.summary = dict( + ok=[], + failures={}, + dark={}, + skipped=[], + ) + self.status = 'starting' + self.finished = False - cols = os.environ.get("TERM_COLS", None) - self._display.columns = 79 - if cols and cols.isdigit(): - self._display.columns = int(cols) - 1 + def is_success(self): + return self.status != 'successful' - def display(self, msg): - self._display.display(msg) - - def gather_result(self, t, result): - self._clean_results(result._result, result._task.action) - host = result._host.get_name() - task_name = result.task_name - task_result = result._result - - self.results_raw[t][host][task_name] = task_result - self.clean_result(t, host, task_name, task_result) - - def close(self): - if hasattr(self._display, 'close'): - self._display.close() - - -class AdHocResultCallback(CallbackMixin, CallbackModule, CMDCallBackModule): - """ - Task result Callback - """ - context = None - events = [ - 'runner_on_failed', 'runner_on_ok', - 'runner_on_skipped', 'runner_on_unreachable', - ] - - def event_handler(self, data): + def event_handler(self, data, **kwargs): event = data.get('event', None) - print("Event: ", event) - print("Event Data: ", json.dumps(data)) + if not event: + return + event_data = data.get('event_data', {}) + host = event_data.get('remote_addr', '') + task = event_data.get('task', '') + res = event_data.get('res', {}) + handler = getattr(self, event, self.on_any) + handler(event_data, host=host, task=task, res=res) - def clean_result(self, t, host, task_name, task_result): - contacted = self.results_summary["contacted"] - dark = self.results_summary["dark"] - - if task_result.get('rc') is not None: - cmd = task_result.get('cmd') - if isinstance(cmd, list): - cmd = " ".join(cmd) - else: - cmd = str(cmd) - detail = { - 'cmd': cmd, - 'stderr': task_result.get('stderr'), - 'stdout': safe_str(str(task_result.get('stdout', ''))), - 'rc': task_result.get('rc'), - 'delta': task_result.get('delta'), - 'msg': task_result.get('msg', '') - } - else: - detail = { - "changed": task_result.get('changed', False), - "msg": task_result.get('msg', '') - } - - if t in ("ok", "skipped"): - contacted[host][task_name] = detail - else: - dark[host][task_name] = detail - - def v2_runner_on_failed(self, result, ignore_errors=False): - self.results_summary['success'] = False - self.gather_result("failed", result) - - if result._task.action in C.MODULE_NO_JSON: - CMDCallBackModule.v2_runner_on_failed(self, - result, ignore_errors=ignore_errors - ) - else: - super().v2_runner_on_failed( - result, ignore_errors=ignore_errors - ) - - def v2_runner_on_ok(self, result): - self.gather_result("ok", result) - if result._task.action in C.MODULE_NO_JSON: - CMDCallBackModule.v2_runner_on_ok(self, result) - else: - super().v2_runner_on_ok(result) - - def v2_runner_on_skipped(self, result): - self.gather_result("skipped", result) - super().v2_runner_on_skipped(result) - - def v2_runner_on_unreachable(self, result): - self.results_summary['success'] = False - self.gather_result("unreachable", result) - super().v2_runner_on_unreachable(result) - - def v2_runner_on_start(self, *args, **kwargs): - pass - - def display_skipped_hosts(self): - pass - - def display_ok_hosts(self): - pass - - def display_failed_stderr(self): - pass - - def set_play_context(self, context): - # for k, v in context._attributes.items(): - # print("{} ==> {}".format(k, v)) - if self.context and isinstance(self.context, dict): - for k, v in self.context.items(): - setattr(context, k, v) - - -class CommandResultCallback(AdHocResultCallback): - """ - Command result callback - - results_command: { - "cmd": "", - "stderr": "", - "stdout": "", - "rc": 0, - "delta": 0:0:0.123 - } - """ - def __init__(self, display=None, **kwargs): - - self.results_command = dict() - super().__init__(display) - - def gather_result(self, t, res): - super().gather_result(t, res) - self.gather_cmd(t, res) - - def v2_playbook_on_play_start(self, play): - now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') - msg = '$ {} ({})'.format(play.name, now) - self._play = play - self._display.banner(msg) - - def v2_runner_on_unreachable(self, result): - self.results_summary['success'] = False - self.gather_result("unreachable", result) - msg = result._result.get("msg") - if not msg: - msg = json.dumps(result._result, indent=4) - self._display.display("%s | FAILED! => \n%s" % ( - result._host.get_name(), - msg, - ), color=C.COLOR_ERROR) - - def v2_runner_on_failed(self, result, ignore_errors=False): - self.results_summary['success'] = False - self.gather_result("failed", result) - msg = result._result.get("msg", '') - stderr = result._result.get("stderr") - if stderr: - msg += '\n' + stderr - module_stdout = result._result.get("module_stdout") - if module_stdout: - msg += '\n' + module_stdout - if not msg: - msg = json.dumps(result._result, indent=4) - self._display.display("%s | FAILED! => \n%s" % ( - result._host.get_name(), - msg, - ), color=C.COLOR_ERROR) - - def v2_playbook_on_stats(self, stats): - pass - - def _print_task_banner(self, task): - pass - - def gather_cmd(self, t, res): - host = res._host.get_name() - cmd = {} - if t == "ok": - cmd['cmd'] = res._result.get('cmd') - cmd['stderr'] = res._result.get('stderr') - cmd['stdout'] = safe_str(str(res._result.get('stdout', ''))) - cmd['rc'] = res._result.get('rc') - cmd['delta'] = res._result.get('delta') - else: - cmd['err'] = "Error: {}".format(res) - self.results_command[host] = cmd - - -class PlaybookResultCallBack(CallbackBase): - """ - Custom callback model for handlering the output data of - execute playbook file, - Base on the build-in callback plugins of ansible which named `json`. - """ - - CALLBACK_VERSION = 2.0 - CALLBACK_TYPE = 'stdout' - CALLBACK_NAME = 'Dict' - - def __init__(self, display=None): - super(PlaybookResultCallBack, self).__init__(display) - self.results = [] - self.output = "" - self.item_results = {} # {"host": []} - - def _new_play(self, play): - return { - 'play': { - 'name': play.name, - 'id': str(play._uuid) - }, - 'tasks': [] + def runner_on_ok(self, event_data, host=None, task=None, res=None): + detail = { + 'action': event_data.get('task_action', ''), + 'res': res, + 'rc': res.get('rc', 0), + 'stdout': res.get('stdout', ''), } + self.result['ok'][host][task] = detail - def _new_task(self, task): - return { - 'task': { - 'name': task.get_name(), - }, - 'hosts': {} + def runer_on_failed(self, event_data, host=None, task=None, res=None, **kwargs): + detail = { + 'action': event_data.get('task_action', ''), + 'res': res, + 'rc': res.get('rc', 0), + 'stdout': res.get('stdout', ''), + 'stderr': ';'.join([res.get('stderr', ''), res.get('msg', '')]).strip(';') } + self.result['failures'][host][task] = detail - def v2_playbook_on_no_hosts_matched(self): - self.output = "skipping: No match hosts." + def runner_on_skipped(self, event_data, host=None, task=None, **kwargs): + detail = { + 'action': event_data.get('task_action', ''), + 'res': {}, + 'rc': 0, + } + self.result['skipped'][host][task] = detail - def v2_playbook_on_no_hosts_remaining(self): + def runner_on_unreachable(self, event_data, host=None, task=None, res=None, **kwargs): + detail = { + 'action': event_data.get('task_action', ''), + 'res': res, + 'rc': 255, + 'stderr': ';'.join([res.get('stderr', ''), res.get('msg', '')]).strip(';') + } + self.result['dark'][host][task] = detail + + def runner_on_start(self, event_data, **kwargs): pass - def v2_playbook_on_task_start(self, task, is_conditional): - self.results[-1]['tasks'].append(self._new_task(task)) + def runer_retry(self, event_data, **kwargs): + pass - def v2_playbook_on_play_start(self, play): - self.results.append(self._new_play(play)) + def runner_on_file_diff(self, event_data, **kwargs): + pass - def v2_playbook_on_stats(self, stats): - hosts = sorted(stats.processed.keys()) - summary = {} - for h in hosts: - s = stats.summarize(h) - summary[h] = s + def runner_item_on_failed(self, event_data, **kwargs): + pass - if self.output: - pass - else: - self.output = { - 'plays': self.results, - 'stats': summary - } + def runner_item_on_skipped(self, event_data, **kwargs): + pass - def gather_result(self, res): - if res._task.loop and "results" in res._result and res._host.name in self.item_results: - res._result.update({"results": self.item_results[res._host.name]}) - del self.item_results[res._host.name] + def playbook_on_play_start(self, event_data, **kwargs): + pass - self.results[-1]['tasks'][-1]['hosts'][res._host.name] = res._result + def playbook_on_stats(self, event_data, **kwargs): + failed = [] + for i in ['dark', 'failures']: + for host, tasks in self.result[i].items(): + failed.append(host) + error = '' + for task, detail in tasks.items(): + error += f'{task}: {detail["stderr"]};' + self.summary[i][host] = error.strip(';') + self.summary['ok'] = list(set(self.result['ok'].keys()) - set(failed)) + self.summary['skipped'] = list(set(self.result['skipped'].keys()) - set(failed)) - def v2_runner_on_ok(self, res, **kwargs): - if "ansible_facts" in res._result: - del res._result["ansible_facts"] + def playbook_on_include(self, event_data, **kwargs): + pass - self.gather_result(res) + def playbook_on_notify(self, event_data, **kwargs): + pass - def v2_runner_on_failed(self, res, **kwargs): - self.gather_result(res) + def playbook_on_vars_prompt(self, event_data, **kwargs): + pass - def v2_runner_on_unreachable(self, res, **kwargs): - self.gather_result(res) + def playbook_on_handler_task_start(self, event_data, **kwargs): + pass - def v2_runner_on_skipped(self, res, **kwargs): - self.gather_result(res) + def playbook_on_no_hosts_matched(self, event_data, **kwargs): + pass - def gather_item_result(self, res): - self.item_results.setdefault(res._host.name, []).append(res._result) - - def v2_runner_item_on_ok(self, res): - self.gather_item_result(res) - - def v2_runner_item_on_failed(self, res): - self.gather_item_result(res) - - def v2_runner_item_on_skipped(self, res): - self.gather_item_result(res) + def playbook_on_no_hosts_remaining(self, event_data, **kwargs): + pass + def warning(self, event_data, **kwargs): + pass + def on_any(self, event_data, **kwargs): + pass + def status_handler(self, data, **kwargs): + self.status = data.get('status', 'unknown') diff --git a/apps/ops/ansible/display.py b/apps/ops/ansible/display.py deleted file mode 100644 index ab93892b2..000000000 --- a/apps/ops/ansible/display.py +++ /dev/null @@ -1,69 +0,0 @@ -import errno -import sys -import os - -from ansible.utils.display import Display -from ansible.utils.color import stringc -from ansible.utils.singleton import Singleton - -from .utils import get_ansible_task_log_path - - -class UnSingleton(Singleton): - def __init__(cls, name, bases, dct): - type.__init__(cls, name, bases, dct) - - def __call__(cls, *args, **kwargs): - return type.__call__(cls, *args, **kwargs) - - -class AdHocDisplay(Display, metaclass=UnSingleton): - def __init__(self, execution_id, verbosity=0): - super().__init__(verbosity=verbosity) - if execution_id: - log_path = get_ansible_task_log_path(execution_id) - else: - log_path = os.devnull - self.log_file = open(log_path, mode='a') - - def close(self): - self.log_file.close() - - def set_cowsay_info(self): - # 中断 cowsay 的测试，会频繁开启子进程 - return - - def _write_to_screen(self, msg, stderr): - if not stderr: - screen = sys.stdout - else: - screen = sys.stderr - - screen.write(msg) - - try: - screen.flush() - except IOError as e: - # Ignore EPIPE in case fileobj has been prematurely closed, eg. - # when piping to "head -n1" - if e.errno != errno.EPIPE: - raise - - def _write_to_log_file(self, msg): - # 这里先不 flush，log 文件不需要那么及时。 - self.log_file.write(msg) - - def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False, newline=True): - if log_only: - return - - if color: - msg = stringc(msg, color) - - if not msg.endswith(u'\n'): - msg2 = msg + u'\n' - else: - msg2 = msg - - self._write_to_log_file(msg2) - self._write_to_screen(msg2, stderr) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index dabda5e4e..2382525ed 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -2,161 +2,12 @@ from collections import defaultdict import json -from ansible.inventory.host import Host -from ansible.vars.manager import VariableManager -from ansible.inventory.manager import InventoryManager -from ansible.parsing.dataloader import DataLoader - __all__ = [ - 'BaseHost', 'BaseInventory' + 'JMSInventory', ] -class BaseHost(Host): - def __init__(self, host_data): - """ - 初始化 - :param host_data: { - "name": "", - "ip": "", - "port": "", - # behind is not must be required - "username": "", - "password": "", - "private_key_path": "", - "become": { - "method": "", - "user": "", - "pass": "", - } - "groups": [], - "vars": {}, - } - """ - self.host_data = host_data - hostname = host_data.get('name') or host_data.get('ip') - port = host_data.get('port') or 22 - super().__init__(hostname, port) - self.__set_required_variables() - self.__set_extra_variables() - - def __set_required_variables(self): - host_data = self.host_data - self.set_variable('ansible_host', host_data['address']) - self.set_variable('ansible_port', host_data['port']) - - if host_data.get('username'): - self.set_variable('ansible_user', host_data['username']) - - # 添加密码和密钥 - if host_data.get('password'): - self.set_variable('ansible_ssh_pass', host_data['password']) - if host_data.get('private_key_path'): - self.set_variable('ansible_ssh_private_key_file', host_data['private_key_path']) - - # 添加become支持 - become = host_data.get("become", False) - if become: - self.set_variable("ansible_become", True) - self.set_variable("ansible_become_method", become.get('method', 'sudo')) - self.set_variable("ansible_become_user", become.get('user', 'root')) - self.set_variable("ansible_become_pass", become.get('pass', '')) - else: - self.set_variable("ansible_become", False) - - def __set_extra_variables(self): - for k, v in self.host_data.get('vars', {}).items(): - self.set_variable(k, v) - - def __repr__(self): - return self.name - - -class BaseInventory(InventoryManager): - """ - 提供生成Ansible inventory对象的方法 - """ - loader_class = DataLoader - variable_manager_class = VariableManager - host_manager_class = BaseHost - - def __init__(self, host_list=None, group_list=None): - """ - 用于生成动态构建Ansible Inventory. super().__init__ 会自动调用 - host_list: [{ - "name": "", - "address": "", - "port": "", - "username": "", - "password": "", - "private_key": "", - "become": { - "method": "", - "user": "", - "pass": "", - }, - "groups": [], - "vars": {}, - }, - ] - group_list: [ - {"name: "", children: [""]}, - ] - :param host_list: - :param group_list - """ - self.host_list = host_list or [] - self.group_list = group_list or [] - assert isinstance(host_list, list) - self.loader = self.loader_class() - self.variable_manager = self.variable_manager_class() - super().__init__(self.loader) - - def get_groups(self): - return self._inventory.groups - - def get_group(self, name): - return self._inventory.groups.get(name, None) - - def get_or_create_group(self, name): - group = self.get_group(name) - if not group: - self.add_group(name) - return self.get_or_create_group(name) - else: - return group - - def parse_groups(self): - for g in self.group_list: - parent = self.get_or_create_group(g.get("name")) - children = [self.get_or_create_group(n) for n in g.get('children', [])] - for child in children: - parent.add_child_group(child) - - def parse_hosts(self): - group_all = self.get_or_create_group('all') - ungrouped = self.get_or_create_group('ungrouped') - for host_data in self.host_list: - host = self.host_manager_class(host_data=host_data) - self.hosts[host_data['name']] = host - groups_data = host_data.get('groups') - if groups_data: - for group_name in groups_data: - group = self.get_or_create_group(group_name) - group.add_host(host) - else: - ungrouped.add_host(host) - group_all.add_host(host) - - def parse_sources(self, cache=False): - self.parse_groups() - self.parse_hosts() - - def get_matched_hosts(self, pattern): - return self.get_hosts(pattern) - - class JMSInventory: def __init__(self, assets, account_username=None, account_policy='smart', host_var_callback=None): """ diff --git a/apps/ops/ansible/new_callback.py b/apps/ops/ansible/new_callback.py deleted file mode 100644 index 90f945f48..000000000 --- a/apps/ops/ansible/new_callback.py +++ /dev/null @@ -1,65 +0,0 @@ - -class JMSCallback: - def event_handler(self, data, runner_config): - event = data.get('event', None) - if not event: - return - event_data = data.get('event_data', {}) - pass - - def runner_on_ok(self, event_data): - pass - - def runer_on_failed(self, event_data): - pass - - def runner_on_skipped(self, event_data): - pass - - def runner_on_unreachable(self, event_data): - pass - - def runner_on_start(self, event_data): - pass - - def runer_retry(self, event_data): - pass - - def runner_on_file_diff(self, event_data): - pass - - def runner_item_on_failed(self, event_data): - pass - - def runner_item_on_skipped(self, event_data): - pass - - def playbook_on_play_start(self, event_data): - pass - - def playbook_on_stats(self, event_data): - pass - - def playbook_on_include(self, event_data): - pass - - def playbook_on_notify(self, event_data): - pass - - def playbook_on_vars_prompt(self, event_data): - pass - - def playbook_on_handler_task_start(self, event_data): - pass - - def playbook_on_no_hosts_matched(self, event_data): - pass - - def playbook_on_no_hosts_remaining(self, event_data): - pass - - def warning(self): - pass - - def status_handler(self): - pass diff --git a/apps/ops/ansible/new_runner.py b/apps/ops/ansible/new_runner.py deleted file mode 100644 index 26cb121a7..000000000 --- a/apps/ops/ansible/new_runner.py +++ /dev/null @@ -1,44 +0,0 @@ -import uuid -import ansible_runner - -from django.conf import settings - - -class AdHocRunner: - cmd_modules_choices = ('shell', 'raw', 'command', 'script', 'win_shell') - cmd_blacklist = [ - "reboot", 'shutdown', 'poweroff', 'halt', 'dd', 'half', 'top' - ] - - def __init__(self, inventory, module, module_args, pattern='*', project_dir='/tmp/'): - self.id = uuid.uuid4() - self.inventory = inventory - self.pattern = pattern - self.module = module - self.module_args = module_args - self.project_dir = project_dir - - def check_module(self): - if self.module not in self.cmd_modules_choices: - return - if self.module_args and self.module_args.split()[0] in self.cmd_blacklist: - raise Exception("command not allowed: {}".format(self.module_args[0])) - - def run(self, verbosity=0, **kwargs): - self.check_module() - if verbosity is None and settings.DEBUG: - verbosity = 1 - - return ansible_runner.run( - host_pattern=self.pattern, - private_data_dir=self.project_dir, - inventory=self.inventory, - module=self.module, - module_args=self.module_args, - verbosity=verbosity, - **kwargs - ) - - -class PlaybookRunner: - pass diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index a25d681b9..6c339eba6 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -1,261 +1,70 @@ -# ~*~ coding: utf-8 ~*~ +import uuid +import ansible_runner -import os - -import shutil -from collections import namedtuple - -from ansible import context -from ansible.playbook import Playbook -from ansible.module_utils.common.collections import ImmutableDict -from ansible.executor.task_queue_manager import TaskQueueManager -from ansible.vars.manager import VariableManager -from ansible.parsing.dataloader import DataLoader -from ansible.executor.playbook_executor import PlaybookExecutor -from ansible.playbook.play import Play -import ansible.constants as C - -from .callback import ( - AdHocResultCallback, PlaybookResultCallBack, CommandResultCallback -) -from common.utils import get_logger -from .exceptions import AnsibleError -from .display import AdHocDisplay - - -__all__ = ["AdHocRunner", "PlayBookRunner", "CommandRunner"] -C.HOST_KEY_CHECKING = False -logger = get_logger(__name__) - - -Options = namedtuple('Options', [ - 'listtags', 'listtasks', 'listhosts', 'syntax', 'connection', - 'module_path', 'forks', 'remote_user', 'private_key_file', 'timeout', - 'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args', - 'scp_extra_args', 'become', 'become_method', 'become_user', - 'verbosity', 'check', 'extra_vars', 'playbook_path', 'passwords', - 'diff', 'gathering', 'remote_tmp', -]) - - -def get_default_options(): - options = dict( - syntax=False, - timeout=30, - connection='ssh', - forks=10, - remote_user='root', - private_key_file=None, - become=None, - become_method=None, - become_user=None, - verbosity=1, - check=False, - diff=False, - gathering='implicit', - remote_tmp='/tmp/.ansible' - ) - return options - - -# JumpServer not use playbook -class PlayBookRunner: - """ - 用于执行AnsiblePlaybook的接口.简化Playbook对象的使用. - """ - - # Default results callback - results_callback_class = PlaybookResultCallBack - loader_class = DataLoader - variable_manager_class = VariableManager - options = get_default_options() - - def __init__(self, inventory=None, options=None): - """ - :param options: Ansible options like ansible.cfg - :param inventory: Ansible inventory - """ - if options: - self.options = options - C.RETRY_FILES_ENABLED = False - self.inventory = inventory - self.loader = self.loader_class() - self.results_callback = self.results_callback_class() - self.playbook_path = options.playbook_path - self.variable_manager = self.variable_manager_class( - loader=self.loader, inventory=self.inventory - ) - self.passwords = options.passwords - self.__check() - - def __check(self): - if self.options.playbook_path is None or \ - not os.path.exists(self.options.playbook_path): - raise AnsibleError( - "Not Found the playbook file: {}.".format(self.options.playbook_path) - ) - if not self.inventory.list_hosts('all'): - raise AnsibleError('Inventory is empty') - - def run(self): - executor = PlaybookExecutor( - playbooks=[self.playbook_path], - inventory=self.inventory, - variable_manager=self.variable_manager, - loader=self.loader, - passwords={"conn_pass": self.passwords} - ) - context.CLIARGS = ImmutableDict(self.options) - - if executor._tqm: - executor._tqm._stdout_callback = self.results_callback - executor.run() - executor._tqm.cleanup() - return self.results_callback.output +from django.conf import settings +from .callback import DefaultCallback class AdHocRunner: - """ - ADHoc Runner接口 - """ - results_callback_class = AdHocResultCallback - results_callback = None - loader_class = DataLoader - variable_manager_class = VariableManager - default_options = get_default_options() - command_modules_choices = ('shell', 'raw', 'command', 'script', 'win_shell') + cmd_modules_choices = ('shell', 'raw', 'command', 'script', 'win_shell') + cmd_blacklist = [ + "reboot", 'shutdown', 'poweroff', 'halt', 'dd', 'half', 'top' + ] - def __init__(self, inventory, options=None): - self.options = self.update_options(options) + def __init__(self, inventory, module, module_args, pattern='*', project_dir='/tmp/'): + self.id = uuid.uuid4() self.inventory = inventory - self.loader = DataLoader() - self.variable_manager = VariableManager( - loader=self.loader, inventory=self.inventory - ) + self.pattern = pattern + self.module = module + self.module_args = module_args + self.project_dir = project_dir + self.cb = DefaultCallback() + self.runner = None - def get_result_callback(self, execution_id=None): - return self.__class__.results_callback_class(display=AdHocDisplay(execution_id)) + def check_module(self): + if self.module not in self.cmd_modules_choices: + return + if self.module_args and self.module_args.split()[0] in self.cmd_blacklist: + raise Exception("command not allowed: {}".format(self.module_args[0])) - @staticmethod - def check_module_args(module_name, module_args=''): - if module_name in C.MODULE_REQUIRE_ARGS and not module_args: - err = "No argument passed to '%s' module." % module_name - raise AnsibleError(err) + def run(self, verbosity=0, **kwargs): + self.check_module() + if verbosity is None and settings.DEBUG: + verbosity = 1 - def check_pattern(self, pattern): - if not pattern: - raise AnsibleError("Pattern `{}` is not valid!".format(pattern)) - if not self.inventory.list_hosts("all"): - raise AnsibleError("Inventory is empty.") - if not self.inventory.list_hosts(pattern): - raise AnsibleError( - "pattern: %s dose not match any hosts." % pattern - ) - - def clean_args(self, module, args): - if not args: - return '' - if module not in self.command_modules_choices: - return args - if isinstance(args, str): - if args.startswith('executable='): - _args = args.split(' ') - executable, command = _args[0].split('=')[1], ' '.join(_args[1:]) - args = {'executable': executable, '_raw_params': command} - else: - args = {'_raw_params': args} - return args - else: - return args - - def clean_tasks(self, tasks): - cleaned_tasks = [] - for task in tasks: - module = task['action']['module'] - args = task['action'].get('args') - cleaned_args = self.clean_args(module, args) - task['action']['args'] = cleaned_args - self.check_module_args(module, cleaned_args) - cleaned_tasks.append(task) - return cleaned_tasks - - def update_options(self, options): - _options = {k: v for k, v in self.default_options.items()} - if options and isinstance(options, dict): - _options.update(options) - return _options - - def set_control_master_if_need(self, cleaned_tasks): - modules = [task.get('action', {}).get('module') for task in cleaned_tasks] - if {'ping', 'win_ping'} & set(modules): - self.results_callback.context = { - 'ssh_args': '-C -o ControlMaster=no' - } - - def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no', execution_id=None): - """ - :param tasks: [{'action': {'module': 'shell', 'args': 'ls'}, ...}, ] - :param pattern: all, *, or others - :param play_name: The play name - :param gather_facts: - :return: - """ - self.check_pattern(pattern) - self.results_callback = self.get_result_callback(execution_id) - cleaned_tasks = self.clean_tasks(tasks) - self.set_control_master_if_need(cleaned_tasks) - context.CLIARGS = ImmutableDict(self.options) - - play_source = dict( - name=play_name, - hosts=pattern, - gather_facts=gather_facts, - tasks=cleaned_tasks - ) - - play = Play().load( - play_source, - variable_manager=self.variable_manager, - loader=self.loader, - ) - loader = DataLoader() - # used in start callback - playbook = Playbook(loader) - playbook._entries.append(play) - playbook._file_name = '__adhoc_playbook__' - - tqm = TaskQueueManager( + ansible_runner.run( + host_pattern=self.pattern, + private_data_dir=self.project_dir, inventory=self.inventory, - variable_manager=self.variable_manager, - loader=self.loader, - stdout_callback=self.results_callback, - passwords={"conn_pass": self.options.get("password", "")} + module=self.module, + module_args=self.module_args, + verbosity=verbosity, + event_handler=self.cb.event_handler, + status_handler=self.cb.status_handler, + **kwargs ) - try: - tqm.send_callback('v2_playbook_on_start', playbook) - tqm.run(play) - tqm.send_callback('v2_playbook_on_stats', tqm._stats) - return self.results_callback - except Exception as e: - raise AnsibleError(e) - finally: - if tqm is not None: - tqm.cleanup() - shutil.rmtree(C.DEFAULT_LOCAL_TMP, True) - - self.results_callback.close() + return self.cb -class CommandRunner(AdHocRunner): - results_callback_class = CommandResultCallback - modules_choices = ('shell', 'raw', 'command', 'script', 'win_shell') +class PlaybookRunner: + def __init__(self, inventory, playbook, project_dir='/tmp/'): + self.id = uuid.uuid4() + self.inventory = inventory + self.playbook = playbook + self.project_dir = project_dir + self.cb = DefaultCallback() - def execute(self, cmd, pattern, module='shell'): - if module and module not in self.modules_choices: - raise AnsibleError("Module should in {}".format(self.modules_choices)) - - tasks = [ - {"action": {"module": module, "args": cmd}} - ] - return self.run(tasks, pattern, play_name=cmd) + def run(self, verbosity=0, **kwargs): + if verbosity is None and settings.DEBUG: + verbosity = 1 + ansible_runner.run( + private_data_dir=self.project_dir, + inventory=self.inventory, + playbook=self.playbook, + verbosity=verbosity, + event_handler=self.cb.event_handler, + status_handler=self.cb.status_handler, + **kwargs + ) + return self.cb diff --git a/apps/ops/ansible/test_inventory.py b/apps/ops/ansible/test_inventory.py deleted file mode 100644 index a03faeaf5..000000000 --- a/apps/ops/ansible/test_inventory.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -# - -import sys -import unittest - - -sys.path.insert(0, '../..') -from ops.ansible.inventory import BaseInventory - - -class TestJMSInventory(unittest.TestCase): - def setUp(self): - host_list = [{ - "name": "testserver1", - "ip": "102.1.1.1", - "port": 22, - "username": "root", - "password": "password", - "private_key": "/tmp/private_key", - "become": { - "method": "sudo", - "user": "root", - "pass": None, - }, - "groups": ["group1", "group2"], - "vars": {"sexy": "yes"}, - }, { - "name": "testserver2", - "ip": "8.8.8.8", - "port": 2222, - "username": "root", - "password": "password", - "private_key": "/tmp/private_key", - "become": { - "method": "su", - "user": "root", - "pass": "123", - }, - "groups": ["group3", "group4"], - "vars": {"love": "yes"}, - }] - - self.inventory = BaseInventory(host_list=host_list) - - def test_hosts(self): - print("#"*10 + "Hosts" + "#"*10) - for host in self.inventory.hosts: - print(host) - - def test_groups(self): - print("#" * 10 + "Groups" + "#" * 10) - for group in self.inventory.groups: - print(group) - - def test_group_all(self): - print("#" * 10 + "all group hosts" + "#" * 10) - group = self.inventory.get_group('all') - print(group.hosts) - - -if __name__ == '__main__': - unittest.main() diff --git a/apps/ops/ansible/test_runner.py b/apps/ops/ansible/test_runner.py deleted file mode 100644 index 6f56985a7..000000000 --- a/apps/ops/ansible/test_runner.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# - -import unittest -import sys - -sys.path.insert(0, "../..") - -from ops.ansible.runner import AdHocRunner, CommandRunner -from ops.ansible.inventory import BaseInventory - - -class TestAdHocRunner(unittest.TestCase): - def setUp(self): - host_data = [ - { - "name": "testserver", - "ip": "192.168.244.185", - "port": 22, - "username": "root", - "password": "redhat", - }, - ] - inventory = BaseInventory(host_data) - self.runner = AdHocRunner(inventory) - - def test_run(self): - tasks = [ - {"action": {"module": "shell", "args": "ls"}, "name": "run_cmd"}, - {"action": {"module": "shell", "args": "whoami"}, "name": "run_whoami"}, - ] - ret = self.runner.run(tasks, "all") - print(ret.results_summary) - print(ret.results_raw) - - -class TestCommandRunner(unittest.TestCase): - def setUp(self): - host_data = [ - { - "name": "testserver", - "ip": "192.168.244.168", - "port": 22, - "username": "root", - "password": "redhat", - }, - ] - inventory = BaseInventory(host_data) - self.runner = CommandRunner(inventory) - - def test_execute(self): - res = self.runner.execute('ls', 'all') - print(res.results_command) - print(res.results_raw) - - -if __name__ == "__main__": - unittest.main() From 0fb4b52232993f9ecf948ce6c8956ee3abb4decf Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 8 Oct 2022 16:55:14 +0800 Subject: [PATCH 03/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20ansible=20?= =?UTF-8?q?=E8=A1=A8=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/automation/base.py | 7 +- apps/assets/models/label.py | 4 - apps/audits/api.py | 110 +++--- apps/audits/filters.py | 39 +-- apps/audits/serializers.py | 79 +++-- apps/audits/urls/api_urls.py | 4 +- apps/jumpserver/settings/base.py | 1 + apps/ops/ansible/callback.py | 2 +- apps/ops/ansible/inventory.py | 10 +- apps/ops/ansible/runner.py | 8 + apps/ops/api/__init__.py | 1 - apps/ops/api/adhoc.py | 50 +-- apps/ops/api/command.py | 76 ---- apps/ops/inventory.py | 149 -------- .../ops/migrations/0024_auto_20221008_1514.py | 58 ++++ .../ops/migrations/0025_auto_20221008_1631.py | 72 ++++ apps/ops/mixin.py | 41 --- apps/ops/models/__init__.py | 1 - apps/ops/models/adhoc.py | 328 +----------------- apps/ops/models/base.py | 106 ++++++ apps/ops/models/command.py | 160 --------- apps/ops/models/playbook.py | 31 +- apps/ops/serializers/adhoc.py | 77 ++-- apps/ops/tasks.py | 2 +- apps/ops/urls/api_urls.py | 5 +- 25 files changed, 438 insertions(+), 983 deletions(-) delete mode 100644 apps/ops/api/command.py delete mode 100644 apps/ops/inventory.py create mode 100644 apps/ops/migrations/0024_auto_20221008_1514.py create mode 100644 apps/ops/migrations/0025_auto_20221008_1631.py create mode 100644 apps/ops/models/base.py delete mode 100644 apps/ops/models/command.py diff --git a/apps/assets/models/automation/base.py b/apps/assets/models/automation/base.py index 27c971e0d..a9b8ab087 100644 --- a/apps/assets/models/automation/base.py +++ b/apps/assets/models/automation/base.py @@ -4,15 +4,14 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from common.const.choices import Trigger -from common.mixins.models import CommonModelMixin from common.db.fields import EncryptJsonDictTextField -from orgs.mixins.models import OrgModelMixin +from orgs.mixins.models import OrgModelMixin, JMSOrgBaseModel from ops.mixin import PeriodTaskModelMixin from ops.tasks import execute_automation_strategy from ops.task_handlers import ExecutionManager -class BaseAutomation(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): +class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): accounts = models.JSONField(default=list, verbose_name=_("Accounts")) nodes = models.ManyToManyField( 'assets.Node', related_name='automation_strategy', blank=True, verbose_name=_("Nodes") @@ -67,7 +66,7 @@ class AutomationStrategyExecution(OrgModelMixin): default=dict, blank=True, null=True, verbose_name=_('Automation snapshot') ) strategy = models.ForeignKey( - 'assets.models.automation.base.BaseAutomation', related_name='execution', on_delete=models.CASCADE, + 'BaseAutomation', related_name='execution', on_delete=models.CASCADE, verbose_name=_('Automation strategy') ) trigger = models.CharField( diff --git a/apps/assets/models/label.py b/apps/assets/models/label.py index f7820ccb1..937d0d95c 100644 --- a/apps/assets/models/label.py +++ b/apps/assets/models/label.py @@ -14,16 +14,12 @@ class Label(OrgModelMixin): ("S", _("System")), ("U", _("User")) ) - id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField(max_length=128, verbose_name=_("Name")) value = models.CharField(max_length=128, verbose_name=_("Value")) category = models.CharField(max_length=128, choices=CATEGORY_CHOICES, default=USER_CATEGORY, verbose_name=_("Category")) is_active = models.BooleanField(default=True, verbose_name=_("Is active")) comment = models.TextField(blank=True, null=True, verbose_name=_("Comment")) - date_created = models.DateTimeField( - auto_now_add=True, null=True, blank=True, verbose_name=_('Date created') - ) @classmethod def get_queryset_group_by_name(cls): diff --git a/apps/audits/api.py b/apps/audits/api.py index 6cb2e1283..a61694e85 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -11,16 +11,14 @@ from common.drf.filters import DatetimeRangeFilter from common.api import CommonGenericViewSet from orgs.mixins.api import OrgGenericViewSet, OrgBulkModelViewSet, OrgRelationMixin from orgs.utils import current_org -from ops.models import CommandExecution +# from ops.models import CommandExecution from . import filters from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog -from .serializers import FTPLogSerializer, UserLoginLogSerializer, CommandExecutionSerializer -from .serializers import OperateLogSerializer, PasswordChangeLogSerializer, CommandExecutionHostsRelationSerializer +from .serializers import FTPLogSerializer, UserLoginLogSerializer +from .serializers import OperateLogSerializer, PasswordChangeLogSerializer -class FTPLogViewSet(CreateModelMixin, - ListModelMixin, - OrgGenericViewSet): +class FTPLogViewSet(CreateModelMixin, ListModelMixin, OrgGenericViewSet): model = FTPLog serializer_class = FTPLogSerializer extra_filter_backends = [DatetimeRangeFilter] @@ -98,53 +96,53 @@ class PasswordChangeLogViewSet(ListModelMixin, CommonGenericViewSet): ) return queryset - -class CommandExecutionViewSet(ListModelMixin, OrgGenericViewSet): - model = CommandExecution - serializer_class = CommandExecutionSerializer - extra_filter_backends = [DatetimeRangeFilter] - date_range_filter_fields = [ - ('date_start', ('date_from', 'date_to')) - ] - filterset_fields = [ - 'user__name', 'user__username', 'command', - 'account', 'is_finished' - ] - search_fields = [ - 'command', 'user__name', 'user__username', - 'account__username', - ] - ordering = ['-date_created'] - - def get_queryset(self): - queryset = super().get_queryset() - if getattr(self, 'swagger_fake_view', False): - return queryset.model.objects.none() - if current_org.is_root(): - return queryset - # queryset = queryset.filter(run_as__org_id=current_org.org_id()) - return queryset - - -class CommandExecutionHostRelationViewSet(OrgRelationMixin, OrgBulkModelViewSet): - serializer_class = CommandExecutionHostsRelationSerializer - m2m_field = CommandExecution.hosts.field - filterset_fields = [ - 'id', 'asset', 'commandexecution' - ] - search_fields = ('asset__name', ) - http_method_names = ['options', 'get'] - rbac_perms = { - 'GET': 'ops.view_commandexecution', - 'list': 'ops.view_commandexecution', - } - - def get_queryset(self): - queryset = super().get_queryset() - queryset = queryset.annotate( - asset_display=Concat( - F('asset__name'), Value('('), - F('asset__address'), Value(')') - ) - ) - return queryset +# Todo: 看看怎么搞 +# class CommandExecutionViewSet(ListModelMixin, OrgGenericViewSet): +# model = CommandExecution +# serializer_class = CommandExecutionSerializer +# extra_filter_backends = [DatetimeRangeFilter] +# date_range_filter_fields = [ +# ('date_start', ('date_from', 'date_to')) +# ] +# filterset_fields = [ +# 'user__name', 'user__username', 'command', +# 'account', 'is_finished' +# ] +# search_fields = [ +# 'command', 'user__name', 'user__username', +# 'account__username', +# ] +# ordering = ['-date_created'] +# +# def get_queryset(self): +# queryset = super().get_queryset() +# if getattr(self, 'swagger_fake_view', False): +# return queryset.model.objects.none() +# if current_org.is_root(): +# return queryset +# # queryset = queryset.filter(run_as__org_id=current_org.org_id()) +# return queryset +# +# +# class CommandExecutionHostRelationViewSet(OrgRelationMixin, OrgBulkModelViewSet): +# serializer_class = CommandExecutionHostsRelationSerializer +# m2m_field = CommandExecution.hosts.field +# filterset_fields = [ +# 'id', 'asset', 'commandexecution' +# ] +# search_fields = ('asset__name', ) +# http_method_names = ['options', 'get'] +# rbac_perms = { +# 'GET': 'ops.view_commandexecution', +# 'list': 'ops.view_commandexecution', +# } +# +# def get_queryset(self): +# queryset = super().get_queryset() +# queryset = queryset.annotate( +# asset_display=Concat( +# F('asset__name'), Value('('), +# F('asset__address'), Value(')') +# ) +# ) +# return queryset diff --git a/apps/audits/filters.py b/apps/audits/filters.py index a6c44b5c5..c15c22b56 100644 --- a/apps/audits/filters.py +++ b/apps/audits/filters.py @@ -5,10 +5,9 @@ from rest_framework import filters from rest_framework.compat import coreapi, coreschema from orgs.utils import current_org -from ops.models import CommandExecution from common.drf.filters import BaseFilterSet -__all__ = ['CurrentOrgMembersFilter', 'CommandExecutionFilter'] +__all__ = ['CurrentOrgMembersFilter'] class CurrentOrgMembersFilter(filters.BaseFilterBackend): @@ -35,21 +34,21 @@ class CurrentOrgMembersFilter(filters.BaseFilterBackend): queryset = queryset.filter(user__in=self._get_user_list()) return queryset - -class CommandExecutionFilter(BaseFilterSet): - hostname_ip = CharFilter(method='filter_hostname_ip') - - class Meta: - model = CommandExecution.hosts.through - fields = ( - 'id', 'asset', 'commandexecution', 'hostname_ip' - ) - - def filter_hostname_ip(self, queryset, name, value): - queryset = queryset.annotate( - hostname_ip=Concat( - F('asset__hostname'), Value('('), - F('asset__address'), Value(')') - ) - ).filter(hostname_ip__icontains=value) - return queryset +# +# class CommandExecutionFilter(BaseFilterSet): +# hostname_ip = CharFilter(method='filter_hostname_ip') +# +# class Meta: +# model = CommandExecution.hosts.through +# fields = ( +# 'id', 'asset', 'commandexecution', 'hostname_ip' +# ) +# +# def filter_hostname_ip(self, queryset, name, value): +# queryset = queryset.annotate( +# hostname_ip=Concat( +# F('asset__hostname'), Value('('), +# F('asset__address'), Value(')') +# ) +# ).filter(hostname_ip__icontains=value) +# return queryset diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index 8b9d28005..0f595be25 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -5,7 +5,6 @@ from rest_framework import serializers from common.drf.serializers import BulkSerializerMixin from terminal.models import Session -from ops.models import CommandExecution from . import models @@ -76,42 +75,42 @@ class SessionAuditSerializer(serializers.ModelSerializer): model = Session fields = '__all__' - -class CommandExecutionSerializer(serializers.ModelSerializer): - is_success = serializers.BooleanField(read_only=True, label=_('Is success')) - hosts_display = serializers.ListSerializer( - child=serializers.CharField(), source='hosts', read_only=True, label=_('Hosts display') - ) - - class Meta: - model = CommandExecution - fields_mini = ['id'] - fields_small = fields_mini + [ - 'command', 'is_finished', 'user', - 'date_start', 'result', 'is_success', 'org_id' - ] - fields = fields_small + ['hosts', 'hosts_display', 'user_display'] - extra_kwargs = { - 'result': {'label': _('Result')}, # model 上的方法，只能在这修改 - 'is_success': {'label': _('Is success')}, - 'hosts': {'label': _('Hosts')}, # 外键，会生成 sql。不在 model 上修改 - 'user': {'label': _('User')}, - 'user_display': {'label': _('User display')}, - } - - @classmethod - def setup_eager_loading(cls, queryset): - """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('user', 'hosts') - return queryset - - -class CommandExecutionHostsRelationSerializer(BulkSerializerMixin, serializers.ModelSerializer): - asset_display = serializers.ReadOnlyField() - commandexecution_display = serializers.ReadOnlyField() - - class Meta: - model = CommandExecution.hosts.through - fields = [ - 'id', 'asset', 'asset_display', 'commandexecution', 'commandexecution_display' - ] +# +# class CommandExecutionSerializer(serializers.ModelSerializer): +# is_success = serializers.BooleanField(read_only=True, label=_('Is success')) +# hosts_display = serializers.ListSerializer( +# child=serializers.CharField(), source='hosts', read_only=True, label=_('Hosts display') +# ) +# +# class Meta: +# model = CommandExecution +# fields_mini = ['id'] +# fields_small = fields_mini + [ +# 'command', 'is_finished', 'user', +# 'date_start', 'result', 'is_success', 'org_id' +# ] +# fields = fields_small + ['hosts', 'hosts_display', 'user_display'] +# extra_kwargs = { +# 'result': {'label': _('Result')}, # model 上的方法，只能在这修改 +# 'is_success': {'label': _('Is success')}, +# 'hosts': {'label': _('Hosts')}, # 外键，会生成 sql。不在 model 上修改 +# 'user': {'label': _('User')}, +# 'user_display': {'label': _('User display')}, +# } +# +# @classmethod +# def setup_eager_loading(cls, queryset): +# """ Perform necessary eager loading of data. """ +# queryset = queryset.prefetch_related('user', 'hosts') +# return queryset +# +# +# class CommandExecutionHostsRelationSerializer(BulkSerializerMixin, serializers.ModelSerializer): +# asset_display = serializers.ReadOnlyField() +# commandexecution_display = serializers.ReadOnlyField() +# +# class Meta: +# model = CommandExecution.hosts.through +# fields = [ +# 'id', 'asset', 'asset_display', 'commandexecution', 'commandexecution_display' +# ] diff --git a/apps/audits/urls/api_urls.py b/apps/audits/urls/api_urls.py index 7301b67fb..902c65fbf 100644 --- a/apps/audits/urls/api_urls.py +++ b/apps/audits/urls/api_urls.py @@ -15,8 +15,8 @@ router.register(r'ftp-logs', api.FTPLogViewSet, 'ftp-log') router.register(r'login-logs', api.UserLoginLogViewSet, 'login-log') router.register(r'operate-logs', api.OperateLogViewSet, 'operate-log') router.register(r'password-change-logs', api.PasswordChangeLogViewSet, 'password-change-log') -router.register(r'command-execution-logs', api.CommandExecutionViewSet, 'command-execution-log') -router.register(r'command-executions-hosts-relations', api.CommandExecutionHostRelationViewSet, 'command-executions-hosts-relation') +# router.register(r'command-execution-logs', api.CommandExecutionViewSet, 'command-execution-log') +# router.register(r'command-executions-hosts-relations', api.CommandExecutionHostRelationViewSet, 'command-executions-hosts-relation') urlpatterns = [ diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index d009238cd..8456d9fa0 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -16,6 +16,7 @@ VERSION = const.VERSION BASE_DIR = const.BASE_DIR PROJECT_DIR = const.PROJECT_DIR DATA_DIR = os.path.join(PROJECT_DIR, 'data') +ANSIBLE_DIR = os.path.join(DATA_DIR, 'ansible') CERTS_DIR = os.path.join(DATA_DIR, 'certs') # Quick-start development settings - unsuitable for production diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index 8b6ad1f8f..59734b07d 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -15,7 +15,7 @@ class DefaultCallback: dark={}, skipped=[], ) - self.status = 'starting' + self.status = 'running' self.finished = False def is_success(self): diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 2382525ed..4da027696 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -3,21 +3,19 @@ from collections import defaultdict import json -__all__ = [ - 'JMSInventory', -] +__all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, assets, account_username=None, account_policy='smart', host_var_callback=None): + def __init__(self, assets, account='', account_policy='smart', host_var_callback=None): """ :param assets: - :param account_username: account username name if not set use account_policy + :param account: account username name if not set use account_policy :param account_policy: :param host_var_callback: """ self.assets = self.clean_assets(assets) - self.account_username = account_username + self.account_username = account self.account_policy = account_policy self.host_var_callback = host_var_callback diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index 6c339eba6..a420fb8a4 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -68,3 +68,11 @@ class PlaybookRunner: **kwargs ) return self.cb + + +class CommandRunner(AdHocRunner): + def __init__(self, inventory, command, pattern='*', project_dir='/tmp/'): + super().__init__(inventory, 'shell', command, pattern, project_dir) + + def run(self, verbosity=0, **kwargs): + return super().run(verbosity, **kwargs) diff --git a/apps/ops/api/__init__.py b/apps/ops/api/__init__.py index e59889cd2..8eb5356e4 100644 --- a/apps/ops/api/__init__.py +++ b/apps/ops/api/__init__.py @@ -2,4 +2,3 @@ # from .adhoc import * from .celery import * -from .command import * diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index 0cc7b6d55..8644ac5d2 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -6,52 +6,18 @@ from rest_framework import viewsets, generics from rest_framework.views import Response from common.drf.serializers import CeleryTaskSerializer -from ..models import Task, AdHoc, AdHocExecution +from ..models import AdHoc, AdHocExecution from ..serializers import ( - TaskSerializer, AdHocSerializer, AdHocExecutionSerializer, - TaskDetailSerializer, AdHocDetailSerializer, ) -from ..tasks import run_ansible_task -from orgs.mixins.api import OrgBulkModelViewSet __all__ = [ - 'TaskViewSet', 'TaskRun', 'AdHocViewSet', 'AdHocRunHistoryViewSet' + 'AdHocViewSet', 'AdHocExecutionViewSet' ] -class TaskViewSet(OrgBulkModelViewSet): - model = Task - filterset_fields = ("name",) - search_fields = filterset_fields - serializer_class = TaskSerializer - - def get_serializer_class(self): - if self.action == 'retrieve': - return TaskDetailSerializer - return super().get_serializer_class() - - def get_queryset(self): - queryset = super().get_queryset() - queryset = queryset.select_related('latest_execution') - return queryset - - -class TaskRun(generics.RetrieveAPIView): - queryset = Task.objects.all() - serializer_class = CeleryTaskSerializer - rbac_perms = { - 'retrieve': 'ops.add_adhoc' - } - - def retrieve(self, request, *args, **kwargs): - task = self.get_object() - t = run_ansible_task.delay(str(task.id)) - return Response({"task": t.id}) - - class AdHocViewSet(viewsets.ModelViewSet): queryset = AdHoc.objects.all() serializer_class = AdHocSerializer @@ -61,23 +27,17 @@ class AdHocViewSet(viewsets.ModelViewSet): return AdHocDetailSerializer return super().get_serializer_class() - def get_queryset(self): - task_id = self.request.query_params.get('task') - if task_id: - task = get_object_or_404(Task, id=task_id) - self.queryset = self.queryset.filter(task=task) - return self.queryset - -class AdHocRunHistoryViewSet(viewsets.ModelViewSet): +class AdHocExecutionViewSet(viewsets.ModelViewSet): queryset = AdHocExecution.objects.all() serializer_class = AdHocExecutionSerializer def get_queryset(self): task_id = self.request.query_params.get('task') adhoc_id = self.request.query_params.get('adhoc') + if task_id: - task = get_object_or_404(Task, id=task_id) + task = get_object_or_404(AdHoc, id=task_id) adhocs = task.adhoc.all() self.queryset = self.queryset.filter(adhoc__in=adhocs) diff --git a/apps/ops/api/command.py b/apps/ops/api/command.py deleted file mode 100644 index 1cf7950a6..000000000 --- a/apps/ops/api/command.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -# -from rest_framework import viewsets -from rest_framework.exceptions import ValidationError -from django.db import transaction -from django.db.models import Q -from django.utils.translation import ugettext as _ -from django.conf import settings - -from assets.models import Asset, Node -from orgs.mixins.api import RootOrgViewMixin -from rbac.permissions import RBACPermission -from ..models import CommandExecution -from ..serializers import CommandExecutionSerializer -from ..tasks import run_command_execution - - -class CommandExecutionViewSet(RootOrgViewMixin, viewsets.ModelViewSet): - serializer_class = CommandExecutionSerializer - permission_classes = (RBACPermission,) - - def get_queryset(self): - return CommandExecution.objects.filter(user_id=str(self.request.user.id)) - - def check_hosts(self, serializer): - data = serializer.validated_data - assets = data["hosts"] - user = self.request.user - - # TOdo: - # Q(granted_by_permissions__system_users__id=system_user.id) & - q = ( - Q(granted_by_permissions__users=user) | - Q(granted_by_permissions__user_groups__users=user) - ) - - permed_assets = set() - permed_assets.update(Asset.objects.filter(id__in=[a.id for a in assets]).filter(q).distinct()) - node_keys = Node.objects.filter(q).distinct().values_list('key', flat=True) - - nodes_assets_q = Q() - for _key in node_keys: - nodes_assets_q |= Q(nodes__key__startswith=f'{_key}:') - nodes_assets_q |= Q(nodes__key=_key) - - permed_assets.update( - Asset.objects.filter( - id__in=[a.id for a in assets] - ).filter( - nodes_assets_q - ).distinct() - ) - - invalid_assets = set(assets) - set(permed_assets) - if invalid_assets: - msg = _("Not has host {} permission").format( - [str(a.id) for a in invalid_assets] - ) - raise ValidationError({"hosts": msg}) - - def check_permissions(self, request): - if not settings.SECURITY_COMMAND_EXECUTION: - return self.permission_denied(request, "Command execution disabled") - return super().check_permissions(request) - - def perform_create(self, serializer): - self.check_hosts(serializer) - instance = serializer.save() - instance.user = self.request.user - instance.save() - cols = self.request.query_params.get("cols", '80') - rows = self.request.query_params.get("rows", '24') - transaction.on_commit(lambda: run_command_execution.apply_async( - args=(instance.id,), kwargs={"cols": cols, "rows": rows}, - task_id=str(instance.id) - )) diff --git a/apps/ops/inventory.py b/apps/ops/inventory.py deleted file mode 100644 index d6943f5c5..000000000 --- a/apps/ops/inventory.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from django.conf import settings -from .ansible.inventory import BaseInventory - -from common.utils import get_logger - -__all__ = [ - 'JMSInventory', 'JMSCustomInventory', -] - - -logger = get_logger(__file__) - - -class JMSBaseInventory(BaseInventory): - def convert_to_ansible(self, asset, run_as_admin=False): - info = { - 'id': asset.id, - 'name': asset.name, - 'ip': asset.address, - 'port': asset.ssh_port, - 'vars': dict(), - 'groups': [], - } - if asset.domain and asset.domain.has_gateway(): - info["vars"].update(self.make_proxy_command(asset)) - if run_as_admin: - info.update(asset.get_auth_info(with_become=True)) - if asset.is_windows(): - info["vars"].update({ - "ansible_connection": "ssh", - "ansible_shell_type": settings.WINDOWS_SSH_DEFAULT_SHELL, - }) - for label in asset.labels.all(): - info["vars"].update({ - label.name: label.value - }) - if asset.domain: - info["vars"].update({ - "domain": asset.domain.name, - }) - return info - - @staticmethod - def make_proxy_command(asset): - gateway = asset.domain.random_gateway() - proxy_command_list = [ - "ssh", "-o", "Port={}".format(gateway.port), - "-o", "StrictHostKeyChecking=no", - "{}@{}".format(gateway.username, gateway.address), - "-W", "%h:%p", "-q", - ] - - if gateway.password: - proxy_command_list.insert( - 0, "sshpass -p '{}'".format(gateway.password) - ) - if gateway.private_key: - proxy_command_list.append("-i {}".format(gateway.private_key_file)) - - proxy_command = "'-o ProxyCommand={}'".format( - " ".join(proxy_command_list) - ) - return {"ansible_ssh_common_args": proxy_command} - - -class JMSInventory(JMSBaseInventory): - """ - JMS Inventory is the inventory with jumpserver assets, so you can - write you own inventory, construct you inventory, - user_info is obtained from admin_user or asset_user - """ - def __init__(self, assets, run_as_admin=False, run_as=None, become_info=None, system_user=None): - """ - :param assets: assets - :param run_as_admin: True 是否使用管理用户去执行, 每台服务器的管理用户可能不同 - :param run_as: 用户名(添加了统一的资产用户管理器之后AssetUserManager加上之后修改为username) - :param become_info: 是否become成某个用户去执行 - """ - self.assets = assets - self.using_admin = run_as_admin - self.run_as = run_as - self.system_user = system_user - self.become_info = become_info - - host_list = [] - - for asset in assets: - host = self.convert_to_ansible(asset, run_as_admin=run_as_admin) - if run_as is not None: - run_user_info = self.get_run_user_info(host) - host.update(run_user_info) - if become_info and asset.is_unixlike(): - host.update(become_info) - host_list.append(host) - - super().__init__(host_list=host_list) - - def get_run_user_info(self, host): - if not self.run_as and not self.system_user: - return {} - - asset_id = host.get('id', '') - asset = self.assets.filter(id=asset_id).first() - if not asset: - logger.error('Host not found: ', asset_id) - return {} - - if self.system_user: - self.system_user.load_asset_special_auth(asset=asset, username=self.run_as) - return self.system_user._to_secret_json() - else: - return {} - - -class JMSCustomInventory(JMSBaseInventory): - """ - JMS Custom Inventory is the inventory with jumpserver assets, - user_info is obtained from custom parameter - """ - - def __init__(self, assets, username, password=None, public_key=None, private_key=None): - """ - """ - self.assets = assets - self.username = username - self.password = password - self.public_key = public_key - self.private_key = private_key - - host_list = [] - - for asset in assets: - host = self.convert_to_ansible(asset) - run_user_info = self.get_run_user_info() - host.update(run_user_info) - host_list.append(host) - - super().__init__(host_list=host_list) - - def get_run_user_info(self): - return { - 'username': self.username, - 'password': self.password, - 'public_key': self.public_key, - 'private_key': self.private_key - } diff --git a/apps/ops/migrations/0024_auto_20221008_1514.py b/apps/ops/migrations/0024_auto_20221008_1514.py new file mode 100644 index 000000000..e208af96e --- /dev/null +++ b/apps/ops/migrations/0024_auto_20221008_1514.py @@ -0,0 +1,58 @@ +# Generated by Django 3.2.14 on 2022-10-08 07:19 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0106_auto_20220916_1556'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('ops', '0023_auto_20220929_2025'), + ] + + operations = [ + migrations.RemoveField( + model_name='adhocexecution', + name='adhoc', + ), + migrations.RemoveField( + model_name='adhocexecution', + name='task', + ), + migrations.RemoveField( + model_name='commandexecution', + name='hosts', + ), + migrations.RemoveField( + model_name='commandexecution', + name='user', + ), + migrations.AlterUniqueTogether( + name='task', + unique_together=None, + ), + migrations.RemoveField( + model_name='task', + name='latest_adhoc', + ), + migrations.RemoveField( + model_name='task', + name='latest_execution', + ), + migrations.DeleteModel( + name='AdHoc', + ), + migrations.DeleteModel( + name='AdHocExecution', + ), + migrations.DeleteModel( + name='CommandExecution', + ), + migrations.DeleteModel( + name='Task', + ), + ] diff --git a/apps/ops/migrations/0025_auto_20221008_1631.py b/apps/ops/migrations/0025_auto_20221008_1631.py new file mode 100644 index 000000000..7e814c3d1 --- /dev/null +++ b/apps/ops/migrations/0025_auto_20221008_1631.py @@ -0,0 +1,72 @@ +# Generated by Django 3.2.14 on 2022-10-08 08:31 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0106_auto_20220916_1556'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('ops', '0024_auto_20221008_1514'), + ] + + operations = [ + migrations.CreateModel( + name='AdHoc', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('is_periodic', models.BooleanField(default=False)), + ('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')), + ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')), + ('account', models.CharField(default='root', max_length=128, verbose_name='Account')), + ('account_policy', models.CharField(default='root', max_length=128, verbose_name='Account policy')), + ('date_last_run', models.DateTimeField(null=True, verbose_name='Date last run')), + ('pattern', models.CharField(default='all', max_length=1024, verbose_name='Pattern')), + ('module', models.CharField(default='shell', max_length=128, verbose_name='Module')), + ('args', models.CharField(default='', max_length=1024, verbose_name='Args')), + ('assets', models.ManyToManyField(to='assets.Asset', verbose_name='Assets')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AdHocExecution', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('status', models.CharField(default='running', max_length=16, verbose_name='Status')), + ('result', models.JSONField(blank=True, null=True, verbose_name='Result')), + ('summary', models.JSONField(default=dict, verbose_name='Summary')), + ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), + ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), + ('date_finished', models.DateTimeField(null=True)), + ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ('task', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='ops.adhoc', verbose_name='Adhoc')), + ], + options={ + 'verbose_name': 'AdHoc execution', + 'db_table': 'ops_adhoc_execution', + 'get_latest_by': 'date_start', + }, + ), + migrations.AddField( + model_name='adhoc', + name='last_execution', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ops.adhocexecution', verbose_name='Last execution'), + ), + migrations.AddField( + model_name='adhoc', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator'), + ), + ] diff --git a/apps/ops/mixin.py b/apps/ops/mixin.py index e64a763fc..4d2fd52a7 100644 --- a/apps/ops/mixin.py +++ b/apps/ops/mixin.py @@ -14,12 +14,10 @@ from .celery.utils import ( __all__ = [ 'PeriodTaskModelMixin', 'PeriodTaskSerializerMixin', - 'PeriodTaskFormMixin', ] class PeriodTaskModelMixin(models.Model): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField( max_length=128, unique=False, verbose_name=_("Name") ) @@ -140,42 +138,3 @@ class PeriodTaskSerializerMixin(serializers.Serializer): msg = _("Require periodic or regularly perform setting") raise serializers.ValidationError(msg) return ok - - -class PeriodTaskFormMixin(forms.Form): - is_periodic = forms.BooleanField( - initial=True, required=False, label=_('Periodic perform') - ) - crontab = forms.CharField( - max_length=128, required=False, label=_('Regularly perform'), - help_text=_("eg: Every Sunday 03:05 run <5 3 * * 0>
" - "Tips: " - "Using 5 digits linux crontab expressions " - " " - "(Online tools)
" - "Note: " - "If both Regularly perform and Cycle perform are set, " - "give priority to Regularly perform"), - ) - interval = forms.IntegerField( - required=False, initial=24, - help_text=_('Unit: hour'), label=_("Cycle perform"), - ) - - def get_initial_for_field(self, field, field_name): - """ - Return initial data for field on form. Use initial data from the form - or the field, in that order. Evaluate callable values. - """ - if field_name not in ['is_periodic', 'crontab', 'interval']: - return super().get_initial_for_field(field, field_name) - instance = getattr(self, 'instance', None) - if instance is None: - return super().get_initial_for_field(field, field_name) - init_attr_name = field_name + '_initial' - value = getattr(self, init_attr_name, None) - if value is None: - return super().get_initial_for_field(field, field_name) - return value - - diff --git a/apps/ops/models/__init__.py b/apps/ops/models/__init__.py index 0a9ed463c..fcd8bd8f7 100644 --- a/apps/ops/models/__init__.py +++ b/apps/ops/models/__init__.py @@ -3,4 +3,3 @@ from .adhoc import * from .celery import * -from .command import * diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index 1d2920206..565df9f3e 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -1,337 +1,41 @@ # ~*~ coding: utf-8 ~*~ -import uuid -import os -import time -import datetime -from celery import current_task from django.db import models -from django.conf import settings -from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from common.utils import get_logger, lazyproperty -from common.utils.translate import translate_value -from common.db.fields import ( - JsonListTextField, JsonDictCharField, EncryptJsonDictCharField, - JsonDictTextField, -) -from orgs.mixins.models import OrgModelMixin -from ..ansible import AdHocRunner, AnsibleError -from ..inventory import JMSInventory -from ..mixin import PeriodTaskModelMixin +from common.utils import get_logger +from .base import BaseAnsibleTask, BaseAnsibleExecution +from ..ansible import AdHocRunner -__all__ = ["Task", "AdHoc", "AdHocExecution"] +__all__ = ["AdHoc", "AdHocExecution"] logger = get_logger(__file__) -class Task(PeriodTaskModelMixin, OrgModelMixin): - """ - This task is different ansible task, Task like 'push system user', 'get asset info' .. - One task can have some versions of adhoc, run a task only run the latest version adhoc - """ - callback = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Callback")) # Callback must be a registered celery task - is_deleted = models.BooleanField(default=False) - comment = models.TextField(blank=True, verbose_name=_("Comment")) - date_created = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name=_("Date created")) - date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) - latest_adhoc = models.ForeignKey('ops.AdHoc', on_delete=models.SET_NULL, - null=True, related_name='task_latest') - latest_execution = models.ForeignKey('ops.AdHocExecution', on_delete=models.SET_NULL, null=True, related_name='task_latest') - total_run_amount = models.IntegerField(default=0) - success_run_amount = models.IntegerField(default=0) - _ignore_auto_created_by = True - - @property - def short_id(self): - return str(self.id).split('-')[-1] - - @lazyproperty - def versions(self): - return self.adhoc.all().count() - - @property - def is_success(self): - if self.latest_execution: - return self.latest_execution.is_success - else: - return False - - @lazyproperty - def display_name(self): - value = translate_value(self.name) - return value - - @property - def timedelta(self): - if self.latest_execution: - return self.latest_execution.timedelta - else: - return 0 - - @property - def date_start(self): - if self.latest_execution: - return self.latest_execution.date_start - else: - return None - - @property - def assets_amount(self): - if self.latest_execution: - return self.latest_execution.hosts_amount - return 0 - - def get_latest_adhoc(self): - if self.latest_adhoc: - return self.latest_adhoc - try: - adhoc = self.adhoc.all().latest() - self.latest_adhoc = adhoc - self.save() - return adhoc - except AdHoc.DoesNotExist: - return None - - @property - def history_summary(self): - total = self.total_run_amount - success = self.success_run_amount - failed = total - success - return {'total': total, 'success': success, 'failed': failed} - - def get_run_execution(self): - return self.execution.all() - - def run(self): - latest_adhoc = self.get_latest_adhoc() - if latest_adhoc: - return latest_adhoc.run() - else: - return {'error': 'No adhoc'} - - @property - def period_key(self): - return self.__str__() - - def get_register_task(self): - from ..tasks import run_ansible_task - name = self.__str__() - task = run_ansible_task.name - args = (str(self.id),) - kwargs = {"callback": self.callback} - return name, task, args, kwargs +class AdHoc(BaseAnsibleTask): + pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all') + module = models.CharField(max_length=128, default='shell', verbose_name=_('Module')) + args = models.CharField(max_length=1024, default='', verbose_name=_('Args')) + last_execution = models.ForeignKey('AdHocExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) def __str__(self): - return self.name + '@' + str(self.org_id) - - class Meta: - db_table = 'ops_task' - unique_together = ('name', 'org_id') - ordering = ('-date_updated',) - verbose_name = _("Task") - get_latest_by = 'date_created' - permissions = [ - ('view_taskmonitor', _('Can view task monitor')) - ] + return "{}: {}".format(self.module, self.args) -class AdHoc(OrgModelMixin): - """ - task: A task reference - _tasks: [{'name': 'task_name', 'action': {'module': '', 'args': ''}, 'other..': ''}, ] - _options: ansible options, more see ops.ansible.runner.Options - run_as_admin: if true, then need get every host admin user run it, because every host may be have different admin user, so we choise host level - run_as: username(Add the uniform AssetUserManager and change it to username) - _become: May be using become [sudo, su] options. {method: "sudo", user: "user", pass: "pass"] - pattern: Even if we set _hosts, We only use that to make inventory, We also can set `patter` to run task on match hosts - """ - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - task = models.ForeignKey(Task, related_name='adhoc', on_delete=models.CASCADE) - tasks = JsonListTextField(verbose_name=_('Tasks')) - pattern = models.CharField(max_length=64, default='{}', verbose_name=_('Pattern')) - options = JsonDictCharField(max_length=1024, default='', verbose_name=_('Options')) - hosts = models.ManyToManyField('assets.Asset', verbose_name=_("Host")) - run_as_admin = models.BooleanField(default=False, verbose_name=_('Run as admin')) - run_as = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Username')) - become = EncryptJsonDictCharField(max_length=1024, default='', blank=True, null=True, verbose_name=_("Become")) - created_by = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Create by')) - date_created = models.DateTimeField(auto_now_add=True, db_index=True) - - @lazyproperty - def run_times(self): - return self.execution.count() - - @property - def inventory(self): - if self.become: - become_info = { - 'become': { - self.become - } - } - else: - become_info = None - - inventory = JMSInventory( - self.hosts.all(), run_as_admin=self.run_as_admin, - run_as=self.run_as, become_info=become_info, system_user=self.run_system_user - ) - return inventory - - @property - def become_display(self): - if self.become: - return self.become.get("user", "") - return "" - - def run(self): - try: - celery_task_id = current_task.request.id - except AttributeError: - celery_task_id = None - - execution = AdHocExecution( - celery_task_id=celery_task_id, - adhoc=self, task=self.task, - task_display=str(self.task)[:128], - date_start=timezone.now(), - hosts_amount=self.hosts.count(), - ) - execution.save() - return execution.start() - - @property - def short_id(self): - return str(self.id).split('-')[-1] - - @property - def latest_execution(self): - try: - return self.execution.all().latest() - except AdHocExecution.DoesNotExist: - return None - - def save(self, **kwargs): - instance = super().save(**kwargs) - self.task.latest_adhoc = instance - self.task.save() - return instance - - def __str__(self): - return "{} of {}".format(self.task.name, self.short_id) - - def same_with(self, other): - if not isinstance(other, self.__class__): - return False - fields_check = [] - for field in self.__class__._meta.fields: - if field.name not in ['id', 'date_created']: - fields_check.append(field) - for field in fields_check: - if getattr(self, field.name) != getattr(other, field.name): - return False - return True - - class Meta: - db_table = "ops_adhoc" - get_latest_by = 'date_created' - verbose_name = _('AdHoc') - - -class AdHocExecution(OrgModelMixin): +class AdHocExecution(BaseAnsibleExecution): """ AdHoc running history. """ - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - task = models.ForeignKey(Task, related_name='execution', on_delete=models.SET_NULL, null=True) - task_display = models.CharField(max_length=128, blank=True, default='', verbose_name=_("Task display")) - celery_task_id = models.UUIDField(default=None, null=True) - hosts_amount = models.IntegerField(default=0, verbose_name=_("Host amount")) - adhoc = models.ForeignKey(AdHoc, related_name='execution', on_delete=models.SET_NULL, null=True) - date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time')) - date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time')) - timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True) - is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) - is_success = models.BooleanField(default=False, verbose_name=_('Is success')) - result = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc raw result')) - summary = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc result summary')) + task = models.ForeignKey('AdHoc', verbose_name=_("Adhoc"), related_name='executions', on_delete=models.CASCADE) - @property - def short_id(self): - return str(self.id).split('-')[-1] - - @property - def adhoc_short_id(self): - return str(self.adhoc_id).split('-')[-1] - - @property - def log_path(self): - dt = datetime.datetime.now().strftime('%Y-%m-%d') - log_dir = os.path.join(settings.PROJECT_DIR, 'data', 'ansible', dt) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - return os.path.join(log_dir, str(self.id) + '.log') - - def start_runner(self): - runner = AdHocRunner(self.adhoc.inventory, options=self.adhoc.options) - try: - result = runner.run( - self.adhoc.tasks, - self.adhoc.pattern, - self.task.name, - execution_id=self.id - ) - return result.results_raw, result.results_summary - except AnsibleError as e: - logger.warn("Failed run adhoc {}, {}".format(self.task.name, e)) - return {}, {} - - def start(self): - self.task.latest_execution = self - self.task.save() - time_start = time.time() - summary = {} - raw = '' - - try: - raw, summary = self.start_runner() - except Exception as e: - logger.error(e, exc_info=True) - raw = {"dark": {"all": str(e)}, "contacted": []} - finally: - self.clean_up(summary, time_start) - return raw, summary - - def clean_up(self, summary, time_start): - is_success = summary.get('success', False) - task = Task.objects.get(id=self.task_id) - task.total_run_amount = models.F('total_run_amount') + 1 - if is_success: - task.success_run_amount = models.F('success_run_amount') + 1 - task.save() - AdHocExecution.objects.filter(id=self.id).update( - is_finished=True, - is_success=is_success, - date_finished=timezone.now(), - timedelta=time.time() - time_start, - summary=summary + def get_runner(self): + return AdHocRunner( + self.task.inventory, self.task.module, self.task.args, + pattern=self.task.pattern, project_dir=self.private_dir ) - @property - def success_hosts(self): - return self.summary.get('contacted', []) - - @property - def failed_hosts(self): - return self.summary.get('dark', {}) - - def __str__(self): - return self.short_id - class Meta: db_table = "ops_adhoc_execution" get_latest_by = 'date_start' diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py new file mode 100644 index 000000000..2992173c3 --- /dev/null +++ b/apps/ops/models/base.py @@ -0,0 +1,106 @@ +import os.path +import uuid + +from django.db import models +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from django.conf import settings + +from orgs.mixins.models import JMSOrgBaseModel +from ..ansible.inventory import JMSInventory +from ..mixin import PeriodTaskModelMixin + + +class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): + owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) + account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) + account_policy = models.CharField(max_length=128, default='root', verbose_name=_('Account policy')) + last_execution = models.ForeignKey('BaseAnsibleExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True) + date_last_run = models.DateTimeField(null=True, verbose_name=_('Date last run')) + + class Meta: + abstract = True + + @property + def inventory(self): + inv = JMSInventory(self.assets.all(), self.account, self.account_policy) + return inv.generate() + + def get_register_task(self): + raise NotImplemented + + def to_json(self): + raise NotImplemented + + +class BaseAnsibleExecution(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4) + status = models.CharField(max_length=16, verbose_name=_('Status'), default='running') + task = models.ForeignKey(BaseAnsibleTask, on_delete=models.CASCADE, null=True) + result = models.JSONField(blank=True, null=True, verbose_name=_('Result')) + summary = models.JSONField(default=dict, verbose_name=_('Summary')) + creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) + date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True) + date_finished = models.DateTimeField(null=True) + + class Meta: + abstract = True + ordering = ["-date_start"] + + def __str__(self): + return str(self.id) + + def private_dir(self): + uniq = self.date_created.strftime('%Y%m%d_%H%M%S') + '_' + self.short_id + return os.path.join(settings.ANSIBLE_DIR, self.task.name, uniq) + + def get_runner(self): + raise NotImplemented + + def update_task(self): + self.task.last_execution = self + self.task.date_last_run = timezone.now() + self.task.save(update_fields=['last_execution', 'date_last_run']) + + def start(self, **kwargs): + runner = self.get_runner() + try: + cb = runner.run(**kwargs) + self.status = cb.status + self.summary = cb.summary + self.result = cb.result + self.date_finished = timezone.now() + except Exception as e: + self.status = 'failed' + self.summary = {'error': str(e)} + finally: + self.save() + self.update_task() + + @property + def is_finished(self): + return self.status in ['succeeded', 'failed'] + + @property + def is_success(self): + return self.status == 'succeeded' + + @property + def time_cost(self): + if self.date_finished and self.date_start: + return (self.date_finished - self.date_start).total_seconds() + return None + + @property + def short_id(self): + return str(self.id).split('-')[-1] + + @property + def timedelta(self): + if self.date_start and self.date_finished: + return self.date_finished - self.date_start + return None + + diff --git a/apps/ops/models/command.py b/apps/ops/models/command.py deleted file mode 100644 index cb6023564..000000000 --- a/apps/ops/models/command.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# -import uuid -import json - -from celery.exceptions import SoftTimeLimitExceeded -from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ -from django.utils.translation import ugettext -from django.db import models - -from terminal.notifications import CommandExecutionAlert -from assets.models import Asset -from common.utils import lazyproperty -from orgs.models import Organization -from orgs.mixins.models import OrgModelMixin -from orgs.utils import tmp_to_org -from ..ansible.runner import CommandRunner -from ..inventory import JMSInventory - - -class CommandExecution(OrgModelMixin): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - hosts = models.ManyToManyField('assets.Asset') - account = models.CharField(max_length=128, default='', verbose_name=_('account')) - command = models.TextField(verbose_name=_("Command")) - _result = models.TextField(blank=True, null=True, verbose_name=_('Result')) - user = models.ForeignKey('users.User', on_delete=models.CASCADE, null=True) - is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) - date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) - date_start = models.DateTimeField(null=True, verbose_name=_('Date start')) - date_finished = models.DateTimeField(null=True, verbose_name=_('Date finished')) - - def __str__(self): - return self.command[:10] - - def save(self, *args, **kwargs): - with tmp_to_org(self.run_as.org_id): - super().save(*args, **kwargs) - - @property - def inventory(self): - if self.run_as.username_same_with_user: - username = self.user.username - else: - username = self.run_as.username - inv = JMSInventory(self.allow_assets, run_as=username, system_user=self.run_as) - return inv - - @lazyproperty - def user_display(self): - return str(self.user) - - @lazyproperty - def hosts_display(self): - return ','.join(self.hosts.all().values_list('name', flat=True)) - - @property - def result(self): - if self._result: - return json.loads(self._result) - else: - return {} - - @result.setter - def result(self, item): - self._result = json.dumps(item) - - @property - def is_success(self): - if 'error' in self.result: - return False - return True - - def get_hosts_names(self): - return ','.join(self.hosts.all().values_list('name', flat=True)) - - def cmd_filter_rules(self, asset_id=None): - from assets.models import CommandFilterRule - user_id = self.user.id - system_user_id = self.run_as.id - rules = CommandFilterRule.get_queryset( - user_id=user_id, - system_user_id=system_user_id, - asset_id=asset_id, - ) - return rules - - def is_command_can_run(self, command, asset_id=None): - for rule in self.cmd_filter_rules(asset_id=asset_id): - action, matched_cmd = rule.match(command) - if action == rule.ActionChoices.allow: - return True, None - elif action == rule.ActionChoices.deny: - return False, matched_cmd - return True, None - - @property - def allow_assets(self): - allow_asset_ids = [] - for asset in self.hosts.all(): - ok, __ = self.is_command_can_run(self.command, asset_id=asset.id) - if ok: - allow_asset_ids.append(asset.id) - allow_assets = Asset.objects.filter(id__in=allow_asset_ids) - return allow_assets - - def run(self): - print('-' * 10 + ' ' + ugettext('Task start') + ' ' + '-' * 10) - org = Organization.get_instance(self.run_as.org_id) - org.change_to() - self.date_start = timezone.now() - ok, msg = self.is_command_can_run(self.command) - if ok: - allow_assets = self.allow_assets - deny_assets = set(list(self.hosts.all())) - set(list(allow_assets)) - for asset in deny_assets: - print(f'资产{asset}: 命令{self.command}不允许执行') - if not allow_assets: - self.result = { - "error": 'There are currently no assets that can be executed' - } - self.save() - return self.result - runner = CommandRunner(self.inventory) - try: - host = allow_assets.first() - if host and host.is_windows(): - shell = 'win_shell' - elif host and host.is_unixlike(): - shell = 'shell' - else: - shell = 'raw' - result = runner.execute(self.command, 'all', module=shell) - self.result = result.results_command - except SoftTimeLimitExceeded as e: - print("Run timeout than 60s") - self.result = {"error": str(e)} - except Exception as e: - print("Error occur: {}".format(e)) - self.result = {"error": str(e)} - else: - msg = _("Command `{}` is forbidden ........").format(self.command) - print('\033[31m' + msg + '\033[0m') - CommandExecutionAlert({ - 'input': self.command, - 'assets': self.hosts.all(), - 'user': str(self.user), - 'risk_level': 5, - }).publish_async() - self.result = {"error": msg} - self.org_id = self.run_as.org_id - self.is_finished = True - self.date_finished = timezone.now() - self.save() - print('-' * 10 + ' ' + ugettext('Task end') + ' ' + '-' * 10) - return self.result - - class Meta: - verbose_name = _("Command execution") diff --git a/apps/ops/models/playbook.py b/apps/ops/models/playbook.py index aaec7a4ef..aec59bfb0 100644 --- a/apps/ops/models/playbook.py +++ b/apps/ops/models/playbook.py @@ -2,15 +2,34 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from orgs.mixins.models import JMSOrgBaseModel -from ..mixin import PeriodTaskModelMixin +from .base import BaseAnsibleExecution, BaseAnsibleTask -class PlaybookTask(PeriodTaskModelMixin, JMSOrgBaseModel): - assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) - account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) - playbook = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) - owner = models.CharField(max_length=1024, verbose_name=_("Owner")) +class PlaybookTemplate(JMSOrgBaseModel): + name = models.CharField(max_length=128, verbose_name=_("Name")) + path = models.FilePathField(verbose_name=_("Path")) + comment = models.TextField(verbose_name=_("Comment"), blank=True) + + def __str__(self): + return self.name + + class Meta: + ordering = ['name'] + verbose_name = _("Playbook template") + unique_together = [('org_id', 'name')] + + +class Playbook(BaseAnsibleTask): + path = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) + owner = models.ForeignKey('users.User', verbose_name=_("Owner"), on_delete=models.SET_NULL, null=True) comment = models.TextField(blank=True, verbose_name=_("Comment")) + template = models.ForeignKey('PlaybookTemplate', verbose_name=_("Template"), on_delete=models.SET_NULL, null=True) + last_execution = models.ForeignKey('PlaybookExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) def get_register_task(self): pass + + +class PlaybookExecution(BaseAnsibleExecution): + task = models.ForeignKey('Playbook', verbose_name=_("Task"), on_delete=models.CASCADE) + path = models.FilePathField(max_length=1024, verbose_name=_("Run dir")) diff --git a/apps/ops/serializers/adhoc.py b/apps/ops/serializers/adhoc.py index 50b1faeba..b6522b85f 100644 --- a/apps/ops/serializers/adhoc.py +++ b/apps/ops/serializers/adhoc.py @@ -3,8 +3,7 @@ from __future__ import unicode_literals from rest_framework import serializers from django.shortcuts import reverse -from orgs.mixins.serializers import BulkOrgResourceModelSerializer -from ..models import Task, AdHoc, AdHocExecution, CommandExecution +from ..models import AdHoc, AdHocExecution class AdHocExecutionSerializer(serializers.ModelSerializer): @@ -50,36 +49,6 @@ class AdHocExecutionExcludeResultSerializer(AdHocExecutionSerializer): ] -class TaskSerializer(BulkOrgResourceModelSerializer): - summary = serializers.ReadOnlyField(source='history_summary') - latest_execution = AdHocExecutionExcludeResultSerializer(read_only=True) - - class Meta: - model = Task - fields_mini = ['id', 'name', 'display_name'] - fields_small = fields_mini + [ - 'interval', 'crontab', - 'is_periodic', 'is_deleted', - 'date_created', 'date_updated', - 'comment', - ] - fields_fk = ['latest_execution'] - fields_custom = ['summary'] - fields = fields_small + fields_fk + fields_custom - read_only_fields = [ - 'is_deleted', 'date_created', 'date_updated', - 'latest_adhoc', 'latest_execution', 'total_run_amount', - 'success_run_amount', 'summary', - ] - - -class TaskDetailSerializer(TaskSerializer): - contents = serializers.ListField(source='latest_adhoc.tasks') - - class Meta(TaskSerializer.Meta): - fields = TaskSerializer.Meta.fields + ['contents'] - - class AdHocSerializer(serializers.ModelSerializer): become_display = serializers.ReadOnlyField() tasks = serializers.ListField() @@ -127,26 +96,26 @@ class AdHocDetailSerializer(AdHocSerializer): ] -class CommandExecutionSerializer(serializers.ModelSerializer): - result = serializers.JSONField(read_only=True) - log_url = serializers.SerializerMethodField() - - class Meta: - model = CommandExecution - fields_mini = ['id'] - fields_small = fields_mini + [ - 'command', 'result', 'log_url', - 'is_finished', 'date_created', 'date_finished' - ] - fields_m2m = ['hosts'] - fields = fields_small + fields_m2m - read_only_fields = [ - 'result', 'is_finished', 'log_url', 'date_created', - 'date_finished' - ] - ref_name = 'OpsCommandExecution' - - @staticmethod - def get_log_url(obj): - return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id}) +# class CommandExecutionSerializer(serializers.ModelSerializer): +# result = serializers.JSONField(read_only=True) +# log_url = serializers.SerializerMethodField() +# +# class Meta: +# model = CommandExecution +# fields_mini = ['id'] +# fields_small = fields_mini + [ +# 'command', 'result', 'log_url', +# 'is_finished', 'date_created', 'date_finished' +# ] +# fields_m2m = ['hosts'] +# fields = fields_small + fields_m2m +# read_only_fields = [ +# 'result', 'is_finished', 'log_url', 'date_created', +# 'date_finished' +# ] +# ref_name = 'OpsCommandExecution' +# +# @staticmethod +# def get_log_url(obj): +# return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id}) diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index cb21b5c3d..0ef430d7a 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -20,7 +20,7 @@ from .celery.utils import ( create_or_update_celery_periodic_tasks, get_celery_periodic_task, disable_celery_periodic_task, delete_celery_periodic_task ) -from .models import Task, CommandExecution, CeleryTask +from .models import CommandExecution, CeleryTask from .notifications import ServerPerformanceCheckUtil logger = get_logger(__file__) diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index a5838073f..49038b9b1 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -12,14 +12,11 @@ app_name = "ops" router = DefaultRouter() bulk_router = BulkRouter() -bulk_router.register(r'tasks', api.TaskViewSet, 'task') router.register(r'adhoc', api.AdHocViewSet, 'adhoc') -router.register(r'adhoc-executions', api.AdHocRunHistoryViewSet, 'execution') -router.register(r'command-executions', api.CommandExecutionViewSet, 'command-execution') +router.register(r'adhoc-executions', api.AdHocExecutionViewSet, 'execution') router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task') urlpatterns = [ - path('tasks//run/', api.TaskRun.as_view(), name='task-run'), path('celery/task//log/', api.CeleryTaskLogApi.as_view(), name='celery-task-log'), path('celery/task//result/', api.CeleryResultApi.as_view(), name='celery-result'), From a543a2ee377ce4d37a8c63b5c5a54ab18211f49b Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 8 Oct 2022 19:12:04 +0800 Subject: [PATCH 04/13] =?UTF-8?q?perf:=20=E5=9F=BA=E6=9C=AC=E5=AE=8C?= =?UTF-8?q?=E6=88=90=20adhoc=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/ansible/inventory.py | 11 ++++++-- apps/ops/ansible/runner.py | 12 ++++++-- apps/ops/models/__init__.py | 1 + apps/ops/models/adhoc.py | 17 ++++++++--- apps/ops/models/base.py | 53 +++++++++++++++++++++++++++-------- apps/ops/models/playbook.py | 6 +++- apps/ops/tasks.py | 45 ++++++++++++++++------------- 7 files changed, 105 insertions(+), 40 deletions(-) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 4da027696..85a3d03ff 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -1,6 +1,7 @@ # ~*~ coding: utf-8 ~*~ from collections import defaultdict import json +import os __all__ = ['JMSInventory'] @@ -136,15 +137,19 @@ class JMSInventory: account = self.select_account(asset) host = self.asset_to_host(asset, account, automation, protocols) hosts.append(host) - return hosts - def write_to_file(self, path): - hosts = self.generate() data = {'all': {'hosts': {}}} for host in hosts: name = host.pop('name') var = host.pop('vars', {}) host.update(var) data['all']['hosts'][name] = host + return data + + def write_to_file(self, path): + data = self.generate() + path_dir = os.path.dirname(path) + if not os.path.exists(path_dir): + os.makedirs(path_dir, 0o700, True) with open(path, 'w') as f: f.write(json.dumps(data, indent=4)) diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index a420fb8a4..36a0e7e8c 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -1,7 +1,9 @@ import uuid -import ansible_runner +import os +import ansible_runner from django.conf import settings + from .callback import DefaultCallback @@ -11,7 +13,7 @@ class AdHocRunner: "reboot", 'shutdown', 'poweroff', 'halt', 'dd', 'half', 'top' ] - def __init__(self, inventory, module, module_args, pattern='*', project_dir='/tmp/'): + def __init__(self, inventory, module, module_args='', pattern='*', project_dir='/tmp/'): self.id = uuid.uuid4() self.inventory = inventory self.pattern = pattern @@ -32,6 +34,12 @@ class AdHocRunner: if verbosity is None and settings.DEBUG: verbosity = 1 + if not os.path.exists(self.project_dir): + os.mkdir(self.project_dir, 0o755) + + print("inventory: ") + print(self.inventory) + ansible_runner.run( host_pattern=self.pattern, private_data_dir=self.project_dir, diff --git a/apps/ops/models/__init__.py b/apps/ops/models/__init__.py index fcd8bd8f7..93b630dd6 100644 --- a/apps/ops/models/__init__.py +++ b/apps/ops/models/__init__.py @@ -3,3 +3,4 @@ from .adhoc import * from .celery import * +from .playbook import * diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index 565df9f3e..c3a7822a9 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -1,5 +1,5 @@ # ~*~ coding: utf-8 ~*~ - +import os.path from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -18,7 +18,12 @@ class AdHoc(BaseAnsibleTask): pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all') module = models.CharField(max_length=128, default='shell', verbose_name=_('Module')) args = models.CharField(max_length=1024, default='', verbose_name=_('Args')) - last_execution = models.ForeignKey('AdHocExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) + last_execution = models.ForeignKey('AdHocExecution', verbose_name=_("Last execution"), + on_delete=models.SET_NULL, null=True, blank=True) + + def get_register_task(self): + from ops.tasks import run_adhoc + return "run_adhoc_{}".format(self.id), run_adhoc, (str(self.id),), {} def __str__(self): return "{}: {}".format(self.module, self.args) @@ -31,10 +36,14 @@ class AdHocExecution(BaseAnsibleExecution): task = models.ForeignKey('AdHoc', verbose_name=_("Adhoc"), related_name='executions', on_delete=models.CASCADE) def get_runner(self): - return AdHocRunner( - self.task.inventory, self.task.module, self.task.args, + inv = self.task.inventory + inv.write_to_file(self.inventory_path) + + runner = AdHocRunner( + self.inventory_path, self.task.module, module_args=self.task.args, pattern=self.task.pattern, project_dir=self.private_dir ) + return runner class Meta: db_table = "ops_adhoc_execution" diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py index 2992173c3..f43e0c584 100644 --- a/apps/ops/models/base.py +++ b/apps/ops/models/base.py @@ -1,5 +1,6 @@ import os.path import uuid +import logging from django.db import models from django.utils.translation import gettext_lazy as _ @@ -25,7 +26,7 @@ class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): @property def inventory(self): inv = JMSInventory(self.assets.all(), self.account, self.account_policy) - return inv.generate() + return inv def get_register_task(self): raise NotImplemented @@ -33,11 +34,19 @@ class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): def to_json(self): raise NotImplemented + def create_execution(self): + execution = self.executions.create() + return execution + + def run(self, *args, **kwargs): + execution = self.create_execution() + return execution.start() + class BaseAnsibleExecution(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4) status = models.CharField(max_length=16, verbose_name=_('Status'), default='running') - task = models.ForeignKey(BaseAnsibleTask, on_delete=models.CASCADE, null=True) + task = models.ForeignKey(BaseAnsibleTask, on_delete=models.CASCADE, related_name='executions', null=True) result = models.JSONField(blank=True, null=True, verbose_name=_('Result')) summary = models.JSONField(default=dict, verbose_name=_('Summary')) creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) @@ -52,13 +61,40 @@ class BaseAnsibleExecution(models.Model): def __str__(self): return str(self.id) + @property def private_dir(self): uniq = self.date_created.strftime('%Y%m%d_%H%M%S') + '_' + self.short_id return os.path.join(settings.ANSIBLE_DIR, self.task.name, uniq) + @property + def inventory_path(self): + return os.path.join(self.private_dir, 'inventory', 'hosts') + def get_runner(self): raise NotImplemented + def finish_task(self): + self.date_finished = timezone.now() + self.save(update_fields=['result', 'status', 'summary', 'date_finished']) + self.update_task() + + def set_error(self, error): + this = self.__class__.objects.get(id=self.id) # 重新获取一次，避免数据库超时连接超时 + this.status = 'failed' + this.summary['error'] = str(error) + this.finish_task() + + def set_result(self, cb): + status_mapper = { + 'successful': 'succeeded', + } + this = self.__class__.objects.get(id=self.id) + this.status = status_mapper.get(cb.status, cb.status) + this.summary = cb.summary + this.result = cb.result + this.finish_task() + print("Finished") + def update_task(self): self.task.last_execution = self self.task.date_last_run = timezone.now() @@ -68,16 +104,11 @@ class BaseAnsibleExecution(models.Model): runner = self.get_runner() try: cb = runner.run(**kwargs) - self.status = cb.status - self.summary = cb.summary - self.result = cb.result - self.date_finished = timezone.now() + self.set_result(cb) + return cb except Exception as e: - self.status = 'failed' - self.summary = {'error': str(e)} - finally: - self.save() - self.update_task() + logging.error(e, exc_info=True) + self.set_error(e) @property def is_finished(self): diff --git a/apps/ops/models/playbook.py b/apps/ops/models/playbook.py index aec59bfb0..0701ed13e 100644 --- a/apps/ops/models/playbook.py +++ b/apps/ops/models/playbook.py @@ -27,7 +27,11 @@ class Playbook(BaseAnsibleTask): last_execution = models.ForeignKey('PlaybookExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) def get_register_task(self): - pass + name = "automation_strategy_period_{}".format(str(self.id)[:8]) + task = execute_automation_strategy.name + args = (str(self.id), Trigger.timing) + kwargs = {} + return name, task, args, kwargs class PlaybookExecution(BaseAnsibleExecution): diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 0ef430d7a..e9ba28eb7 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -20,7 +20,7 @@ from .celery.utils import ( create_or_update_celery_periodic_tasks, get_celery_periodic_task, disable_celery_periodic_task, delete_celery_periodic_task ) -from .models import CommandExecution, CeleryTask +from .models import CeleryTask, AdHoc, Playbook from .notifications import ServerPerformanceCheckUtil logger = get_logger(__file__) @@ -30,41 +30,48 @@ def rerun_task(): pass -@shared_task(queue="ansible", verbose_name=_("Run ansible task")) -def run_ansible_task(tid, callback=None, **kwargs): +@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task")) +def run_adhoc(tid, **kwargs): """ :param tid: is the tasks serialized data :param callback: callback function name :return: """ with tmp_to_root_org(): - task = get_object_or_none(Task, id=tid) + task = get_object_or_none(AdHoc, id=tid) if not task: logger.error("No task found") return with tmp_to_org(task.org): - result = task.run() - if callback is not None: - subtask(callback).delay(result, task_name=task.name) - return result + execution = task.create_execution() + try: + execution.start(**kwargs) + except SoftTimeLimitExceeded: + execution.set_error('Run timeout') + logger.error("Run adhoc timeout") + except Exception as e: + execution.set_error(e) + logger.error("Start adhoc execution error: {}".format(e)) @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible command")) -def run_command_execution(cid, **kwargs): +def run_playbook(pid, **kwargs): with tmp_to_root_org(): - execution = get_object_or_none(CommandExecution, id=cid) - if not execution: - logger.error("Not found the execution id: {}".format(cid)) + task = get_object_or_none(Playbook, id=pid) + if not task: + logger.error("No task found") return - with tmp_to_org(execution.run_as.org): + + with tmp_to_org(task.org): + execution = task.create_execution() try: - os.environ.update({ - "TERM_ROWS": kwargs.get("rows", ""), - "TERM_COLS": kwargs.get("cols", ""), - }) - execution.run() + execution.start(**kwargs) except SoftTimeLimitExceeded: - logger.error("Run time out") + execution.set_error('Run timeout') + logger.error("Run playbook timeout") + except Exception as e: + execution.set_error(e) + logger.error("Run playbook execution error: {}".format(e)) @shared_task From f921f12171e87ff88d5c71dad144692f68c0704b Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 8 Oct 2022 19:51:29 +0800 Subject: [PATCH 05/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20adhoc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/ansible/inventory.py | 29 +++++++++++++++++++---------- apps/ops/ansible/runner.py | 3 --- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 85a3d03ff..6a7e3c5aa 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -3,6 +3,8 @@ from collections import defaultdict import json import os +from django.utils.translation import gettext as _ + __all__ = ['JMSInventory'] @@ -60,6 +62,7 @@ class JMSInventory: host = {'name': asset.name, 'vars': { 'asset_id': str(asset.id), 'asset_name': asset.name, 'asset_type': asset.type, 'asset_category': asset.category, + 'exclude': '' }} ansible_connection = automation.ansible_config.get('ansible_connection', 'ssh') gateway = None @@ -87,6 +90,8 @@ class JMSInventory: host['ansible_password'] = account.secret elif account.secret_type == 'private_key' and account.secret: host['ssh_private_key'] = account.private_key_file + else: + host['vars']['exclude'] = _("No account found") if gateway: host['vars'].update(self.make_proxy_command(gateway)) @@ -99,28 +104,26 @@ class JMSInventory: def select_account(self, asset): accounts = list(asset.accounts.all()) - if not accounts: - return None - account_selected = None account_username = self.account_username if isinstance(self.account_username, str): account_username = [self.account_username] + if account_username: for username in account_username: account_matched = list(filter(lambda account: account.username == username, accounts)) if account_matched: account_selected = account_matched[0] - return account_selected + break if not account_selected: if self.account_policy in ['privileged_must', 'privileged_first']: - account_selected = list(filter(lambda account: account.is_privileged, accounts)) - account_selected = account_selected[0] if account_selected else None + account_matched = list(filter(lambda account: account.is_privileged, accounts)) + account_selected = account_matched[0] if account_matched else None if not account_selected and self.account_policy == 'privileged_first': - account_selected = accounts[0] + account_selected = accounts[0] if accounts else None return account_selected def generate(self): @@ -130,14 +133,20 @@ class JMSInventory: automation = platform.automation protocols = platform.protocols.all() - if not automation.ansible_enabled: - continue - for asset in self.assets: account = self.select_account(asset) host = self.asset_to_host(asset, account, automation, protocols) + if not automation.ansible_enabled: + host['vars']['exclude'] = _('Ansible disabled') hosts.append(host) + exclude_hosts = list(filter(lambda x: x.get('exclude'), hosts)) + if exclude_hosts: + print(_("Skip hosts below:")) + for host in exclude_hosts: + print(" {}:\t{}".format(host['name'], host['exclude'])) + + hosts = list(filter(lambda x: not x.get('exclude'), hosts)) data = {'all': {'hosts': {}}} for host in hosts: name = host.pop('name') diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index 36a0e7e8c..e8f232e74 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -37,9 +37,6 @@ class AdHocRunner: if not os.path.exists(self.project_dir): os.mkdir(self.project_dir, 0o755) - print("inventory: ") - print(self.inventory) - ansible_runner.run( host_pattern=self.pattern, private_data_dir=self.project_dir, From 4e5a7a0a252e4ba091dbb524390ba0edcb04e6ef Mon Sep 17 00:00:00 2001 From: ibuler Date: Sun, 9 Oct 2022 20:54:11 +0800 Subject: [PATCH 06/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=E6=94=B9?= =?UTF-8?q?=E5=AF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/automations/__init__.py | 1 + .../base => automations/backup}/__init__.py | 0 .../backup/handlers.py | 0 .../backup/manager.py | 0 .../base}/__init__.py | 0 .../automations/base/base_inventory.txt | 14 +++ apps/assets/automations/base/manager.py | 70 ++++++++++++++ .../change_password}/__init__.py | 0 .../database/change_password_mysql/main.yml | 29 ++++++ .../change_password_mysql/manifest.yml | 0 .../database/change_password_oracle/main.yml | 29 ++++++ .../change_password_oracle/manifest.yml | 0 .../change_password_postgresql}/main.yml | 0 .../change_password_postgresql/manifest.yml | 0 .../roles/change_password/tasks/main.yml | 0 .../change_password_sqlserver}/main.yml | 0 .../change_password_sqlserver/manifest.yml | 0 .../roles/change_password/tasks/main.yml | 0 .../change_password/demo_inventory.txt | 2 + .../host/change_password_aix/main.yml | 29 ++++++ .../host/change_password_aix/manifest.yml | 0 .../host/change_password_linux/main.yml | 29 ++++++ .../host/change_password_linux/manifest.yml | 0 .../change_password_local_windows/main.yml | 29 ++++++ .../manifest.yml | 0 .../automations/change_password/manager.py | 91 +++++++++++++++++++ apps/assets/automations/endpoint.py | 7 ++ .../generate_playbook}/__init__.py | 0 .../generate_playbook/change_password.py | 0 .../generate_playbook/verify.py | 0 .../__init__.py => automations/methods.py} | 3 +- apps/assets/const/types.py | 2 +- apps/assets/models/__init__.py | 1 + apps/assets/models/asset/common.py | 23 +++++ .../{automation => automations}/__init__.py | 0 .../account_discovery.py | 1 + .../account_reconcile.py | 0 .../account_verify.py | 0 .../{automation => automations}/base.py | 73 +++++++-------- .../change_secret.py | 2 +- apps/assets/playbooks/base/generator.py | 33 ------- apps/assets/playbooks/base/runner.py | 47 ---------- .../change_password_postgresql/main.yml | 10 -- .../roles/change_password/tasks/main.yml | 27 ------ .../change_password_sqlserver/main.yml | 10 -- .../roles/change_password/tasks/main.yml | 27 ------ .../host/change_password_aix/main.yml | 10 -- .../roles/change_password/tasks/main.yml | 27 ------ .../host/change_password_linux/main.yml | 8 -- .../roles/change_password/tasks/main.yml | 23 ----- .../change_password_local_windows/main.yml | 10 -- .../roles/change_password/tasks/main.yml | 27 ------ .../host/ansible_posix_ping/main.yml | 13 --- .../host/ansible_posix_ping/manifest.yml | 10 -- .../playbooks/host/ansible_win_ping/main.yml | 13 --- .../host/ansible_win_ping/manifest.yml | 6 -- .../change_password/roles/linux/main.yml | 12 --- .../roles/linux/tasks/main.yml | 36 -------- .../strategy/verify/roles/linux/main.yml | 5 - .../verify/roles/linux/tasks/main.yml | 8 -- apps/assets/task_handlers/__init__.py | 1 - apps/assets/task_handlers/endpoint.py | 10 -- apps/ops/ansible/callback.py | 2 +- apps/ops/ansible/inventory.py | 33 ++++--- apps/ops/models/base.py | 2 +- apps/ops/task_handlers/__init__.py | 1 - apps/ops/task_handlers/base/__init__.py | 2 - apps/ops/task_handlers/base/handlers.py | 16 ---- apps/ops/task_handlers/base/manager.py | 78 ---------------- .../ops/task_handlers/change_auth/__init__.py | 2 - .../ops/task_handlers/change_auth/handlers.py | 10 -- apps/ops/task_handlers/change_auth/manager.py | 12 --- apps/ops/task_handlers/collect/__init__.py | 2 - apps/ops/task_handlers/collect/handlers.py | 10 -- apps/ops/task_handlers/collect/manager.py | 10 -- apps/ops/task_handlers/endpoint.py | 31 ------- apps/ops/task_handlers/push/__init__.py | 2 - apps/ops/task_handlers/push/handlers.py | 10 -- apps/ops/task_handlers/push/manager.py | 10 -- apps/ops/task_handlers/verify/__init__.py | 2 - apps/ops/task_handlers/verify/handlers.py | 10 -- apps/ops/task_handlers/verify/manager.py | 10 -- apps/ops/utils.py | 2 +- 83 files changed, 413 insertions(+), 652 deletions(-) create mode 100644 apps/assets/automations/__init__.py rename apps/assets/{playbooks/base => automations/backup}/__init__.py (100%) rename apps/assets/{task_handlers => automations}/backup/handlers.py (100%) rename apps/assets/{task_handlers => automations}/backup/manager.py (100%) rename apps/assets/{playbooks/change_password => automations/base}/__init__.py (100%) create mode 100644 apps/assets/automations/base/base_inventory.txt create mode 100644 apps/assets/automations/base/manager.py rename apps/assets/{playbooks/generate_playbook => automations/change_password}/__init__.py (100%) create mode 100644 apps/assets/automations/change_password/database/change_password_mysql/main.yml rename apps/assets/{playbooks => automations}/change_password/database/change_password_mysql/manifest.yml (100%) create mode 100644 apps/assets/automations/change_password/database/change_password_oracle/main.yml rename apps/assets/{playbooks => automations}/change_password/database/change_password_oracle/manifest.yml (100%) rename apps/assets/{playbooks/change_password/database/change_password_mysql => automations/change_password/database/change_password_postgresql}/main.yml (100%) rename apps/assets/{playbooks => automations}/change_password/database/change_password_postgresql/manifest.yml (100%) rename apps/assets/{playbooks/change_password/database/change_password_mysql => automations/change_password/database/change_password_postgresql}/roles/change_password/tasks/main.yml (100%) rename apps/assets/{playbooks/change_password/database/change_password_oracle => automations/change_password/database/change_password_sqlserver}/main.yml (100%) rename apps/assets/{playbooks => automations}/change_password/database/change_password_sqlserver/manifest.yml (100%) rename apps/assets/{playbooks/change_password/database/change_password_oracle => automations/change_password/database/change_password_sqlserver}/roles/change_password/tasks/main.yml (100%) create mode 100644 apps/assets/automations/change_password/demo_inventory.txt create mode 100644 apps/assets/automations/change_password/host/change_password_aix/main.yml rename apps/assets/{playbooks => automations}/change_password/host/change_password_aix/manifest.yml (100%) create mode 100644 apps/assets/automations/change_password/host/change_password_linux/main.yml rename apps/assets/{playbooks => automations}/change_password/host/change_password_linux/manifest.yml (100%) create mode 100644 apps/assets/automations/change_password/host/change_password_local_windows/main.yml rename apps/assets/{playbooks => automations}/change_password/host/change_password_local_windows/manifest.yml (100%) create mode 100644 apps/assets/automations/change_password/manager.py create mode 100644 apps/assets/automations/endpoint.py rename apps/assets/{task_handlers/backup => automations/generate_playbook}/__init__.py (100%) rename apps/assets/{playbooks => automations}/generate_playbook/change_password.py (100%) rename apps/assets/{playbooks => automations}/generate_playbook/verify.py (100%) rename apps/assets/{playbooks/__init__.py => automations/methods.py} (96%) rename apps/assets/models/{automation => automations}/__init__.py (100%) rename apps/assets/models/{automation => automations}/account_discovery.py (89%) rename apps/assets/models/{automation => automations}/account_reconcile.py (100%) rename apps/assets/models/{automation => automations}/account_verify.py (100%) rename apps/assets/models/{automation => automations}/base.py (54%) rename apps/assets/models/{automation => automations}/change_secret.py (96%) delete mode 100644 apps/assets/playbooks/base/generator.py delete mode 100644 apps/assets/playbooks/base/runner.py delete mode 100644 apps/assets/playbooks/change_password/database/change_password_postgresql/main.yml delete mode 100644 apps/assets/playbooks/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml delete mode 100644 apps/assets/playbooks/change_password/database/change_password_sqlserver/main.yml delete mode 100644 apps/assets/playbooks/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml delete mode 100644 apps/assets/playbooks/change_password/host/change_password_aix/main.yml delete mode 100644 apps/assets/playbooks/change_password/host/change_password_aix/roles/change_password/tasks/main.yml delete mode 100644 apps/assets/playbooks/change_password/host/change_password_linux/main.yml delete mode 100644 apps/assets/playbooks/change_password/host/change_password_linux/roles/change_password/tasks/main.yml delete mode 100644 apps/assets/playbooks/change_password/host/change_password_local_windows/main.yml delete mode 100644 apps/assets/playbooks/change_password/host/change_password_local_windows/roles/change_password/tasks/main.yml delete mode 100644 apps/assets/playbooks/host/ansible_posix_ping/main.yml delete mode 100644 apps/assets/playbooks/host/ansible_posix_ping/manifest.yml delete mode 100644 apps/assets/playbooks/host/ansible_win_ping/main.yml delete mode 100644 apps/assets/playbooks/host/ansible_win_ping/manifest.yml delete mode 100644 apps/assets/playbooks/strategy/change_password/roles/linux/main.yml delete mode 100644 apps/assets/playbooks/strategy/change_password/roles/linux/tasks/main.yml delete mode 100644 apps/assets/playbooks/strategy/verify/roles/linux/main.yml delete mode 100644 apps/assets/playbooks/strategy/verify/roles/linux/tasks/main.yml delete mode 100644 apps/assets/task_handlers/__init__.py delete mode 100644 apps/assets/task_handlers/endpoint.py delete mode 100644 apps/ops/task_handlers/__init__.py delete mode 100644 apps/ops/task_handlers/base/__init__.py delete mode 100644 apps/ops/task_handlers/base/handlers.py delete mode 100644 apps/ops/task_handlers/base/manager.py delete mode 100644 apps/ops/task_handlers/change_auth/__init__.py delete mode 100644 apps/ops/task_handlers/change_auth/handlers.py delete mode 100644 apps/ops/task_handlers/change_auth/manager.py delete mode 100644 apps/ops/task_handlers/collect/__init__.py delete mode 100644 apps/ops/task_handlers/collect/handlers.py delete mode 100644 apps/ops/task_handlers/collect/manager.py delete mode 100644 apps/ops/task_handlers/endpoint.py delete mode 100644 apps/ops/task_handlers/push/__init__.py delete mode 100644 apps/ops/task_handlers/push/handlers.py delete mode 100644 apps/ops/task_handlers/push/manager.py delete mode 100644 apps/ops/task_handlers/verify/__init__.py delete mode 100644 apps/ops/task_handlers/verify/handlers.py delete mode 100644 apps/ops/task_handlers/verify/manager.py diff --git a/apps/assets/automations/__init__.py b/apps/assets/automations/__init__.py new file mode 100644 index 000000000..478e9740d --- /dev/null +++ b/apps/assets/automations/__init__.py @@ -0,0 +1 @@ +from .methods import platform_automation_methods diff --git a/apps/assets/playbooks/base/__init__.py b/apps/assets/automations/backup/__init__.py similarity index 100% rename from apps/assets/playbooks/base/__init__.py rename to apps/assets/automations/backup/__init__.py diff --git a/apps/assets/task_handlers/backup/handlers.py b/apps/assets/automations/backup/handlers.py similarity index 100% rename from apps/assets/task_handlers/backup/handlers.py rename to apps/assets/automations/backup/handlers.py diff --git a/apps/assets/task_handlers/backup/manager.py b/apps/assets/automations/backup/manager.py similarity index 100% rename from apps/assets/task_handlers/backup/manager.py rename to apps/assets/automations/backup/manager.py diff --git a/apps/assets/playbooks/change_password/__init__.py b/apps/assets/automations/base/__init__.py similarity index 100% rename from apps/assets/playbooks/change_password/__init__.py rename to apps/assets/automations/base/__init__.py diff --git a/apps/assets/automations/base/base_inventory.txt b/apps/assets/automations/base/base_inventory.txt new file mode 100644 index 000000000..a2b73db16 --- /dev/null +++ b/apps/assets/automations/base/base_inventory.txt @@ -0,0 +1,14 @@ +## all connection vars +hostname asset_name=name asset_type=type asset_primary_protocol=ssh asset_primary_port=22 asset_protocols=[] + +## local connection +hostname ansible_connection=local + +## local connection with gateway +hostname ansible_connection=ssh ansible_user=gateway.username ansible_port=gateway.port ansible_host=gateway.host ansible_ssh_private_key_file=gateway.key + +## ssh connection for windows +hostname ansible_connection=ssh ansible_shell_type=powershell/cmd ansible_user=windows.username ansible_port=windows.port ansible_host=windows.host ansible_ssh_private_key_file=windows.key + +## ssh connection +hostname ansible_user=user ansible_password=pass ansible_host=host ansible_port=port ansible_ssh_private_key_file=key ssh_args="-o StrictHostKeyChecking=no" diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py new file mode 100644 index 000000000..83097d0eb --- /dev/null +++ b/apps/assets/automations/base/manager.py @@ -0,0 +1,70 @@ +import os + +from django.conf import settings +from django.utils import timezone + +from ops.ansible import JMSInventory + + +class BasePlaybookManager: + ansible_account_policy = 'privileged_first' + + def __init__(self, execution): + self.execution = execution + self.automation = execution.automation + + def get_grouped_assets(self): + return self.automation.all_assets_group_by_platform() + + @property + def playbook_dir_path(self): + ansible_dir = settings.ANSIBLE_DIR + path = os.path.join( + ansible_dir, self.automation.type, self.automation.name, + timezone.now().strftime('%Y%m%d_%H%M%S') + ) + return path + + @property + def inventory_path(self): + return os.path.join(self.playbook_dir_path, 'inventory', 'hosts.json') + + @property + def playbook_path(self): + return os.path.join(self.playbook_dir_path, 'project', 'main.yml') + + def generate(self): + self.prepare_playbook_dir() + self.generate_inventory() + self.generate_playbook() + + def prepare_playbook_dir(self): + inventory_dir = os.path.dirname(self.inventory_path) + playbook_dir = os.path.dirname(self.playbook_path) + for d in [inventory_dir, playbook_dir]: + if not os.path.exists(d): + os.makedirs(d, exist_ok=True, mode=0o755) + + def inventory_kwargs(self): + raise NotImplemented + + def generate_inventory(self): + inventory = JMSInventory( + assets=self.automation.get_all_assets(), + account_policy=self.ansible_account_policy, + **self.inventory_kwargs() + ) + inventory.write_to_file(self.inventory_path) + print("Generate inventory done: {}".format(self.inventory_path)) + + def generate_playbook(self): + raise NotImplemented + + def get_runner(self): + raise NotImplemented + + def run(self, **kwargs): + self.generate() + runner = self.get_runner() + return runner.run(**kwargs) + diff --git a/apps/assets/playbooks/generate_playbook/__init__.py b/apps/assets/automations/change_password/__init__.py similarity index 100% rename from apps/assets/playbooks/generate_playbook/__init__.py rename to apps/assets/automations/change_password/__init__.py diff --git a/apps/assets/automations/change_password/database/change_password_mysql/main.yml b/apps/assets/automations/change_password/database/change_password_mysql/main.yml new file mode 100644 index 000000000..483554a1e --- /dev/null +++ b/apps/assets/automations/change_password/database/change_password_mysql/main.yml @@ -0,0 +1,29 @@ +- hosts: demo + tasks: + - name: ping + ping: + + #- name: print variables + # debug: + # msg: "Username: {{ account.username }}, Password: {{ account.password }}" + + - name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.password + + - name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + + - name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/database/change_password_mysql/manifest.yml b/apps/assets/automations/change_password/database/change_password_mysql/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_mysql/manifest.yml rename to apps/assets/automations/change_password/database/change_password_mysql/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_oracle/main.yml b/apps/assets/automations/change_password/database/change_password_oracle/main.yml new file mode 100644 index 000000000..483554a1e --- /dev/null +++ b/apps/assets/automations/change_password/database/change_password_oracle/main.yml @@ -0,0 +1,29 @@ +- hosts: demo + tasks: + - name: ping + ping: + + #- name: print variables + # debug: + # msg: "Username: {{ account.username }}, Password: {{ account.password }}" + + - name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.password + + - name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + + - name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/database/change_password_oracle/manifest.yml b/apps/assets/automations/change_password/database/change_password_oracle/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_oracle/manifest.yml rename to apps/assets/automations/change_password/database/change_password_oracle/manifest.yml diff --git a/apps/assets/playbooks/change_password/database/change_password_mysql/main.yml b/apps/assets/automations/change_password/database/change_password_postgresql/main.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_mysql/main.yml rename to apps/assets/automations/change_password/database/change_password_postgresql/main.yml diff --git a/apps/assets/playbooks/change_password/database/change_password_postgresql/manifest.yml b/apps/assets/automations/change_password/database/change_password_postgresql/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_postgresql/manifest.yml rename to apps/assets/automations/change_password/database/change_password_postgresql/manifest.yml diff --git a/apps/assets/playbooks/change_password/database/change_password_mysql/roles/change_password/tasks/main.yml b/apps/assets/automations/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_mysql/roles/change_password/tasks/main.yml rename to apps/assets/automations/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml diff --git a/apps/assets/playbooks/change_password/database/change_password_oracle/main.yml b/apps/assets/automations/change_password/database/change_password_sqlserver/main.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_oracle/main.yml rename to apps/assets/automations/change_password/database/change_password_sqlserver/main.yml diff --git a/apps/assets/playbooks/change_password/database/change_password_sqlserver/manifest.yml b/apps/assets/automations/change_password/database/change_password_sqlserver/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_sqlserver/manifest.yml rename to apps/assets/automations/change_password/database/change_password_sqlserver/manifest.yml diff --git a/apps/assets/playbooks/change_password/database/change_password_oracle/roles/change_password/tasks/main.yml b/apps/assets/automations/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml similarity index 100% rename from apps/assets/playbooks/change_password/database/change_password_oracle/roles/change_password/tasks/main.yml rename to apps/assets/automations/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml diff --git a/apps/assets/automations/change_password/demo_inventory.txt b/apps/assets/automations/change_password/demo_inventory.txt new file mode 100644 index 000000000..dcc7d1b6d --- /dev/null +++ b/apps/assets/automations/change_password/demo_inventory.txt @@ -0,0 +1,2 @@ +# all base inventory in base/base_inventory.txt +asset_name(ip)_account_username account={"username": "", "password": "xxx"} ...base_inventory_vars diff --git a/apps/assets/automations/change_password/host/change_password_aix/main.yml b/apps/assets/automations/change_password/host/change_password_aix/main.yml new file mode 100644 index 000000000..483554a1e --- /dev/null +++ b/apps/assets/automations/change_password/host/change_password_aix/main.yml @@ -0,0 +1,29 @@ +- hosts: demo + tasks: + - name: ping + ping: + + #- name: print variables + # debug: + # msg: "Username: {{ account.username }}, Password: {{ account.password }}" + + - name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.password + + - name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + + - name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/host/change_password_aix/manifest.yml b/apps/assets/automations/change_password/host/change_password_aix/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/host/change_password_aix/manifest.yml rename to apps/assets/automations/change_password/host/change_password_aix/manifest.yml diff --git a/apps/assets/automations/change_password/host/change_password_linux/main.yml b/apps/assets/automations/change_password/host/change_password_linux/main.yml new file mode 100644 index 000000000..1ad590cff --- /dev/null +++ b/apps/assets/automations/change_password/host/change_password_linux/main.yml @@ -0,0 +1,29 @@ +- hosts: demo + tasks: + - name: Test privileged account + ping: + + #- name: print variables + # debug: + # msg: "Username: {{ account.username }}, Password: {{ account.password }}" + + - name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.secret_type == 'password' + + - name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + + - name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/host/change_password_linux/manifest.yml b/apps/assets/automations/change_password/host/change_password_linux/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/host/change_password_linux/manifest.yml rename to apps/assets/automations/change_password/host/change_password_linux/manifest.yml diff --git a/apps/assets/automations/change_password/host/change_password_local_windows/main.yml b/apps/assets/automations/change_password/host/change_password_local_windows/main.yml new file mode 100644 index 000000000..483554a1e --- /dev/null +++ b/apps/assets/automations/change_password/host/change_password_local_windows/main.yml @@ -0,0 +1,29 @@ +- hosts: demo + tasks: + - name: ping + ping: + + #- name: print variables + # debug: + # msg: "Username: {{ account.username }}, Password: {{ account.password }}" + + - name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.password + + - name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + + - name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/host/change_password_local_windows/manifest.yml b/apps/assets/automations/change_password/host/change_password_local_windows/manifest.yml similarity index 100% rename from apps/assets/playbooks/change_password/host/change_password_local_windows/manifest.yml rename to apps/assets/automations/change_password/host/change_password_local_windows/manifest.yml diff --git a/apps/assets/automations/change_password/manager.py b/apps/assets/automations/change_password/manager.py new file mode 100644 index 000000000..ea0bb9a26 --- /dev/null +++ b/apps/assets/automations/change_password/manager.py @@ -0,0 +1,91 @@ +import os +import shutil +from copy import deepcopy +from collections import defaultdict + +import yaml +from django.utils.translation import gettext as _ + +from ops.ansible import PlaybookRunner, JMSInventory +from ..base.manager import BasePlaybookManager +from assets.automations.methods import platform_automation_methods + + +class ChangePasswordManager(BasePlaybookManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.id_method_mapper = { + method['id']: method + for method in platform_automation_methods + } + self.method_hosts_mapper = defaultdict(list) + + def host_duplicator(self, host, asset=None, account=None, platform=None, **kwargs): + accounts = asset.accounts.all() + if account: + accounts = accounts.exclude(id=account.id) + if '*' not in self.automation.accounts: + accounts = accounts.filter(username__in=self.automation.accounts) + + automation = platform.automation + change_password_enabled = automation and \ + automation.change_password_enabled and \ + automation.change_password_method and \ + automation.change_password_method in self.id_method_mapper + + if not change_password_enabled: + host.exclude = _('Change password disabled') + return [host] + + hosts = [] + for account in accounts: + h = deepcopy(host) + h['name'] += '_' + account.username + h['account'] = { + 'name': account.name, + 'username': account.username, + 'secret_type': account.secret_type, + 'secret': account.secret, + } + hosts.append(h) + self.method_hosts_mapper[automation.change_password_method].append(h['name']) + return hosts + + def inventory_kwargs(self): + return { + 'host_duplicator': self.host_duplicator + } + + def generate_playbook(self): + playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.id_method_mapper[method_id] + playbook_dir_path = method['dir'] + playbook_dir_name = os.path.dirname(playbook_dir_path) + shutil.copytree(playbook_dir_path, self.playbook_dir_path) + sub_playbook_path = os.path.join(self.playbook_dir_path, playbook_dir_name, 'main.yml') + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + plays = [] + for name in host_names: + play = deepcopy(host_playbook_play) + play['hosts'] = name + plays.append(play) + + with open(sub_playbook_path, 'w') as f: + yaml.safe_dump(plays, f, default_flow_style=False) + + playbook.append({ + 'name': method['name'], + 'import_playbook': playbook_dir_name + '/' + 'main.yml' + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(playbook, f, default_flow_style=False) + + print("Generate playbook done: " + self.playbook_path) + + + diff --git a/apps/assets/automations/endpoint.py b/apps/assets/automations/endpoint.py new file mode 100644 index 000000000..f99defdd8 --- /dev/null +++ b/apps/assets/automations/endpoint.py @@ -0,0 +1,7 @@ +# from .backup.manager import AccountBackupExecutionManager +# +# +class ExecutionManager: + manager_type = { + } + diff --git a/apps/assets/task_handlers/backup/__init__.py b/apps/assets/automations/generate_playbook/__init__.py similarity index 100% rename from apps/assets/task_handlers/backup/__init__.py rename to apps/assets/automations/generate_playbook/__init__.py diff --git a/apps/assets/playbooks/generate_playbook/change_password.py b/apps/assets/automations/generate_playbook/change_password.py similarity index 100% rename from apps/assets/playbooks/generate_playbook/change_password.py rename to apps/assets/automations/generate_playbook/change_password.py diff --git a/apps/assets/playbooks/generate_playbook/verify.py b/apps/assets/automations/generate_playbook/verify.py similarity index 100% rename from apps/assets/playbooks/generate_playbook/verify.py rename to apps/assets/automations/generate_playbook/verify.py diff --git a/apps/assets/playbooks/__init__.py b/apps/assets/automations/methods.py similarity index 96% rename from apps/assets/playbooks/__init__.py rename to apps/assets/automations/methods.py index 59f705fb9..811d0c77d 100644 --- a/apps/assets/playbooks/__init__.py +++ b/apps/assets/automations/methods.py @@ -1,5 +1,6 @@ import os import yaml +import json from functools import partial BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -62,4 +63,4 @@ platform_automation_methods = get_platform_automation_methods() if __name__ == '__main__': - print(get_platform_automation_methods()) + print(json.dumps(platform_automation_methods, indent=4)) diff --git a/apps/assets/const/types.py b/apps/assets/const/types.py index 454a5d358..0d8b86afa 100644 --- a/apps/assets/const/types.py +++ b/apps/assets/const/types.py @@ -38,7 +38,7 @@ class AllTypes(ChoicesMixin): @classmethod def set_automation_methods(cls, category, tp, constraints): - from assets.playbooks import filter_platform_methods + from assets.automations import filter_platform_methods automation = constraints.get('automation', {}) automation_methods = {} for item, enabled in automation.items(): diff --git a/apps/assets/models/__init__.py b/apps/assets/models/__init__.py index e79b2fff5..376355657 100644 --- a/apps/assets/models/__init__.py +++ b/apps/assets/models/__init__.py @@ -10,6 +10,7 @@ from .gathered_user import * from .favorite_asset import * from .account import * from .backup import * +from .automations import * from ._user import * # 废弃以下 # from ._authbook import * diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index ef64e5fb6..5703c82dc 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -4,6 +4,7 @@ import logging import uuid +from collections import defaultdict from django.db import models from django.db.models import Q @@ -41,6 +42,12 @@ class AssetQuerySet(models.QuerySet): def has_protocol(self, name): return self.filter(protocols__contains=name) + def group_by_platform(self) -> dict: + groups = defaultdict(list) + for asset in self.all(): + groups[asset.platform].append(asset) + return groups + class NodesRelationMixin: NODES_CACHE_KEY = 'ASSET_NODES_{}' @@ -126,6 +133,22 @@ class Asset(AbsConnectivity, NodesRelationMixin, JMSOrgBaseModel): names.append(n.name + ':' + n.value) return names + @lazyproperty + def primary_protocol(self): + return self.protocols.first() + + @lazyproperty + def protocol(self): + if not self.primary_protocol: + return 'none' + return self.primary_protocol.name + + @lazyproperty + def port(self): + if not self.primary_protocol: + return 0 + return self.primary_protocol.port + @property def protocols_as_list(self): return [{'name': p.name, 'port': p.port} for p in self.protocols.all()] diff --git a/apps/assets/models/automation/__init__.py b/apps/assets/models/automations/__init__.py similarity index 100% rename from apps/assets/models/automation/__init__.py rename to apps/assets/models/automations/__init__.py diff --git a/apps/assets/models/automation/account_discovery.py b/apps/assets/models/automations/account_discovery.py similarity index 89% rename from apps/assets/models/automation/account_discovery.py rename to apps/assets/models/automations/account_discovery.py index bfe9d0d80..9572986f3 100644 --- a/apps/assets/models/automation/account_discovery.py +++ b/apps/assets/models/automations/account_discovery.py @@ -1,6 +1,7 @@ from django.utils.translation import ugettext_lazy as _ from ops.const import StrategyChoice +from ops.ansible.runner import PlaybookRunner from .base import BaseAutomation diff --git a/apps/assets/models/automation/account_reconcile.py b/apps/assets/models/automations/account_reconcile.py similarity index 100% rename from apps/assets/models/automation/account_reconcile.py rename to apps/assets/models/automations/account_reconcile.py diff --git a/apps/assets/models/automation/account_verify.py b/apps/assets/models/automations/account_verify.py similarity index 100% rename from apps/assets/models/automation/account_verify.py rename to apps/assets/models/automations/account_verify.py diff --git a/apps/assets/models/automation/base.py b/apps/assets/models/automations/base.py similarity index 54% rename from apps/assets/models/automation/base.py rename to apps/assets/models/automations/base.py index a9b8ab087..ba0d393ea 100644 --- a/apps/assets/models/automation/base.py +++ b/apps/assets/models/automations/base.py @@ -8,16 +8,17 @@ from common.db.fields import EncryptJsonDictTextField from orgs.mixins.models import OrgModelMixin, JMSOrgBaseModel from ops.mixin import PeriodTaskModelMixin from ops.tasks import execute_automation_strategy -from ops.task_handlers import ExecutionManager +from assets.models import Node, Asset +from assets.automations.endpoint import ExecutionManager class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): accounts = models.JSONField(default=list, verbose_name=_("Accounts")) nodes = models.ManyToManyField( - 'assets.Node', related_name='automation_strategy', blank=True, verbose_name=_("Nodes") + 'assets.Node', blank=True, verbose_name=_("Nodes") ) assets = models.ManyToManyField( - 'assets.Asset', related_name='automation_strategy', blank=True, verbose_name=_("Assets") + 'assets.Asset', blank=True, verbose_name=_("Assets") ) type = models.CharField(max_length=16, verbose_name=_('Type')) comment = models.TextField(blank=True, verbose_name=_('Comment')) @@ -25,6 +26,17 @@ class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): def __str__(self): return self.name + '@' + str(self.created_by) + def get_all_assets(self): + nodes = self.nodes.all() + node_asset_ids = Node.get_nodes_all_assets(*nodes).values_list('id', flat=True) + direct_asset_ids = self.assets.all().values_list('id', flat=True) + asset_ids = set(list(direct_asset_ids) + list(node_asset_ids)) + return Asset.objects.filter(id__in=asset_ids) + + def all_assets_group_by_platform(self): + assets = self.get_all_assets().prefetch_related('platform') + return assets.group_by_platform() + def get_register_task(self): name = "automation_strategy_period_{}".format(str(self.id)[:8]) task = execute_automation_strategy.name @@ -40,13 +52,15 @@ class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): 'nodes': list(self.assets.all().values_list('id', flat=True)), } - def execute(self, trigger): + def execute(self, trigger=Trigger.manual): try: eid = current_task.request.id except AttributeError: eid = str(uuid.uuid4()) - execution = AutomationStrategyExecution.objects.create( - id=eid, strategy=self, snapshot=self.to_attr_json(), trigger=trigger + + execution = AutomationExecution.objects.create( + id=eid, strategy=self, trigger=trigger, + snapshot=self.to_attr_json(), ) return execution.start() @@ -55,22 +69,22 @@ class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): verbose_name = _("Automation plan") -class AutomationStrategyExecution(OrgModelMixin): +class AutomationExecution(OrgModelMixin): id = models.UUIDField(default=uuid.uuid4, primary_key=True) - - date_created = models.DateTimeField(auto_now_add=True) - timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True) - date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Date start')) - + automation = models.ForeignKey( + 'BaseAutomation', related_name='executions', on_delete=models.CASCADE, + verbose_name=_('Automation strategy') + ) + status = models.CharField(max_length=16, default='pending') + date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) + date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True) + date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished")) snapshot = EncryptJsonDictTextField( default=dict, blank=True, null=True, verbose_name=_('Automation snapshot') ) - strategy = models.ForeignKey( - 'BaseAutomation', related_name='execution', on_delete=models.CASCADE, - verbose_name=_('Automation strategy') - ) trigger = models.CharField( - max_length=128, default=Trigger.manual, choices=Trigger.choices, verbose_name=_('Trigger mode') + max_length=128, default=Trigger.manual, choices=Trigger.choices, + verbose_name=_('Trigger mode') ) class Meta: @@ -83,28 +97,3 @@ class AutomationStrategyExecution(OrgModelMixin): def start(self): manager = ExecutionManager(execution=self) return manager.run() - - -class AutomationStrategyTask(OrgModelMixin): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - asset = models.ForeignKey( - 'assets.Asset', on_delete=models.CASCADE, verbose_name=_('Asset') - ) - account = models.ForeignKey( - 'assets.Account', on_delete=models.CASCADE, verbose_name=_('Account') - ) - is_success = models.BooleanField(default=False, verbose_name=_('Is success')) - timedelta = models.FloatField(default=0.0, null=True, verbose_name=_('Time')) - date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Date start')) - reason = models.CharField(max_length=1024, blank=True, null=True, verbose_name=_('Reason')) - execution = models.ForeignKey( - 'AutomationStrategyExecution', related_name='task', on_delete=models.CASCADE, - verbose_name=_('Automation strategy execution') - ) - - class Meta: - verbose_name = _('Automation strategy task') - - @property - def handler_type(self): - return self.execution.snapshot['type'] diff --git a/apps/assets/models/automation/change_secret.py b/apps/assets/models/automations/change_secret.py similarity index 96% rename from apps/assets/models/automation/change_secret.py rename to apps/assets/models/automations/change_secret.py index d176f5c6a..4a2c304c2 100644 --- a/apps/assets/models/automation/change_secret.py +++ b/apps/assets/models/automations/change_secret.py @@ -2,7 +2,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from common.db import fields -from ops.const import SSHKeyStrategy, PasswordStrategy, StrategyChoice +from ops.const import PasswordStrategy, StrategyChoice from ops.utils import generate_random_password from .base import BaseAutomation diff --git a/apps/assets/playbooks/base/generator.py b/apps/assets/playbooks/base/generator.py deleted file mode 100644 index 972d12409..000000000 --- a/apps/assets/playbooks/base/generator.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import time -import shutil -from typing import List - -from django.conf import settings - -from assets.models import Asset - - -class BaseRunner: - src_filepath: str - - def __init__(self, assets: List[Asset], strategy): - self.assets = assets - self.strategy = strategy - self.temp_folder = self.temp_folder_path() - - @staticmethod - def temp_folder_path(): - project_dir = settings.PROJECT_DIR - tmp_dir = os.path.join(project_dir, 'tmp') - filepath = os.path.join(tmp_dir, str(time.time())) - return filepath - - def del_temp_folder(self): - shutil.rmtree(self.temp_folder) - - def generate_temp_playbook(self): - src = self.src_filepath - dst = os.path.join(self.temp_folder, self.strategy) - shutil.copytree(src, dst) - return dst diff --git a/apps/assets/playbooks/base/runner.py b/apps/assets/playbooks/base/runner.py deleted file mode 100644 index 1370db6ba..000000000 --- a/apps/assets/playbooks/base/runner.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import tempfile -import shutil -from typing import List - -from django.conf import settings - -from assets.models import Asset - - -class BasePlaybookGenerator: - def __init__(self, assets: list[Asset], strategy, ansible_connection='ssh'): - self.assets = assets - self.strategy = strategy - self.playbook_dir = self.temp_folder_path() - - def generate(self): - self.prepare_playbook_dir() - self.generate_inventory() - self.generate_playbook() - - def prepare_playbook_dir(self): - pass - - def generate_inventory(self): - pass - - def generate_playbook(self): - pass - - @property - def base_dir(self): - tmp_dir = os.path.join(settings.PROJECT_DIR, 'tmp') - path = os.path.join(tmp_dir, self.strategy) - return path - - def temp_folder_path(self): - return tempfile.mkdtemp(dir=self.base_dir) - - def del_temp_folder(self): - shutil.rmtree(self.playbook_dir) - - def generate_temp_playbook(self): - src = self.src_filepath - dst = os.path.join(self.temp_folder, self.strategy) - shutil.copytree(src, dst) - return dst diff --git a/apps/assets/playbooks/change_password/database/change_password_postgresql/main.yml b/apps/assets/playbooks/change_password/database/change_password_postgresql/main.yml deleted file mode 100644 index 402c7fa8d..000000000 --- a/apps/assets/playbooks/change_password/database/change_password_postgresql/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -{% for account in accounts %} -- hosts: {{ account.asset.name }} - vars: - account: - username: {{ account.username }} - password: {{ account.password }} - public_key: {{ account.public_key }} - roles: - - change_password -{% endfor %} diff --git a/apps/assets/playbooks/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml b/apps/assets/playbooks/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml deleted file mode 100644 index 903cd9115..000000000 --- a/apps/assets/playbooks/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: ping - ping: - -#- name: print variables -# debug: -# msg: "Username: {{ account.username }}, Password: {{ account.password }}" - -- name: Change password - user: - name: "{{ account.username }}" - password: "{{ account.password | password_hash('des') }}" - update_password: always - when: account.password - -- name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key - -- name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/database/change_password_sqlserver/main.yml b/apps/assets/playbooks/change_password/database/change_password_sqlserver/main.yml deleted file mode 100644 index 402c7fa8d..000000000 --- a/apps/assets/playbooks/change_password/database/change_password_sqlserver/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -{% for account in accounts %} -- hosts: {{ account.asset.name }} - vars: - account: - username: {{ account.username }} - password: {{ account.password }} - public_key: {{ account.public_key }} - roles: - - change_password -{% endfor %} diff --git a/apps/assets/playbooks/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml b/apps/assets/playbooks/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml deleted file mode 100644 index 903cd9115..000000000 --- a/apps/assets/playbooks/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: ping - ping: - -#- name: print variables -# debug: -# msg: "Username: {{ account.username }}, Password: {{ account.password }}" - -- name: Change password - user: - name: "{{ account.username }}" - password: "{{ account.password | password_hash('des') }}" - update_password: always - when: account.password - -- name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key - -- name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/host/change_password_aix/main.yml b/apps/assets/playbooks/change_password/host/change_password_aix/main.yml deleted file mode 100644 index 402c7fa8d..000000000 --- a/apps/assets/playbooks/change_password/host/change_password_aix/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -{% for account in accounts %} -- hosts: {{ account.asset.name }} - vars: - account: - username: {{ account.username }} - password: {{ account.password }} - public_key: {{ account.public_key }} - roles: - - change_password -{% endfor %} diff --git a/apps/assets/playbooks/change_password/host/change_password_aix/roles/change_password/tasks/main.yml b/apps/assets/playbooks/change_password/host/change_password_aix/roles/change_password/tasks/main.yml deleted file mode 100644 index 903cd9115..000000000 --- a/apps/assets/playbooks/change_password/host/change_password_aix/roles/change_password/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: ping - ping: - -#- name: print variables -# debug: -# msg: "Username: {{ account.username }}, Password: {{ account.password }}" - -- name: Change password - user: - name: "{{ account.username }}" - password: "{{ account.password | password_hash('des') }}" - update_password: always - when: account.password - -- name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key - -- name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/host/change_password_linux/main.yml b/apps/assets/playbooks/change_password/host/change_password_linux/main.yml deleted file mode 100644 index a7d0f9417..000000000 --- a/apps/assets/playbooks/change_password/host/change_password_linux/main.yml +++ /dev/null @@ -1,8 +0,0 @@ -- hosts: all - vars: - account: - username: {{ account.username }} - password: {{ account.password }} - public_key: {{ account.public_key }} - roles: - - change_password diff --git a/apps/assets/playbooks/change_password/host/change_password_linux/roles/change_password/tasks/main.yml b/apps/assets/playbooks/change_password/host/change_password_linux/roles/change_password/tasks/main.yml deleted file mode 100644 index e0ba9c73f..000000000 --- a/apps/assets/playbooks/change_password/host/change_password_linux/roles/change_password/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -- name: Check connection - ping: - -- name: Change password - user: - name: "{{ account.username }}" - password: "{{ account.password | password_hash('sha512') }}" - update_password: always - when: account.password - -- name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key - -- name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/change_password/host/change_password_local_windows/main.yml b/apps/assets/playbooks/change_password/host/change_password_local_windows/main.yml deleted file mode 100644 index 402c7fa8d..000000000 --- a/apps/assets/playbooks/change_password/host/change_password_local_windows/main.yml +++ /dev/null @@ -1,10 +0,0 @@ -{% for account in accounts %} -- hosts: {{ account.asset.name }} - vars: - account: - username: {{ account.username }} - password: {{ account.password }} - public_key: {{ account.public_key }} - roles: - - change_password -{% endfor %} diff --git a/apps/assets/playbooks/change_password/host/change_password_local_windows/roles/change_password/tasks/main.yml b/apps/assets/playbooks/change_password/host/change_password_local_windows/roles/change_password/tasks/main.yml deleted file mode 100644 index 903cd9115..000000000 --- a/apps/assets/playbooks/change_password/host/change_password_local_windows/roles/change_password/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: ping - ping: - -#- name: print variables -# debug: -# msg: "Username: {{ account.username }}, Password: {{ account.password }}" - -- name: Change password - user: - name: "{{ account.username }}" - password: "{{ account.password | password_hash('des') }}" - update_password: always - when: account.password - -- name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key - -- name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko diff --git a/apps/assets/playbooks/host/ansible_posix_ping/main.yml b/apps/assets/playbooks/host/ansible_posix_ping/main.yml deleted file mode 100644 index 4ccdb3074..000000000 --- a/apps/assets/playbooks/host/ansible_posix_ping/main.yml +++ /dev/null @@ -1,13 +0,0 @@ -- hosts: centos - gather_facts: no - vars: - account: - username: web - password: test123 - - tasks: - - name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" diff --git a/apps/assets/playbooks/host/ansible_posix_ping/manifest.yml b/apps/assets/playbooks/host/ansible_posix_ping/manifest.yml deleted file mode 100644 index 6cd223f1c..000000000 --- a/apps/assets/playbooks/host/ansible_posix_ping/manifest.yml +++ /dev/null @@ -1,10 +0,0 @@ -id: ansible_posix_ping -name: Ansible posix ping -description: Ansible ping -category: host -type: - - linux - - unix - - macos - - bsd -method: verify_account diff --git a/apps/assets/playbooks/host/ansible_win_ping/main.yml b/apps/assets/playbooks/host/ansible_win_ping/main.yml deleted file mode 100644 index 726d04a53..000000000 --- a/apps/assets/playbooks/host/ansible_win_ping/main.yml +++ /dev/null @@ -1,13 +0,0 @@ -- hosts: centos - gather_facts: no - vars: - account: - username: web - password: test123 - - tasks: - - name: Verify password - win_ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" diff --git a/apps/assets/playbooks/host/ansible_win_ping/manifest.yml b/apps/assets/playbooks/host/ansible_win_ping/manifest.yml deleted file mode 100644 index fe881de3b..000000000 --- a/apps/assets/playbooks/host/ansible_win_ping/manifest.yml +++ /dev/null @@ -1,6 +0,0 @@ -id: ansible_win_ping -name: Ansible win ping -category: host -type: - - windows -method: verify_account diff --git a/apps/assets/playbooks/strategy/change_password/roles/linux/main.yml b/apps/assets/playbooks/strategy/change_password/roles/linux/main.yml deleted file mode 100644 index 16f0d1037..000000000 --- a/apps/assets/playbooks/strategy/change_password/roles/linux/main.yml +++ /dev/null @@ -1,12 +0,0 @@ -- hosts: all - vars: - connection_type: ssh - password: - value: {{ password }} - public_key: - value: {{ jms_key }} - exclusive: {{ exclusive }} - key_strategy: {{ key_strategy }} - private_key_file: {{ private_key_file }} - roles: - - linux diff --git a/apps/assets/playbooks/strategy/change_password/roles/linux/tasks/main.yml b/apps/assets/playbooks/strategy/change_password/roles/linux/tasks/main.yml deleted file mode 100644 index cc9467ca1..000000000 --- a/apps/assets/playbooks/strategy/change_password/roles/linux/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ -- name: Check connection - ping: - -- name: Change password - user: - name: "{{ item }}" - password: "{{ password.value | password_hash('sha512') }}" - update_password: always - with_items: "{{ usernames }}" - when: "{{ password.value }}" - -- name: Change public key - authorized_key: - user: "{{ item }}" - key: "{{ lookup('file', id_rsa.pub) }}" - state: present - exclusive: "{{ public_key.exclusive }}" - with_items: "{{ usernames }}" - when: "{{ public_key.value and key_strategy != 'set_jms' }}" - -- name: Change public key - lineinfile: - user: "{{ item }}" - dest: /home/{{ item }}/.ssh/authorized_keys regexp='.*{{ public_key.value }}$ - state: absent - with_items: "{{ usernames }}" - when: "{{ public_key.value and key_strategy == 'set_jms' }}" - -- name: Verify user - ping: - vars: - ansible_user: "{{ item }}" - ansible_pass: "{{ password.value }}" - ansible_ssh_private_key_file: "{{ private_key_file }}" - ansible_connection: "{{ connection_type | default('ssh') }}" - with_items: "{{ usernames }}" diff --git a/apps/assets/playbooks/strategy/verify/roles/linux/main.yml b/apps/assets/playbooks/strategy/verify/roles/linux/main.yml deleted file mode 100644 index 03c666df7..000000000 --- a/apps/assets/playbooks/strategy/verify/roles/linux/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -- hosts: all - vars: - connection_type: ssh - roles: - - linux diff --git a/apps/assets/playbooks/strategy/verify/roles/linux/tasks/main.yml b/apps/assets/playbooks/strategy/verify/roles/linux/tasks/main.yml deleted file mode 100644 index ff9e1eb99..000000000 --- a/apps/assets/playbooks/strategy/verify/roles/linux/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: Verify user - ping: - vars: - ansible_user: "{{ item.username }}" - ansible_pass: "{{ item.username }}" - ansible_connection: "{{ connection_type | default('ssh') }}" - ansible_ssh_private_key_file: "{{ item.private_key_file }}" - with_items: "{{ account_info }}" diff --git a/apps/assets/task_handlers/__init__.py b/apps/assets/task_handlers/__init__.py deleted file mode 100644 index d557c449e..000000000 --- a/apps/assets/task_handlers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .endpoint import * diff --git a/apps/assets/task_handlers/endpoint.py b/apps/assets/task_handlers/endpoint.py deleted file mode 100644 index 729fc8648..000000000 --- a/apps/assets/task_handlers/endpoint.py +++ /dev/null @@ -1,10 +0,0 @@ -from .backup.manager import AccountBackupExecutionManager - - -class ExecutionManager: - manager_type = { - 'backup': AccountBackupExecutionManager - } - - def __new__(cls, execution): - return AccountBackupExecutionManager(execution) diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index 59734b07d..84e106e64 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -71,7 +71,7 @@ class DefaultCallback: def runner_on_start(self, event_data, **kwargs): pass - def runer_retry(self, event_data, **kwargs): + def runner_retry(self, event_data, **kwargs): pass def runner_on_file_diff(self, event_data, **kwargs): diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 6a7e3c5aa..5528f1a14 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -1,7 +1,7 @@ # ~*~ coding: utf-8 ~*~ -from collections import defaultdict import json import os +from collections import defaultdict from django.utils.translation import gettext as _ @@ -10,7 +10,7 @@ __all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, assets, account='', account_policy='smart', host_var_callback=None): + def __init__(self, assets, account='', account_policy='smart', host_var_callback=None, host_duplicator=None): """ :param assets: :param account: account username name if not set use account_policy @@ -21,6 +21,7 @@ class JMSInventory: self.account_username = account self.account_policy = account_policy self.host_var_callback = host_var_callback + self.host_duplicator = host_duplicator @staticmethod def clean_assets(assets): @@ -59,11 +60,16 @@ class JMSInventory: return {"ansible_ssh_common_args": proxy_command} def asset_to_host(self, asset, account, automation, protocols): - host = {'name': asset.name, 'vars': { - 'asset_id': str(asset.id), 'asset_name': asset.name, - 'asset_type': asset.type, 'asset_category': asset.category, + host = { + 'name': asset.name, + 'asset': { + 'id': asset.id, 'name': asset.name, 'ip': asset.ip, + 'type': asset.type, 'category': asset.category, + 'protocol': asset.protocol, 'port': asset.port, + 'protocols': [{'name': p.name, 'port': p.port} for p in protocols], + }, 'exclude': '' - }} + } ansible_connection = automation.ansible_config.get('ansible_connection', 'ssh') gateway = None if asset.domain: @@ -91,15 +97,15 @@ class JMSInventory: elif account.secret_type == 'private_key' and account.secret: host['ssh_private_key'] = account.private_key_file else: - host['vars']['exclude'] = _("No account found") + host['exclude'] = _("No account found") if gateway: - host['vars'].update(self.make_proxy_command(gateway)) + host.update(self.make_proxy_command(gateway)) if self.host_var_callback: callback_var = self.host_var_callback(asset) if isinstance(callback_var, dict): - host['vars'].update(callback_var) + host.update(callback_var) return host def select_account(self, asset): @@ -137,8 +143,11 @@ class JMSInventory: account = self.select_account(asset) host = self.asset_to_host(asset, account, automation, protocols) if not automation.ansible_enabled: - host['vars']['exclude'] = _('Ansible disabled') - hosts.append(host) + host['exclude'] = _('Ansible disabled') + if self.host_duplicator: + hosts.extend(self.host_duplicator(host, asset=asset, account=account, platform=platform)) + else: + hosts.append(host) exclude_hosts = list(filter(lambda x: x.get('exclude'), hosts)) if exclude_hosts: @@ -150,8 +159,6 @@ class JMSInventory: data = {'all': {'hosts': {}}} for host in hosts: name = host.pop('name') - var = host.pop('vars', {}) - host.update(var) data['all']['hosts'][name] = host return data diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py index f43e0c584..e91af13a6 100644 --- a/apps/ops/models/base.py +++ b/apps/ops/models/base.py @@ -52,7 +52,7 @@ class BaseAnsibleExecution(models.Model): creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True) - date_finished = models.DateTimeField(null=True) + date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished")) class Meta: abstract = True diff --git a/apps/ops/task_handlers/__init__.py b/apps/ops/task_handlers/__init__.py deleted file mode 100644 index d557c449e..000000000 --- a/apps/ops/task_handlers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .endpoint import * diff --git a/apps/ops/task_handlers/base/__init__.py b/apps/ops/task_handlers/base/__init__.py deleted file mode 100644 index 2dc0b1a17..000000000 --- a/apps/ops/task_handlers/base/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .manager import * -from .handlers import * diff --git a/apps/ops/task_handlers/base/handlers.py b/apps/ops/task_handlers/base/handlers.py deleted file mode 100644 index 1df6d946c..000000000 --- a/apps/ops/task_handlers/base/handlers.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -执行改密计划的基类 -""" -from common.utils import get_logger - -logger = get_logger(__file__) - - -class BaseHandler: - def __init__(self, task, show_step_info=True): - self.task = task - self.conn = None - self.retry_times = 3 - self.current_step = 0 - self.is_frozen = False # 任务状态冻结标志 - self.show_step_info = show_step_info diff --git a/apps/ops/task_handlers/base/manager.py b/apps/ops/task_handlers/base/manager.py deleted file mode 100644 index 080bf866c..000000000 --- a/apps/ops/task_handlers/base/manager.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- -# -import time -from openpyxl import Workbook -from django.utils import timezone - -from common.utils import get_logger -from common.utils.timezone import local_now_display - -logger = get_logger(__file__) - - -class BaseExecutionManager: - task_back_up_serializer: None - - def __init__(self, execution): - self.execution = execution - self.date_start = timezone.now() - self.time_start = time.time() - self.date_end = None - self.time_end = None - self.timedelta = 0 - self.total_tasks = [] - - def on_tasks_pre_run(self, tasks): - raise NotImplementedError - - def on_per_task_pre_run(self, task, total, index): - raise NotImplementedError - - def create_csv_file(self, tasks, file_name): - raise NotImplementedError - - def get_handler_cls(self): - raise NotImplemented - - def do_run(self): - tasks = self.total_tasks = self.execution.create_plan_tasks() - self.on_tasks_pre_run(tasks) - total = len(tasks) - - for index, task in enumerate(tasks, start=1): - self.on_per_task_pre_run(task, total, index) - task.start(show_step_info=False) - - def pre_run(self): - self.execution.date_start = self.date_start - self.execution.save() - self.show_execution_steps() - - def show_execution_steps(self): - pass - - def show_summary(self): - split_line = '#' * 40 - summary = self.execution.result_summary - logger.info(f'\n{split_line} 改密计划执行结果汇总 {split_line}') - logger.info( - '\n成功: {succeed}, 失败: {failed}, 总数: {total}\n' - ''.format(**summary) - ) - - def post_run(self): - self.time_end = time.time() - self.date_end = timezone.now() - - logger.info('\n\n' + '-' * 80) - logger.info('任务执行结束 {}\n'.format(local_now_display())) - self.timedelta = int(self.time_end - self.time_start) - logger.info('用时: {}s'.format(self.timedelta)) - self.execution.timedelta = self.timedelta - self.execution.save() - self.show_summary() - - def run(self): - self.pre_run() - self.do_run() - self.post_run() diff --git a/apps/ops/task_handlers/change_auth/__init__.py b/apps/ops/task_handlers/change_auth/__init__.py deleted file mode 100644 index 2dc0b1a17..000000000 --- a/apps/ops/task_handlers/change_auth/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .manager import * -from .handlers import * diff --git a/apps/ops/task_handlers/change_auth/handlers.py b/apps/ops/task_handlers/change_auth/handlers.py deleted file mode 100644 index c982a089d..000000000 --- a/apps/ops/task_handlers/change_auth/handlers.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseHandler - -logger = get_logger(__name__) - - -class ChangeAuthHandler(BaseHandler): - pass diff --git a/apps/ops/task_handlers/change_auth/manager.py b/apps/ops/task_handlers/change_auth/manager.py deleted file mode 100644 index a9f77927c..000000000 --- a/apps/ops/task_handlers/change_auth/manager.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseExecutionManager -from .handlers import ChangeAuthHandler - -logger = get_logger(__name__) - - -class ChangeAuthExecutionManager(BaseExecutionManager): - def get_handler_cls(self): - return ChangeAuthHandler diff --git a/apps/ops/task_handlers/collect/__init__.py b/apps/ops/task_handlers/collect/__init__.py deleted file mode 100644 index 2dc0b1a17..000000000 --- a/apps/ops/task_handlers/collect/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .manager import * -from .handlers import * diff --git a/apps/ops/task_handlers/collect/handlers.py b/apps/ops/task_handlers/collect/handlers.py deleted file mode 100644 index 81b1e748d..000000000 --- a/apps/ops/task_handlers/collect/handlers.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseHandler - -logger = get_logger(__name__) - - -class CollectHandler(BaseHandler): - pass diff --git a/apps/ops/task_handlers/collect/manager.py b/apps/ops/task_handlers/collect/manager.py deleted file mode 100644 index 18a685d7f..000000000 --- a/apps/ops/task_handlers/collect/manager.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseExecutionManager - -logger = get_logger(__name__) - - -class CollectExecutionManager(object): - pass diff --git a/apps/ops/task_handlers/endpoint.py b/apps/ops/task_handlers/endpoint.py deleted file mode 100644 index 2d95dcad5..000000000 --- a/apps/ops/task_handlers/endpoint.py +++ /dev/null @@ -1,31 +0,0 @@ -from ops.const import StrategyChoice -from .push import PushExecutionManager, PushHandler -from .verify import VerifyExecutionManager, VerifyHandler -from .collect import CollectExecutionManager, CollectHandler -from .change_auth import ChangeAuthExecutionManager, ChangeAuthHandler - - -class ExecutionManager: - manager_type = { - StrategyChoice.push: PushExecutionManager, - StrategyChoice.verify: VerifyExecutionManager, - StrategyChoice.collect: CollectExecutionManager, - StrategyChoice.change_password: ChangeAuthExecutionManager, - } - - def __new__(cls, execution): - manager = cls.manager_type[execution.manager_type] - return manager(execution) - - -class TaskHandler: - handler_type = { - StrategyChoice.push: PushHandler, - StrategyChoice.verify: VerifyHandler, - StrategyChoice.collect: CollectHandler, - StrategyChoice.change_password: ChangeAuthHandler, - } - - def __new__(cls, task, show_step_info): - handler = cls.handler_type[task.handler_type] - return handler(task, show_step_info) diff --git a/apps/ops/task_handlers/push/__init__.py b/apps/ops/task_handlers/push/__init__.py deleted file mode 100644 index 2dc0b1a17..000000000 --- a/apps/ops/task_handlers/push/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .manager import * -from .handlers import * diff --git a/apps/ops/task_handlers/push/handlers.py b/apps/ops/task_handlers/push/handlers.py deleted file mode 100644 index 891ac4bb6..000000000 --- a/apps/ops/task_handlers/push/handlers.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseHandler - -logger = get_logger(__name__) - - -class PushHandler(BaseHandler): - pass diff --git a/apps/ops/task_handlers/push/manager.py b/apps/ops/task_handlers/push/manager.py deleted file mode 100644 index 933f9a0cc..000000000 --- a/apps/ops/task_handlers/push/manager.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseExecutionManager - -logger = get_logger(__name__) - - -class PushExecutionManager(BaseExecutionManager): - pass diff --git a/apps/ops/task_handlers/verify/__init__.py b/apps/ops/task_handlers/verify/__init__.py deleted file mode 100644 index 2dc0b1a17..000000000 --- a/apps/ops/task_handlers/verify/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .manager import * -from .handlers import * diff --git a/apps/ops/task_handlers/verify/handlers.py b/apps/ops/task_handlers/verify/handlers.py deleted file mode 100644 index 7a7f69881..000000000 --- a/apps/ops/task_handlers/verify/handlers.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseHandler - -logger = get_logger(__name__) - - -class VerifyHandler(BaseHandler): - pass diff --git a/apps/ops/task_handlers/verify/manager.py b/apps/ops/task_handlers/verify/manager.py deleted file mode 100644 index a3727f1de..000000000 --- a/apps/ops/task_handlers/verify/manager.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# -from common.utils import get_logger -from ..base import BaseExecutionManager - -logger = get_logger(__name__) - - -class VerifyExecutionManager(BaseExecutionManager): - pass diff --git a/apps/ops/utils.py b/apps/ops/utils.py index ea9d74cbc..44c486ded 100644 --- a/apps/ops/utils.py +++ b/apps/ops/utils.py @@ -8,7 +8,7 @@ from common.utils import get_logger, get_object_or_none from orgs.utils import org_aware_func from jumpserver.const import PROJECT_DIR -from .models import Task, AdHoc +from .models import AdHoc from .const import DEFAULT_PASSWORD_RULES logger = get_logger(__file__) From 9a0bae5bfd75fb5396cbdb0ea3a55e522e85378e Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 10 Oct 2022 13:56:42 +0800 Subject: [PATCH 07/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20ansible=20?= =?UTF-8?q?=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/automations/base/manager.py | 11 ++++---- .../host/change_password_linux/main.yml | 1 + .../change_password_local_windows/main.yml | 1 + .../automations/change_password/manager.py | 27 +++++++++++++------ apps/assets/automations/endpoint.py | 13 ++++++++- apps/assets/models/automations/base.py | 5 ++-- apps/assets/models/label.py | 6 ++--- apps/common/drf/parsers/base.py | 2 +- apps/ops/ansible/inventory.py | 8 +++--- apps/ops/models/base.py | 6 ++--- apps/orgs/signal_handlers/common.py | 8 +++--- 11 files changed, 57 insertions(+), 31 deletions(-) diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 83097d0eb..e55a564bf 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -20,7 +20,7 @@ class BasePlaybookManager: def playbook_dir_path(self): ansible_dir = settings.ANSIBLE_DIR path = os.path.join( - ansible_dir, self.automation.type, self.automation.name, + ansible_dir, self.automation.type, self.automation.name.replace(' ', '_'), timezone.now().strftime('%Y%m%d_%H%M%S') ) return path @@ -41,12 +41,13 @@ class BasePlaybookManager: def prepare_playbook_dir(self): inventory_dir = os.path.dirname(self.inventory_path) playbook_dir = os.path.dirname(self.playbook_path) - for d in [inventory_dir, playbook_dir]: + for d in [inventory_dir, playbook_dir, self.playbook_dir_path]: + print("Create dir: {}".format(d)) if not os.path.exists(d): os.makedirs(d, exist_ok=True, mode=0o755) def inventory_kwargs(self): - raise NotImplemented + raise NotImplementedError def generate_inventory(self): inventory = JMSInventory( @@ -58,10 +59,10 @@ class BasePlaybookManager: print("Generate inventory done: {}".format(self.inventory_path)) def generate_playbook(self): - raise NotImplemented + raise NotImplementedError def get_runner(self): - raise NotImplemented + raise NotImplementedError def run(self, **kwargs): self.generate() diff --git a/apps/assets/automations/change_password/host/change_password_linux/main.yml b/apps/assets/automations/change_password/host/change_password_linux/main.yml index 1ad590cff..8bdaa5cdc 100644 --- a/apps/assets/automations/change_password/host/change_password_linux/main.yml +++ b/apps/assets/automations/change_password/host/change_password_linux/main.yml @@ -1,4 +1,5 @@ - hosts: demo + gather_facts: no tasks: - name: Test privileged account ping: diff --git a/apps/assets/automations/change_password/host/change_password_local_windows/main.yml b/apps/assets/automations/change_password/host/change_password_local_windows/main.yml index 483554a1e..d10c3681e 100644 --- a/apps/assets/automations/change_password/host/change_password_local_windows/main.yml +++ b/apps/assets/automations/change_password/host/change_password_local_windows/main.yml @@ -1,4 +1,5 @@ - hosts: demo + gather_facts: no tasks: - name: ping ping: diff --git a/apps/assets/automations/change_password/manager.py b/apps/assets/automations/change_password/manager.py index ea0bb9a26..aa86c6d08 100644 --- a/apps/assets/automations/change_password/manager.py +++ b/apps/assets/automations/change_password/manager.py @@ -34,7 +34,7 @@ class ChangePasswordManager(BasePlaybookManager): automation.change_password_method in self.id_method_mapper if not change_password_enabled: - host.exclude = _('Change password disabled') + host['exclude'] = _('Change password disabled') return [host] hosts = [] @@ -60,14 +60,18 @@ class ChangePasswordManager(BasePlaybookManager): playbook = [] for method_id, host_names in self.method_hosts_mapper.items(): method = self.id_method_mapper[method_id] - playbook_dir_path = method['dir'] - playbook_dir_name = os.path.dirname(playbook_dir_path) - shutil.copytree(playbook_dir_path, self.playbook_dir_path) - sub_playbook_path = os.path.join(self.playbook_dir_path, playbook_dir_name, 'main.yml') + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') with open(sub_playbook_path, 'r') as f: host_playbook_play = yaml.safe_load(f) + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + plays = [] for name in host_names: play = deepcopy(host_playbook_play) @@ -75,17 +79,24 @@ class ChangePasswordManager(BasePlaybookManager): plays.append(play) with open(sub_playbook_path, 'w') as f: - yaml.safe_dump(plays, f, default_flow_style=False) + yaml.safe_dump(plays, f) playbook.append({ 'name': method['name'], - 'import_playbook': playbook_dir_name + '/' + 'main.yml' + 'import_playbook': os.path.join(method_playbook_dir_name, 'main.yml') }) with open(self.playbook_path, 'w') as f: - yaml.safe_dump(playbook, f, default_flow_style=False) + yaml.safe_dump(playbook, f) print("Generate playbook done: " + self.playbook_path) + def get_runner(self): + return PlaybookRunner( + self.inventory_path, + self.playbook_path, + self.playbook_dir_path + ) + diff --git a/apps/assets/automations/endpoint.py b/apps/assets/automations/endpoint.py index f99defdd8..8064ae58f 100644 --- a/apps/assets/automations/endpoint.py +++ b/apps/assets/automations/endpoint.py @@ -1,7 +1,18 @@ # from .backup.manager import AccountBackupExecutionManager # # +from .change_password.manager import ChangePasswordManager + + class ExecutionManager: - manager_type = { + manager_type_mapper = { + 'change_password': ChangePasswordManager, } + def __init__(self, execution): + self.execution = execution + self._runner = self.manager_type_mapper[execution.automation.type](execution) + + def run(self, **kwargs): + return self._runner.run(**kwargs) + diff --git a/apps/assets/models/automations/base.py b/apps/assets/models/automations/base.py index ba0d393ea..e440429f4 100644 --- a/apps/assets/models/automations/base.py +++ b/apps/assets/models/automations/base.py @@ -58,9 +58,8 @@ class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): except AttributeError: eid = str(uuid.uuid4()) - execution = AutomationExecution.objects.create( - id=eid, strategy=self, trigger=trigger, - snapshot=self.to_attr_json(), + execution = self.executions.create( + id=eid, trigger=trigger, ) return execution.start() diff --git a/apps/assets/models/label.py b/apps/assets/models/label.py index 937d0d95c..afad1f069 100644 --- a/apps/assets/models/label.py +++ b/apps/assets/models/label.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- # -import uuid from django.db import models from django.utils.translation import ugettext_lazy as _ -from orgs.mixins.models import OrgModelMixin + +from orgs.mixins.models import JMSOrgBaseModel -class Label(OrgModelMixin): +class Label(JMSOrgBaseModel): SYSTEM_CATEGORY = "S" USER_CATEGORY = "U" CATEGORY_CHOICES = ( diff --git a/apps/common/drf/parsers/base.py b/apps/common/drf/parsers/base.py index 1f15ea72d..ac75ff645 100644 --- a/apps/common/drf/parsers/base.py +++ b/apps/common/drf/parsers/base.py @@ -39,7 +39,7 @@ class BaseFileParser(BaseParser): @abc.abstractmethod def generate_rows(self, stream_data): - raise NotImplemented + raise NotImplementedError def get_column_titles(self, rows): return next(rows) diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 5528f1a14..42cd7320c 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -63,7 +63,7 @@ class JMSInventory: host = { 'name': asset.name, 'asset': { - 'id': asset.id, 'name': asset.name, 'ip': asset.ip, + 'id': str(asset.id), 'name': asset.name, 'address': asset.address, 'type': asset.type, 'category': asset.category, 'protocol': asset.protocol, 'port': asset.port, 'protocols': [{'name': p.name, 'port': p.port} for p in protocols], @@ -125,7 +125,7 @@ class JMSInventory: if not account_selected: if self.account_policy in ['privileged_must', 'privileged_first']: - account_matched = list(filter(lambda account: account.is_privileged, accounts)) + account_matched = list(filter(lambda account: account.privileged, accounts)) account_selected = account_matched[0] if account_matched else None if not account_selected and self.account_policy == 'privileged_first': @@ -152,8 +152,8 @@ class JMSInventory: exclude_hosts = list(filter(lambda x: x.get('exclude'), hosts)) if exclude_hosts: print(_("Skip hosts below:")) - for host in exclude_hosts: - print(" {}:\t{}".format(host['name'], host['exclude'])) + for i, host in enumerate(exclude_hosts, start=1): + print("{}: [{}] \t{}".format(i, host['name'], host['exclude'])) hosts = list(filter(lambda x: not x.get('exclude'), hosts)) data = {'all': {'hosts': {}}} diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py index e91af13a6..4ff69277f 100644 --- a/apps/ops/models/base.py +++ b/apps/ops/models/base.py @@ -29,10 +29,10 @@ class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): return inv def get_register_task(self): - raise NotImplemented + raise NotImplementedError def to_json(self): - raise NotImplemented + raise NotImplementedError def create_execution(self): execution = self.executions.create() @@ -71,7 +71,7 @@ class BaseAnsibleExecution(models.Model): return os.path.join(self.private_dir, 'inventory', 'hosts') def get_runner(self): - raise NotImplemented + raise NotImplementedError def finish_task(self): self.date_finished = timezone.now() diff --git a/apps/orgs/signal_handlers/common.py b/apps/orgs/signal_handlers/common.py index 7b825b748..a4e1c7085 100644 --- a/apps/orgs/signal_handlers/common.py +++ b/apps/orgs/signal_handlers/common.py @@ -5,11 +5,11 @@ from collections import defaultdict from functools import partial from django.dispatch import receiver +from django.conf import settings from django.utils.functional import LazyObject -from django.db.models.signals import m2m_changed -from django.db.models.signals import post_save, pre_delete +from django.db.models.signals import post_save, pre_delete, m2m_changed -from orgs.utils import tmp_to_org +from orgs.utils import tmp_to_org, set_to_default_org from orgs.models import Organization from orgs.hands import set_current_org, Node, get_current_org from perms.models import AssetPermission @@ -44,6 +44,8 @@ def expire_orgs_mapping_for_memory(org_id): @receiver(django_ready) def subscribe_orgs_mapping_expire(sender, **kwargs): logger.debug("Start subscribe for expire orgs mapping from memory") + if settings.DEBUG: + set_to_default_org() def keep_subscribe_org_mapping(): orgs_mapping_for_memory_pub_sub.subscribe( From 1d757ec19a46cd47a0a2b06b58542a277bb22a90 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 10 Oct 2022 15:07:36 +0800 Subject: [PATCH 08/13] =?UTF-8?q?pref:=20=E4=BF=AE=E6=94=B9=E6=94=B9?= =?UTF-8?q?=E5=AF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../automations/change_password/manager.py | 26 ++-- .../migrations/0107_auto_20221010_0959.py | 116 ++++++++++++++++++ .../ops/migrations/0026_auto_20221009_2050.py | 100 +++++++++++++++ 3 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 apps/assets/migrations/0107_auto_20221010_0959.py create mode 100644 apps/ops/migrations/0026_auto_20221009_2050.py diff --git a/apps/assets/automations/change_password/manager.py b/apps/assets/automations/change_password/manager.py index aa86c6d08..6c315cb82 100644 --- a/apps/assets/automations/change_password/manager.py +++ b/apps/assets/automations/change_password/manager.py @@ -6,7 +6,7 @@ from collections import defaultdict import yaml from django.utils.translation import gettext as _ -from ops.ansible import PlaybookRunner, JMSInventory +from ops.ansible import PlaybookRunner from ..base.manager import BasePlaybookManager from assets.automations.methods import platform_automation_methods @@ -19,6 +19,7 @@ class ChangePasswordManager(BasePlaybookManager): for method in platform_automation_methods } self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] def host_duplicator(self, host, asset=None, account=None, platform=None, **kwargs): accounts = asset.accounts.all() @@ -72,19 +73,23 @@ class ChangePasswordManager(BasePlaybookManager): if isinstance(host_playbook_play, list): host_playbook_play = host_playbook_play[0] - plays = [] - for name in host_names: + step = 10 + hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] + for i, hosts in enumerate(hosts_grouped): + plays = [] play = deepcopy(host_playbook_play) - play['hosts'] = name + play['hosts'] = ':'.join(hosts) plays.append(play) - with open(sub_playbook_path, 'w') as f: - yaml.safe_dump(plays, f) + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) - playbook.append({ - 'name': method['name'], - 'import_playbook': os.path.join(method_playbook_dir_name, 'main.yml') - }) + playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) with open(self.playbook_path, 'w') as f: yaml.safe_dump(playbook, f) @@ -99,4 +104,3 @@ class ChangePasswordManager(BasePlaybookManager): ) - diff --git a/apps/assets/migrations/0107_auto_20221010_0959.py b/apps/assets/migrations/0107_auto_20221010_0959.py new file mode 100644 index 000000000..f9cace60f --- /dev/null +++ b/apps/assets/migrations/0107_auto_20221010_0959.py @@ -0,0 +1,116 @@ +# Generated by Django 3.2.14 on 2022-10-10 01:59 + +import common.db.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('assets', '0106_auto_20220916_1556'), + ] + + operations = [ + migrations.CreateModel( + name='BaseAutomation', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('is_periodic', models.BooleanField(default=False)), + ('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')), + ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')), + ('accounts', models.JSONField(default=list, verbose_name='Accounts')), + ('type', models.CharField(max_length=16, verbose_name='Type')), + ('comment', models.TextField(blank=True, verbose_name='Comment')), + ('assets', models.ManyToManyField(blank=True, to='assets.Asset', verbose_name='Assets')), + ('nodes', models.ManyToManyField(blank=True, to='assets.Node', verbose_name='Nodes')), + ], + options={ + 'verbose_name': 'Automation plan', + 'unique_together': {('org_id', 'name')}, + }, + ), + migrations.AddField( + model_name='label', + name='created_by', + field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by'), + ), + migrations.AddField( + model_name='label', + name='date_updated', + field=models.DateTimeField(auto_now=True, verbose_name='Date updated'), + ), + migrations.AddField( + model_name='label', + name='updated_by', + field=models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by'), + ), + migrations.CreateModel( + name='DiscoveryAutomation', + fields=[ + ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), + ], + options={ + 'verbose_name': 'Discovery strategy', + }, + bases=('assets.baseautomation',), + ), + migrations.CreateModel( + name='ReconcileAutomation', + fields=[ + ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), + ], + options={ + 'verbose_name': 'Reconcile strategy', + }, + bases=('assets.baseautomation',), + ), + migrations.CreateModel( + name='VerifyAutomation', + fields=[ + ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), + ], + options={ + 'verbose_name': 'Verify strategy', + }, + bases=('assets.baseautomation',), + ), + migrations.CreateModel( + name='AutomationExecution', + fields=[ + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('status', models.CharField(default='pending', max_length=16)), + ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), + ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), + ('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')), + ('snapshot', common.db.fields.EncryptJsonDictTextField(blank=True, default=dict, null=True, verbose_name='Automation snapshot')), + ('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')], default='manual', max_length=128, verbose_name='Trigger mode')), + ('automation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='assets.baseautomation', verbose_name='Automation strategy')), + ], + options={ + 'verbose_name': 'Automation strategy execution', + }, + ), + migrations.CreateModel( + name='ChangePasswordAutomation', + fields=[ + ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), + ('password', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')), + ('recipients', models.ManyToManyField(blank=True, related_name='recipients_change_auth_strategy', to=settings.AUTH_USER_MODEL, verbose_name='Recipient')), + ], + options={ + 'verbose_name': 'Change auth strategy', + }, + bases=('assets.baseautomation',), + ), + ] diff --git a/apps/ops/migrations/0026_auto_20221009_2050.py b/apps/ops/migrations/0026_auto_20221009_2050.py new file mode 100644 index 000000000..699246531 --- /dev/null +++ b/apps/ops/migrations/0026_auto_20221009_2050.py @@ -0,0 +1,100 @@ +# Generated by Django 3.2.14 on 2022-10-09 12:50 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('assets', '0106_auto_20220916_1556'), + ('ops', '0025_auto_20221008_1631'), + ] + + operations = [ + migrations.CreateModel( + name='Playbook', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('is_periodic', models.BooleanField(default=False)), + ('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')), + ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')), + ('account', models.CharField(default='root', max_length=128, verbose_name='Account')), + ('account_policy', models.CharField(default='root', max_length=128, verbose_name='Account policy')), + ('date_last_run', models.DateTimeField(null=True, verbose_name='Date last run')), + ('path', models.FilePathField(max_length=1024, verbose_name='Playbook')), + ('comment', models.TextField(blank=True, verbose_name='Comment')), + ('assets', models.ManyToManyField(to='assets.Asset', verbose_name='Assets')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AlterField( + model_name='adhocexecution', + name='date_finished', + field=models.DateTimeField(null=True, verbose_name='Date finished'), + ), + migrations.CreateModel( + name='PlaybookTemplate', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('path', models.FilePathField(verbose_name='Path')), + ('comment', models.TextField(blank=True, verbose_name='Comment')), + ], + options={ + 'verbose_name': 'Playbook template', + 'ordering': ['name'], + 'unique_together': {('org_id', 'name')}, + }, + ), + migrations.CreateModel( + name='PlaybookExecution', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('status', models.CharField(default='running', max_length=16, verbose_name='Status')), + ('result', models.JSONField(blank=True, null=True, verbose_name='Result')), + ('summary', models.JSONField(default=dict, verbose_name='Summary')), + ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), + ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), + ('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')), + ('path', models.FilePathField(max_length=1024, verbose_name='Run dir')), + ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ('task', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ops.playbook', verbose_name='Task')), + ], + options={ + 'ordering': ['-date_start'], + 'abstract': False, + }, + ), + migrations.AddField( + model_name='playbook', + name='last_execution', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ops.playbookexecution', verbose_name='Last execution'), + ), + migrations.AddField( + model_name='playbook', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Owner'), + ), + migrations.AddField( + model_name='playbook', + name='template', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='ops.playbooktemplate', verbose_name='Template'), + ), + ] From 6e0d211645df57ca696732b39a4713cb6f24c82e Mon Sep 17 00:00:00 2001 From: feng626 <1304903146@qq.com> Date: Mon, 10 Oct 2022 17:08:06 +0800 Subject: [PATCH 09/13] perf: automation migrate --- .../{0107_auto_20221010_0959.py => 0108_migrate_automation.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename apps/assets/migrations/{0107_auto_20221010_0959.py => 0108_migrate_automation.py} (99%) diff --git a/apps/assets/migrations/0107_auto_20221010_0959.py b/apps/assets/migrations/0108_migrate_automation.py similarity index 99% rename from apps/assets/migrations/0107_auto_20221010_0959.py rename to apps/assets/migrations/0108_migrate_automation.py index f9cace60f..3bce64e4e 100644 --- a/apps/assets/migrations/0107_auto_20221010_0959.py +++ b/apps/assets/migrations/0108_migrate_automation.py @@ -11,7 +11,7 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('assets', '0106_auto_20220916_1556'), + ('assets', '0107_account_history'), ] operations = [ From 9198c93fcf27d1cc64b39a22e31f6e8bd74b08d8 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 10 Oct 2022 20:56:13 +0800 Subject: [PATCH 10/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20ansible=20ch?= =?UTF-8?q?ange=20password?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../database/change_password_mysql/main.yml | 63 ++++++++++++------- .../change_password_postgresql/main.yml | 51 ++++++++++++--- .../roles/change_password/tasks/main.yml | 27 -------- apps/ops/ansible/inventory.py | 7 ++- apps/orgs/models.py | 4 +- apps/perms/urls/asset_permission.py | 2 +- 6 files changed, 92 insertions(+), 62 deletions(-) delete mode 100644 apps/assets/automations/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_mysql/main.yml b/apps/assets/automations/change_password/database/change_password_mysql/main.yml index 483554a1e..b42251300 100644 --- a/apps/assets/automations/change_password/database/change_password_mysql/main.yml +++ b/apps/assets/automations/change_password/database/change_password_mysql/main.yml @@ -1,29 +1,46 @@ -- hosts: demo +- hosts: mysql + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: root + password: redhat + jms_asset: + address: 127.0.0.1 + port: 3306 + account: + username: web1 + password: jumpserver + tasks: - - name: ping - ping: + - name: Test MySQL connection + community.mysql.mysql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + filter: version + register: db_info - #- name: print variables - # debug: - # msg: "Username: {{ account.username }}, Password: {{ account.password }}" + - name: MySQL version + debug: + var: db_info.version.full - - name: Change password - user: + - name: Change MySQL password + community.mysql.mysql_user: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" name: "{{ account.username }}" - password: "{{ account.password | password_hash('des') }}" - update_password: always - when: account.password - - - name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key + password: "{{ account.secret }}" + host: "%" + when: db_info is succeeded - name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko + community.mysql.mysql_info: + login_user: "{{ account.username }}" + login_password: "{{ account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + filter: version diff --git a/apps/assets/automations/change_password/database/change_password_postgresql/main.yml b/apps/assets/automations/change_password/database/change_password_postgresql/main.yml index 402c7fa8d..0180c559c 100644 --- a/apps/assets/automations/change_password/database/change_password_postgresql/main.yml +++ b/apps/assets/automations/change_password/database/change_password_postgresql/main.yml @@ -1,10 +1,45 @@ -{% for account in accounts %} -- hosts: {{ account.asset.name }} +- hosts: mysql + gather_facts: no vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: postgre + password: postgre + jms_asset: + address: 127.0.0.1 + port: 5432 account: - username: {{ account.username }} - password: {{ account.password }} - public_key: {{ account.public_key }} - roles: - - change_password -{% endfor %} + username: web1 + secret: jumpserver + + tasks: + - name: Test PostgreSQL connection + community.postgresql.postgresql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + register: db_info + + - name: Display PostgreSQL version + debug: + var: db_info.version.full + + - name: Change PostgreSQL password + community.postgresql.postgresql_user: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + name: "{{ account.username }}" + password: "{{ account.secret }}" + comment: Updated by jumpserver + state: present + when: db_info is succeeded + + - name: Verify password + community.postgresql.postgresql_info: + login_user: "{{ account.username }}" + login_password: "{{ account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" diff --git a/apps/assets/automations/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml b/apps/assets/automations/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml deleted file mode 100644 index 903cd9115..000000000 --- a/apps/assets/automations/change_password/database/change_password_postgresql/roles/change_password/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: ping - ping: - -#- name: print variables -# debug: -# msg: "Username: {{ account.username }}, Password: {{ account.password }}" - -- name: Change password - user: - name: "{{ account.username }}" - password: "{{ account.password | password_hash('des') }}" - update_password: always - when: account.password - -- name: Change public key - authorized_key: - user: "{{ account.username }}" - key: "{{ account.public_key }}" - state: present - when: account.public_key - -- name: Verify password - ping: - vars: - ansible_user: "{{ account.username }}" - ansible_pass: "{{ account.password }}" - ansible_ssh_connection: paramiko diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 42cd7320c..9d4498515 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -62,13 +62,16 @@ class JMSInventory: def asset_to_host(self, asset, account, automation, protocols): host = { 'name': asset.name, - 'asset': { + 'jms_asset': { 'id': str(asset.id), 'name': asset.name, 'address': asset.address, 'type': asset.type, 'category': asset.category, 'protocol': asset.protocol, 'port': asset.port, 'protocols': [{'name': p.name, 'port': p.port} for p in protocols], }, - 'exclude': '' + 'jms_account': { + 'id': str(account.id), 'username': account.username, + 'secret': account.secret, 'secret_type': account.secret_type + } if account else None } ansible_connection = automation.ansible_config.get('ansible_connection', 'ssh') gateway = None diff --git a/apps/orgs/models.py b/apps/orgs/models.py index d5e3ae617..7c7babd76 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -78,7 +78,9 @@ class Organization(OrgRoleMixin, models.Model): ROOT_ID = '00000000-0000-0000-0000-000000000000' ROOT_NAME = _('GLOBAL') DEFAULT_ID = '00000000-0000-0000-0000-000000000002' - DEFAULT_NAME = 'Default' + DEFAULT_NAME = _('DEFAULT') + SYSTEM_ID = '00000000-0000-0000-0000-000000000004' + SYSTEM_NAME = _('SYSTEM') orgs_mapping = None class Meta: diff --git a/apps/perms/urls/asset_permission.py b/apps/perms/urls/asset_permission.py index 0ef87f606..cc8e10f36 100644 --- a/apps/perms/urls/asset_permission.py +++ b/apps/perms/urls/asset_permission.py @@ -65,7 +65,7 @@ user_permission_urlpatterns = [ path('assets//accounts/', api.MyGrantedAssetAccountsApi.as_view(), name='my-asset-accounts'), # 用户登录资产的特殊账号, @INPUT, @USER 等 path('/assets/special-accounts/', api.UserGrantedAssetSpecialAccountsApi.as_view(), name='user-special-accounts'), - path('/assets/special-accounts/', api.MyGrantedAssetSpecialAccountsApi.as_view(), name='my-special-accounts'), + path('assets/special-accounts/', api.MyGrantedAssetSpecialAccountsApi.as_view(), name='my-special-accounts'), ] user_group_permission_urlpatterns = [ From 22e211625ede30e945be013f4fe11d04d6d901f9 Mon Sep 17 00:00:00 2001 From: feng626 <1304903146@qq.com> Date: Tue, 11 Oct 2022 10:50:39 +0800 Subject: [PATCH 11/13] fix: platform 500 --- apps/assets/automations/__init__.py | 2 +- apps/perms/urls/asset_permission.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/assets/automations/__init__.py b/apps/assets/automations/__init__.py index 478e9740d..ae4bc0377 100644 --- a/apps/assets/automations/__init__.py +++ b/apps/assets/automations/__init__.py @@ -1 +1 @@ -from .methods import platform_automation_methods +from .methods import * diff --git a/apps/perms/urls/asset_permission.py b/apps/perms/urls/asset_permission.py index 0ef87f606..cc8e10f36 100644 --- a/apps/perms/urls/asset_permission.py +++ b/apps/perms/urls/asset_permission.py @@ -65,7 +65,7 @@ user_permission_urlpatterns = [ path('assets//accounts/', api.MyGrantedAssetAccountsApi.as_view(), name='my-asset-accounts'), # 用户登录资产的特殊账号, @INPUT, @USER 等 path('/assets/special-accounts/', api.UserGrantedAssetSpecialAccountsApi.as_view(), name='user-special-accounts'), - path('/assets/special-accounts/', api.MyGrantedAssetSpecialAccountsApi.as_view(), name='my-special-accounts'), + path('assets/special-accounts/', api.MyGrantedAssetSpecialAccountsApi.as_view(), name='my-special-accounts'), ] user_group_permission_urlpatterns = [ From 85a6f29a0aecd999cd71178f74571f74f031a13a Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 12 Oct 2022 18:08:57 +0800 Subject: [PATCH 12/13] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20playbook=20?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=89=A7=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/automations/__init__.py | 2 +- apps/assets/automations/base/manager.py | 139 +++++++++++++++--- .../{change_password_mysql => mysql}/main.yml | 0 .../manifest.yml | 0 .../main.yml | 0 .../manifest.yml | 0 .../main.yml | 16 +- .../manifest.yml | 0 .../main.yml | 0 .../manifest.yml | 0 .../roles/change_password/tasks/main.yml | 0 .../{change_password_aix => aix}/main.yml | 0 .../{change_password_aix => aix}/manifest.yml | 0 .../{change_password_linux => linux}/main.yml | 0 .../manifest.yml | 0 .../main.yml | 0 .../manifest.yml | 0 .../automations/change_password/manager.py | 97 +++--------- .../__init__.py | 0 .../gather_facts/database/mysql/main.yml | 28 ++++ .../gather_facts/database/mysql/manifest.yml | 6 + .../gather_facts/database/postgresql/main.yml | 28 ++++ .../database/postgresql/manifest.yml | 6 + .../gather_facts/database/sqlserver/main.yml | 10 ++ .../database/sqlserver/manifest.yml | 8 + .../roles/change_password/tasks/main.yml | 27 ++++ .../gather_facts/demo_inventory.txt | 2 + .../gather_facts/host/posix/main.yml | 19 +++ .../gather_facts/host/posix/manifest.yml | 8 + .../gather_facts/host/windows/main.yml | 24 +++ .../gather_facts/host/windows/manifest.yml | 7 + .../automations/gather_facts/manager.py | 77 ++++++++++ .../generate_playbook/change_password.py | 106 ------------- .../automations/generate_playbook/verify.py | 86 ----------- apps/assets/automations/ping/__init__.py | 0 .../automations/ping/database/mysql/main.yml | 20 +++ .../ping/database/mysql/manifest.yml | 6 + .../ping/database/postgresql/main.yml | 23 +++ .../ping/database/postgresql/manifest.yml | 6 + .../automations/ping/demo_inventory.txt | 2 + .../automations/ping/host/posix/main.yml | 5 + .../automations/ping/host/posix/manifest.yml | 8 + .../automations/ping/host/windows/main.yml | 5 + .../ping/host/windows/manifest.yml | 7 + apps/assets/automations/ping/manager.py | 75 ++++++++++ .../models/automations/change_secret.py | 4 + apps/ops/ansible/inventory.py | 30 ++-- apps/ops/ansible/runner.py | 6 +- 48 files changed, 585 insertions(+), 308 deletions(-) rename apps/assets/automations/change_password/database/{change_password_mysql => mysql}/main.yml (100%) rename apps/assets/automations/change_password/database/{change_password_mysql => mysql}/manifest.yml (100%) rename apps/assets/automations/change_password/database/{change_password_oracle => oracle}/main.yml (100%) rename apps/assets/automations/change_password/database/{change_password_oracle => oracle}/manifest.yml (100%) rename apps/assets/automations/change_password/database/{change_password_postgresql => postgresql}/main.yml (79%) rename apps/assets/automations/change_password/database/{change_password_postgresql => postgresql}/manifest.yml (100%) rename apps/assets/automations/change_password/database/{change_password_sqlserver => sqlserver}/main.yml (100%) rename apps/assets/automations/change_password/database/{change_password_sqlserver => sqlserver}/manifest.yml (100%) rename apps/assets/automations/change_password/database/{change_password_sqlserver => sqlserver}/roles/change_password/tasks/main.yml (100%) rename apps/assets/automations/change_password/host/{change_password_aix => aix}/main.yml (100%) rename apps/assets/automations/change_password/host/{change_password_aix => aix}/manifest.yml (100%) rename apps/assets/automations/change_password/host/{change_password_linux => linux}/main.yml (100%) rename apps/assets/automations/change_password/host/{change_password_linux => linux}/manifest.yml (100%) rename apps/assets/automations/change_password/host/{change_password_local_windows => windows}/main.yml (100%) rename apps/assets/automations/change_password/host/{change_password_local_windows => windows}/manifest.yml (100%) rename apps/assets/automations/{generate_playbook => gather_facts}/__init__.py (100%) create mode 100644 apps/assets/automations/gather_facts/database/mysql/main.yml create mode 100644 apps/assets/automations/gather_facts/database/mysql/manifest.yml create mode 100644 apps/assets/automations/gather_facts/database/postgresql/main.yml create mode 100644 apps/assets/automations/gather_facts/database/postgresql/manifest.yml create mode 100644 apps/assets/automations/gather_facts/database/sqlserver/main.yml create mode 100644 apps/assets/automations/gather_facts/database/sqlserver/manifest.yml create mode 100644 apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml create mode 100644 apps/assets/automations/gather_facts/demo_inventory.txt create mode 100644 apps/assets/automations/gather_facts/host/posix/main.yml create mode 100644 apps/assets/automations/gather_facts/host/posix/manifest.yml create mode 100644 apps/assets/automations/gather_facts/host/windows/main.yml create mode 100644 apps/assets/automations/gather_facts/host/windows/manifest.yml create mode 100644 apps/assets/automations/gather_facts/manager.py delete mode 100644 apps/assets/automations/generate_playbook/change_password.py delete mode 100644 apps/assets/automations/generate_playbook/verify.py create mode 100644 apps/assets/automations/ping/__init__.py create mode 100644 apps/assets/automations/ping/database/mysql/main.yml create mode 100644 apps/assets/automations/ping/database/mysql/manifest.yml create mode 100644 apps/assets/automations/ping/database/postgresql/main.yml create mode 100644 apps/assets/automations/ping/database/postgresql/manifest.yml create mode 100644 apps/assets/automations/ping/demo_inventory.txt create mode 100644 apps/assets/automations/ping/host/posix/main.yml create mode 100644 apps/assets/automations/ping/host/posix/manifest.yml create mode 100644 apps/assets/automations/ping/host/windows/main.yml create mode 100644 apps/assets/automations/ping/host/windows/manifest.yml create mode 100644 apps/assets/automations/ping/manager.py diff --git a/apps/assets/automations/__init__.py b/apps/assets/automations/__init__.py index 478e9740d..7c63c916a 100644 --- a/apps/assets/automations/__init__.py +++ b/apps/assets/automations/__init__.py @@ -1 +1 @@ -from .methods import platform_automation_methods +from .methods import platform_automation_methods, filter_platform_methods diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index e55a564bf..626ced70a 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -1,37 +1,65 @@ import os +import shutil +import yaml +from copy import deepcopy +from collections import defaultdict from django.conf import settings from django.utils import timezone +from django.utils.translation import gettext as _ -from ops.ansible import JMSInventory +from common.utils import get_logger +from assets.automations.methods import platform_automation_methods +from ops.ansible import JMSInventory, PlaybookRunner, DefaultCallback + +logger = get_logger(__name__) + + +class PlaybookCallback(DefaultCallback): + def playbook_on_stats(self, event_data, **kwargs): + print("\n*** 分任务结果") + super().playbook_on_stats(event_data, **kwargs) class BasePlaybookManager: + bulk_size = 100 ansible_account_policy = 'privileged_first' def __init__(self, execution): self.execution = execution self.automation = execution.automation + self.method_id_meta_mapper = { + method['id']: method + for method in platform_automation_methods + if method['method'] == self.__class__.method_type() + } + # 根据执行方式就行分组, 不同资产的改密、推送等操作可能会使用不同的执行方式 + # 然后根据执行方式分组, 再根据 bulk_size 分组, 生成不同的 playbook + # 避免一个 playbook 中包含太多的主机 + self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] - def get_grouped_assets(self): - return self.automation.all_assets_group_by_platform() + @classmethod + def method_type(cls): + raise NotImplementedError @property - def playbook_dir_path(self): + def runtime_dir(self): ansible_dir = settings.ANSIBLE_DIR path = os.path.join( - ansible_dir, self.automation.type, self.automation.name.replace(' ', '_'), + ansible_dir, self.automation.type, + self.automation.name.replace(' ', '_'), timezone.now().strftime('%Y%m%d_%H%M%S') ) return path @property def inventory_path(self): - return os.path.join(self.playbook_dir_path, 'inventory', 'hosts.json') + return os.path.join(self.runtime_dir, 'inventory', 'hosts.json') @property def playbook_path(self): - return os.path.join(self.playbook_dir_path, 'project', 'main.yml') + return os.path.join(self.runtime_dir, 'project', 'main.yml') def generate(self): self.prepare_playbook_dir() @@ -41,31 +69,108 @@ class BasePlaybookManager: def prepare_playbook_dir(self): inventory_dir = os.path.dirname(self.inventory_path) playbook_dir = os.path.dirname(self.playbook_path) - for d in [inventory_dir, playbook_dir, self.playbook_dir_path]: - print("Create dir: {}".format(d)) + for d in [inventory_dir, playbook_dir]: if not os.path.exists(d): os.makedirs(d, exist_ok=True, mode=0o755) - def inventory_kwargs(self): - raise NotImplementedError + def host_callback(self, host, automation=None, **kwargs): + enabled_attr = '{}_enabled'.format(self.__class__.method_type()) + method_attr = '{}_method'.format(self.__class__.method_type()) + + method_enabled = automation and \ + getattr(automation, enabled_attr) and \ + getattr(automation, method_attr) and \ + getattr(automation, method_attr) in self.method_id_meta_mapper + + if not method_enabled: + host['error'] = _('Change password disabled') + return host + + self.method_hosts_mapper[getattr(automation, method_attr)].append(host['name']) + return host def generate_inventory(self): inventory = JMSInventory( assets=self.automation.get_all_assets(), account_policy=self.ansible_account_policy, - **self.inventory_kwargs() + host_callback=self.host_callback ) inventory.write_to_file(self.inventory_path) - print("Generate inventory done: {}".format(self.inventory_path)) + logger.debug("Generate inventory done: {}".format(self.inventory_path)) def generate_playbook(self): + main_playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.method_id_meta_mapper.get(method_id) + if not method: + logger.error("Method not found: {}".format(method_id)) + continue + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + + hosts_bulked = [host_names[i:i+self.bulk_size] for i in range(0, len(host_names), self.bulk_size)] + for i, hosts in enumerate(hosts_bulked): + plays = [] + play = deepcopy(host_playbook_play) + play['hosts'] = ':'.join(hosts) + plays.append(play) + + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) + + main_playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(main_playbook, f) + + logger.debug("Generate playbook done: " + self.playbook_path) + + def get_runners(self): + runners = [] + for playbook_path in self.playbooks: + runer = PlaybookRunner( + self.inventory_path, + playbook_path, + self.runtime_dir, + callback=PlaybookCallback(), + ) + runners.append(runer) + return runners + + def on_runner_done(self, runner, cb): raise NotImplementedError - def get_runner(self): - raise NotImplementedError + def on_runner_failed(self, runner, e): + print("Runner failed: {} {}".format(e, self)) def run(self, **kwargs): self.generate() - runner = self.get_runner() - return runner.run(**kwargs) + runners = self.get_runners() + if len(runners) > 1: + print("### 分批次执行开始任务, 总共 {}\n".format(len(runners))) + else: + print(">>> 开始执行任务\n") + for i, runner in enumerate(runners, start=1): + if len(runners) > 1: + print(">>> 开始执行第 {} 批任务".format(i)) + try: + cb = runner.run(**kwargs) + self.on_runner_done(runner, cb) + except Exception as e: + self.on_runner_failed(runner, e) + print('\n\n') diff --git a/apps/assets/automations/change_password/database/change_password_mysql/main.yml b/apps/assets/automations/change_password/database/mysql/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_mysql/main.yml rename to apps/assets/automations/change_password/database/mysql/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_mysql/manifest.yml b/apps/assets/automations/change_password/database/mysql/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_mysql/manifest.yml rename to apps/assets/automations/change_password/database/mysql/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_oracle/main.yml b/apps/assets/automations/change_password/database/oracle/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_oracle/main.yml rename to apps/assets/automations/change_password/database/oracle/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_oracle/manifest.yml b/apps/assets/automations/change_password/database/oracle/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_oracle/manifest.yml rename to apps/assets/automations/change_password/database/oracle/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_postgresql/main.yml b/apps/assets/automations/change_password/database/postgresql/main.yml similarity index 79% rename from apps/assets/automations/change_password/database/change_password_postgresql/main.yml rename to apps/assets/automations/change_password/database/postgresql/main.yml index 0180c559c..ed4e60abf 100644 --- a/apps/assets/automations/change_password/database/change_password_postgresql/main.yml +++ b/apps/assets/automations/change_password/database/postgresql/main.yml @@ -1,24 +1,26 @@ -- hosts: mysql +- hosts: postgre gather_facts: no vars: ansible_python_interpreter: /usr/local/bin/python jms_account: username: postgre - password: postgre + secret: postgre jms_asset: address: 127.0.0.1 port: 5432 + database: testdb account: - username: web1 + username: test secret: jumpserver tasks: - name: Test PostgreSQL connection - community.postgresql.postgresql_info: + community.postgresql.postgresql_ping: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" + login_db: "{{ jms_asset.database }}" register: db_info - name: Display PostgreSQL version @@ -31,15 +33,15 @@ login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" + db: "{{ jms_asset.database }}" name: "{{ account.username }}" password: "{{ account.secret }}" - comment: Updated by jumpserver - state: present when: db_info is succeeded - name: Verify password - community.postgresql.postgresql_info: + community.postgresql.postgresql_ping: login_user: "{{ account.username }}" login_password: "{{ account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" + db: "{{ jms_asset.database }}" diff --git a/apps/assets/automations/change_password/database/change_password_postgresql/manifest.yml b/apps/assets/automations/change_password/database/postgresql/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_postgresql/manifest.yml rename to apps/assets/automations/change_password/database/postgresql/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_sqlserver/main.yml b/apps/assets/automations/change_password/database/sqlserver/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_sqlserver/main.yml rename to apps/assets/automations/change_password/database/sqlserver/main.yml diff --git a/apps/assets/automations/change_password/database/change_password_sqlserver/manifest.yml b/apps/assets/automations/change_password/database/sqlserver/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_sqlserver/manifest.yml rename to apps/assets/automations/change_password/database/sqlserver/manifest.yml diff --git a/apps/assets/automations/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml b/apps/assets/automations/change_password/database/sqlserver/roles/change_password/tasks/main.yml similarity index 100% rename from apps/assets/automations/change_password/database/change_password_sqlserver/roles/change_password/tasks/main.yml rename to apps/assets/automations/change_password/database/sqlserver/roles/change_password/tasks/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_aix/main.yml b/apps/assets/automations/change_password/host/aix/main.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_aix/main.yml rename to apps/assets/automations/change_password/host/aix/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_aix/manifest.yml b/apps/assets/automations/change_password/host/aix/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_aix/manifest.yml rename to apps/assets/automations/change_password/host/aix/manifest.yml diff --git a/apps/assets/automations/change_password/host/change_password_linux/main.yml b/apps/assets/automations/change_password/host/linux/main.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_linux/main.yml rename to apps/assets/automations/change_password/host/linux/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_linux/manifest.yml b/apps/assets/automations/change_password/host/linux/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_linux/manifest.yml rename to apps/assets/automations/change_password/host/linux/manifest.yml diff --git a/apps/assets/automations/change_password/host/change_password_local_windows/main.yml b/apps/assets/automations/change_password/host/windows/main.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_local_windows/main.yml rename to apps/assets/automations/change_password/host/windows/main.yml diff --git a/apps/assets/automations/change_password/host/change_password_local_windows/manifest.yml b/apps/assets/automations/change_password/host/windows/manifest.yml similarity index 100% rename from apps/assets/automations/change_password/host/change_password_local_windows/manifest.yml rename to apps/assets/automations/change_password/host/windows/manifest.yml diff --git a/apps/assets/automations/change_password/manager.py b/apps/assets/automations/change_password/manager.py index 6c315cb82..1991d6854 100644 --- a/apps/assets/automations/change_password/manager.py +++ b/apps/assets/automations/change_password/manager.py @@ -1,44 +1,35 @@ -import os -import shutil from copy import deepcopy from collections import defaultdict -import yaml -from django.utils.translation import gettext as _ - -from ops.ansible import PlaybookRunner from ..base.manager import BasePlaybookManager -from assets.automations.methods import platform_automation_methods class ChangePasswordManager(BasePlaybookManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.id_method_mapper = { - method['id']: method - for method in platform_automation_methods - } self.method_hosts_mapper = defaultdict(list) self.playbooks = [] - def host_duplicator(self, host, asset=None, account=None, platform=None, **kwargs): + @classmethod + def method_type(cls): + return 'change_password' + + def host_callback(self, host, asset=None, account=None, automation=None, **kwargs): + host = super().host_callback(host, asset=asset, account=account, automation=automation, **kwargs) + if host.get('exclude'): + return host + accounts = asset.accounts.all() if account: accounts = accounts.exclude(id=account.id) + if '*' not in self.automation.accounts: accounts = accounts.filter(username__in=self.automation.accounts) - automation = platform.automation - change_password_enabled = automation and \ - automation.change_password_enabled and \ - automation.change_password_method and \ - automation.change_password_method in self.id_method_mapper - - if not change_password_enabled: - host['exclude'] = _('Change password disabled') - return [host] - - hosts = [] + method_attr = getattr(automation, self.method_type() + '_method') + method_hosts = self.method_hosts_mapper[method_attr] + method_hosts = [h for h in method_hosts if h != host['name']] + inventory_hosts = [] for account in accounts: h = deepcopy(host) h['name'] += '_' + account.username @@ -48,59 +39,15 @@ class ChangePasswordManager(BasePlaybookManager): 'secret_type': account.secret_type, 'secret': account.secret, } - hosts.append(h) - self.method_hosts_mapper[automation.change_password_method].append(h['name']) - return hosts + inventory_hosts.append(h) + method_hosts.append(h['name']) + self.method_hosts_mapper[method_attr] = method_hosts + return inventory_hosts - def inventory_kwargs(self): - return { - 'host_duplicator': self.host_duplicator - } + def on_runner_done(self, runner, cb): + pass - def generate_playbook(self): - playbook = [] - for method_id, host_names in self.method_hosts_mapper.items(): - method = self.id_method_mapper[method_id] - method_playbook_dir_path = method['dir'] - method_playbook_dir_name = os.path.basename(method_playbook_dir_path) - sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) - shutil.copytree(method_playbook_dir_path, sub_playbook_dir) - sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') - - with open(sub_playbook_path, 'r') as f: - host_playbook_play = yaml.safe_load(f) - - if isinstance(host_playbook_play, list): - host_playbook_play = host_playbook_play[0] - - step = 10 - hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] - for i, hosts in enumerate(hosts_grouped): - plays = [] - play = deepcopy(host_playbook_play) - play['hosts'] = ':'.join(hosts) - plays.append(play) - - playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) - with open(playbook_path, 'w') as f: - yaml.safe_dump(plays, f) - self.playbooks.append(playbook_path) - - playbook.append({ - 'name': method['name'] + ' for part {}'.format(i), - 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) - }) - - with open(self.playbook_path, 'w') as f: - yaml.safe_dump(playbook, f) - - print("Generate playbook done: " + self.playbook_path) - - def get_runner(self): - return PlaybookRunner( - self.inventory_path, - self.playbook_path, - self.playbook_dir_path - ) + def on_runner_failed(self, runner, e): + pass diff --git a/apps/assets/automations/generate_playbook/__init__.py b/apps/assets/automations/gather_facts/__init__.py similarity index 100% rename from apps/assets/automations/generate_playbook/__init__.py rename to apps/assets/automations/gather_facts/__init__.py diff --git a/apps/assets/automations/gather_facts/database/mysql/main.yml b/apps/assets/automations/gather_facts/database/mysql/main.yml new file mode 100644 index 000000000..e7ba00880 --- /dev/null +++ b/apps/assets/automations/gather_facts/database/mysql/main.yml @@ -0,0 +1,28 @@ +- hosts: mysql + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: root + secret: redhat + jms_asset: + address: 127.0.0.1 + port: 3306 + + tasks: + - name: Gather facts info + community.mysql.mysql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + register: db_info + + - name: Get info + set_fact: + info: + version: "{{ db_info.version.full }}" + + - debug: + var: db_info + diff --git a/apps/assets/automations/gather_facts/database/mysql/manifest.yml b/apps/assets/automations/gather_facts/database/mysql/manifest.yml new file mode 100644 index 000000000..33109b29b --- /dev/null +++ b/apps/assets/automations/gather_facts/database/mysql/manifest.yml @@ -0,0 +1,6 @@ +id: gather_facts_mysql +name: Gather facts from MySQL +category: database +type: + - mysql +method: gather_facts diff --git a/apps/assets/automations/gather_facts/database/postgresql/main.yml b/apps/assets/automations/gather_facts/database/postgresql/main.yml new file mode 100644 index 000000000..6af98366b --- /dev/null +++ b/apps/assets/automations/gather_facts/database/postgresql/main.yml @@ -0,0 +1,28 @@ +- hosts: postgre + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: postgre + secret: postgre + jms_asset: + address: 127.0.0.1 + port: 5432 + database: testdb + account: + username: test + secret: jumpserver + + tasks: + - name: Test PostgreSQL connection + community.postgresql.postgresql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + login_db: "{{ jms_asset.database }}" + register: db_info + + - name: Debug it + debug: + var: db_info diff --git a/apps/assets/automations/gather_facts/database/postgresql/manifest.yml b/apps/assets/automations/gather_facts/database/postgresql/manifest.yml new file mode 100644 index 000000000..19bf255de --- /dev/null +++ b/apps/assets/automations/gather_facts/database/postgresql/manifest.yml @@ -0,0 +1,6 @@ +id: gather_facts_postgresql +name: Gather facts for PostgreSQL +category: database +type: + - postgresql +method: gather_facts diff --git a/apps/assets/automations/gather_facts/database/sqlserver/main.yml b/apps/assets/automations/gather_facts/database/sqlserver/main.yml new file mode 100644 index 000000000..402c7fa8d --- /dev/null +++ b/apps/assets/automations/gather_facts/database/sqlserver/main.yml @@ -0,0 +1,10 @@ +{% for account in accounts %} +- hosts: {{ account.asset.name }} + vars: + account: + username: {{ account.username }} + password: {{ account.password }} + public_key: {{ account.public_key }} + roles: + - change_password +{% endfor %} diff --git a/apps/assets/automations/gather_facts/database/sqlserver/manifest.yml b/apps/assets/automations/gather_facts/database/sqlserver/manifest.yml new file mode 100644 index 000000000..3c4c82de4 --- /dev/null +++ b/apps/assets/automations/gather_facts/database/sqlserver/manifest.yml @@ -0,0 +1,8 @@ +id: gather_facts_sqlserver +name: Change password for SQLServer +version: 1 +category: database +type: + - sqlserver +method: gather_facts + diff --git a/apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml b/apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml new file mode 100644 index 000000000..903cd9115 --- /dev/null +++ b/apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml @@ -0,0 +1,27 @@ +- name: ping + ping: + +#- name: print variables +# debug: +# msg: "Username: {{ account.username }}, Password: {{ account.password }}" + +- name: Change password + user: + name: "{{ account.username }}" + password: "{{ account.password | password_hash('des') }}" + update_password: always + when: account.password + +- name: Change public key + authorized_key: + user: "{{ account.username }}" + key: "{{ account.public_key }}" + state: present + when: account.public_key + +- name: Verify password + ping: + vars: + ansible_user: "{{ account.username }}" + ansible_pass: "{{ account.password }}" + ansible_ssh_connection: paramiko diff --git a/apps/assets/automations/gather_facts/demo_inventory.txt b/apps/assets/automations/gather_facts/demo_inventory.txt new file mode 100644 index 000000000..ed011eae2 --- /dev/null +++ b/apps/assets/automations/gather_facts/demo_inventory.txt @@ -0,0 +1,2 @@ +# all base inventory in base/base_inventory.txt +asset_name(ip) ...base_inventory_vars diff --git a/apps/assets/automations/gather_facts/host/posix/main.yml b/apps/assets/automations/gather_facts/host/posix/main.yml new file mode 100644 index 000000000..6e900fccb --- /dev/null +++ b/apps/assets/automations/gather_facts/host/posix/main.yml @@ -0,0 +1,19 @@ +- hosts: website + gather_facts: yes + tasks: + - name: Get info + set_fact: + info: + arch: "{{ ansible_architecture }}" + distribution: "{{ ansible_distribution }}" + distribution_version: "{{ ansible_distribution_version }}" + kernel: "{{ ansible_kernel }}" + vendor: "{{ ansible_system_vendor }}" + model: "{{ ansible_product_name }}" + sn: "{{ ansible_product_serial }}" + cpu_vcpus: "{{ ansible_processor_vcpus }}" + memory: "{{ ansible_memtotal_mb }}" + disk_total: "{{ (ansible_mounts | map(attribute='size_total') | sum / 1024 / 1024 / 1024) | round(2) }}" + + - debug: + var: info diff --git a/apps/assets/automations/gather_facts/host/posix/manifest.yml b/apps/assets/automations/gather_facts/host/posix/manifest.yml new file mode 100644 index 000000000..e5622cf28 --- /dev/null +++ b/apps/assets/automations/gather_facts/host/posix/manifest.yml @@ -0,0 +1,8 @@ +id: gather_facts_posix +name: Gather posix facts +category: host +type: + - linux + - windows + - unix +method: gather_facts diff --git a/apps/assets/automations/gather_facts/host/windows/main.yml b/apps/assets/automations/gather_facts/host/windows/main.yml new file mode 100644 index 000000000..723aa7720 --- /dev/null +++ b/apps/assets/automations/gather_facts/host/windows/main.yml @@ -0,0 +1,24 @@ +- hosts: windows + gather_facts: yes + tasks: +# - name: Gather facts windows +# setup: +# register: facts +# +# - debug: +# var: facts + - name: Get info + set_fact: + info: + arch: "{{ ansible_architecture2 }}" + distribution: "{{ ansible_distribution }}" + distribution_version: "{{ ansible_distribution_version }}" + kernel: "{{ ansible_kernel }}" + vendor: "{{ ansible_system_vendor }}" + model: "{{ ansible_product_name }}" + sn: "{{ ansible_product_serial }}" + cpu_vcpus: "{{ ansible_processor_vcpus }}" + memory: "{{ ansible_memtotal_mb }}" +t + - debug: + var: info diff --git a/apps/assets/automations/gather_facts/host/windows/manifest.yml b/apps/assets/automations/gather_facts/host/windows/manifest.yml new file mode 100644 index 000000000..929a6626f --- /dev/null +++ b/apps/assets/automations/gather_facts/host/windows/manifest.yml @@ -0,0 +1,7 @@ +id: gather_facts_windows +name: Gather facts windows +version: 1 +method: gather_facts +category: host +type: + - windows diff --git a/apps/assets/automations/gather_facts/manager.py b/apps/assets/automations/gather_facts/manager.py new file mode 100644 index 000000000..7b56d728e --- /dev/null +++ b/apps/assets/automations/gather_facts/manager.py @@ -0,0 +1,77 @@ +import os +import shutil +from copy import deepcopy +from collections import defaultdict + +import yaml +from django.utils.translation import gettext as _ + +from ops.ansible import PlaybookRunner +from ..base.manager import BasePlaybookManager +from assets.automations.methods import platform_automation_methods + + +class GatherFactsManager(BasePlaybookManager): + method_name = 'gather_facts' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.id_method_mapper = { + method['id']: method + for method in platform_automation_methods + if method['method'] == self.method_name + } + self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] + + def inventory_kwargs(self): + return { + } + + def generate_playbook(self): + playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.id_method_mapper[method_id] + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + + step = 10 + hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] + for i, hosts in enumerate(hosts_grouped): + plays = [] + play = deepcopy(host_playbook_play) + play['hosts'] = ':'.join(hosts) + plays.append(play) + + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) + + playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(playbook, f) + + print("Generate playbook done: " + self.playbook_path) + + def get_runner(self): + return PlaybookRunner( + self.inventory_path, + self.playbook_path, + self.runtime_dir + ) + + diff --git a/apps/assets/automations/generate_playbook/change_password.py b/apps/assets/automations/generate_playbook/change_password.py deleted file mode 100644 index 20c3f0889..000000000 --- a/apps/assets/automations/generate_playbook/change_password.py +++ /dev/null @@ -1,106 +0,0 @@ -import os -import yaml -import jinja2 -from typing import List - -from django.conf import settings -from assets.models import Asset -from .base import BaseGeneratePlaybook - - -class GenerateChangePasswordPlaybook(BaseGeneratePlaybook): - - def __init__( - self, assets: List[Asset], strategy, usernames, password='', - private_key='', public_key='', key_strategy='' - ): - super().__init__(assets, strategy) - self.password = password - self.public_key = public_key - self.private_key = private_key - self.key_strategy = key_strategy - self.relation_asset_map = self.get_username_relation_asset_map(usernames) - - def get_username_relation_asset_map(self, usernames): - # TODO 没特权用户的资产 要考虑网关 - - complete_map = { - asset: list(asset.accounts.value_list('username', flat=True)) - for asset in self.assets - } - - if '*' in usernames: - return complete_map - - relation_map = {} - for asset, usernames in complete_map.items(): - usernames = list(set(usernames) & set(usernames)) - if not usernames: - continue - relation_map[asset] = list(set(usernames) & set(usernames)) - return relation_map - - @property - def src_filepath(self): - return os.path.join( - settings.BASE_DIR, 'assets', 'playbooks', 'strategy', - 'change_password', 'roles', self.strategy - ) - - def generate_hosts(self): - host_pathname = os.path.join(self.temp_folder, 'hosts') - with open(host_pathname, 'w', encoding='utf8') as f: - for asset in self.relation_asset_map.keys(): - f.write(f'{asset.name}\n') - - def generate_host_vars(self): - host_vars_pathname = os.path.join(self.temp_folder, 'hosts', 'host_vars') - os.makedirs(host_vars_pathname, exist_ok=True) - for asset, usernames in self.relation_asset_map.items(): - host_vars = { - 'ansible_host': asset.get_target_ip(), - 'ansible_port': asset.get_target_ssh_port(), # TODO 需要根绝协议取端口号 - 'ansible_user': asset.admin_user.username, - 'ansible_pass': asset.admin_user.username, - 'usernames': usernames, - } - pathname = os.path.join(host_vars_pathname, f'{asset.name}.yml') - with open(pathname, 'w', encoding='utf8') as f: - f.write(yaml.dump(host_vars, allow_unicode=True)) - - def generate_secret_key_files(self): - if not self.private_key and not self.public_key: - return - - file_pathname = os.path.join(self.temp_folder, self.strategy, 'files') - public_pathname = os.path.join(file_pathname, 'id_rsa.pub') - private_pathname = os.path.join(file_pathname, 'id_rsa') - - os.makedirs(file_pathname, exist_ok=True) - with open(public_pathname, 'w', encoding='utf8') as f: - f.write(self.public_key) - with open(private_pathname, 'w', encoding='utf8') as f: - f.write(self.private_key) - - def generate_role_main(self): - task_main_pathname = os.path.join(self.temp_folder, 'main.yaml') - context = { - 'password': self.password, - 'key_strategy': self.key_strategy, - 'private_key_file': 'id_rsa' if self.private_key else '', - 'exclusive': 'no' if self.key_strategy == 'all' else 'yes', - 'jms_key': self.public_key.split()[2].strip() if self.public_key else '', - } - with open(task_main_pathname, 'r+', encoding='utf8') as f: - string_var = f.read() - f.seek(0, 0) - response = jinja2.Template(string_var).render(context) - results = yaml.safe_load(response) - f.write(yaml.dump(results, allow_unicode=True)) - - def execute(self): - self.generate_temp_playbook() - self.generate_hosts() - self.generate_host_vars() - self.generate_secret_key_files() - self.generate_role_main() diff --git a/apps/assets/automations/generate_playbook/verify.py b/apps/assets/automations/generate_playbook/verify.py deleted file mode 100644 index 88695b814..000000000 --- a/apps/assets/automations/generate_playbook/verify.py +++ /dev/null @@ -1,86 +0,0 @@ -import os -import yaml -from typing import List - -from django.conf import settings -from assets.models import Asset -from .base import BaseGeneratePlaybook - - -class GenerateVerifyPlaybook(BaseGeneratePlaybook): - - def __init__( - self, assets: List[Asset], strategy, usernames - ): - super().__init__(assets, strategy) - self.relation_asset_map = self.get_account_relation_asset_map(usernames) - - def get_account_relation_asset_map(self, usernames): - # TODO 没特权用户的资产 要考虑网关 - complete_map = { - asset: list(asset.accounts.all()) - for asset in self.assets - } - - if '*' in usernames: - return complete_map - - relation_map = {} - for asset, accounts in complete_map.items(): - account_map = {account.username: account for account in accounts} - accounts = [account_map[i] for i in (set(usernames) & set(account_map))] - if not accounts: - continue - relation_map[asset] = accounts - return relation_map - - @property - def src_filepath(self): - return os.path.join( - settings.BASE_DIR, 'assets', 'playbooks', 'strategy', - 'verify', 'roles', self.strategy - ) - - def generate_hosts(self): - host_pathname = os.path.join(self.temp_folder, 'hosts') - with open(host_pathname, 'w', encoding='utf8') as f: - for asset in self.relation_asset_map.keys(): - f.write(f'{asset.name}\n') - - def generate_host_vars(self): - host_vars_pathname = os.path.join(self.temp_folder, 'hosts', 'host_vars') - os.makedirs(host_vars_pathname, exist_ok=True) - for asset, accounts in self.relation_asset_map.items(): - account_info = [] - for account in accounts: - private_key_filename = f'{asset.name}_{account.username}' if account.private_key else '' - account_info.append({ - 'username': account.username, - 'password': account.password, - 'private_key_filename': private_key_filename, - }) - host_vars = { - 'ansible_host': asset.get_target_ip(), - 'ansible_port': asset.get_target_ssh_port(), # TODO 需要根绝协议取端口号 - 'account_info': account_info, - } - pathname = os.path.join(host_vars_pathname, f'{asset.name}.yml') - with open(pathname, 'w', encoding='utf8') as f: - f.write(yaml.dump(host_vars, allow_unicode=True)) - - def generate_secret_key_files(self): - file_pathname = os.path.join(self.temp_folder, self.strategy, 'files') - os.makedirs(file_pathname, exist_ok=True) - for asset, accounts in self.relation_asset_map.items(): - for account in accounts: - if account.private_key: - path_name = os.path.join(file_pathname, f'{asset.name}_{account.username}') - with open(path_name, 'w', encoding='utf8') as f: - f.write(account.private_key) - - def execute(self): - self.generate_temp_playbook() - self.generate_hosts() - self.generate_host_vars() - self.generate_secret_key_files() - # self.generate_role_main() # TODO Linux 暂时不需要 diff --git a/apps/assets/automations/ping/__init__.py b/apps/assets/automations/ping/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/assets/automations/ping/database/mysql/main.yml b/apps/assets/automations/ping/database/mysql/main.yml new file mode 100644 index 000000000..fab498c76 --- /dev/null +++ b/apps/assets/automations/ping/database/mysql/main.yml @@ -0,0 +1,20 @@ +- hosts: mysql + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: root + password: redhat + jms_asset: + address: 127.0.0.1 + port: 3306 + + tasks: + - name: Test MySQL connection + community.mysql.mysql_info: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + filter: version + register: db_info diff --git a/apps/assets/automations/ping/database/mysql/manifest.yml b/apps/assets/automations/ping/database/mysql/manifest.yml new file mode 100644 index 000000000..aded00b1f --- /dev/null +++ b/apps/assets/automations/ping/database/mysql/manifest.yml @@ -0,0 +1,6 @@ +id: mysql_ping +name: Ping MySQL +category: database +type: + - mysql +method: ping diff --git a/apps/assets/automations/ping/database/postgresql/main.yml b/apps/assets/automations/ping/database/postgresql/main.yml new file mode 100644 index 000000000..3bc2f7957 --- /dev/null +++ b/apps/assets/automations/ping/database/postgresql/main.yml @@ -0,0 +1,23 @@ +- hosts: postgre + gather_facts: no + vars: + ansible_python_interpreter: /usr/local/bin/python + jms_account: + username: postgre + secret: postgre + jms_asset: + address: 127.0.0.1 + port: 5432 + database: testdb + account: + username: test + secret: jumpserver + + tasks: + - name: Test PostgreSQL connection + community.postgresql.postgresql_ping: + login_user: "{{ jms_account.username }}" + login_password: "{{ jms_account.secret }}" + login_host: "{{ jms_asset.address }}" + login_port: "{{ jms_asset.port }}" + login_db: "{{ jms_asset.database }}" diff --git a/apps/assets/automations/ping/database/postgresql/manifest.yml b/apps/assets/automations/ping/database/postgresql/manifest.yml new file mode 100644 index 000000000..337b2b50d --- /dev/null +++ b/apps/assets/automations/ping/database/postgresql/manifest.yml @@ -0,0 +1,6 @@ +id: ping_postgresql +name: Ping PostgreSQL +category: database +type: + - postgresql +method: ping diff --git a/apps/assets/automations/ping/demo_inventory.txt b/apps/assets/automations/ping/demo_inventory.txt new file mode 100644 index 000000000..dcc7d1b6d --- /dev/null +++ b/apps/assets/automations/ping/demo_inventory.txt @@ -0,0 +1,2 @@ +# all base inventory in base/base_inventory.txt +asset_name(ip)_account_username account={"username": "", "password": "xxx"} ...base_inventory_vars diff --git a/apps/assets/automations/ping/host/posix/main.yml b/apps/assets/automations/ping/host/posix/main.yml new file mode 100644 index 000000000..c4c740367 --- /dev/null +++ b/apps/assets/automations/ping/host/posix/main.yml @@ -0,0 +1,5 @@ +- hosts: demo + gather_facts: no + tasks: + - name: Posix ping + ping: diff --git a/apps/assets/automations/ping/host/posix/manifest.yml b/apps/assets/automations/ping/host/posix/manifest.yml new file mode 100644 index 000000000..b07caa7f8 --- /dev/null +++ b/apps/assets/automations/ping/host/posix/manifest.yml @@ -0,0 +1,8 @@ +id: posix_ping +name: Posix ping +category: host +type: + - linux + - windows + - unix +method: ping diff --git a/apps/assets/automations/ping/host/windows/main.yml b/apps/assets/automations/ping/host/windows/main.yml new file mode 100644 index 000000000..495b82a3d --- /dev/null +++ b/apps/assets/automations/ping/host/windows/main.yml @@ -0,0 +1,5 @@ +- hosts: windows + gather_facts: no + tasks: + - name: Windows ping + win_ping: diff --git a/apps/assets/automations/ping/host/windows/manifest.yml b/apps/assets/automations/ping/host/windows/manifest.yml new file mode 100644 index 000000000..55e336f19 --- /dev/null +++ b/apps/assets/automations/ping/host/windows/manifest.yml @@ -0,0 +1,7 @@ +id: win_ping +name: Windows ping +version: 1 +method: change_password +category: host +type: + - windows diff --git a/apps/assets/automations/ping/manager.py b/apps/assets/automations/ping/manager.py new file mode 100644 index 000000000..36017438b --- /dev/null +++ b/apps/assets/automations/ping/manager.py @@ -0,0 +1,75 @@ +import os +import shutil +from copy import deepcopy +from collections import defaultdict + +import yaml +from django.utils.translation import gettext as _ + +from ops.ansible import PlaybookRunner +from ..base.manager import BasePlaybookManager +from assets.automations.methods import platform_automation_methods + + +class ChangePasswordManager(BasePlaybookManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.id_method_mapper = { + method['id']: method + for method in platform_automation_methods + } + self.method_hosts_mapper = defaultdict(list) + self.playbooks = [] + + def inventory_kwargs(self): + return { + 'host_callback': self.host_duplicator + } + + def generate_playbook(self): + playbook = [] + for method_id, host_names in self.method_hosts_mapper.items(): + method = self.id_method_mapper[method_id] + method_playbook_dir_path = method['dir'] + method_playbook_dir_name = os.path.basename(method_playbook_dir_path) + sub_playbook_dir = os.path.join(os.path.dirname(self.playbook_path), method_playbook_dir_name) + shutil.copytree(method_playbook_dir_path, sub_playbook_dir) + sub_playbook_path = os.path.join(sub_playbook_dir, 'main.yml') + + with open(sub_playbook_path, 'r') as f: + host_playbook_play = yaml.safe_load(f) + + if isinstance(host_playbook_play, list): + host_playbook_play = host_playbook_play[0] + + step = 10 + hosts_grouped = [host_names[i:i+step] for i in range(0, len(host_names), step)] + for i, hosts in enumerate(hosts_grouped): + plays = [] + play = deepcopy(host_playbook_play) + play['hosts'] = ':'.join(hosts) + plays.append(play) + + playbook_path = os.path.join(sub_playbook_dir, 'part_{}.yml'.format(i)) + with open(playbook_path, 'w') as f: + yaml.safe_dump(plays, f) + self.playbooks.append(playbook_path) + + playbook.append({ + 'name': method['name'] + ' for part {}'.format(i), + 'import_playbook': os.path.join(method_playbook_dir_name, 'part_{}.yml'.format(i)) + }) + + with open(self.playbook_path, 'w') as f: + yaml.safe_dump(playbook, f) + + print("Generate playbook done: " + self.playbook_path) + + def get_runner(self): + return PlaybookRunner( + self.inventory_path, + self.playbook_path, + self.runtime_dir + ) + + diff --git a/apps/assets/models/automations/change_secret.py b/apps/assets/models/automations/change_secret.py index 4a2c304c2..dc0403a12 100644 --- a/apps/assets/models/automations/change_secret.py +++ b/apps/assets/models/automations/change_secret.py @@ -19,6 +19,10 @@ class ChangePasswordAutomation(BaseAutomation): verbose_name=_("Recipient") ) + def save(self, *args, **kwargs): + self.type = 'change_password' + super().save(*args, **kwargs) + class Meta: verbose_name = _("Change auth strategy") diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 9d4498515..a91126b5f 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -10,18 +10,17 @@ __all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, assets, account='', account_policy='smart', host_var_callback=None, host_duplicator=None): + def __init__(self, assets, account='', account_policy='smart', host_callback=None): """ :param assets: :param account: account username name if not set use account_policy :param account_policy: - :param host_var_callback: + :param host_callback: after generate host, call this callback to modify host """ self.assets = self.clean_assets(assets) self.account_username = account self.account_policy = account_policy - self.host_var_callback = host_var_callback - self.host_duplicator = host_duplicator + self.host_callback = host_callback @staticmethod def clean_assets(assets): @@ -100,15 +99,10 @@ class JMSInventory: elif account.secret_type == 'private_key' and account.secret: host['ssh_private_key'] = account.private_key_file else: - host['exclude'] = _("No account found") + host['error'] = _("No account found") if gateway: host.update(self.make_proxy_command(gateway)) - - if self.host_var_callback: - callback_var = self.host_var_callback(asset) - if isinstance(callback_var, dict): - host.update(callback_var) return host def select_account(self, asset): @@ -145,10 +139,18 @@ class JMSInventory: for asset in self.assets: account = self.select_account(asset) host = self.asset_to_host(asset, account, automation, protocols) + if not automation.ansible_enabled: - host['exclude'] = _('Ansible disabled') - if self.host_duplicator: - hosts.extend(self.host_duplicator(host, asset=asset, account=account, platform=platform)) + host['error'] = _('Ansible disabled') + + if self.host_callback is not None: + host = self.host_callback( + host, asset=asset, account=account, + platform=platform, automation=automation + ) + + if isinstance(host, list): + hosts.extend(host) else: hosts.append(host) @@ -156,7 +158,7 @@ class JMSInventory: if exclude_hosts: print(_("Skip hosts below:")) for i, host in enumerate(exclude_hosts, start=1): - print("{}: [{}] \t{}".format(i, host['name'], host['exclude'])) + print("{}: [{}] \t{}".format(i, host['name'], host['error'])) hosts = list(filter(lambda x: not x.get('exclude'), hosts)) data = {'all': {'hosts': {}}} diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index e8f232e74..fbf3245ae 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -52,12 +52,14 @@ class AdHocRunner: class PlaybookRunner: - def __init__(self, inventory, playbook, project_dir='/tmp/'): + def __init__(self, inventory, playbook, project_dir='/tmp/', callback=None): self.id = uuid.uuid4() self.inventory = inventory self.playbook = playbook self.project_dir = project_dir - self.cb = DefaultCallback() + if not callback: + callback = DefaultCallback() + self.cb = callback def run(self, verbosity=0, **kwargs): if verbosity is None and settings.DEBUG: From 52fb55e806ec4ae14576c215ce118fd2e867250b Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 13 Oct 2022 17:52:25 +0800 Subject: [PATCH 13/13] =?UTF-8?q?pref:=20=E4=BF=AE=E6=94=B9=E6=94=B9?= =?UTF-8?q?=E5=AF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...013_1429.py => 0108_migrate_automation.py} | 57 +++---------- .../migrations/0109_auto_20221013_1751.py | 83 +++++++++++++++++++ 2 files changed, 96 insertions(+), 44 deletions(-) rename apps/assets/migrations/{0108_auto_20221013_1429.py => 0108_migrate_automation.py} (64%) create mode 100644 apps/assets/migrations/0109_auto_20221013_1751.py diff --git a/apps/assets/migrations/0108_auto_20221013_1429.py b/apps/assets/migrations/0108_migrate_automation.py similarity index 64% rename from apps/assets/migrations/0108_auto_20221013_1429.py rename to apps/assets/migrations/0108_migrate_automation.py index df19a59d5..86a29c193 100644 --- a/apps/assets/migrations/0108_auto_20221013_1429.py +++ b/apps/assets/migrations/0108_migrate_automation.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.14 on 2022-10-13 06:29 +# Generated by Django 3.2.14 on 2022-10-10 01:59 import common.db.fields from django.conf import settings @@ -15,22 +15,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.CreateModel( - name='AutomationExecution', - fields=[ - ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), - ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), - ('status', models.CharField(default='pending', max_length=16)), - ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), - ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), - ('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')), - ('snapshot', common.db.fields.EncryptJsonDictTextField(blank=True, default=dict, null=True, verbose_name='Automation snapshot')), - ('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')], default='manual', max_length=128, verbose_name='Trigger mode')), - ], - options={ - 'verbose_name': 'Automation strategy execution', - }, - ), migrations.CreateModel( name='BaseAutomation', fields=[ @@ -46,7 +30,6 @@ class Migration(migrations.Migration): ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')), ('accounts', models.JSONField(default=list, verbose_name='Accounts')), ('type', models.CharField(max_length=16, verbose_name='Type')), - ('is_active', models.BooleanField(default=True, verbose_name='Is active')), ('comment', models.TextField(blank=True, verbose_name='Comment')), ('assets', models.ManyToManyField(blank=True, to='assets.Asset', verbose_name='Assets')), ('nodes', models.ManyToManyField(blank=True, to='assets.Node', verbose_name='Nodes')), @@ -102,47 +85,33 @@ class Migration(migrations.Migration): bases=('assets.baseautomation',), ), migrations.CreateModel( - name='ChangeSecretRecord', + name='AutomationExecution', fields=[ - ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), - ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), - ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), - ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), - ('old_secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Old secret')), - ('new_secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')), - ('date_started', models.DateTimeField(blank=True, null=True, verbose_name='Date started')), - ('date_finished', models.DateTimeField(blank=True, null=True, verbose_name='Date finished')), ('status', models.CharField(default='pending', max_length=16)), - ('error', models.TextField(blank=True, null=True, verbose_name='Error')), - ('account', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='assets.account')), - ('execution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.automationexecution')), + ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), + ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), + ('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')), + ('snapshot', common.db.fields.EncryptJsonDictTextField(blank=True, default=dict, null=True, verbose_name='Automation snapshot')), + ('trigger', models.CharField(choices=[('manual', 'Manual trigger'), ('timing', 'Timing trigger')], default='manual', max_length=128, verbose_name='Trigger mode')), + ('automation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='assets.baseautomation', verbose_name='Automation strategy')), ], options={ - 'verbose_name': 'Change secret', + 'verbose_name': 'Automation strategy execution', }, ), - migrations.AddField( - model_name='automationexecution', - name='automation', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='assets.baseautomation', verbose_name='Automation strategy'), - ), migrations.CreateModel( - name='ChangeSecretAutomation', + name='ChangePasswordAutomation', fields=[ ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), - ('secret_types', models.JSONField(default=list, verbose_name='Secret types')), - ('password_strategy', models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='specific', max_length=16, verbose_name='Password strategy')), ('password', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')), - ('password_rules', models.JSONField(default=dict, verbose_name='Password rules')), - ('ssh_key_strategy', models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='specific', max_length=16)), - ('ssh_key', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH key')), - ('ssh_key_change_strategy', models.CharField(choices=[('add', 'Append SSH KEY'), ('set', 'Empty and append SSH KEY'), ('set_jms', 'Replace (The key generated by JumpServer) ')], default='add', max_length=16, verbose_name='SSH key strategy')), - ('recipients', models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL, verbose_name='Recipient')), + ('recipients', models.ManyToManyField(blank=True, related_name='recipients_change_auth_strategy', to=settings.AUTH_USER_MODEL, verbose_name='Recipient')), ], options={ 'verbose_name': 'Change auth strategy', }, bases=('assets.baseautomation',), ), + ] diff --git a/apps/assets/migrations/0109_auto_20221013_1751.py b/apps/assets/migrations/0109_auto_20221013_1751.py new file mode 100644 index 000000000..c00a11b34 --- /dev/null +++ b/apps/assets/migrations/0109_auto_20221013_1751.py @@ -0,0 +1,83 @@ +# Generated by Django 3.2.14 on 2022-10-13 09:51 + +import common.db.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('assets', '0108_migrate_automation'), + ] + + operations = [ + migrations.RenameModel( + old_name='ChangePasswordAutomation', + new_name='ChangeSecretAutomation', + ), + migrations.AddField( + model_name='baseautomation', + name='is_active', + field=models.BooleanField(default=True, verbose_name='Is active'), + ), + migrations.AddField( + model_name='changesecretautomation', + name='password_rules', + field=models.JSONField(default=dict, verbose_name='Password rules'), + ), + migrations.AddField( + model_name='changesecretautomation', + name='password_strategy', + field=models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='random_one', max_length=16, verbose_name='Password strategy'), + ), + migrations.AddField( + model_name='changesecretautomation', + name='secret_types', + field=models.JSONField(default=list, verbose_name='Secret types'), + ), + migrations.AddField( + model_name='changesecretautomation', + name='ssh_key', + field=common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='SSH key'), + ), + migrations.AddField( + model_name='changesecretautomation', + name='ssh_key_change_strategy', + field=models.CharField(choices=[('add', 'Append SSH KEY'), ('set', 'Empty and append SSH KEY'), ('set_jms', 'Replace (The key generated by JumpServer) ')], default='add', max_length=16, verbose_name='SSH key strategy'), + ), + migrations.AddField( + model_name='changesecretautomation', + name='ssh_key_strategy', + field=models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='random_one', max_length=16), + ), + migrations.AlterField( + model_name='changesecretautomation', + name='recipients', + field=models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL, verbose_name='Recipient'), + ), + migrations.CreateModel( + name='ChangeSecretRecord', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('old_secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Old secret')), + ('new_secret', common.db.fields.EncryptTextField(blank=True, null=True, verbose_name='Secret')), + ('date_started', models.DateTimeField(blank=True, null=True, verbose_name='Date started')), + ('date_finished', models.DateTimeField(blank=True, null=True, verbose_name='Date finished')), + ('status', models.CharField(default='pending', max_length=16)), + ('error', models.TextField(blank=True, null=True, verbose_name='Error')), + ('account', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='assets.account')), + ('execution', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='assets.automationexecution')), + ], + options={ + 'verbose_name': 'Change secret', + }, + ), + ]