From 0fb4b52232993f9ecf948ce6c8956ee3abb4decf Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 8 Oct 2022 16:55:14 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20ansible=20?= =?UTF-8?q?=E8=A1=A8=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/automation/base.py | 7 +- apps/assets/models/label.py | 4 - apps/audits/api.py | 110 +++--- apps/audits/filters.py | 39 +-- apps/audits/serializers.py | 79 +++-- apps/audits/urls/api_urls.py | 4 +- apps/jumpserver/settings/base.py | 1 + apps/ops/ansible/callback.py | 2 +- apps/ops/ansible/inventory.py | 10 +- apps/ops/ansible/runner.py | 8 + apps/ops/api/__init__.py | 1 - apps/ops/api/adhoc.py | 50 +-- apps/ops/api/command.py | 76 ---- apps/ops/inventory.py | 149 -------- .../ops/migrations/0024_auto_20221008_1514.py | 58 ++++ .../ops/migrations/0025_auto_20221008_1631.py | 72 ++++ apps/ops/mixin.py | 41 --- apps/ops/models/__init__.py | 1 - apps/ops/models/adhoc.py | 328 +----------------- apps/ops/models/base.py | 106 ++++++ apps/ops/models/command.py | 160 --------- apps/ops/models/playbook.py | 31 +- apps/ops/serializers/adhoc.py | 77 ++-- apps/ops/tasks.py | 2 +- apps/ops/urls/api_urls.py | 5 +- 25 files changed, 438 insertions(+), 983 deletions(-) delete mode 100644 apps/ops/api/command.py delete mode 100644 apps/ops/inventory.py create mode 100644 apps/ops/migrations/0024_auto_20221008_1514.py create mode 100644 apps/ops/migrations/0025_auto_20221008_1631.py create mode 100644 apps/ops/models/base.py delete mode 100644 apps/ops/models/command.py diff --git a/apps/assets/models/automation/base.py b/apps/assets/models/automation/base.py index 27c971e0d..a9b8ab087 100644 --- a/apps/assets/models/automation/base.py +++ b/apps/assets/models/automation/base.py @@ -4,15 +4,14 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from common.const.choices import Trigger -from common.mixins.models import CommonModelMixin from common.db.fields import EncryptJsonDictTextField -from orgs.mixins.models import OrgModelMixin +from orgs.mixins.models import OrgModelMixin, JMSOrgBaseModel from ops.mixin import PeriodTaskModelMixin from ops.tasks import execute_automation_strategy from ops.task_handlers import ExecutionManager -class BaseAutomation(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin): +class BaseAutomation(JMSOrgBaseModel, PeriodTaskModelMixin): accounts = models.JSONField(default=list, verbose_name=_("Accounts")) nodes = models.ManyToManyField( 'assets.Node', related_name='automation_strategy', blank=True, verbose_name=_("Nodes") @@ -67,7 +66,7 @@ class AutomationStrategyExecution(OrgModelMixin): default=dict, blank=True, null=True, verbose_name=_('Automation snapshot') ) strategy = models.ForeignKey( - 'assets.models.automation.base.BaseAutomation', related_name='execution', on_delete=models.CASCADE, + 'BaseAutomation', related_name='execution', on_delete=models.CASCADE, verbose_name=_('Automation strategy') ) trigger = models.CharField( diff --git a/apps/assets/models/label.py b/apps/assets/models/label.py index f7820ccb1..937d0d95c 100644 --- a/apps/assets/models/label.py +++ b/apps/assets/models/label.py @@ -14,16 +14,12 @@ class Label(OrgModelMixin): ("S", _("System")), ("U", _("User")) ) - id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField(max_length=128, verbose_name=_("Name")) value = models.CharField(max_length=128, verbose_name=_("Value")) category = models.CharField(max_length=128, choices=CATEGORY_CHOICES, default=USER_CATEGORY, verbose_name=_("Category")) is_active = models.BooleanField(default=True, verbose_name=_("Is active")) comment = models.TextField(blank=True, null=True, verbose_name=_("Comment")) - date_created = models.DateTimeField( - auto_now_add=True, null=True, blank=True, verbose_name=_('Date created') - ) @classmethod def get_queryset_group_by_name(cls): diff --git a/apps/audits/api.py b/apps/audits/api.py index 6cb2e1283..a61694e85 100644 --- a/apps/audits/api.py +++ b/apps/audits/api.py @@ -11,16 +11,14 @@ from common.drf.filters import DatetimeRangeFilter from common.api import CommonGenericViewSet from orgs.mixins.api import OrgGenericViewSet, OrgBulkModelViewSet, OrgRelationMixin from orgs.utils import current_org -from ops.models import CommandExecution +# from ops.models import CommandExecution from . import filters from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog -from .serializers import FTPLogSerializer, UserLoginLogSerializer, CommandExecutionSerializer -from .serializers import OperateLogSerializer, PasswordChangeLogSerializer, CommandExecutionHostsRelationSerializer +from .serializers import FTPLogSerializer, UserLoginLogSerializer +from .serializers import OperateLogSerializer, PasswordChangeLogSerializer -class FTPLogViewSet(CreateModelMixin, - ListModelMixin, - OrgGenericViewSet): +class FTPLogViewSet(CreateModelMixin, ListModelMixin, OrgGenericViewSet): model = FTPLog serializer_class = FTPLogSerializer extra_filter_backends = [DatetimeRangeFilter] @@ -98,53 +96,53 @@ class PasswordChangeLogViewSet(ListModelMixin, CommonGenericViewSet): ) return queryset - -class CommandExecutionViewSet(ListModelMixin, OrgGenericViewSet): - model = CommandExecution - serializer_class = CommandExecutionSerializer - extra_filter_backends = [DatetimeRangeFilter] - date_range_filter_fields = [ - ('date_start', ('date_from', 'date_to')) - ] - filterset_fields = [ - 'user__name', 'user__username', 'command', - 'account', 'is_finished' - ] - search_fields = [ - 'command', 'user__name', 'user__username', - 'account__username', - ] - ordering = ['-date_created'] - - def get_queryset(self): - queryset = super().get_queryset() - if getattr(self, 'swagger_fake_view', False): - return queryset.model.objects.none() - if current_org.is_root(): - return queryset - # queryset = queryset.filter(run_as__org_id=current_org.org_id()) - return queryset - - -class CommandExecutionHostRelationViewSet(OrgRelationMixin, OrgBulkModelViewSet): - serializer_class = CommandExecutionHostsRelationSerializer - m2m_field = CommandExecution.hosts.field - filterset_fields = [ - 'id', 'asset', 'commandexecution' - ] - search_fields = ('asset__name', ) - http_method_names = ['options', 'get'] - rbac_perms = { - 'GET': 'ops.view_commandexecution', - 'list': 'ops.view_commandexecution', - } - - def get_queryset(self): - queryset = super().get_queryset() - queryset = queryset.annotate( - asset_display=Concat( - F('asset__name'), Value('('), - F('asset__address'), Value(')') - ) - ) - return queryset +# Todo: 看看怎么搞 +# class CommandExecutionViewSet(ListModelMixin, OrgGenericViewSet): +# model = CommandExecution +# serializer_class = CommandExecutionSerializer +# extra_filter_backends = [DatetimeRangeFilter] +# date_range_filter_fields = [ +# ('date_start', ('date_from', 'date_to')) +# ] +# filterset_fields = [ +# 'user__name', 'user__username', 'command', +# 'account', 'is_finished' +# ] +# search_fields = [ +# 'command', 'user__name', 'user__username', +# 'account__username', +# ] +# ordering = ['-date_created'] +# +# def get_queryset(self): +# queryset = super().get_queryset() +# if getattr(self, 'swagger_fake_view', False): +# return queryset.model.objects.none() +# if current_org.is_root(): +# return queryset +# # queryset = queryset.filter(run_as__org_id=current_org.org_id()) +# return queryset +# +# +# class CommandExecutionHostRelationViewSet(OrgRelationMixin, OrgBulkModelViewSet): +# serializer_class = CommandExecutionHostsRelationSerializer +# m2m_field = CommandExecution.hosts.field +# filterset_fields = [ +# 'id', 'asset', 'commandexecution' +# ] +# search_fields = ('asset__name', ) +# http_method_names = ['options', 'get'] +# rbac_perms = { +# 'GET': 'ops.view_commandexecution', +# 'list': 'ops.view_commandexecution', +# } +# +# def get_queryset(self): +# queryset = super().get_queryset() +# queryset = queryset.annotate( +# asset_display=Concat( +# F('asset__name'), Value('('), +# F('asset__address'), Value(')') +# ) +# ) +# return queryset diff --git a/apps/audits/filters.py b/apps/audits/filters.py index a6c44b5c5..c15c22b56 100644 --- a/apps/audits/filters.py +++ b/apps/audits/filters.py @@ -5,10 +5,9 @@ from rest_framework import filters from rest_framework.compat import coreapi, coreschema from orgs.utils import current_org -from ops.models import CommandExecution from common.drf.filters import BaseFilterSet -__all__ = ['CurrentOrgMembersFilter', 'CommandExecutionFilter'] +__all__ = ['CurrentOrgMembersFilter'] class CurrentOrgMembersFilter(filters.BaseFilterBackend): @@ -35,21 +34,21 @@ class CurrentOrgMembersFilter(filters.BaseFilterBackend): queryset = queryset.filter(user__in=self._get_user_list()) return queryset - -class CommandExecutionFilter(BaseFilterSet): - hostname_ip = CharFilter(method='filter_hostname_ip') - - class Meta: - model = CommandExecution.hosts.through - fields = ( - 'id', 'asset', 'commandexecution', 'hostname_ip' - ) - - def filter_hostname_ip(self, queryset, name, value): - queryset = queryset.annotate( - hostname_ip=Concat( - F('asset__hostname'), Value('('), - F('asset__address'), Value(')') - ) - ).filter(hostname_ip__icontains=value) - return queryset +# +# class CommandExecutionFilter(BaseFilterSet): +# hostname_ip = CharFilter(method='filter_hostname_ip') +# +# class Meta: +# model = CommandExecution.hosts.through +# fields = ( +# 'id', 'asset', 'commandexecution', 'hostname_ip' +# ) +# +# def filter_hostname_ip(self, queryset, name, value): +# queryset = queryset.annotate( +# hostname_ip=Concat( +# F('asset__hostname'), Value('('), +# F('asset__address'), Value(')') +# ) +# ).filter(hostname_ip__icontains=value) +# return queryset diff --git a/apps/audits/serializers.py b/apps/audits/serializers.py index 8b9d28005..0f595be25 100644 --- a/apps/audits/serializers.py +++ b/apps/audits/serializers.py @@ -5,7 +5,6 @@ from rest_framework import serializers from common.drf.serializers import BulkSerializerMixin from terminal.models import Session -from ops.models import CommandExecution from . import models @@ -76,42 +75,42 @@ class SessionAuditSerializer(serializers.ModelSerializer): model = Session fields = '__all__' - -class CommandExecutionSerializer(serializers.ModelSerializer): - is_success = serializers.BooleanField(read_only=True, label=_('Is success')) - hosts_display = serializers.ListSerializer( - child=serializers.CharField(), source='hosts', read_only=True, label=_('Hosts display') - ) - - class Meta: - model = CommandExecution - fields_mini = ['id'] - fields_small = fields_mini + [ - 'command', 'is_finished', 'user', - 'date_start', 'result', 'is_success', 'org_id' - ] - fields = fields_small + ['hosts', 'hosts_display', 'user_display'] - extra_kwargs = { - 'result': {'label': _('Result')}, # model 上的方法，只能在这修改 - 'is_success': {'label': _('Is success')}, - 'hosts': {'label': _('Hosts')}, # 外键，会生成 sql。不在 model 上修改 - 'user': {'label': _('User')}, - 'user_display': {'label': _('User display')}, - } - - @classmethod - def setup_eager_loading(cls, queryset): - """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('user', 'hosts') - return queryset - - -class CommandExecutionHostsRelationSerializer(BulkSerializerMixin, serializers.ModelSerializer): - asset_display = serializers.ReadOnlyField() - commandexecution_display = serializers.ReadOnlyField() - - class Meta: - model = CommandExecution.hosts.through - fields = [ - 'id', 'asset', 'asset_display', 'commandexecution', 'commandexecution_display' - ] +# +# class CommandExecutionSerializer(serializers.ModelSerializer): +# is_success = serializers.BooleanField(read_only=True, label=_('Is success')) +# hosts_display = serializers.ListSerializer( +# child=serializers.CharField(), source='hosts', read_only=True, label=_('Hosts display') +# ) +# +# class Meta: +# model = CommandExecution +# fields_mini = ['id'] +# fields_small = fields_mini + [ +# 'command', 'is_finished', 'user', +# 'date_start', 'result', 'is_success', 'org_id' +# ] +# fields = fields_small + ['hosts', 'hosts_display', 'user_display'] +# extra_kwargs = { +# 'result': {'label': _('Result')}, # model 上的方法，只能在这修改 +# 'is_success': {'label': _('Is success')}, +# 'hosts': {'label': _('Hosts')}, # 外键，会生成 sql。不在 model 上修改 +# 'user': {'label': _('User')}, +# 'user_display': {'label': _('User display')}, +# } +# +# @classmethod +# def setup_eager_loading(cls, queryset): +# """ Perform necessary eager loading of data. """ +# queryset = queryset.prefetch_related('user', 'hosts') +# return queryset +# +# +# class CommandExecutionHostsRelationSerializer(BulkSerializerMixin, serializers.ModelSerializer): +# asset_display = serializers.ReadOnlyField() +# commandexecution_display = serializers.ReadOnlyField() +# +# class Meta: +# model = CommandExecution.hosts.through +# fields = [ +# 'id', 'asset', 'asset_display', 'commandexecution', 'commandexecution_display' +# ] diff --git a/apps/audits/urls/api_urls.py b/apps/audits/urls/api_urls.py index 7301b67fb..902c65fbf 100644 --- a/apps/audits/urls/api_urls.py +++ b/apps/audits/urls/api_urls.py @@ -15,8 +15,8 @@ router.register(r'ftp-logs', api.FTPLogViewSet, 'ftp-log') router.register(r'login-logs', api.UserLoginLogViewSet, 'login-log') router.register(r'operate-logs', api.OperateLogViewSet, 'operate-log') router.register(r'password-change-logs', api.PasswordChangeLogViewSet, 'password-change-log') -router.register(r'command-execution-logs', api.CommandExecutionViewSet, 'command-execution-log') -router.register(r'command-executions-hosts-relations', api.CommandExecutionHostRelationViewSet, 'command-executions-hosts-relation') +# router.register(r'command-execution-logs', api.CommandExecutionViewSet, 'command-execution-log') +# router.register(r'command-executions-hosts-relations', api.CommandExecutionHostRelationViewSet, 'command-executions-hosts-relation') urlpatterns = [ diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index d009238cd..8456d9fa0 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -16,6 +16,7 @@ VERSION = const.VERSION BASE_DIR = const.BASE_DIR PROJECT_DIR = const.PROJECT_DIR DATA_DIR = os.path.join(PROJECT_DIR, 'data') +ANSIBLE_DIR = os.path.join(DATA_DIR, 'ansible') CERTS_DIR = os.path.join(DATA_DIR, 'certs') # Quick-start development settings - unsuitable for production diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index 8b6ad1f8f..59734b07d 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -15,7 +15,7 @@ class DefaultCallback: dark={}, skipped=[], ) - self.status = 'starting' + self.status = 'running' self.finished = False def is_success(self): diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 2382525ed..4da027696 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -3,21 +3,19 @@ from collections import defaultdict import json -__all__ = [ - 'JMSInventory', -] +__all__ = ['JMSInventory'] class JMSInventory: - def __init__(self, assets, account_username=None, account_policy='smart', host_var_callback=None): + def __init__(self, assets, account='', account_policy='smart', host_var_callback=None): """ :param assets: - :param account_username: account username name if not set use account_policy + :param account: account username name if not set use account_policy :param account_policy: :param host_var_callback: """ self.assets = self.clean_assets(assets) - self.account_username = account_username + self.account_username = account self.account_policy = account_policy self.host_var_callback = host_var_callback diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index 6c339eba6..a420fb8a4 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -68,3 +68,11 @@ class PlaybookRunner: **kwargs ) return self.cb + + +class CommandRunner(AdHocRunner): + def __init__(self, inventory, command, pattern='*', project_dir='/tmp/'): + super().__init__(inventory, 'shell', command, pattern, project_dir) + + def run(self, verbosity=0, **kwargs): + return super().run(verbosity, **kwargs) diff --git a/apps/ops/api/__init__.py b/apps/ops/api/__init__.py index e59889cd2..8eb5356e4 100644 --- a/apps/ops/api/__init__.py +++ b/apps/ops/api/__init__.py @@ -2,4 +2,3 @@ # from .adhoc import * from .celery import * -from .command import * diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index 0cc7b6d55..8644ac5d2 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -6,52 +6,18 @@ from rest_framework import viewsets, generics from rest_framework.views import Response from common.drf.serializers import CeleryTaskSerializer -from ..models import Task, AdHoc, AdHocExecution +from ..models import AdHoc, AdHocExecution from ..serializers import ( - TaskSerializer, AdHocSerializer, AdHocExecutionSerializer, - TaskDetailSerializer, AdHocDetailSerializer, ) -from ..tasks import run_ansible_task -from orgs.mixins.api import OrgBulkModelViewSet __all__ = [ - 'TaskViewSet', 'TaskRun', 'AdHocViewSet', 'AdHocRunHistoryViewSet' + 'AdHocViewSet', 'AdHocExecutionViewSet' ] -class TaskViewSet(OrgBulkModelViewSet): - model = Task - filterset_fields = ("name",) - search_fields = filterset_fields - serializer_class = TaskSerializer - - def get_serializer_class(self): - if self.action == 'retrieve': - return TaskDetailSerializer - return super().get_serializer_class() - - def get_queryset(self): - queryset = super().get_queryset() - queryset = queryset.select_related('latest_execution') - return queryset - - -class TaskRun(generics.RetrieveAPIView): - queryset = Task.objects.all() - serializer_class = CeleryTaskSerializer - rbac_perms = { - 'retrieve': 'ops.add_adhoc' - } - - def retrieve(self, request, *args, **kwargs): - task = self.get_object() - t = run_ansible_task.delay(str(task.id)) - return Response({"task": t.id}) - - class AdHocViewSet(viewsets.ModelViewSet): queryset = AdHoc.objects.all() serializer_class = AdHocSerializer @@ -61,23 +27,17 @@ class AdHocViewSet(viewsets.ModelViewSet): return AdHocDetailSerializer return super().get_serializer_class() - def get_queryset(self): - task_id = self.request.query_params.get('task') - if task_id: - task = get_object_or_404(Task, id=task_id) - self.queryset = self.queryset.filter(task=task) - return self.queryset - -class AdHocRunHistoryViewSet(viewsets.ModelViewSet): +class AdHocExecutionViewSet(viewsets.ModelViewSet): queryset = AdHocExecution.objects.all() serializer_class = AdHocExecutionSerializer def get_queryset(self): task_id = self.request.query_params.get('task') adhoc_id = self.request.query_params.get('adhoc') + if task_id: - task = get_object_or_404(Task, id=task_id) + task = get_object_or_404(AdHoc, id=task_id) adhocs = task.adhoc.all() self.queryset = self.queryset.filter(adhoc__in=adhocs) diff --git a/apps/ops/api/command.py b/apps/ops/api/command.py deleted file mode 100644 index 1cf7950a6..000000000 --- a/apps/ops/api/command.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- -# -from rest_framework import viewsets -from rest_framework.exceptions import ValidationError -from django.db import transaction -from django.db.models import Q -from django.utils.translation import ugettext as _ -from django.conf import settings - -from assets.models import Asset, Node -from orgs.mixins.api import RootOrgViewMixin -from rbac.permissions import RBACPermission -from ..models import CommandExecution -from ..serializers import CommandExecutionSerializer -from ..tasks import run_command_execution - - -class CommandExecutionViewSet(RootOrgViewMixin, viewsets.ModelViewSet): - serializer_class = CommandExecutionSerializer - permission_classes = (RBACPermission,) - - def get_queryset(self): - return CommandExecution.objects.filter(user_id=str(self.request.user.id)) - - def check_hosts(self, serializer): - data = serializer.validated_data - assets = data["hosts"] - user = self.request.user - - # TOdo: - # Q(granted_by_permissions__system_users__id=system_user.id) & - q = ( - Q(granted_by_permissions__users=user) | - Q(granted_by_permissions__user_groups__users=user) - ) - - permed_assets = set() - permed_assets.update(Asset.objects.filter(id__in=[a.id for a in assets]).filter(q).distinct()) - node_keys = Node.objects.filter(q).distinct().values_list('key', flat=True) - - nodes_assets_q = Q() - for _key in node_keys: - nodes_assets_q |= Q(nodes__key__startswith=f'{_key}:') - nodes_assets_q |= Q(nodes__key=_key) - - permed_assets.update( - Asset.objects.filter( - id__in=[a.id for a in assets] - ).filter( - nodes_assets_q - ).distinct() - ) - - invalid_assets = set(assets) - set(permed_assets) - if invalid_assets: - msg = _("Not has host {} permission").format( - [str(a.id) for a in invalid_assets] - ) - raise ValidationError({"hosts": msg}) - - def check_permissions(self, request): - if not settings.SECURITY_COMMAND_EXECUTION: - return self.permission_denied(request, "Command execution disabled") - return super().check_permissions(request) - - def perform_create(self, serializer): - self.check_hosts(serializer) - instance = serializer.save() - instance.user = self.request.user - instance.save() - cols = self.request.query_params.get("cols", '80') - rows = self.request.query_params.get("rows", '24') - transaction.on_commit(lambda: run_command_execution.apply_async( - args=(instance.id,), kwargs={"cols": cols, "rows": rows}, - task_id=str(instance.id) - )) diff --git a/apps/ops/inventory.py b/apps/ops/inventory.py deleted file mode 100644 index d6943f5c5..000000000 --- a/apps/ops/inventory.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from django.conf import settings -from .ansible.inventory import BaseInventory - -from common.utils import get_logger - -__all__ = [ - 'JMSInventory', 'JMSCustomInventory', -] - - -logger = get_logger(__file__) - - -class JMSBaseInventory(BaseInventory): - def convert_to_ansible(self, asset, run_as_admin=False): - info = { - 'id': asset.id, - 'name': asset.name, - 'ip': asset.address, - 'port': asset.ssh_port, - 'vars': dict(), - 'groups': [], - } - if asset.domain and asset.domain.has_gateway(): - info["vars"].update(self.make_proxy_command(asset)) - if run_as_admin: - info.update(asset.get_auth_info(with_become=True)) - if asset.is_windows(): - info["vars"].update({ - "ansible_connection": "ssh", - "ansible_shell_type": settings.WINDOWS_SSH_DEFAULT_SHELL, - }) - for label in asset.labels.all(): - info["vars"].update({ - label.name: label.value - }) - if asset.domain: - info["vars"].update({ - "domain": asset.domain.name, - }) - return info - - @staticmethod - def make_proxy_command(asset): - gateway = asset.domain.random_gateway() - proxy_command_list = [ - "ssh", "-o", "Port={}".format(gateway.port), - "-o", "StrictHostKeyChecking=no", - "{}@{}".format(gateway.username, gateway.address), - "-W", "%h:%p", "-q", - ] - - if gateway.password: - proxy_command_list.insert( - 0, "sshpass -p '{}'".format(gateway.password) - ) - if gateway.private_key: - proxy_command_list.append("-i {}".format(gateway.private_key_file)) - - proxy_command = "'-o ProxyCommand={}'".format( - " ".join(proxy_command_list) - ) - return {"ansible_ssh_common_args": proxy_command} - - -class JMSInventory(JMSBaseInventory): - """ - JMS Inventory is the inventory with jumpserver assets, so you can - write you own inventory, construct you inventory, - user_info is obtained from admin_user or asset_user - """ - def __init__(self, assets, run_as_admin=False, run_as=None, become_info=None, system_user=None): - """ - :param assets: assets - :param run_as_admin: True 是否使用管理用户去执行, 每台服务器的管理用户可能不同 - :param run_as: 用户名(添加了统一的资产用户管理器之后AssetUserManager加上之后修改为username) - :param become_info: 是否become成某个用户去执行 - """ - self.assets = assets - self.using_admin = run_as_admin - self.run_as = run_as - self.system_user = system_user - self.become_info = become_info - - host_list = [] - - for asset in assets: - host = self.convert_to_ansible(asset, run_as_admin=run_as_admin) - if run_as is not None: - run_user_info = self.get_run_user_info(host) - host.update(run_user_info) - if become_info and asset.is_unixlike(): - host.update(become_info) - host_list.append(host) - - super().__init__(host_list=host_list) - - def get_run_user_info(self, host): - if not self.run_as and not self.system_user: - return {} - - asset_id = host.get('id', '') - asset = self.assets.filter(id=asset_id).first() - if not asset: - logger.error('Host not found: ', asset_id) - return {} - - if self.system_user: - self.system_user.load_asset_special_auth(asset=asset, username=self.run_as) - return self.system_user._to_secret_json() - else: - return {} - - -class JMSCustomInventory(JMSBaseInventory): - """ - JMS Custom Inventory is the inventory with jumpserver assets, - user_info is obtained from custom parameter - """ - - def __init__(self, assets, username, password=None, public_key=None, private_key=None): - """ - """ - self.assets = assets - self.username = username - self.password = password - self.public_key = public_key - self.private_key = private_key - - host_list = [] - - for asset in assets: - host = self.convert_to_ansible(asset) - run_user_info = self.get_run_user_info() - host.update(run_user_info) - host_list.append(host) - - super().__init__(host_list=host_list) - - def get_run_user_info(self): - return { - 'username': self.username, - 'password': self.password, - 'public_key': self.public_key, - 'private_key': self.private_key - } diff --git a/apps/ops/migrations/0024_auto_20221008_1514.py b/apps/ops/migrations/0024_auto_20221008_1514.py new file mode 100644 index 000000000..e208af96e --- /dev/null +++ b/apps/ops/migrations/0024_auto_20221008_1514.py @@ -0,0 +1,58 @@ +# Generated by Django 3.2.14 on 2022-10-08 07:19 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0106_auto_20220916_1556'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('ops', '0023_auto_20220929_2025'), + ] + + operations = [ + migrations.RemoveField( + model_name='adhocexecution', + name='adhoc', + ), + migrations.RemoveField( + model_name='adhocexecution', + name='task', + ), + migrations.RemoveField( + model_name='commandexecution', + name='hosts', + ), + migrations.RemoveField( + model_name='commandexecution', + name='user', + ), + migrations.AlterUniqueTogether( + name='task', + unique_together=None, + ), + migrations.RemoveField( + model_name='task', + name='latest_adhoc', + ), + migrations.RemoveField( + model_name='task', + name='latest_execution', + ), + migrations.DeleteModel( + name='AdHoc', + ), + migrations.DeleteModel( + name='AdHocExecution', + ), + migrations.DeleteModel( + name='CommandExecution', + ), + migrations.DeleteModel( + name='Task', + ), + ] diff --git a/apps/ops/migrations/0025_auto_20221008_1631.py b/apps/ops/migrations/0025_auto_20221008_1631.py new file mode 100644 index 000000000..7e814c3d1 --- /dev/null +++ b/apps/ops/migrations/0025_auto_20221008_1631.py @@ -0,0 +1,72 @@ +# Generated by Django 3.2.14 on 2022-10-08 08:31 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0106_auto_20220916_1556'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('ops', '0024_auto_20221008_1514'), + ] + + operations = [ + migrations.CreateModel( + name='AdHoc', + fields=[ + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('is_periodic', models.BooleanField(default=False)), + ('interval', models.IntegerField(blank=True, default=24, null=True, verbose_name='Cycle perform')), + ('crontab', models.CharField(blank=True, max_length=128, null=True, verbose_name='Regularly perform')), + ('account', models.CharField(default='root', max_length=128, verbose_name='Account')), + ('account_policy', models.CharField(default='root', max_length=128, verbose_name='Account policy')), + ('date_last_run', models.DateTimeField(null=True, verbose_name='Date last run')), + ('pattern', models.CharField(default='all', max_length=1024, verbose_name='Pattern')), + ('module', models.CharField(default='shell', max_length=128, verbose_name='Module')), + ('args', models.CharField(default='', max_length=1024, verbose_name='Args')), + ('assets', models.ManyToManyField(to='assets.Asset', verbose_name='Assets')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='AdHocExecution', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('status', models.CharField(default='running', max_length=16, verbose_name='Status')), + ('result', models.JSONField(blank=True, null=True, verbose_name='Result')), + ('summary', models.JSONField(default=dict, verbose_name='Summary')), + ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')), + ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')), + ('date_finished', models.DateTimeField(null=True)), + ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ('task', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='ops.adhoc', verbose_name='Adhoc')), + ], + options={ + 'verbose_name': 'AdHoc execution', + 'db_table': 'ops_adhoc_execution', + 'get_latest_by': 'date_start', + }, + ), + migrations.AddField( + model_name='adhoc', + name='last_execution', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ops.adhocexecution', verbose_name='Last execution'), + ), + migrations.AddField( + model_name='adhoc', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator'), + ), + ] diff --git a/apps/ops/mixin.py b/apps/ops/mixin.py index e64a763fc..4d2fd52a7 100644 --- a/apps/ops/mixin.py +++ b/apps/ops/mixin.py @@ -14,12 +14,10 @@ from .celery.utils import ( __all__ = [ 'PeriodTaskModelMixin', 'PeriodTaskSerializerMixin', - 'PeriodTaskFormMixin', ] class PeriodTaskModelMixin(models.Model): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField( max_length=128, unique=False, verbose_name=_("Name") ) @@ -140,42 +138,3 @@ class PeriodTaskSerializerMixin(serializers.Serializer): msg = _("Require periodic or regularly perform setting") raise serializers.ValidationError(msg) return ok - - -class PeriodTaskFormMixin(forms.Form): - is_periodic = forms.BooleanField( - initial=True, required=False, label=_('Periodic perform') - ) - crontab = forms.CharField( - max_length=128, required=False, label=_('Regularly perform'), - help_text=_("eg: Every Sunday 03:05 run <5 3 * * 0>
" - "Tips: " - "Using 5 digits linux crontab expressions " - " " - "(Online tools)
" - "Note: " - "If both Regularly perform and Cycle perform are set, " - "give priority to Regularly perform"), - ) - interval = forms.IntegerField( - required=False, initial=24, - help_text=_('Unit: hour'), label=_("Cycle perform"), - ) - - def get_initial_for_field(self, field, field_name): - """ - Return initial data for field on form. Use initial data from the form - or the field, in that order. Evaluate callable values. - """ - if field_name not in ['is_periodic', 'crontab', 'interval']: - return super().get_initial_for_field(field, field_name) - instance = getattr(self, 'instance', None) - if instance is None: - return super().get_initial_for_field(field, field_name) - init_attr_name = field_name + '_initial' - value = getattr(self, init_attr_name, None) - if value is None: - return super().get_initial_for_field(field, field_name) - return value - - diff --git a/apps/ops/models/__init__.py b/apps/ops/models/__init__.py index 0a9ed463c..fcd8bd8f7 100644 --- a/apps/ops/models/__init__.py +++ b/apps/ops/models/__init__.py @@ -3,4 +3,3 @@ from .adhoc import * from .celery import * -from .command import * diff --git a/apps/ops/models/adhoc.py b/apps/ops/models/adhoc.py index 1d2920206..565df9f3e 100644 --- a/apps/ops/models/adhoc.py +++ b/apps/ops/models/adhoc.py @@ -1,337 +1,41 @@ # ~*~ coding: utf-8 ~*~ -import uuid -import os -import time -import datetime -from celery import current_task from django.db import models -from django.conf import settings -from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from common.utils import get_logger, lazyproperty -from common.utils.translate import translate_value -from common.db.fields import ( - JsonListTextField, JsonDictCharField, EncryptJsonDictCharField, - JsonDictTextField, -) -from orgs.mixins.models import OrgModelMixin -from ..ansible import AdHocRunner, AnsibleError -from ..inventory import JMSInventory -from ..mixin import PeriodTaskModelMixin +from common.utils import get_logger +from .base import BaseAnsibleTask, BaseAnsibleExecution +from ..ansible import AdHocRunner -__all__ = ["Task", "AdHoc", "AdHocExecution"] +__all__ = ["AdHoc", "AdHocExecution"] logger = get_logger(__file__) -class Task(PeriodTaskModelMixin, OrgModelMixin): - """ - This task is different ansible task, Task like 'push system user', 'get asset info' .. - One task can have some versions of adhoc, run a task only run the latest version adhoc - """ - callback = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Callback")) # Callback must be a registered celery task - is_deleted = models.BooleanField(default=False) - comment = models.TextField(blank=True, verbose_name=_("Comment")) - date_created = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name=_("Date created")) - date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) - latest_adhoc = models.ForeignKey('ops.AdHoc', on_delete=models.SET_NULL, - null=True, related_name='task_latest') - latest_execution = models.ForeignKey('ops.AdHocExecution', on_delete=models.SET_NULL, null=True, related_name='task_latest') - total_run_amount = models.IntegerField(default=0) - success_run_amount = models.IntegerField(default=0) - _ignore_auto_created_by = True - - @property - def short_id(self): - return str(self.id).split('-')[-1] - - @lazyproperty - def versions(self): - return self.adhoc.all().count() - - @property - def is_success(self): - if self.latest_execution: - return self.latest_execution.is_success - else: - return False - - @lazyproperty - def display_name(self): - value = translate_value(self.name) - return value - - @property - def timedelta(self): - if self.latest_execution: - return self.latest_execution.timedelta - else: - return 0 - - @property - def date_start(self): - if self.latest_execution: - return self.latest_execution.date_start - else: - return None - - @property - def assets_amount(self): - if self.latest_execution: - return self.latest_execution.hosts_amount - return 0 - - def get_latest_adhoc(self): - if self.latest_adhoc: - return self.latest_adhoc - try: - adhoc = self.adhoc.all().latest() - self.latest_adhoc = adhoc - self.save() - return adhoc - except AdHoc.DoesNotExist: - return None - - @property - def history_summary(self): - total = self.total_run_amount - success = self.success_run_amount - failed = total - success - return {'total': total, 'success': success, 'failed': failed} - - def get_run_execution(self): - return self.execution.all() - - def run(self): - latest_adhoc = self.get_latest_adhoc() - if latest_adhoc: - return latest_adhoc.run() - else: - return {'error': 'No adhoc'} - - @property - def period_key(self): - return self.__str__() - - def get_register_task(self): - from ..tasks import run_ansible_task - name = self.__str__() - task = run_ansible_task.name - args = (str(self.id),) - kwargs = {"callback": self.callback} - return name, task, args, kwargs +class AdHoc(BaseAnsibleTask): + pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all') + module = models.CharField(max_length=128, default='shell', verbose_name=_('Module')) + args = models.CharField(max_length=1024, default='', verbose_name=_('Args')) + last_execution = models.ForeignKey('AdHocExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) def __str__(self): - return self.name + '@' + str(self.org_id) - - class Meta: - db_table = 'ops_task' - unique_together = ('name', 'org_id') - ordering = ('-date_updated',) - verbose_name = _("Task") - get_latest_by = 'date_created' - permissions = [ - ('view_taskmonitor', _('Can view task monitor')) - ] + return "{}: {}".format(self.module, self.args) -class AdHoc(OrgModelMixin): - """ - task: A task reference - _tasks: [{'name': 'task_name', 'action': {'module': '', 'args': ''}, 'other..': ''}, ] - _options: ansible options, more see ops.ansible.runner.Options - run_as_admin: if true, then need get every host admin user run it, because every host may be have different admin user, so we choise host level - run_as: username(Add the uniform AssetUserManager and change it to username) - _become: May be using become [sudo, su] options. {method: "sudo", user: "user", pass: "pass"] - pattern: Even if we set _hosts, We only use that to make inventory, We also can set `patter` to run task on match hosts - """ - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - task = models.ForeignKey(Task, related_name='adhoc', on_delete=models.CASCADE) - tasks = JsonListTextField(verbose_name=_('Tasks')) - pattern = models.CharField(max_length=64, default='{}', verbose_name=_('Pattern')) - options = JsonDictCharField(max_length=1024, default='', verbose_name=_('Options')) - hosts = models.ManyToManyField('assets.Asset', verbose_name=_("Host")) - run_as_admin = models.BooleanField(default=False, verbose_name=_('Run as admin')) - run_as = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Username')) - become = EncryptJsonDictCharField(max_length=1024, default='', blank=True, null=True, verbose_name=_("Become")) - created_by = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Create by')) - date_created = models.DateTimeField(auto_now_add=True, db_index=True) - - @lazyproperty - def run_times(self): - return self.execution.count() - - @property - def inventory(self): - if self.become: - become_info = { - 'become': { - self.become - } - } - else: - become_info = None - - inventory = JMSInventory( - self.hosts.all(), run_as_admin=self.run_as_admin, - run_as=self.run_as, become_info=become_info, system_user=self.run_system_user - ) - return inventory - - @property - def become_display(self): - if self.become: - return self.become.get("user", "") - return "" - - def run(self): - try: - celery_task_id = current_task.request.id - except AttributeError: - celery_task_id = None - - execution = AdHocExecution( - celery_task_id=celery_task_id, - adhoc=self, task=self.task, - task_display=str(self.task)[:128], - date_start=timezone.now(), - hosts_amount=self.hosts.count(), - ) - execution.save() - return execution.start() - - @property - def short_id(self): - return str(self.id).split('-')[-1] - - @property - def latest_execution(self): - try: - return self.execution.all().latest() - except AdHocExecution.DoesNotExist: - return None - - def save(self, **kwargs): - instance = super().save(**kwargs) - self.task.latest_adhoc = instance - self.task.save() - return instance - - def __str__(self): - return "{} of {}".format(self.task.name, self.short_id) - - def same_with(self, other): - if not isinstance(other, self.__class__): - return False - fields_check = [] - for field in self.__class__._meta.fields: - if field.name not in ['id', 'date_created']: - fields_check.append(field) - for field in fields_check: - if getattr(self, field.name) != getattr(other, field.name): - return False - return True - - class Meta: - db_table = "ops_adhoc" - get_latest_by = 'date_created' - verbose_name = _('AdHoc') - - -class AdHocExecution(OrgModelMixin): +class AdHocExecution(BaseAnsibleExecution): """ AdHoc running history. """ - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - task = models.ForeignKey(Task, related_name='execution', on_delete=models.SET_NULL, null=True) - task_display = models.CharField(max_length=128, blank=True, default='', verbose_name=_("Task display")) - celery_task_id = models.UUIDField(default=None, null=True) - hosts_amount = models.IntegerField(default=0, verbose_name=_("Host amount")) - adhoc = models.ForeignKey(AdHoc, related_name='execution', on_delete=models.SET_NULL, null=True) - date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time')) - date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time')) - timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True) - is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) - is_success = models.BooleanField(default=False, verbose_name=_('Is success')) - result = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc raw result')) - summary = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc result summary')) + task = models.ForeignKey('AdHoc', verbose_name=_("Adhoc"), related_name='executions', on_delete=models.CASCADE) - @property - def short_id(self): - return str(self.id).split('-')[-1] - - @property - def adhoc_short_id(self): - return str(self.adhoc_id).split('-')[-1] - - @property - def log_path(self): - dt = datetime.datetime.now().strftime('%Y-%m-%d') - log_dir = os.path.join(settings.PROJECT_DIR, 'data', 'ansible', dt) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - return os.path.join(log_dir, str(self.id) + '.log') - - def start_runner(self): - runner = AdHocRunner(self.adhoc.inventory, options=self.adhoc.options) - try: - result = runner.run( - self.adhoc.tasks, - self.adhoc.pattern, - self.task.name, - execution_id=self.id - ) - return result.results_raw, result.results_summary - except AnsibleError as e: - logger.warn("Failed run adhoc {}, {}".format(self.task.name, e)) - return {}, {} - - def start(self): - self.task.latest_execution = self - self.task.save() - time_start = time.time() - summary = {} - raw = '' - - try: - raw, summary = self.start_runner() - except Exception as e: - logger.error(e, exc_info=True) - raw = {"dark": {"all": str(e)}, "contacted": []} - finally: - self.clean_up(summary, time_start) - return raw, summary - - def clean_up(self, summary, time_start): - is_success = summary.get('success', False) - task = Task.objects.get(id=self.task_id) - task.total_run_amount = models.F('total_run_amount') + 1 - if is_success: - task.success_run_amount = models.F('success_run_amount') + 1 - task.save() - AdHocExecution.objects.filter(id=self.id).update( - is_finished=True, - is_success=is_success, - date_finished=timezone.now(), - timedelta=time.time() - time_start, - summary=summary + def get_runner(self): + return AdHocRunner( + self.task.inventory, self.task.module, self.task.args, + pattern=self.task.pattern, project_dir=self.private_dir ) - @property - def success_hosts(self): - return self.summary.get('contacted', []) - - @property - def failed_hosts(self): - return self.summary.get('dark', {}) - - def __str__(self): - return self.short_id - class Meta: db_table = "ops_adhoc_execution" get_latest_by = 'date_start' diff --git a/apps/ops/models/base.py b/apps/ops/models/base.py new file mode 100644 index 000000000..2992173c3 --- /dev/null +++ b/apps/ops/models/base.py @@ -0,0 +1,106 @@ +import os.path +import uuid + +from django.db import models +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from django.conf import settings + +from orgs.mixins.models import JMSOrgBaseModel +from ..ansible.inventory import JMSInventory +from ..mixin import PeriodTaskModelMixin + + +class BaseAnsibleTask(PeriodTaskModelMixin, JMSOrgBaseModel): + owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) + account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) + account_policy = models.CharField(max_length=128, default='root', verbose_name=_('Account policy')) + last_execution = models.ForeignKey('BaseAnsibleExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True) + date_last_run = models.DateTimeField(null=True, verbose_name=_('Date last run')) + + class Meta: + abstract = True + + @property + def inventory(self): + inv = JMSInventory(self.assets.all(), self.account, self.account_policy) + return inv.generate() + + def get_register_task(self): + raise NotImplemented + + def to_json(self): + raise NotImplemented + + +class BaseAnsibleExecution(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4) + status = models.CharField(max_length=16, verbose_name=_('Status'), default='running') + task = models.ForeignKey(BaseAnsibleTask, on_delete=models.CASCADE, null=True) + result = models.JSONField(blank=True, null=True, verbose_name=_('Result')) + summary = models.JSONField(default=dict, verbose_name=_('Summary')) + creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True) + date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) + date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True) + date_finished = models.DateTimeField(null=True) + + class Meta: + abstract = True + ordering = ["-date_start"] + + def __str__(self): + return str(self.id) + + def private_dir(self): + uniq = self.date_created.strftime('%Y%m%d_%H%M%S') + '_' + self.short_id + return os.path.join(settings.ANSIBLE_DIR, self.task.name, uniq) + + def get_runner(self): + raise NotImplemented + + def update_task(self): + self.task.last_execution = self + self.task.date_last_run = timezone.now() + self.task.save(update_fields=['last_execution', 'date_last_run']) + + def start(self, **kwargs): + runner = self.get_runner() + try: + cb = runner.run(**kwargs) + self.status = cb.status + self.summary = cb.summary + self.result = cb.result + self.date_finished = timezone.now() + except Exception as e: + self.status = 'failed' + self.summary = {'error': str(e)} + finally: + self.save() + self.update_task() + + @property + def is_finished(self): + return self.status in ['succeeded', 'failed'] + + @property + def is_success(self): + return self.status == 'succeeded' + + @property + def time_cost(self): + if self.date_finished and self.date_start: + return (self.date_finished - self.date_start).total_seconds() + return None + + @property + def short_id(self): + return str(self.id).split('-')[-1] + + @property + def timedelta(self): + if self.date_start and self.date_finished: + return self.date_finished - self.date_start + return None + + diff --git a/apps/ops/models/command.py b/apps/ops/models/command.py deleted file mode 100644 index cb6023564..000000000 --- a/apps/ops/models/command.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -# -import uuid -import json - -from celery.exceptions import SoftTimeLimitExceeded -from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ -from django.utils.translation import ugettext -from django.db import models - -from terminal.notifications import CommandExecutionAlert -from assets.models import Asset -from common.utils import lazyproperty -from orgs.models import Organization -from orgs.mixins.models import OrgModelMixin -from orgs.utils import tmp_to_org -from ..ansible.runner import CommandRunner -from ..inventory import JMSInventory - - -class CommandExecution(OrgModelMixin): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - hosts = models.ManyToManyField('assets.Asset') - account = models.CharField(max_length=128, default='', verbose_name=_('account')) - command = models.TextField(verbose_name=_("Command")) - _result = models.TextField(blank=True, null=True, verbose_name=_('Result')) - user = models.ForeignKey('users.User', on_delete=models.CASCADE, null=True) - is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) - date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created')) - date_start = models.DateTimeField(null=True, verbose_name=_('Date start')) - date_finished = models.DateTimeField(null=True, verbose_name=_('Date finished')) - - def __str__(self): - return self.command[:10] - - def save(self, *args, **kwargs): - with tmp_to_org(self.run_as.org_id): - super().save(*args, **kwargs) - - @property - def inventory(self): - if self.run_as.username_same_with_user: - username = self.user.username - else: - username = self.run_as.username - inv = JMSInventory(self.allow_assets, run_as=username, system_user=self.run_as) - return inv - - @lazyproperty - def user_display(self): - return str(self.user) - - @lazyproperty - def hosts_display(self): - return ','.join(self.hosts.all().values_list('name', flat=True)) - - @property - def result(self): - if self._result: - return json.loads(self._result) - else: - return {} - - @result.setter - def result(self, item): - self._result = json.dumps(item) - - @property - def is_success(self): - if 'error' in self.result: - return False - return True - - def get_hosts_names(self): - return ','.join(self.hosts.all().values_list('name', flat=True)) - - def cmd_filter_rules(self, asset_id=None): - from assets.models import CommandFilterRule - user_id = self.user.id - system_user_id = self.run_as.id - rules = CommandFilterRule.get_queryset( - user_id=user_id, - system_user_id=system_user_id, - asset_id=asset_id, - ) - return rules - - def is_command_can_run(self, command, asset_id=None): - for rule in self.cmd_filter_rules(asset_id=asset_id): - action, matched_cmd = rule.match(command) - if action == rule.ActionChoices.allow: - return True, None - elif action == rule.ActionChoices.deny: - return False, matched_cmd - return True, None - - @property - def allow_assets(self): - allow_asset_ids = [] - for asset in self.hosts.all(): - ok, __ = self.is_command_can_run(self.command, asset_id=asset.id) - if ok: - allow_asset_ids.append(asset.id) - allow_assets = Asset.objects.filter(id__in=allow_asset_ids) - return allow_assets - - def run(self): - print('-' * 10 + ' ' + ugettext('Task start') + ' ' + '-' * 10) - org = Organization.get_instance(self.run_as.org_id) - org.change_to() - self.date_start = timezone.now() - ok, msg = self.is_command_can_run(self.command) - if ok: - allow_assets = self.allow_assets - deny_assets = set(list(self.hosts.all())) - set(list(allow_assets)) - for asset in deny_assets: - print(f'资产{asset}: 命令{self.command}不允许执行') - if not allow_assets: - self.result = { - "error": 'There are currently no assets that can be executed' - } - self.save() - return self.result - runner = CommandRunner(self.inventory) - try: - host = allow_assets.first() - if host and host.is_windows(): - shell = 'win_shell' - elif host and host.is_unixlike(): - shell = 'shell' - else: - shell = 'raw' - result = runner.execute(self.command, 'all', module=shell) - self.result = result.results_command - except SoftTimeLimitExceeded as e: - print("Run timeout than 60s") - self.result = {"error": str(e)} - except Exception as e: - print("Error occur: {}".format(e)) - self.result = {"error": str(e)} - else: - msg = _("Command `{}` is forbidden ........").format(self.command) - print('\033[31m' + msg + '\033[0m') - CommandExecutionAlert({ - 'input': self.command, - 'assets': self.hosts.all(), - 'user': str(self.user), - 'risk_level': 5, - }).publish_async() - self.result = {"error": msg} - self.org_id = self.run_as.org_id - self.is_finished = True - self.date_finished = timezone.now() - self.save() - print('-' * 10 + ' ' + ugettext('Task end') + ' ' + '-' * 10) - return self.result - - class Meta: - verbose_name = _("Command execution") diff --git a/apps/ops/models/playbook.py b/apps/ops/models/playbook.py index aaec7a4ef..aec59bfb0 100644 --- a/apps/ops/models/playbook.py +++ b/apps/ops/models/playbook.py @@ -2,15 +2,34 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from orgs.mixins.models import JMSOrgBaseModel -from ..mixin import PeriodTaskModelMixin +from .base import BaseAnsibleExecution, BaseAnsibleTask -class PlaybookTask(PeriodTaskModelMixin, JMSOrgBaseModel): - assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets")) - account = models.CharField(max_length=128, default='root', verbose_name=_('Account')) - playbook = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) - owner = models.CharField(max_length=1024, verbose_name=_("Owner")) +class PlaybookTemplate(JMSOrgBaseModel): + name = models.CharField(max_length=128, verbose_name=_("Name")) + path = models.FilePathField(verbose_name=_("Path")) + comment = models.TextField(verbose_name=_("Comment"), blank=True) + + def __str__(self): + return self.name + + class Meta: + ordering = ['name'] + verbose_name = _("Playbook template") + unique_together = [('org_id', 'name')] + + +class Playbook(BaseAnsibleTask): + path = models.FilePathField(max_length=1024, verbose_name=_("Playbook")) + owner = models.ForeignKey('users.User', verbose_name=_("Owner"), on_delete=models.SET_NULL, null=True) comment = models.TextField(blank=True, verbose_name=_("Comment")) + template = models.ForeignKey('PlaybookTemplate', verbose_name=_("Template"), on_delete=models.SET_NULL, null=True) + last_execution = models.ForeignKey('PlaybookExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True) def get_register_task(self): pass + + +class PlaybookExecution(BaseAnsibleExecution): + task = models.ForeignKey('Playbook', verbose_name=_("Task"), on_delete=models.CASCADE) + path = models.FilePathField(max_length=1024, verbose_name=_("Run dir")) diff --git a/apps/ops/serializers/adhoc.py b/apps/ops/serializers/adhoc.py index 50b1faeba..b6522b85f 100644 --- a/apps/ops/serializers/adhoc.py +++ b/apps/ops/serializers/adhoc.py @@ -3,8 +3,7 @@ from __future__ import unicode_literals from rest_framework import serializers from django.shortcuts import reverse -from orgs.mixins.serializers import BulkOrgResourceModelSerializer -from ..models import Task, AdHoc, AdHocExecution, CommandExecution +from ..models import AdHoc, AdHocExecution class AdHocExecutionSerializer(serializers.ModelSerializer): @@ -50,36 +49,6 @@ class AdHocExecutionExcludeResultSerializer(AdHocExecutionSerializer): ] -class TaskSerializer(BulkOrgResourceModelSerializer): - summary = serializers.ReadOnlyField(source='history_summary') - latest_execution = AdHocExecutionExcludeResultSerializer(read_only=True) - - class Meta: - model = Task - fields_mini = ['id', 'name', 'display_name'] - fields_small = fields_mini + [ - 'interval', 'crontab', - 'is_periodic', 'is_deleted', - 'date_created', 'date_updated', - 'comment', - ] - fields_fk = ['latest_execution'] - fields_custom = ['summary'] - fields = fields_small + fields_fk + fields_custom - read_only_fields = [ - 'is_deleted', 'date_created', 'date_updated', - 'latest_adhoc', 'latest_execution', 'total_run_amount', - 'success_run_amount', 'summary', - ] - - -class TaskDetailSerializer(TaskSerializer): - contents = serializers.ListField(source='latest_adhoc.tasks') - - class Meta(TaskSerializer.Meta): - fields = TaskSerializer.Meta.fields + ['contents'] - - class AdHocSerializer(serializers.ModelSerializer): become_display = serializers.ReadOnlyField() tasks = serializers.ListField() @@ -127,26 +96,26 @@ class AdHocDetailSerializer(AdHocSerializer): ] -class CommandExecutionSerializer(serializers.ModelSerializer): - result = serializers.JSONField(read_only=True) - log_url = serializers.SerializerMethodField() - - class Meta: - model = CommandExecution - fields_mini = ['id'] - fields_small = fields_mini + [ - 'command', 'result', 'log_url', - 'is_finished', 'date_created', 'date_finished' - ] - fields_m2m = ['hosts'] - fields = fields_small + fields_m2m - read_only_fields = [ - 'result', 'is_finished', 'log_url', 'date_created', - 'date_finished' - ] - ref_name = 'OpsCommandExecution' - - @staticmethod - def get_log_url(obj): - return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id}) +# class CommandExecutionSerializer(serializers.ModelSerializer): +# result = serializers.JSONField(read_only=True) +# log_url = serializers.SerializerMethodField() +# +# class Meta: +# model = CommandExecution +# fields_mini = ['id'] +# fields_small = fields_mini + [ +# 'command', 'result', 'log_url', +# 'is_finished', 'date_created', 'date_finished' +# ] +# fields_m2m = ['hosts'] +# fields = fields_small + fields_m2m +# read_only_fields = [ +# 'result', 'is_finished', 'log_url', 'date_created', +# 'date_finished' +# ] +# ref_name = 'OpsCommandExecution' +# +# @staticmethod +# def get_log_url(obj): +# return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id}) diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index cb21b5c3d..0ef430d7a 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -20,7 +20,7 @@ from .celery.utils import ( create_or_update_celery_periodic_tasks, get_celery_periodic_task, disable_celery_periodic_task, delete_celery_periodic_task ) -from .models import Task, CommandExecution, CeleryTask +from .models import CommandExecution, CeleryTask from .notifications import ServerPerformanceCheckUtil logger = get_logger(__file__) diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index a5838073f..49038b9b1 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -12,14 +12,11 @@ app_name = "ops" router = DefaultRouter() bulk_router = BulkRouter() -bulk_router.register(r'tasks', api.TaskViewSet, 'task') router.register(r'adhoc', api.AdHocViewSet, 'adhoc') -router.register(r'adhoc-executions', api.AdHocRunHistoryViewSet, 'execution') -router.register(r'command-executions', api.CommandExecutionViewSet, 'command-execution') +router.register(r'adhoc-executions', api.AdHocExecutionViewSet, 'execution') router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task') urlpatterns = [ - path('tasks//run/', api.TaskRun.as_view(), name='task-run'), path('celery/task//log/', api.CeleryTaskLogApi.as_view(), name='celery-task-log'), path('celery/task//result/', api.CeleryResultApi.as_view(), name='celery-result'),