feat: 执行adhoc和playbook

apps/assets/migrations/0111_alter_automationexecution_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.14 on 2022-11-11 11:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0110_changesecretrecord_asset'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='automationexecution',
+            name='status',
+            field=models.CharField(default='pending', max_length=16, verbose_name='Status'),
+        ),
+    ]

apps/audits/migrations/0015_auto_20221111_1919.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.14 on 2022-11-11 11:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0014_auto_20220505_1902'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ftplog',
+            name='operate',
+            field=models.CharField(choices=[('mkdir', 'Mkdir'), ('rmdir', 'Rmdir'), ('delete', 'Delete'), ('upload', 'Upload'), ('rename', 'Rename'), ('symlink', 'Symlink'), ('download', 'Download')], max_length=16, verbose_name='Operate'),
+        ),
+        migrations.AlterField(
+            model_name='operatelog',
+            name='action',
+            field=models.CharField(choices=[('view', 'View'), ('update', 'Update'), ('delete', 'Delete'), ('create', 'Create')], max_length=16, verbose_name='Action'),
+        ),
+        migrations.AlterField(
+            model_name='userloginlog',
+            name='status',
+            field=models.BooleanField(choices=[(1, 'Success'), (0, 'Failed')], default=1, verbose_name='Status'),
+        ),
+    ]

apps/ops/ansible/utils.py

@@ -3,4 +3,4 @@ from django.conf import settings
 
 def get_ansible_task_log_path(task_id):
     from ops.utils import get_task_log_path
-    return get_task_log_path(settings.ANSIBLE_LOG_DIR, task_id, level=3)
+    return get_task_log_path(settings.CELERY_LOG_DIR, task_id, level=2)

apps/ops/api/__init__.py

@@ -2,3 +2,5 @@
 #
 from .adhoc import *
 from .celery import *
+from .job import *
+from .playbook import *

apps/ops/api/adhoc.py

@@ -1,52 +1,22 @@
 # -*- coding: utf-8 -*-
 #
 
-from django.shortcuts import get_object_or_404
+from rest_framework import viewsets
-from rest_framework import viewsets, generics
+from ..models import AdHoc
-from rest_framework.views import Response
-
-from common.drf.serializers import CeleryTaskExecutionSerializer
-from ..models import AdHoc, AdHocExecution
 from ..serializers import (
-    AdHocSerializer,
+    AdHocSerializer, AdhocListSerializer,
-    AdHocExecutionSerializer,
-    AdHocDetailSerializer,
 )
 
 __all__ = [
-    'AdHocViewSet', 'AdHocExecutionViewSet'
+    'AdHocViewSet'
 ]
 
 
 class AdHocViewSet(viewsets.ModelViewSet):
     queryset = AdHoc.objects.all()
-    serializer_class = AdHocSerializer
 
     def get_serializer_class(self):
-        if self.action == 'retrieve':
+        if self.action != 'list':
-            return AdHocDetailSerializer
+            return AdhocListSerializer
-        return super().get_serializer_class()
+        return AdHocSerializer
-
-
-class AdHocExecutionViewSet(viewsets.ModelViewSet):
-    queryset = AdHocExecution.objects.all()
-    serializer_class = AdHocExecutionSerializer
-
-    def get_queryset(self):
-        task_id = self.request.query_params.get('task')
-        adhoc_id = self.request.query_params.get('adhoc')
-
-        if task_id:
-            task = get_object_or_404(AdHoc, id=task_id)
-            adhocs = task.adhoc.all()
-            self.queryset = self.queryset.filter(adhoc__in=adhocs)
-
-        if adhoc_id:
-            adhoc = get_object_or_404(AdHoc, id=adhoc_id)
-            self.queryset = self.queryset.filter(adhoc=adhoc)
-        return self.queryset
-
-
-
-
 

apps/ops/api/celery.py

@@ -98,6 +98,11 @@ class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet):
         return queryset
 
 
+class CelerySummaryAPIView(generics.RetrieveAPIView):
+    def get(self, request, *args, **kwargs):
+        pass
+
+
 class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
     queryset = CeleryTask.objects.all()
     serializer_class = CeleryTaskSerializer
@@ -107,11 +112,11 @@ class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
 class CeleryTaskExecutionViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
     serializer_class = CeleryTaskExecutionSerializer
     http_method_names = ('get', 'head', 'options',)
+    queryset = CeleryTaskExecution.objects.all()
 
     def get_queryset(self):
-        task_id = self.kwargs.get("task_pk")
+        task_id = self.request.query_params.get('task_id')
         if task_id:
-            task = CeleryTask.objects.get(pk=task_id)
+            task = get_object_or_404(CeleryTask, id=task_id)
-            return CeleryTaskExecution.objects.filter(name=task.name)
+            self.queryset = self.queryset.filter(name=task.name)
-        else:
+        return self.queryset
-            return CeleryTaskExecution.objects.none()

apps/ops/api/job.py

@@ -0,0 +1,41 @@
+from django.shortcuts import get_object_or_404
+from rest_framework import viewsets
+
+from ops.models import Job, JobExecution
+from ops.serializers.job import JobSerializer, JobExecutionSerializer
+
+__all__ = ['JobViewSet', 'JobExecutionViewSet']
+
+from ops.tasks import run_ops_job, run_ops_job_executions
+
+
+class JobViewSet(viewsets.ModelViewSet):
+    serializer_class = JobSerializer
+    queryset = Job.objects.all()
+
+    def get_queryset(self):
+        return self.queryset.filter(instant=False)
+
+    def perform_create(self, serializer):
+        instance = serializer.save()
+        if instance.instant:
+            run_ops_job.delay(instance.id)
+
+
+class JobExecutionViewSet(viewsets.ModelViewSet):
+    serializer_class = JobExecutionSerializer
+    queryset = JobExecution.objects.all()
+    http_method_names = ('get', 'post', 'head', 'options',)
+
+    def perform_create(self, serializer):
+        instance = serializer.save()
+        run_ops_job_executions.delay(instance.id)
+
+    def get_queryset(self):
+        job_id = self.request.query_params.get('job_id')
+        job_type = self.request.query_params.get('type')
+        if job_id:
+            self.queryset = self.queryset.filter(job_id=job_id)
+        if job_type:
+            self.queryset = self.queryset.filter(job__type=job_type)
+        return self.queryset

apps/ops/api/playbook.py

@@ -0,0 +1,28 @@
+import os
+import zipfile
+
+from django.conf import settings
+from rest_framework import viewsets
+from ..models import Playbook
+from ..serializers.playbook import PlaybookSerializer
+
+__all__ = ["PlaybookViewSet"]
+
+
+def unzip_playbook(src, dist):
+    fz = zipfile.ZipFile(src, 'r')
+    for file in fz.namelist():
+        fz.extract(file, dist)
+
+
+class PlaybookViewSet(viewsets.ModelViewSet):
+    queryset = Playbook.objects.all()
+    serializer_class = PlaybookSerializer
+
+    def perform_create(self, serializer):
+        instance = serializer.save()
+        src_path = os.path.join(settings.MEDIA_ROOT, instance.path.name)
+        dest_path = os.path.join(settings.DATA_DIR, "ops", "playbook", instance.id.__str__())
+        if os.path.exists(dest_path):
+            os.makedirs(dest_path)
+        unzip_playbook(src_path, dest_path)

apps/ops/migrations/0029_auto_20221111_1919.py

@@ -0,0 +1,171 @@
+# Generated by Django 3.2.14 on 2022-11-11 11:19
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('assets', '0111_alter_automationexecution_status'),
+        ('ops', '0028_celerytask_last_published_time'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Job',
+            fields=[
+                ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
+                ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')),
+                ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=128, null=True, verbose_name='Name')),
+                ('instant', models.BooleanField(default=False)),
+                ('args', models.CharField(blank=True, default='', max_length=1024, null=True, verbose_name='Args')),
+                ('module', models.CharField(choices=[('shell', 'Shell'), ('win_shell', 'Powershell')], default='shell', max_length=128, null=True, verbose_name='Module')),
+                ('type', models.CharField(choices=[('adhoc', 'Adhoc'), ('playbook', 'Playbook')], default='adhoc', max_length=128, verbose_name='Type')),
+                ('runas', models.CharField(default='root', max_length=128, verbose_name='Runas')),
+                ('runas_policy', models.CharField(choices=[('privileged_only', 'Privileged Only'), ('privileged_first', 'Privileged First'), ('skip', 'Skip')], default='skip', max_length=128, verbose_name='Runas policy')),
+                ('assets', models.ManyToManyField(to='assets.Asset', verbose_name='Assets')),
+                ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='JobExecution',
+            fields=[
+                ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
+                ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('task_id', models.UUIDField(null=True)),
+                ('status', models.CharField(default='running', max_length=16, verbose_name='Status')),
+                ('result', models.JSONField(blank=True, null=True, verbose_name='Result')),
+                ('summary', models.JSONField(default=dict, verbose_name='Summary')),
+                ('date_created', models.DateTimeField(auto_now_add=True, verbose_name='Date created')),
+                ('date_start', models.DateTimeField(db_index=True, null=True, verbose_name='Date start')),
+                ('date_finished', models.DateTimeField(null=True, verbose_name='Date finished')),
+                ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='Creator')),
+                ('job', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='ops.job')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name='playbooktemplate',
+            unique_together=None,
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='account',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='account_policy',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='assets',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='crontab',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='date_last_run',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='interval',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='is_periodic',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='last_execution',
+        ),
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='org_id',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='account',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='account_policy',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='assets',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='comment',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='crontab',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='date_last_run',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='interval',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='is_periodic',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='last_execution',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='org_id',
+        ),
+        migrations.RemoveField(
+            model_name='playbook',
+            name='template',
+        ),
+        migrations.AlterField(
+            model_name='adhoc',
+            name='module',
+            field=models.CharField(choices=[('shell', 'Shell'), ('win_shell', 'Powershell')], default='shell', max_length=128, verbose_name='Module'),
+        ),
+        migrations.AlterField(
+            model_name='playbook',
+            name='name',
+            field=models.CharField(max_length=128, null=True, verbose_name='Name'),
+        ),
+        migrations.AlterField(
+            model_name='playbook',
+            name='path',
+            field=models.FileField(upload_to='playbooks/'),
+        ),
+        migrations.DeleteModel(
+            name='PlaybookExecution',
+        ),
+        migrations.DeleteModel(
+            name='PlaybookTemplate',
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='playbook',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='ops.playbook', verbose_name='Playbook'),
+        ),
+    ]

apps/ops/models/__init__.py

@@ -4,3 +4,4 @@
 from .adhoc import *
 from .celery import *
 from .playbook import *
+from .job import *

apps/ops/models/adhoc.py

@@ -1,29 +1,43 @@
 # ~*~ coding: utf-8 ~*~
 import os.path
+import uuid
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
+from common.db.models import BaseCreateUpdateModel
 from common.utils import get_logger
 from .base import BaseAnsibleJob, BaseAnsibleExecution
 from ..ansible import AdHocRunner
 
 __all__ = ["AdHoc", "AdHocExecution"]
 
-
 logger = get_logger(__file__)
 
 
-class AdHoc(BaseAnsibleJob):
+class AdHoc(BaseCreateUpdateModel):
-    pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all')
+    class Modules(models.TextChoices):
-    module = models.CharField(max_length=128, default='shell', verbose_name=_('Module'))
+        shell = 'shell', _('Shell')
-    args = models.CharField(max_length=1024, default='', verbose_name=_('Args'))
+        winshell = 'win_shell', _('Powershell')
-    last_execution = models.ForeignKey('AdHocExecution', verbose_name=_("Last execution"),
-                                       on_delete=models.SET_NULL, null=True, blank=True)
 
-    def get_register_task(self):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-        from ops.tasks import run_adhoc
+    name = models.CharField(max_length=128, verbose_name=_('Name'))
-        return "run_adhoc_{}".format(self.id), run_adhoc, (str(self.id),), {}
+    pattern = models.CharField(max_length=1024, verbose_name=_("Pattern"), default='all')
+    module = models.CharField(max_length=128, choices=Modules.choices, default=Modules.shell,
+                              verbose_name=_('Module'))
+    args = models.CharField(max_length=1024, default='', verbose_name=_('Args'))
+    owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True)
+
+    @property
+    def row_count(self):
+        if len(self.args) == 0:
+            return 0
+        count = str(self.args).count('\n')
+        return count + 1
+
+    @property
+    def size(self):
+        return len(self.args)
 
     def __str__(self):
         return "{}: {}".format(self.module, self.args)

apps/ops/models/base.py

@@ -17,7 +17,8 @@ class BaseAnsibleJob(PeriodTaskModelMixin, JMSOrgBaseModel):
     assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets"))
     account = models.CharField(max_length=128, default='root', verbose_name=_('Account'))
     account_policy = models.CharField(max_length=128, default='root', verbose_name=_('Account policy'))
-    last_execution = models.ForeignKey('BaseAnsibleExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True)
+    last_execution = models.ForeignKey('BaseAnsibleExecution', verbose_name=_("Last execution"),
+                                       on_delete=models.SET_NULL, null=True)
     date_last_run = models.DateTimeField(null=True, verbose_name=_('Date last run'))
 
     class Meta:
@@ -118,12 +119,6 @@ class BaseAnsibleExecution(models.Model):
     def is_success(self):
         return self.status == 'success'
 
-    @property
-    def time_cost(self):
-        if self.date_finished and self.date_start:
-            return (self.date_finished - self.date_start).total_seconds()
-        return None
-
     @property
     def short_id(self):
         return str(self.id).split('-')[-1]
@@ -134,4 +129,8 @@ class BaseAnsibleExecution(models.Model):
             return self.date_finished - self.date_start
         return None
 
-
+    @property
+    def time_cost(self):
+        if self.date_finished and self.date_start:
+            return (self.date_finished - self.date_start).total_seconds()
+        return None

apps/ops/models/celery.py

@@ -19,7 +19,7 @@ class CeleryTask(models.Model):
     def meta(self):
         task = app.tasks.get(self.name, None)
         return {
-            "verbose_name": getattr(task, 'verbose_name', None),
+            "display_name": getattr(task, 'verbose_name', None),
             "comment": getattr(task, 'comment', None),
             "queue": getattr(task, 'queue', 'default')
         }

apps/ops/models/job.py

@@ -0,0 +1,154 @@
+import os
+import uuid
+import logging
+
+from django.conf import settings
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
+from celery import current_task
+
+from common.db.models import BaseCreateUpdateModel
+
+__all__ = ["Job", "JobExecution"]
+
+from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner
+
+
+class Job(BaseCreateUpdateModel):
+    class Types(models.TextChoices):
+        adhoc = 'adhoc', _('Adhoc')
+        playbook = 'playbook', _('Playbook')
+
+    class RunasPolicies(models.TextChoices):
+        privileged_only = 'privileged_only', _('Privileged Only')
+        privileged_first = 'privileged_first', _('Privileged First')
+        skip = 'skip', _('Skip')
+
+    class Modules(models.TextChoices):
+        shell = 'shell', _('Shell')
+        winshell = 'win_shell', _('Powershell')
+
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    name = models.CharField(max_length=128, null=True, verbose_name=_('Name'))
+    instant = models.BooleanField(default=False)
+    args = models.CharField(max_length=1024, default='', verbose_name=_('Args'), null=True, blank=True)
+    module = models.CharField(max_length=128, choices=Modules.choices, default=Modules.shell,
+                              verbose_name=_('Module'), null=True)
+    playbook = models.ForeignKey('ops.Playbook', verbose_name=_("Playbook"), null=True, on_delete=models.SET_NULL)
+    type = models.CharField(max_length=128, choices=Types.choices, default=Types.adhoc, verbose_name=_("Type"))
+    owner = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True)
+    assets = models.ManyToManyField('assets.Asset', verbose_name=_("Assets"))
+    runas = models.CharField(max_length=128, default='root', verbose_name=_('Runas'))
+    runas_policy = models.CharField(max_length=128, choices=RunasPolicies.choices, default=RunasPolicies.skip,
+                                    verbose_name=_('Runas policy'))
+
+    @property
+    def inventory(self):
+        return JMSInventory(self.assets.all(), self.runas_policy, self.runas)
+
+    def create_execution(self):
+        return self.executions.create()
+
+
+class JobExecution(BaseCreateUpdateModel):
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
+    task_id = models.UUIDField(null=True)
+    status = models.CharField(max_length=16, verbose_name=_('Status'), default='running')
+    job = models.ForeignKey(Job, on_delete=models.CASCADE, related_name='executions', null=True)
+    result = models.JSONField(blank=True, null=True, verbose_name=_('Result'))
+    summary = models.JSONField(default=dict, verbose_name=_('Summary'))
+    creator = models.ForeignKey('users.User', verbose_name=_("Creator"), on_delete=models.SET_NULL, null=True)
+    date_created = models.DateTimeField(auto_now_add=True, verbose_name=_('Date created'))
+    date_start = models.DateTimeField(null=True, verbose_name=_('Date start'), db_index=True)
+    date_finished = models.DateTimeField(null=True, verbose_name=_("Date finished"))
+
+    def get_runner(self):
+        inv = self.job.inventory
+        inv.write_to_file(self.inventory_path)
+
+        if self.job.type == 'adhoc':
+            runner = AdHocRunner(
+                self.inventory_path, self.job.module, module_args=self.job.args,
+                pattern="all", project_dir=self.private_dir
+            )
+        elif self.job.type == 'playbook':
+            runner = PlaybookRunner(
+                self.inventory_path, self.job.playbook.work_path
+            )
+        else:
+            raise Exception("unsupported job type")
+        return runner
+
+    @property
+    def short_id(self):
+        return str(self.id).split('-')[-1]
+
+    @property
+    def timedelta(self):
+        if self.date_start and self.date_finished:
+            return self.date_finished - self.date_start
+        return None
+
+    @property
+    def is_finished(self):
+        return self.status in ['success', 'failed']
+
+    @property
+    def is_success(self):
+        return self.status == 'success'
+
+    @property
+    def time_cost(self):
+        if self.date_finished and self.date_start:
+            return (self.date_finished - self.date_start).total_seconds()
+        return None
+
+    @property
+    def inventory_path(self):
+        return os.path.join(self.private_dir, 'inventory', 'hosts')
+
+    @property
+    def private_dir(self):
+        uniq = self.date_created.strftime('%Y%m%d_%H%M%S') + '_' + self.short_id
+        job_name = self.job.name if self.job.name else 'instant'
+        return os.path.join(settings.ANSIBLE_DIR, job_name, uniq)
+
+    def set_error(self, error):
+        this = self.__class__.objects.get(id=self.id)  # 重新获取一次，避免数据库超时连接超时
+        this.status = 'failed'
+        this.summary['error'] = str(error)
+        this.finish_task()
+
+    def set_result(self, cb):
+        status_mapper = {
+            'successful': 'success',
+        }
+        this = self.__class__.objects.get(id=self.id)
+        this.status = status_mapper.get(cb.status, cb.status)
+        this.summary = cb.summary
+        this.result = cb.result
+        this.finish_task()
+
+    def finish_task(self):
+        self.date_finished = timezone.now()
+        self.save(update_fields=['result', 'status', 'summary', 'date_finished'])
+
+    def set_celery_id(self):
+        if not current_task:
+            return
+        task_id = current_task.request.root_id
+        self.task_id = task_id
+
+    def start(self, **kwargs):
+        self.date_start = timezone.now()
+        self.set_celery_id()
+        self.save()
+        runner = self.get_runner()
+        try:
+            cb = runner.run(**kwargs)
+            self.set_result(cb)
+            return cb
+        except Exception as e:
+            logging.error(e, exc_info=True)
+            self.set_error(e)

apps/ops/models/playbook.py

@@ -1,39 +1,19 @@
+import os.path
+import uuid
+
+from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from orgs.mixins.models import JMSOrgBaseModel
+from common.db.models import BaseCreateUpdateModel
-from .base import BaseAnsibleExecution, BaseAnsibleJob
 
 
-class PlaybookTemplate(JMSOrgBaseModel):
+class Playbook(BaseCreateUpdateModel):
-    name = models.CharField(max_length=128, verbose_name=_("Name"))
+    id = models.UUIDField(default=uuid.uuid4, primary_key=True)
-    path = models.FilePathField(verbose_name=_("Path"))
+    name = models.CharField(max_length=128, verbose_name=_('Name'), null=True)
-    comment = models.TextField(verbose_name=_("Comment"), blank=True)
+    path = models.FileField(upload_to='playbooks/')
-
-    def __str__(self):
-        return self.name
-
-    class Meta:
-        ordering = ['name']
-        verbose_name = _("Playbook template")
-        unique_together = [('org_id', 'name')]
-
-
-class Playbook(BaseAnsibleJob):
-    path = models.FilePathField(max_length=1024, verbose_name=_("Playbook"))
     owner = models.ForeignKey('users.User', verbose_name=_("Owner"), on_delete=models.SET_NULL, null=True)
-    comment = models.TextField(blank=True, verbose_name=_("Comment"))
-    template = models.ForeignKey('PlaybookTemplate', verbose_name=_("Template"), on_delete=models.SET_NULL, null=True)
-    last_execution = models.ForeignKey('PlaybookExecution', verbose_name=_("Last execution"), on_delete=models.SET_NULL, null=True, blank=True)
 
-    def get_register_task(self):
+    @property
-        name = "automation_strategy_period_{}".format(str(self.id)[:8])
+    def work_path(self):
-        task = execute_automation_strategy.name
+        return os.path.join(settings.DATA_DIR, "ops", "playbook", self.id.__str__())
-        args = (str(self.id), Trigger.timing)
-        kwargs = {}
-        return name, task, args, kwargs
-
-
-class PlaybookExecution(BaseAnsibleExecution):
-    task = models.ForeignKey('Playbook', verbose_name=_("Task"), on_delete=models.CASCADE)
-    path = models.FilePathField(max_length=1024, verbose_name=_("Run dir"))

apps/ops/serializers/adhoc.py

@@ -1,11 +1,33 @@
 # ~*~ coding: utf-8 ~*~
 from __future__ import unicode_literals
-from rest_framework import serializers
-from django.shortcuts import reverse
 
+import datetime
+
+from rest_framework import serializers
+
+from common.drf.fields import ReadableHiddenField
 from ..models import AdHoc, AdHocExecution
 
 
+class AdHocSerializer(serializers.ModelSerializer):
+    owner = ReadableHiddenField(default=serializers.CurrentUserDefault())
+    row_count = serializers.IntegerField(read_only=True)
+    size = serializers.IntegerField(read_only=True)
+
+    class Meta:
+        model = AdHoc
+        fields = ["id", "name", "module", "owner", "row_count", "size", "date_created", "date_updated"]
+
+
+class AdhocListSerializer(AdHocSerializer):
+    row_count = serializers.IntegerField(read_only=True)
+    size = serializers.IntegerField(read_only=True)
+
+    class Meta:
+        model = AdHoc
+        fields = ["id", "name", "module", "row_count", "size", "args", "owner", "date_created", "date_updated"]
+
+
 class AdHocExecutionSerializer(serializers.ModelSerializer):
     stat = serializers.SerializerMethodField()
     last_success = serializers.ListField(source='success_hosts')
@@ -49,26 +71,6 @@ class AdHocExecutionExcludeResultSerializer(AdHocExecutionSerializer):
         ]
 
 
-class AdHocSerializer(serializers.ModelSerializer):
-    tasks = serializers.ListField()
-
-    class Meta:
-        model = AdHoc
-        fields_mini = ['id']
-        fields_small = fields_mini + [
-            'tasks', "pattern", "args", "date_created",
-        ]
-        fields_fk = ["last_execution"]
-        fields_m2m = ["assets"]
-        fields = fields_small + fields_fk + fields_m2m
-        read_only_fields = [
-            'date_created'
-        ]
-        extra_kwargs = {
-            "become": {'write_only': True}
-        }
-
-
 class AdHocExecutionNestSerializer(serializers.ModelSerializer):
     last_success = serializers.ListField(source='success_hosts')
     last_failure = serializers.DictField(source='failed_hosts')
@@ -80,38 +82,3 @@ class AdHocExecutionNestSerializer(serializers.ModelSerializer):
             'last_success', 'last_failure', 'last_run', 'timedelta',
             'is_finished', 'is_success'
         )
-
-
-class AdHocDetailSerializer(AdHocSerializer):
-    latest_execution = AdHocExecutionNestSerializer(allow_null=True)
-    task_name = serializers.CharField(source='task.name')
-
-    class Meta(AdHocSerializer.Meta):
-        fields = AdHocSerializer.Meta.fields + [
-            'latest_execution', 'created_by', 'task_name'
-        ]
-
-
-# class CommandExecutionSerializer(serializers.ModelSerializer):
-#     result = serializers.JSONField(read_only=True)
-#     log_url = serializers.SerializerMethodField()
-#
-#     class Meta:
-#         model = CommandExecution
-#         fields_mini = ['id']
-#         fields_small = fields_mini + [
-#             'command', 'result', 'log_url',
-#             'is_finished', 'date_created', 'date_finished'
-#         ]
-#         fields_m2m = ['hosts']
-#         fields = fields_small + fields_m2m
-#         read_only_fields = [
-#             'result', 'is_finished', 'log_url', 'date_created',
-#             'date_finished'
-#         ]
-#         ref_name = 'OpsCommandExecution'
-#
-#     @staticmethod
-#     def get_log_url(obj):
-#         return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id})
-

apps/ops/serializers/job.py

@@ -0,0 +1,29 @@
+from django.db import transaction
+from rest_framework import serializers
+
+from common.drf.fields import ReadableHiddenField
+from ops.models import Job, JobExecution
+
+_all_ = []
+
+
+class JobSerializer(serializers.ModelSerializer):
+    owner = ReadableHiddenField(default=serializers.CurrentUserDefault())
+
+    class Meta:
+        model = Job
+        fields = [
+            "id", "name", "instant", "type", "module", "args", "playbook", "assets", "runas_policy", "runas", "owner",
+            "date_created",
+            "date_updated"
+        ]
+
+
+class JobExecutionSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = JobExecution
+        read_only_fields = ["id", "task_id", "timedelta", "time_cost", 'is_finished', 'date_start', 'date_created',
+                            'is_success', 'task_id', 'short_id']
+        fields = read_only_fields + [
+            "job"
+        ]

apps/ops/serializers/playbook.py

@@ -0,0 +1,28 @@
+import os
+
+from rest_framework import serializers
+
+from common.drf.fields import ReadableHiddenField
+from ops.models import Playbook
+
+
+def parse_playbook_name(path):
+    file_name = os.path.split(path)[-1]
+    return file_name.split(".")[-2]
+
+
+class PlaybookSerializer(serializers.ModelSerializer):
+    owner = ReadableHiddenField(default=serializers.CurrentUserDefault())
+
+    def create(self, validated_data):
+        name = validated_data.get('name')
+        if not name:
+            path = validated_data.get('path').name
+            validated_data['name'] = parse_playbook_name(path)
+        return super().create(validated_data)
+
+    class Meta:
+        model = Playbook
+        fields = [
+            "id", "name", "path", "date_created", "owner", "date_updated"
+        ]

apps/ops/tasks.py

@@ -2,6 +2,7 @@
 import os
 import random
 import subprocess
+import time
 
 from django.conf import settings
 from celery import shared_task, subtask
@@ -21,7 +22,7 @@ from .celery.utils import (
     create_or_update_celery_periodic_tasks, get_celery_periodic_task,
     disable_celery_periodic_task, delete_celery_periodic_task
 )
-from .models import CeleryTaskExecution, AdHoc, Playbook
+from .models import CeleryTaskExecution, Playbook, Job, JobExecution
 from .notifications import ServerPerformanceCheckUtil
 
 logger = get_logger(__file__)
@@ -31,6 +32,33 @@ def rerun_task():
     pass
 
 
+@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task"))
+def run_ops_job(job_id, **kwargs):
+    job = get_object_or_none(Job, id=job_id)
+    execution = job.create_execution()
+    try:
+        execution.start()
+    except SoftTimeLimitExceeded:
+        execution.set_error('Run timeout')
+        logger.error("Run adhoc timeout")
+    except Exception as e:
+        execution.set_error(e)
+        logger.error("Start adhoc execution error: {}".format(e))
+
+
+@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task execution"))
+def run_ops_job_executions(execution_id, **kwargs):
+    execution = get_object_or_none(JobExecution, id=execution_id)
+    try:
+        execution.start()
+    except SoftTimeLimitExceeded:
+        execution.set_error('Run timeout')
+        logger.error("Run adhoc timeout")
+    except Exception as e:
+        execution.set_error(e)
+        logger.error("Start adhoc execution error: {}".format(e))
+
+
 @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task"))
 def run_adhoc(tid, **kwargs):
     """
@@ -156,18 +184,23 @@ def hello(name, callback=None):
     return gettext("Hello")
 
 
-@shared_task(verbose_name="Hello Error", comment="an test shared task error")
+@shared_task(verbose_name=_("Hello Error"), comment="an test shared task error")
 def hello_error():
     raise Exception("must be error")
 
 
-@shared_task(verbose_name="Hello Random", comment="some time error and some time success")
+@shared_task(verbose_name=_("Hello Random"), comment="some time error and some time success")
 def hello_random():
     i = random.randint(0, 1)
     if i == 1:
         raise Exception("must be error")
 
 
+@shared_task(verbose_name="Hello Running", comment="an task running 1m")
+def hello_running(sec=60):
+    time.sleep(sec)
+
+
 @shared_task
 def hello_callback(result):
     print(result)

apps/ops/urls/api_urls.py

@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from django.urls import path
 from rest_framework.routers import DefaultRouter
 from rest_framework_bulk.routes import BulkRouter
-from rest_framework_nested import routers
 
 from .. import api
 
@@ -13,23 +12,25 @@ app_name = "ops"
 router = DefaultRouter()
 bulk_router = BulkRouter()
 
-router.register(r'adhoc', api.AdHocViewSet, 'adhoc')
+router.register(r'adhocs', api.AdHocViewSet, 'adhoc')
-router.register(r'adhoc-executions', api.AdHocExecutionViewSet, 'execution')
+router.register(r'playbooks', api.PlaybookViewSet, 'playbook')
+router.register(r'jobs', api.JobViewSet, 'job')
+router.register(r'job-executions', api.JobExecutionViewSet, 'job-execution')
+
 router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task')
 
 router.register(r'tasks', api.CeleryTaskViewSet, 'task')
-
+router.register(r'task-executions', api.CeleryTaskExecutionViewSet, 'task-executions')
-task_router = routers.NestedDefaultRouter(router, r'tasks', lookup='task')
-task_router.register(r'executions', api.CeleryTaskExecutionViewSet, 'task-execution')
 
 urlpatterns = [
 
+    path('ansible/job-execution/<uuid:pk>/log/', api.AnsibleTaskLogApi.as_view(), name='job-execution-log'),
+
     path('celery/task/<uuid:name>/task-execution/<uuid:pk>/log/', api.CeleryTaskExecutionLogApi.as_view(),
          name='celery-task-execution-log'),
     path('celery/task/<uuid:name>/task-execution/<uuid:pk>/result/', api.CeleryResultApi.as_view(),
          name='celery-task-execution-result'),
 
-    path('ansible/task-execution/<uuid:pk>/log/', api.AnsibleTaskLogApi.as_view(), name='ansible-task-log'),
 ]
 
-urlpatterns += (router.urls + bulk_router.urls + task_router.urls)
+urlpatterns += (router.urls + bulk_router.urls)

apps/perms/migrations/0032_auto_20221111_1919.py

@@ -0,0 +1,31 @@
+# Generated by Django 3.2.14 on 2022-11-11 11:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0111_alter_automationexecution_status'),
+        ('perms', '0031_auto_20220816_1600'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PermedAccount',
+            fields=[
+            ],
+            options={
+                'verbose_name': 'Permed account',
+                'proxy': True,
+                'indexes': [],
+                'constraints': [],
+            },
+            bases=('assets.account',),
+        ),
+        migrations.AlterField(
+            model_name='assetpermission',
+            name='actions',
+            field=models.IntegerField(default=0, verbose_name='Actions'),
+        ),
+    ]

apps/tickets/migrations/0022_alter_applyassetticket_apply_actions.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.14 on 2022-11-11 11:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tickets', '0021_auto_20220921_1814'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='applyassetticket',
+            name='apply_actions',
+            field=models.IntegerField(default=1, verbose_name='Actions'),
+        ),
+    ]