Add new model to operate log (#3546)

* [Update] 添加一下model到operate log, [platform,remoteapppermission,changeauthplan,gatherusertask]

* [Bugfix] 修改了返回platform的几个位置，修改了command execution的url

* [Update] 优化ops task表结构，避免列表页查询几十次sql, 优化了基础的encryptjsonfield

* [Update] 修改adhoc 返回的become字段，避免密码泄露

* [Update] 修改变量名称

apps/assets/migrations/0044_platform.py

@@ -40,6 +40,9 @@ class Migration(migrations.Migration):
                 ('internal', models.BooleanField(default=False, verbose_name='Internal')),
                 ('comment', models.TextField(blank=True, null=True, verbose_name='Comment')),
             ],
+            options={
+                'verbose_name': 'Platform'
+            }
         ),
         migrations.RunPython(create_internal_platform)
     ]

apps/assets/models/base.py

@@ -11,14 +11,13 @@ from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
 
 from common.utils import (
-    get_signer, ssh_key_string_to_obj, ssh_key_gen, get_logger
+    signer, ssh_key_string_to_obj, ssh_key_gen, get_logger
 )
 from common.validators import alphanumeric
 from common import fields
 from orgs.mixins.models import OrgModelMixin
 from .utils import private_key_validator, Connectivity
 
-signer = get_signer()
 
 logger = get_logger(__file__)
 

apps/assets/models/user.py

@@ -10,14 +10,13 @@ from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 from django.core.validators import MinValueValidator, MaxValueValidator
 
-from common.utils import get_signer
+from common.utils import signer
 from .base import AssetUser
 from .asset import Asset
 
 
 __all__ = ['AdminUser', 'SystemUser']
 logger = logging.getLogger(__name__)
-signer = get_signer()
 
 
 class AdminUser(AssetUser):

apps/audits/signals_handler.py

@@ -27,6 +27,7 @@ MODELS_NEED_RECORD = (
     'User', 'UserGroup', 'Asset', 'Node', 'AdminUser', 'SystemUser',
     'Domain', 'Gateway', 'Organization', 'AssetPermission', 'CommandFilter',
     'CommandFilterRule', 'License', 'Setting', 'Account', 'SyncInstanceTask',
+    'Platform', 'RemoteAppPermission', 'ChangeAuthPlan', 'GatherUserTask',
 )
 
 

apps/common/fields/form.py

@@ -6,9 +6,8 @@ from django import forms
 from django.utils import six
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext as _
-from ..utils import get_signer
+from ..utils import signer
 
-signer = get_signer()
 
 __all__ = [
     'FormDictField', 'FormEncryptCharField', 'FormEncryptDictField',

apps/common/fields/model.py

@@ -4,7 +4,7 @@ import json
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
-from ..utils import get_signer
+from ..utils import signer
 
 
 __all__ = [
@@ -12,8 +12,8 @@ __all__ = [
     'JsonCharField', 'JsonTextField', 'JsonListCharField', 'JsonListTextField',
     'JsonDictCharField', 'JsonDictTextField', 'EncryptCharField',
     'EncryptTextField', 'EncryptMixin', 'EncryptJsonDictTextField',
+    'EncryptJsonDictCharField',
 ]
-signer = get_signer()
 
 
 class JsonMixin:
@@ -108,14 +108,24 @@ class JsonTextField(JsonMixin, models.TextField):
 
 
 class EncryptMixin:
+    """
+    EncryptMixin要放在最前面
+    """
     def from_db_value(self, value, expression, connection, context):
-        if value is not None:
+        if value is None:
-            return signer.unsign(value)
+            return value
-        return None
+        value = signer.unsign(value)
+        sp = super()
+        if hasattr(sp, 'from_db_value'):
+            return sp.from_db_value(value, expression, connection, context)
+        return value
 
     def get_prep_value(self, value):
         if value is None:
             return value
+        sp = super()
+        if hasattr(sp, 'get_prep_value'):
+            value = sp.get_prep_value(value)
         return signer.sign(value)
 
 
@@ -150,3 +160,6 @@ class EncryptJsonDictTextField(EncryptMixin, JsonDictTextField):
     pass
 
 
+class EncryptJsonDictCharField(EncryptMixin, JsonDictCharField):
+    pass
+

apps/common/tests.py

@@ -2,11 +2,10 @@ from django.test import TestCase
 
 # Create your tests here.
 
-from .utils import random_string, get_signer
+from .utils import random_string, signer
 
 
 def test_signer_len():
-    signer = get_signer()
     results = {}
     for i in range(1, 4096):
         s = random_string(i)

apps/common/utils/encode.py

@@ -184,8 +184,11 @@ def encrypt_password(password, salt=None):
 
 
 def get_signer():
-    signer = Signer(settings.SECRET_KEY)
+    s = Signer(settings.SECRET_KEY)
-    return signer
+    return s
+
+
+signer = get_signer()
 
 
 def ensure_last_char_is_ascii(data):

apps/jumpserver/settings/libs.py

@@ -105,7 +105,7 @@ CELERY_TASK_SERIALIZER = 'pickle'
 CELERY_RESULT_SERIALIZER = 'pickle'
 CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 CELERY_ACCEPT_CONTENT = ['json', 'pickle']
-CELERY_RESULT_EXPIRES = 3600
+CELERY_RESULT_EXPIRES = 600
 # CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
 # CELERY_WORKER_LOG_FORMAT = '%(message)s'
 # CELERY_WORKER_TASK_LOG_FORMAT = '%(task_id)s %(task_name)s %(message)s'

apps/ops/api/adhoc.py

@@ -4,6 +4,7 @@
 from django.shortcuts import get_object_or_404
 from rest_framework import viewsets, generics
 from rest_framework.views import Response
+from django.db.models import Count, Q
 
 from common.permissions import IsOrgAdmin
 from common.serializers import CeleryTaskSerializer
@@ -31,6 +32,7 @@ class TaskViewSet(viewsets.ModelViewSet):
             queryset = queryset.filter(created_by=current_org.id)
         else:
             queryset = queryset.filter(created_by='')
+        queryset = queryset.select_related('latest_history')
         return queryset
 
 

apps/ops/celery/decorator.py

@@ -33,11 +33,14 @@ def get_after_app_ready_tasks():
 
 def register_as_period_task(
         crontab=None, interval=None, name=None,
+        args=(), kwargs=None,
         description=''):
     """
     Warning: Task must be have not any args and kwargs
     :param crontab:  "* * * * *"
     :param interval:  60*60*60
+    :param args: ()
+    :param kwargs: {}
     :param description: "
     :param name: ""
     :return:
@@ -58,7 +61,8 @@ def register_as_period_task(
                'task': task,
                'interval': interval,
                'crontab': crontab,
-               'args': (),
+               'args': args,
+               'kwargs': kwargs if kwargs else {},
                'enabled': True,
                'description': description
            }

apps/ops/celery/utils.py

@@ -74,8 +74,6 @@ def create_or_update_celery_periodic_tasks(tasks):
             kwargs=json.dumps(detail.get('kwargs', {})),
             description=detail.get('description') or ''
         )
-        print(defaults)
-
         task = PeriodicTask.objects.update_or_create(
             defaults=defaults, name=name,
         )
@@ -101,4 +99,3 @@ def get_celery_task_log_path(task_id):
     path = os.path.join(settings.CELERY_LOG_DIR, rel_path)
     os.makedirs(os.path.dirname(path), exist_ok=True)
     return path
-

apps/ops/migrations/0009_auto_20191217_1713.py

@@ -0,0 +1,72 @@
+# Generated by Django 2.2.7 on 2019-12-17 09:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+from django.core.exceptions import ObjectDoesNotExist
+
+
+def migrate_task_data(apps, schema_editor):
+    task_model = apps.get_model("ops", "Task")
+    db_alias = schema_editor.connection.alias
+    tasks = task_model.objects.using(db_alias).all()
+    for task in tasks:
+        try:
+            latest_history = task.history.latest()
+        except ObjectDoesNotExist:
+            latest_history = None
+        try:
+            latest_adhoc = task.adhoc.latest()
+        except ObjectDoesNotExist:
+            latest_adhoc = None
+        if latest_history and latest_history.adhoc:
+            latest_history.hosts_amount = latest_history.adhoc.hosts.count()
+            latest_history.save()
+        total_run_amount = task.history.all().count()
+        success_run_amount = task.history.filter(is_success=True).count()
+        task.latest_history = latest_history
+        task.latest_adhoc = latest_adhoc
+        task.total_run_amount = total_run_amount
+        task.success_run_amount = success_run_amount
+        task.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0008_auto_20190919_2100'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='latest_adhoc',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='task_latest', to='ops.AdHoc'),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='latest_history',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='task_latest', to='ops.AdHocRunHistory'),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='success_run_amount',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='task',
+            name='total_run_amount',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='adhocrunhistory',
+            name='hosts_amount',
+            field=models.IntegerField(default=0, verbose_name='Host amount'),
+        ),
+        migrations.AddField(
+            model_name='adhocrunhistory',
+            name='task_display',
+            field=models.CharField(blank=True, default='', max_length=128,
+                                   verbose_name='Task display'),
+        ),
+        migrations.RunPython(migrate_task_data),
+    ]

apps/ops/migrations/0010_auto_20191217_1758.py

@@ -0,0 +1,68 @@
+# Generated by Django 2.2.7 on 2019-12-17 09:58
+
+import common.fields.model
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0009_auto_20191217_1713'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='adhoc',
+            name='_hosts',
+        ),
+        migrations.AlterField(
+            model_name='adhoc',
+            name='_become',
+            field=common.fields.model.EncryptJsonDictCharField(blank=True, default='', max_length=1024, verbose_name='Become'),
+        ),
+        migrations.AlterField(
+            model_name='adhoc',
+            name='_options',
+            field=common.fields.model.JsonDictCharField(default='', max_length=1024, verbose_name='Options'),
+        ),
+        migrations.AlterField(
+            model_name='adhoc',
+            name='_tasks',
+            field=common.fields.model.JsonListTextField(verbose_name='Tasks'),
+        ),
+        migrations.RenameField(
+            model_name='adhoc',
+            old_name='_become',
+            new_name='become',
+        ),
+        migrations.RenameField(
+            model_name='adhoc',
+            old_name='_options',
+            new_name='options',
+        ),
+        migrations.RenameField(
+            model_name='adhoc',
+            old_name='_tasks',
+            new_name='tasks',
+        ),
+        migrations.AlterField(
+            model_name='adhocrunhistory',
+            name='_result',
+            field=common.fields.model.JsonDictTextField(blank=True, null=True, verbose_name='Adhoc raw result'),
+        ),
+        migrations.AlterField(
+            model_name='adhocrunhistory',
+            name='_summary',
+            field=common.fields.model.JsonDictTextField(blank=True, null=True, verbose_name='Adhoc result summary'),
+        ),
+        migrations.RenameField(
+            model_name='adhocrunhistory',
+            old_name='_result',
+            new_name='result',
+        ),
+        migrations.RenameField(
+            model_name='adhocrunhistory',
+            old_name='_summary',
+            new_name='summary',
+        ),
+    ]

apps/ops/models/adhoc.py

@@ -1,6 +1,5 @@
 # ~*~ coding: utf-8 ~*~
 
-import json
 import uuid
 import os
 import time
@@ -13,11 +12,16 @@ from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django_celery_beat.models import PeriodicTask
 
-from common.utils import get_signer, get_logger, lazyproperty
+from common.utils import get_logger, lazyproperty
-from orgs.utils import set_to_root_org
+from common.fields.model import (
-from ..celery.utils import delete_celery_periodic_task, \
+    JsonListTextField, JsonDictCharField, EncryptJsonDictCharField,
-    create_or_update_celery_periodic_tasks, \
+    JsonDictTextField,
+)
+from orgs.utils import set_to_root_org, get_current_org, set_current_org
+from ..celery.utils import (
+    delete_celery_periodic_task, create_or_update_celery_periodic_tasks,
     disable_celery_periodic_task
+)
 from ..ansible import AdHocRunner, AnsibleError
 from ..inventory import JMSInventory
 
@@ -25,7 +29,6 @@ __all__ = ["Task", "AdHoc", "AdHocRunHistory"]
 
 
 logger = get_logger(__file__)
-signer = get_signer()
 
 
 class Task(models.Model):
@@ -44,14 +47,17 @@ class Task(models.Model):
     created_by = models.CharField(max_length=128, blank=True, default='')
     date_created = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name=_("Date created"))
     date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated"))
-    __latest_adhoc = None
+    latest_adhoc = models.ForeignKey('ops.AdHoc', on_delete=models.SET_NULL, null=True, related_name='task_latest')
+    latest_history = models.ForeignKey('ops.AdHocRunHistory', on_delete=models.SET_NULL, null=True, related_name='task_latest')
+    total_run_amount = models.IntegerField(default=0)
+    success_run_amount = models.IntegerField(default=0)
     _ignore_auto_created_by = True
 
     @property
     def short_id(self):
         return str(self.id).split('-')[-1]
 
-    @property
+    @lazyproperty
     def versions(self):
         return self.adhoc.all().count()
 
@@ -78,52 +84,40 @@ class Task(models.Model):
 
     @property
     def assets_amount(self):
-        return self.latest_adhoc.hosts.count()
+        if self.latest_history:
-
+            return self.latest_history.hosts_amount
-    @lazyproperty
+        return 0
-    def latest_adhoc(self):
-        return self.get_latest_adhoc()
-
-    @lazyproperty
-    def latest_history(self):
-        try:
-            return self.history.all().latest()
-        except AdHocRunHistory.DoesNotExist:
-            return None
 
     def get_latest_adhoc(self):
+        if self.latest_adhoc:
+            return self.latest_adhoc
         try:
-            return self.adhoc.all().latest()
+            adhoc = self.adhoc.all().latest()
+            self.latest_adhoc = adhoc
+            self.save()
+            return adhoc
         except AdHoc.DoesNotExist:
             return None
 
     @property
     def history_summary(self):
-        history = self.get_run_history()
+        total = self.total_run_amount
-        total = len(history)
+        success = self.success_run_amount
-        success = len([history for history in history if history.is_success])
+        failed = total - success
-        failed = len([history for history in history if not history.is_success])
         return {'total': total, 'success': success, 'failed': failed}
 
     def get_run_history(self):
         return self.history.all()
 
-    def run(self, record=True):
+    def run(self):
-        set_to_root_org()
+        latest_adhoc = self.get_latest_adhoc()
-        if self.latest_adhoc:
+        if latest_adhoc:
-            return self.latest_adhoc.run(record=record)
+            return latest_adhoc.run()
         else:
             return {'error': 'No adhoc'}
 
-    def save(self, force_insert=False, force_update=False, using=None,
+    def register_as_period_task(self):
-             update_fields=None):
         from ..tasks import run_ansible_task
-        super().save(
-            force_insert=force_insert, force_update=force_update,
-            using=using, update_fields=update_fields,
-        )
-
-        if self.is_periodic:
         interval = None
         crontab = None
 
@@ -143,8 +137,14 @@ class Task(models.Model):
             }
         }
         create_or_update_celery_periodic_tasks(tasks)
+
+    def save(self, **kwargs):
+        instance = super().save(**kwargs)
+        if self.is_periodic:
+            self.register_as_period_task()
         else:
             disable_celery_periodic_task(self.__str__())
+        return instance
 
     def delete(self, using=None, keep_parents=False):
         super().delete(using=using, keep_parents=keep_parents)
@@ -153,7 +153,7 @@ class Task(models.Model):
     @property
     def schedule(self):
         try:
-            return PeriodicTask.objects.get(name=self.name)
+            return PeriodicTask.objects.get(name=str(self))
         except PeriodicTask.DoesNotExist:
             return None
 
@@ -172,7 +172,6 @@ class AdHoc(models.Model):
     task: A task reference
     _tasks: [{'name': 'task_name', 'action': {'module': '', 'args': ''}, 'other..': ''}, ]
     _options: ansible options, more see ops.ansible.runner.Options
-    _hosts: ["hostname1", "hostname2"], hostname must be unique key of cmdb
     run_as_admin: if true, then need get every host admin user run it, because every host may be have different admin user, so we choise host level
     run_as: username(Add the uniform AssetUserManager <AssetUserManager> and change it to username)
     _become: May be using become [sudo, su] options. {method: "sudo", user: "user", pass: "pass"]
@@ -180,31 +179,16 @@ class AdHoc(models.Model):
     """
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     task = models.ForeignKey(Task, related_name='adhoc', on_delete=models.CASCADE)
-    _tasks = models.TextField(verbose_name=_('Tasks'))
+    tasks = JsonListTextField(verbose_name=_('Tasks'))
     pattern = models.CharField(max_length=64, default='{}', verbose_name=_('Pattern'))
-    _options = models.CharField(max_length=1024, default='', verbose_name=_('Options'))
+    options = JsonDictCharField(max_length=1024, default='', verbose_name=_('Options'))
-    _hosts = models.TextField(blank=True, verbose_name=_('Hosts'))  # ['hostname1', 'hostname2']
     hosts = models.ManyToManyField('assets.Asset', verbose_name=_("Host"))
     run_as_admin = models.BooleanField(default=False, verbose_name=_('Run as admin'))
     run_as = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Username'))
-    _become = models.CharField(max_length=1024, default='', blank=True, verbose_name=_("Become"))
+    become = EncryptJsonDictCharField(max_length=1024, default='', blank=True, verbose_name=_("Become"))
     created_by = models.CharField(max_length=64, default='', blank=True, null=True, verbose_name=_('Create by'))
     date_created = models.DateTimeField(auto_now_add=True, db_index=True)
 
-    @property
-    def tasks(self):
-        try:
-            return json.loads(self._tasks)
-        except:
-            return []
-
-    @tasks.setter
-    def tasks(self, item):
-        if item and isinstance(item, list):
-            self._tasks = json.dumps(item)
-        else:
-            raise SyntaxError('Tasks should be a list: {}'.format(item))
-
     @property
     def inventory(self):
         if self.become:
@@ -223,97 +207,22 @@ class AdHoc(models.Model):
         return inventory
 
     @property
-    def become(self):
+    def become_display(self):
-        if self._become:
+        if self.become:
-            return json.loads(signer.unsign(self._become))
+            return self.become.get("user", "")
-        else:
+        return ""
-            return {}
 
-    def run(self, record=True):
+    def run(self):
-        set_to_root_org()
-        if record:
-            return self._run_and_record()
-        else:
-            return self._run_only()
-
-    def _run_and_record(self):
         try:
             hid = current_task.request.id
         except AttributeError:
             hid = str(uuid.uuid4())
-        history = AdHocRunHistory(id=hid, adhoc=self, task=self.task)
+        history = AdHocRunHistory(
+            id=hid, adhoc=self, task=self.task,
+            task_display=str(self.task)
+        )
         history.save()
-        time_start = time.time()
+        return history.start()
-        date_start = timezone.now()
-        is_success = False
-        summary = {}
-        raw = ''
-
-        try:
-            date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-            print(_("{} Start task: {}").format(date_start_s, self.task.name))
-            raw, summary = self._run_only()
-            is_success = summary.get('success', False)
-        except Exception as e:
-            logger.error(e, exc_info=True)
-            raw = {"dark": {"all": str(e)}, "contacted": []}
-        finally:
-            date_end = timezone.now()
-            date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S')
-            print(_("{} Task finish").format(date_end_s))
-            print('.\n\n.')
-            try:
-                summary_text = json.dumps(summary)
-            except json.JSONDecodeError:
-                summary_text = '{}'
-            AdHocRunHistory.objects.filter(id=history.id).update(
-                date_start=date_start,
-                is_finished=True,
-                is_success=is_success,
-                date_finished=timezone.now(),
-                timedelta=time.time() - time_start,
-                _summary=summary_text
-            )
-            return raw, summary
-
-    def _run_only(self):
-        Task.objects.filter(id=self.task.id).update(date_updated=timezone.now())
-        runner = AdHocRunner(self.inventory, options=self.options)
-        try:
-            result = runner.run(
-                self.tasks,
-                self.pattern,
-                self.task.name,
-            )
-            return result.results_raw, result.results_summary
-        except AnsibleError as e:
-            logger.warn("Failed run adhoc {}, {}".format(self.task.name, e))
-            pass
-
-    @become.setter
-    def become(self, item):
-        """
-        :param item:  {
-            method: "sudo",
-            user: "user",
-            pass: "pass",
-        }
-        :return:
-        """
-        # self._become = signer.sign(json.dumps(item)).decode('utf-8')
-        self._become = signer.sign(json.dumps(item))
-
-    @property
-    def options(self):
-        if self._options:
-            _options = json.loads(self._options)
-            if isinstance(_options, dict):
-                return _options
-        return {}
-
-    @options.setter
-    def options(self, item):
-        self._options = json.dumps(item)
 
     @property
     def short_id(self):
@@ -328,6 +237,8 @@ class AdHoc(models.Model):
 
     def save(self, **kwargs):
         instance = super().save(**kwargs)
+        self.task.latest_adhoc = instance
+        self.task.save()
         return instance
 
     def __str__(self):
@@ -356,19 +267,25 @@ class AdHocRunHistory(models.Model):
     """
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     task = models.ForeignKey(Task, related_name='history', on_delete=models.SET_NULL, null=True)
+    task_display = models.CharField(max_length=128, blank=True, default='', verbose_name=_("Task display"))
+    hosts_amount = models.IntegerField(default=0, verbose_name=_("Host amount"))
     adhoc = models.ForeignKey(AdHoc, related_name='history', on_delete=models.SET_NULL, null=True)
     date_start = models.DateTimeField(auto_now_add=True, verbose_name=_('Start time'))
     date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('End time'))
     timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True)
     is_finished = models.BooleanField(default=False, verbose_name=_('Is finished'))
     is_success = models.BooleanField(default=False, verbose_name=_('Is success'))
-    _result = models.TextField(blank=True, null=True, verbose_name=_('Adhoc raw result'))
+    result = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc raw result'))
-    _summary = models.TextField(blank=True, null=True, verbose_name=_('Adhoc result summary'))
+    summary = JsonDictTextField(blank=True, null=True, verbose_name=_('Adhoc result summary'))
 
     @property
     def short_id(self):
         return str(self.id).split('-')[-1]
 
+    @property
+    def adhoc_short_id(self):
+        return str(self.adhoc_id).split('-')[-1]
+
     @property
     def log_path(self):
         dt = datetime.datetime.now().strftime('%Y-%m-%d')
@@ -377,30 +294,58 @@ class AdHocRunHistory(models.Model):
             os.makedirs(log_dir)
         return os.path.join(log_dir, str(self.id) + '.log')
 
-    @property
+    def start_runner(self):
-    def result(self):
+        runner = AdHocRunner(self.adhoc.inventory, options=self.adhoc.options)
-        if self._result:
-            return json.loads(self._result)
-        else:
-            return {}
-
-    @result.setter
-    def result(self, item):
-        self._result = json.dumps(item)
-
-    @property
-    def summary(self):
-        if self._summary:
-            return json.loads(self._summary)
-        else:
-            return {"ok": {}, "dark": {}}
-
-    @summary.setter
-    def summary(self, item):
         try:
-            self._summary = json.dumps(item)
+            result = runner.run(
-        except json.JSONDecodeError:
+                self.adhoc.tasks,
-            self._summary = json.dumps({})
+                self.adhoc.pattern,
+                self.task.name,
+            )
+            return result.results_raw, result.results_summary
+        except AnsibleError as e:
+            logger.warn("Failed run adhoc {}, {}".format(self.task.name, e))
+            return {}, {}
+
+    def start(self):
+        self.task.latest_history = self
+        self.task.save()
+        current_org = get_current_org()
+        set_to_root_org()
+        time_start = time.time()
+        date_start = timezone.now()
+        is_success = False
+        summary = {}
+        raw = ''
+
+        try:
+            date_start_s = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+            print(_("{} Start task: {}").format(date_start_s, self.task.name))
+            raw, summary = self.start_runner()
+            is_success = summary.get('success', False)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raw = {"dark": {"all": str(e)}, "contacted": []}
+        finally:
+            date_end = timezone.now()
+            date_end_s = date_end.strftime('%Y-%m-%d %H:%M:%S')
+            print(_("{} Task finish").format(date_end_s))
+            print('.\n\n.')
+            task = Task.objects.get(id=self.task_id)
+            task.total_run_amount = models.F('total_run_amount') + 1
+            if is_success:
+                task.success_run_amount = models.F('success_run_amount') + 1
+            task.save()
+            AdHocRunHistory.objects.filter(id=self.id).update(
+                date_start=date_start,
+                is_finished=True,
+                is_success=is_success,
+                date_finished=timezone.now(),
+                timedelta=time.time() - time_start,
+                summary=summary
+            )
+            set_current_org(current_org)
+            return raw, summary
 
     @property
     def success_hosts(self):

apps/ops/serializers/adhoc.py

@@ -6,59 +6,74 @@ from django.shortcuts import reverse
 from ..models import Task, AdHoc, AdHocRunHistory, CommandExecution
 
 
-class TaskSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = Task
-        fields = [
-            'id', 'name', 'interval', 'crontab', 'is_periodic',
-            'is_deleted', 'comment', 'created_by', 'date_created',
-            'versions', 'is_success', 'timedelta', 'assets_amount',
-            'date_updated', 'history_summary',
-        ]
-
-
-class AdHocSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = AdHoc
-        exclude = ('_tasks', '_options', '_hosts', '_become')
-
-    def get_field_names(self, declared_fields, info):
-        fields = super().get_field_names(declared_fields, info)
-        fields.extend(['tasks', 'options', 'hosts', 'become', 'short_id'])
-        return fields
-
-
 class AdHocRunHistorySerializer(serializers.ModelSerializer):
-    task = serializers.SerializerMethodField()
-    adhoc_short_id = serializers.SerializerMethodField()
     stat = serializers.SerializerMethodField()
 
     class Meta:
         model = AdHocRunHistory
-        exclude = ('_result', '_summary')
+        fields = '__all__'
-
-    @staticmethod
-    def get_adhoc_short_id(obj):
-        return obj.adhoc.short_id
 
     @staticmethod
     def get_task(obj):
-        return obj.adhoc.task.id
+        return obj.task.id
 
     @staticmethod
     def get_stat(obj):
         return {
-            "total": obj.adhoc.hosts.count(),
+            "total": obj.hosts_amount,
             "success": len(obj.summary.get("contacted", [])),
             "failed": len(obj.summary.get("dark", [])),
         }
 
     def get_field_names(self, declared_fields, info):
         fields = super().get_field_names(declared_fields, info)
-        fields.extend(['summary', 'short_id'])
+        fields.extend(['short_id', 'adhoc_short_id'])
         return fields
 
 
+class AdHocRunHistoryExcludeResultSerializer(AdHocRunHistorySerializer):
+    def get_field_names(self, declared_fields, info):
+        fields = super().get_field_names(declared_fields, info)
+        fields = [i for i in fields if i not in ['result', 'summary']]
+        return fields
+
+
+class TaskSerializer(serializers.ModelSerializer):
+    latest_history = AdHocRunHistoryExcludeResultSerializer(read_only=True)
+
+    class Meta:
+        model = Task
+        fields = [
+            'id', 'name', 'interval', 'crontab', 'is_periodic',
+            'is_deleted', 'comment', 'created_by', 'date_created',
+            'date_updated', 'latest_history',
+        ]
+        read_only_fields = [
+            'is_deleted', 'created_by', 'date_created', 'date_updated',
+            'latest_adhoc', 'latest_history', 'total_run_amount',
+            'success_run_amount',
+        ]
+
+
+class AdHocSerializer(serializers.ModelSerializer):
+    become_display = serializers.ReadOnlyField()
+
+    class Meta:
+        model = AdHoc
+        fields = [
+            "id", "task", 'tasks', "pattern", "options",
+            "hosts", "run_as_admin", "run_as", "become",
+            "created_by", "date_created", "short_id",
+            "become_display",
+        ]
+        read_only_fields = [
+            'created_by', 'date_created'
+        ]
+        extra_kwargs = {
+            "become": {'write_only': True}
+        }
+
+
 class CommandExecutionSerializer(serializers.ModelSerializer):
     result = serializers.JSONField(read_only=True)
     log_url = serializers.SerializerMethodField()

apps/ops/templates/ops/adhoc_detail.html

@@ -78,7 +78,7 @@
                                         {% endif %}
                                         <tr>
                                             <td>{% trans 'Become' %}</td>
-                                            <td><b>{{ object.become.user }}</b></td>
+                                            <td><b>{{ object.become_display }}</b></td>
                                         </tr>
                                         <tr>
                                             <td>{% trans 'Created by' %}</td>

apps/ops/templates/ops/command_execution_create.html

@@ -4,17 +4,12 @@
 {% load bootstrap3 %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/ztree/awesomeStyle/awesome.css' %}"
+    <link href="{% static 'css/plugins/ztree/awesomeStyle/awesome.css' %}" rel="stylesheet">
-          rel="stylesheet">
     <link rel="stylesheet" href="{% static 'js/plugins/xterm/xterm.css' %}"/>
-    <link href="{% static 'css/plugins/codemirror/codemirror.css' %}"
+    <link href="{% static 'css/plugins/codemirror/codemirror.css' %}" rel="stylesheet">
-          rel="stylesheet">
+    <link href="{% static 'css/plugins/codemirror/ambiance.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/codemirror/ambiance.css' %}"
+    <script type="text/javascript" src="{% static 'js/plugins/ztree/jquery.ztree.all.min.js' %}"></script>
-          rel="stylesheet">
+    <script type="text/javascript" src="{% static 'js/plugins/ztree/jquery.ztree.exhide.min.js' %}"></script>
-    <script type="text/javascript"
-            src="{% static 'js/plugins/ztree/jquery.ztree.all.min.js' %}"></script>
-    <script type="text/javascript"
-            src="{% static 'js/plugins/ztree/jquery.ztree.exhide.min.js' %}"></script>
     <script src="{% static 'js/jquery.form.min.js' %}"></script>
     <script src="{% static 'js/plugins/xterm/xterm.js' %}"></script>
     <script src="{% static 'js/plugins/xterm/addons/fit/fit.js' %}"></script>
@@ -37,18 +32,18 @@
             overflow: auto;
         }
 
-        body ::-webkit-scrollbar-track {
+        #term ::-webkit-scrollbar-track {
             -webkit-box-shadow: inset 0 0 2px rgba(0, 0, 0, 0.3);
             background-color: #272323;
             border-radius: 6px;
         }
 
-        body ::-webkit-scrollbar {
+        #term ::-webkit-scrollbar {
             width: 8px;
             height: 8px;
         }
 
-        body ::-webkit-scrollbar-thumb {
+        #term ::-webkit-scrollbar-thumb {
             background-color: #494141;
             border-radius: 6px;
         }
@@ -58,8 +53,8 @@
 {% block content %}
     <div class="wrapper wrapper-content">
         <div class="row">
-            <div class="col-sm-3" id="split-left" style="padding-left: 3px">
+            <div class="col-sm-3" id="split-left" style="padding-left: 3px;overflow:auto">
-                <div class="ibox float-e-margins">
+                <div class="ibox treebox float-e-margins">
                     <div class="ibox-content mailbox-content"
                          style="padding-top: 0;padding-left: 1px">
                         <div class="file-manager ">
@@ -73,37 +68,30 @@
             </div>
             <div class="col-sm-9 animated fadeInRight" id="split-right">
                 <div class="tree-toggle">
-                    <div class="btn btn-sm btn-primary tree-toggle-btn"
+                    <div class="btn btn-sm btn-primary tree-toggle-btn" onclick="toggle()">
-                         onclick="toggle()">
                         <i class="fa fa-angle-left fa-x" id="toggle-icon"></i>
                     </div>
                 </div>
                 <div class="mail-box-header" style="padding-top: 5px;">
-                    <form enctype="multipart/form-data" method="post"
+                    <form enctype="multipart/form-data" method="post" class="form-horizontal" action="" onsubmit="return execute()">
-                          class="form-horizontal" action=""
-                          onsubmit="return execute()">
                         <div class="form-group">
-                            <div id="term"
+                            <div id="term" style="height: 100%;width: 100%"></div>
-                                 style="height: 100%;width: 100%"></div>
                         </div>
                         <div class="row">
                             <div class="col-sm-10">
-                                <div class="input-group"
+                                <div class="input-group" style="height: 100%; width: 100%">
-                                     style="height: 100%; width: 100%">
+                                    <textarea class="form-control" id="command-text"></textarea>
-                                    <textarea class="form-control"
-                                              id="command-text"></textarea>
                                 </div>
                             </div>
                             <div class="col-sm-2">
-                                <select class="select2 form-control"
+                                <select class="select2 form-control" id="system-users-select">
-                                        id="system-users-select">
                                     {% for s in system_users %}
                                         <option value="{{ s.id }}" {% if s.protocol != 'ssh' or s.login_mode != 'auto' %}disabled{% endif %}>{{ s }}</option>
                                     {% endfor %}
                                 </select>
-                                <button type="button"
+                                <button type="button" class="btn btn-primary btn-execute" style="margin-top: 30px; width: 100%">
-                                        class="btn btn-primary btn-execute"
+                                    {% trans 'Go' %}
-                                        style="margin-top: 30px; width: 100%">{% trans 'Go' %}</button>
+                                </button>
                             </div>
                         </div>
                     </form>
@@ -114,7 +102,7 @@
 {% endblock %}
 
 {% block custom_foot_js %}
-    <script>
+<script>
     var zTree, show = 0;
     var systemUserId = null;
     var url = null;
@@ -131,6 +119,12 @@
             check: {
                 enable: true
             },
+            async: {
+                enable: true,
+                url: url,
+                autoParam: ["id=key", "name=n", "level=lv"],
+                type: 'get'
+            },
             view: {
                 dblClickExpand: false,
                 showLine: true
@@ -311,9 +305,9 @@
 
     var editor;
     $(document).ready(function () {
+        $('.treebox').css('height', window.innerHeight - 60);
         systemUserId = $('#system-users-select').val();
 
-
         $(".select2").select2({
             dropdownAutoWidth: true,
         }).on('select2:select', function (evt) {
@@ -339,5 +333,5 @@
     }).on('click', '.btn-execute', function () {
         execute()
     })
-    </script>
+</script>
 {% endblock %}

apps/ops/templates/ops/task_adhoc.html

@@ -103,7 +103,7 @@ $(document).ready(function () {
                 if (!cellData) {
                     $(td).html("")
                 } else {
-                    $(td).html(cellData.user)
+                    $(td).html(cellData)
                 }
             }},
             {targets: 6, createdCell: function (td, cellData) {
@@ -118,8 +118,12 @@ $(document).ready(function () {
             }}
         ],
         ajax_url: '{% url "api-ops:adhoc-list" %}?task={{ object.pk }}',
-        columns: [{data: function(){return ""}}, {data: "short_id" }, {data: "hosts", orderable:false}, {data: "pattern", orderable:false},
+        columns: [
-                  {data: "run_as"}, {data: "become", orderable:false}, {data: "date_created"}, {data: "id", orderable:false}]
+            {data: function(){return ""}}, {data: "short_id"},
+            {data: "hosts", orderable:false}, {data: "pattern", orderable:false},
+            {data: "run_as"}, {data: "become_display", orderable:false},
+            {data: "date_created"}, {data: "id", orderable:false}
+        ]
     };
     jumpserver.initDataTable(options);
 }).on('click', '.celery-task-log', function () {

apps/ops/templates/ops/task_detail.html

@@ -80,11 +80,23 @@
                                         </tr>
                                         <tr>
                                             <td>{% trans 'Is finished' %}:</td>
-                                            <td><b>{{ object.latest_history.is_finished|yesno:"Yes,No,Unkown" }}</b></td>
+                                            <td><b>
+                                                {% if object.latest_history.is_finished %}
+                                                    {% trans 'Yes' %}
+                                                {% else %}
+                                                    {% trans 'No' %}
+                                                {% endif %}
+                                            </b></td>
                                         </tr>
                                         <tr>
                                             <td>{% trans 'Is success ' %}:</td>
-                                            <td><b>{{ object.latest_history.is_success|yesno:"Yes,No,Unkown" }}</b></td>
+                                            <td><b>
+                                                {% if object.latest_history.is_success %}
+                                                    {% trans 'Yes' %}
+                                                {% else %}
+                                                    {% trans 'No' %}
+                                                {% endif %}
+                                            </b></td>
                                         </tr>
                                         <tr>
                                             <td>{% trans 'Contents' %}:</td>

apps/ops/templates/ops/task_list.html

@@ -10,7 +10,6 @@
             </th>
             <th class="text-left">{% trans 'Name' %}</th>
             <th class="text-center">{% trans 'Run times' %}</th>
-            <th class="text-center">{% trans 'Versions' %}</th>
             <th class="text-center">{% trans 'Hosts' %}</th>
             <th class="text-center">{% trans 'Success' %}</th>
             <th class="text-center">{% trans 'Date' %}</th>
@@ -36,34 +35,40 @@ $(document).ready(function () {
                 $(td).html(innerHtml);
             }},
             {targets: 2, createdCell: function (td, cellData) {
+                var summary = cellData ? cellData.stat : {failed: 0, success: 0, total: 0};
                 var innerHtml = '<span class="text-danger">failed</span>/<span class="text-navy">success</span>/total';
-                if (cellData) {
+                innerHtml = innerHtml.replace('failed', summary.failed)
-                    innerHtml = innerHtml.replace('failed', cellData.failed)
+                    .replace('success', summary.success)
-                        .replace('success', cellData.success)
+                    .replace('total', summary.total);
-                        .replace('total', cellData.total);
                 $(td).html(innerHtml);
-                } else {
-                    $(td).html('')
-                }
             }},
-            {targets: 5, createdCell: function (td, cellData) {
+            {targets: 3, createdCell: function (td, cellData) {
+                var hostsAmount = cellData ? cellData.hosts_amount : 0;
+                 $(td).html(hostsAmount)
+            }},
+            {targets: 4, createdCell: function (td, cellData) {
                 var successBtn = '<i class="fa fa-check text-navy"></i>';
                 var failedBtn = '<i class="fa fa-times text-danger"></i>';
-                if (cellData) {
+                if (cellData && cellData.is_success) {
                     $(td).html(successBtn)
                 } else {
                     $(td).html(failedBtn)
                 }
             }},
-            {targets: 6, createdCell: function (td, cellData) {
+            {targets: 5, createdCell: function (td, cellData) {
-                $(td).html(toSafeLocalDateStr(cellData));
+                if (cellData) {
+                    $(td).html(toSafeLocalDateStr(cellData.date_start));
+                } else {
+                    $(td).html('');
+                }
             }},
-            {targets: 7, createdCell: function (td, cellData) {
+            {targets: 6, createdCell: function (td, cellData) {
+                cellData = cellData ? cellData.timedelta : 0;
                 var delta = readableSecond(cellData);
                 $(td).html(delta);
             }},
             {
-                targets: 8,
+                targets: 7,
                 createdCell: function (td, cellData, rowData) {
                     var runBtn = '<a data-uid="ID" class="btn btn-xs btn-primary btn-run">{% trans "Run" %}</a> '.replace('ID', cellData);
                     var delBtn = '<a data-uid="ID" class="btn btn-xs btn-danger btn-del">{% trans "Delete" %}</a>'.replace('ID', cellData);
@@ -73,10 +78,11 @@ $(document).ready(function () {
         ],
         ajax_url: '{% url "api-ops:task-list" %}',
         columns: [
-            {data: "id"}, {data: "name", className: "text-left"}, {data: "history_summary", orderable: false},
+            {data: "id"}, {data: "name", className: "text-left"},
-            {data: "versions", orderable: false}, {data: "assets_amount", orderable: false},
+            {data: "latest_history", orderable: false},
-            {data: "is_success", orderable: false}, {data: "date_updated"},
+            {data: "latest_history", orderable: false},
-            {data: "timedelta", orderable:false}, {data: "id", orderable: false},
+            {data: "latest_history", orderable: false}, {data: "latest_history"},
+            {data: "latest_history", orderable:false}, {data: "id", orderable: false},
         ],
         order: [],
         op_html: $('#actions').html()

apps/ops/urls/view_urls.py

@@ -20,5 +20,5 @@ urlpatterns = [
     path('celery/task/<uuid:pk>/log/', views.CeleryTaskLogView.as_view(), name='celery-task-log'),
 
     path('command-execution/', views.CommandExecutionListView.as_view(), name='command-execution-list'),
-    path('command-execution/start/', views.CommandExecutionStartView.as_view(), name='command-execution-start'),
+    path('command-execution/create/', views.CommandExecutionCreateView.as_view(), name='command-execution-create'),
 ]

apps/ops/views/command.py

@@ -15,7 +15,7 @@ from ..forms import CommandExecutionForm
 
 
 __all__ = [
-    'CommandExecutionListView', 'CommandExecutionStartView'
+    'CommandExecutionListView', 'CommandExecutionCreateView'
 ]
 
 
@@ -55,7 +55,7 @@ class CommandExecutionListView(PermissionsMixin, DatetimeSearchMixin, ListView):
         return super().get_context_data(**kwargs)
 
 
-class CommandExecutionStartView(PermissionsMixin, TemplateView):
+class CommandExecutionCreateView(PermissionsMixin, TemplateView):
     template_name = 'ops/command_execution_create.html'
     form_class = CommandExecutionForm
     permission_classes = [IsValidUser]

apps/perms/serializers/user_permission.py

@@ -46,6 +46,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer):
     被授权资产的数据结构
     """
     protocols = ProtocolsField(label=_('Protocols'), required=False, read_only=True)
+    platform = serializers.ReadOnlyField(source='platform_base')
 
     class Meta:
         model = Asset

apps/perms/utils/asset_permission.py

@@ -437,7 +437,7 @@ def sort_assets(assets, order_by='hostname', reverse=False):
 
 class ParserNode:
     nodes_only_fields = ("key", "value", "id")
-    assets_only_fields = ("platform", "hostname", "id", "ip", "protocols")
+    assets_only_fields = ("hostname", "id", "ip", "protocols", "org_id")
     system_users_only_fields = (
         "id", "name", "username", "protocol", "priority", "login_mode",
     )
@@ -445,7 +445,6 @@ class ParserNode:
     @staticmethod
     def parse_node_to_tree_node(node):
         name = '{} ({})'.format(node.value, node.assets_amount)
-        # name = node.value
         data = {
             'id': node.key,
             'name': name,
@@ -468,7 +467,7 @@ class ParserNode:
     @staticmethod
     def parse_asset_to_tree_node(node, asset):
         icon_skin = 'file'
-        platform = asset.platform.lower()
+        platform = asset.platform_base.lower()
         if platform == 'windows':
             icon_skin = 'windows'
         elif platform == 'linux':
@@ -489,8 +488,8 @@ class ParserNode:
                     'hostname': asset.hostname,
                     'ip': asset.ip,
                     'protocols': asset.protocols_as_list,
-                    'platform': asset.platform,
+                    'platform': asset.platform_base,
-                    "org_name": asset.org_name,
+                    'org_name': asset.org_name,
                 },
             }
         }

apps/settings/models.py

@@ -5,9 +5,7 @@ from django.db.utils import ProgrammingError, OperationalError
 from django.utils.translation import ugettext_lazy as _
 from django.core.cache import cache
 
-from common.utils import get_signer
+from common.utils import signer
-
-signer = get_signer()
 
 
 class SettingQuerySet(models.QuerySet):

apps/templates/_base_asset_tree_list.html

@@ -44,7 +44,6 @@ function toggleSpliter() {
             showTree = 1;
         });
     } else {
-        console.log("hide")
         $("#split-right").attr("class", "col-sm-9");
         $("#toggle-icon").attr("class", "fa fa-angle-left fa-x");
         $("#split-left").show(500);

apps/templates/_nav.html

@@ -116,7 +116,7 @@
         </a>
         <ul class="nav nav-second-level">
             <li id="task"><a href="{% url 'ops:task-list' %}">{% trans 'Task list' %}</a></li>
-            <li id="command-execution"><a href="{% url 'ops:command-execution-start' %}">{% trans 'Batch command' %}</a></li>
+            <li id="command-execution"><a href="{% url 'ops:command-execution-create' %}">{% trans 'Batch command' %}</a></li>
             {% if request.user.is_superuser %}
             <li><a href="{% url 'flower-view' path='' %}" target="_blank" >{% trans 'Task monitor' %}</a></li>
             {% endif %}

apps/templates/_nav_user.html

@@ -22,7 +22,7 @@
 
 {% if SECURITY_COMMAND_EXECUTION %}
 <li id="ops">
-    <a href="{% url 'ops:command-execution-start' %}">
+    <a href="{% url 'ops:command-execution-create' %}">
         <i class="fa fa-terminal" style="width: 14px"></i> <span class="nav-label">{% trans 'Command execution' %}</span><span class="label label-info pull-right"></span>
     </a>
 </li>

apps/terminal/migrations/0017_auto_20191125_0931.py

@@ -4,9 +4,8 @@ from django.db import migrations
 
 
 def get_storage_data(s):
-    from common.utils import get_signer
+    from common.utils import signer
     import json
-    signer = get_signer()
     value = s.value
     encrypted = s.encrypted
     if encrypted:

apps/users/models/user.py

@@ -17,15 +17,13 @@ from django.utils import timezone
 from django.shortcuts import reverse
 
 from orgs.utils import current_org
-from common.utils import get_signer, date_expired_default, get_logger, lazyproperty
+from common.utils import signer, date_expired_default, get_logger, lazyproperty
 from common import fields
 from ..signals import post_user_change_password
 
 
 __all__ = ['User']
 
-signer = get_signer()
-
 logger = get_logger(__file__)