[Update] Support history view

apps/ops/ansible/callback.py

@@ -1,14 +1,18 @@
 # ~*~ coding: utf-8 ~*~
 
+import sys
+
 from ansible.plugins.callback import CallbackBase
 from ansible.plugins.callback.default import CallbackModule
 
+from .display import TeeObj
+
 
 class AdHocResultCallback(CallbackModule):
     """
     Task result Callback
     """
-    def __init__(self, display=None, options=None):
+    def __init__(self, display=None, options=None, file_obj=None):
         # result_raw example: {
         #   "ok": {"hostname": {"task_name": {}，...},..},
         #   "failed": {"hostname": {"task_name": {}..}, ..},
@@ -22,6 +26,8 @@ class AdHocResultCallback(CallbackModule):
         self.results_raw = dict(ok={}, failed={}, unreachable={}, skipped={})
         self.results_summary = dict(contacted=[], dark={})
         super().__init__()
+        if file_obj is not None:
+            sys.stdout = TeeObj(file_obj)
 
     def gather_result(self, t, res):
         self._clean_results(res._result, res._task.action)

apps/ops/ansible/display.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+#
+
+import sys
+
+
+class TeeObj:
+    origin_stdout = sys.stdout
+
+    def __init__(self, file_obj):
+        self.file_obj = file_obj
+
+    def write(self, msg):
+        self.origin_stdout.write(msg)
+        self.file_obj.write(msg.replace('*', ''))
+
+    def flush(self):
+        self.origin_stdout.flush()
+        self.file_obj.flush()

apps/ops/ansible/inventory.py

@@ -132,6 +132,8 @@ class BaseInventory(InventoryManager):
                 parent.add_child_group(child)
 
     def parse_hosts(self):
+        group_all = self.get_or_create_group('all')
+        ungrouped = self.get_or_create_group('ungrouped')
         for host_data in self.host_list:
             host = self.host_manager_class(host_data=host_data)
             self.hosts[host_data['hostname']] = host
@@ -140,6 +142,9 @@ class BaseInventory(InventoryManager):
                 for group_name in groups_data:
                     group = self.get_or_create_group(group_name)
                     group.add_host(host)
+            else:
+                ungrouped.add_host(host)
+            group_all.add_host(host)
 
     def parse_sources(self, cache=False):
         self.parse_groups()

apps/ops/ansible/runner.py

@@ -9,6 +9,7 @@ from ansible.parsing.dataloader import DataLoader
 from ansible.executor.playbook_executor import PlaybookExecutor
 from ansible.playbook.play import Play
 import ansible.constants as C
+from ansible.utils.display import Display
 
 from .callback import AdHocResultCallback, PlaybookResultCallBack, \
     CommandResultCallback
@@ -21,6 +22,13 @@ C.HOST_KEY_CHECKING = False
 logger = get_logger(__name__)
 
 
+class CustomDisplay(Display):
+    def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False):
+        pass
+
+display = CustomDisplay()
+
+
 Options = namedtuple('Options', [
     'listtags', 'listtasks', 'listhosts', 'syntax', 'connection',
     'module_path', 'forks', 'remote_user', 'private_key_file', 'timeout',
@@ -123,20 +131,22 @@ class AdHocRunner:
     ADHoc Runner接口
     """
     results_callback_class = AdHocResultCallback
+    results_callback = None
     loader_class = DataLoader
     variable_manager_class = VariableManager
-    options = get_default_options()
     default_options = get_default_options()
 
     def __init__(self, inventory, options=None):
-        if options:
+        self.options = self.update_options(options)
-            self.options = options
         self.inventory = inventory
         self.loader = DataLoader()
         self.variable_manager = VariableManager(
             loader=self.loader, inventory=self.inventory
         )
 
+    def get_result_callback(self, file_obj=None):
+        return self.__class__.results_callback_class(file_obj=file_obj)
+
     @staticmethod
     def check_module_args(module_name, module_args=''):
         if module_name in C.MODULE_REQUIRE_ARGS and not module_args:
@@ -160,19 +170,24 @@ class AdHocRunner:
             cleaned_tasks.append(task)
         return cleaned_tasks
 
-    def set_option(self, k, v):
+    def update_options(self, options):
-        kwargs = {k: v}
+        if options and isinstance(options, dict):
-        self.options = self.options._replace(**kwargs)
+            options = self.__class__.default_options._replace(**options)
+        else:
+            options = self.__class__.default_options
+        return options
 
-    def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no'):
+    def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no', file_obj=None):
         """
         :param tasks: [{'action': {'module': 'shell', 'args': 'ls'}, ...}, ]
         :param pattern: all, *, or others
         :param play_name: The play name
+        :param gather_facts:
+        :param file_obj: logging to file_obj
         :return:
         """
         self.check_pattern(pattern)
-        results_callback = self.results_callback_class()
+        self.results_callback = self.get_result_callback(file_obj)
         cleaned_tasks = self.clean_tasks(tasks)
 
         play_source = dict(
@@ -193,16 +208,16 @@ class AdHocRunner:
             variable_manager=self.variable_manager,
             loader=self.loader,
             options=self.options,
-            stdout_callback=results_callback,
+            stdout_callback=self.results_callback,
             passwords=self.options.passwords,
         )
-        logger.debug("Get inventory matched hosts: {}".format(
+        print("Get matched hosts: {}".format(
             self.inventory.get_matched_hosts(pattern)
         ))
 
         try:
             tqm.run(play)
-            return results_callback
+            return self.results_callback
         except Exception as e:
             raise AnsibleError(e)
         finally:

apps/ops/api.py

@@ -1,8 +1,11 @@
 # ~*~ coding: utf-8 ~*~
+import uuid
+import re
 
-
+from django.core.cache import cache
 from django.shortcuts import get_object_or_404
 from rest_framework import viewsets, generics
+from rest_framework.generics import RetrieveAPIView
 from rest_framework.views import Response
 
 from .hands import IsSuperUser
@@ -58,3 +61,26 @@ class AdHocRunHistorySet(viewsets.ModelViewSet):
             adhoc = get_object_or_404(AdHoc, id=adhoc_id)
             self.queryset = self.queryset.filter(adhoc=adhoc)
         return self.queryset
+
+
+class AdHocHistoryOutputAPI(RetrieveAPIView):
+    queryset = AdHocRunHistory.objects.all()
+    permission_classes = (IsSuperUser,)
+    buff_size = 1024 * 10
+    end = False
+
+    def retrieve(self, request, *args, **kwargs):
+        history = self.get_object()
+        mark = request.query_params.get("mark") or str(uuid.uuid4())
+
+        with open(history.log_path, 'r') as f:
+            offset = cache.get(mark, 0)
+            f.seek(offset)
+            data = f.read(self.buff_size).replace('\n', '\r\n')
+            print(repr(data))
+            mark = str(uuid.uuid4())
+            cache.set(mark, f.tell(), 5)
+
+            if history.is_finished and data == '':
+                self.end = True
+            return Response({"data": data, 'end': self.end, 'mark': mark})

apps/ops/models.py

@@ -2,16 +2,21 @@
 
 import json
 import uuid
-
+import os
 import time
+import datetime
+
 from django.db import models
+from django.conf import settings
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask
+from django_celery_beat.models import CrontabSchedule, IntervalSchedule, \
+    PeriodicTask
 
 from common.utils import get_signer, get_logger
-from common.celery import delete_celery_periodic_task, create_or_update_celery_periodic_tasks, \
+from common.celery import delete_celery_periodic_task, \
-     disable_celery_periodic_task
+    create_or_update_celery_periodic_tasks, \
+    disable_celery_periodic_task
 from .ansible import AdHocRunner, AnsibleError
 from .inventory import JMSInventory
 
@@ -209,7 +214,8 @@ class AdHoc(models.Model):
         history = AdHocRunHistory(adhoc=self, task=self.task)
         time_start = time.time()
         try:
-            raw, summary = self._run_only()
+            with open(history.log_path, 'w') as f:
+                raw, summary = self._run_only(file_obj=f)
             history.is_finished = True
             if summary.get('dark'):
                 history.is_success = False
@@ -225,13 +231,15 @@ class AdHoc(models.Model):
             history.timedelta = time.time() - time_start
             history.save()
 
-    def _run_only(self):
+    def _run_only(self, file_obj=None):
-        runner = AdHocRunner(self.inventory)
+        runner = AdHocRunner(self.inventory, options=self.options)
-        for k, v in self.options.items():
-            runner.set_option(k, v)
-
         try:
-            result = runner.run(self.tasks, self.pattern, self.task.name)
+            result = runner.run(
+                self.tasks,
+                self.pattern,
+                self.task.name,
+                file_obj=file_obj,
+            )
             return result.results_raw, result.results_summary
         except AnsibleError as e:
             logger.warn("Failed run adhoc {}, {}".format(self.task.name, e))
@@ -316,6 +324,14 @@ class AdHocRunHistory(models.Model):
     def short_id(self):
         return str(self.id).split('-')[-1]
 
+    @property
+    def log_path(self):
+        dt = datetime.datetime.now().strftime('%Y-%m-%d')
+        log_dir = os.path.join(settings.PROJECT_DIR, 'data', 'ansible', dt)
+        if not os.path.exists(log_dir):
+            os.makedirs(log_dir)
+        return os.path.join(log_dir, str(self.id) + '.log')
+
     @property
     def result(self):
         if self._result:

apps/ops/templates/ops/adhoc_history_output.html

@@ -0,0 +1,93 @@
+{% load static %}
+<!doctype html>
+<html>
+<head>
+    <title>term.js</title>
+    <script src="{% static 'js/jquery-2.1.1.js' %}"></script>
+    <style>
+      html {
+        background: #000;
+      }
+      h1 {
+        margin-bottom: 20px;
+        font: 20px/1.5 sans-serif;
+      }
+      .terminal {
+        float: left;
+        font-family: 'Monaco', 'Consolas', "DejaVu Sans Mono", "Liberation Mono", monospace;
+        font-size: 14px;
+        color: #f0f0f0;
+        background-color: #555;
+        padding: 20px 20px 20px;
+      }
+      .terminal-cursor {
+        color: #000;
+        background: #f0f0f0;
+      }
+    </style>
+</head>
+<body>
+<div class="container">
+    <div id="term">
+    </div>
+</div>
+</body>
+
+
+<script src="{% static 'js/term.js' %}"></script>
+<script>
+    var rowHeight = 1;
+    var colWidth = 1;
+    var mark = '';
+    var url = "{% url 'api-ops:history-output' pk=object.id %}";
+    var term;
+    var end = false;
+
+    function calWinSize() {
+        var t = $('.terminal');
+        console.log(t.height());
+        rowHeight = 1.00 * t.height() / 24;
+        colWidth = 1.00 * t.width() / 80;
+    }
+    function resize() {
+       var rows = Math.floor(window.innerHeight / rowHeight) - 2;
+       var cols = Math.floor(window.innerWidth / colWidth) - 5;
+       term.resize(cols, rows);
+    }
+    function requestAndWrite() {
+        if (!end) {
+            $.ajax({
+                url: url + '?mark=' + mark,
+                method: "GET",
+                contentType: "application/json; charset=utf-8"
+            }).done(function(data, textStatue, jqXHR) {
+                term.write(data.data);
+                mark = data.mark;
+                if (data.end){
+                    end = true
+                }
+            }).fail(function(jqXHR, textStatus, errorThrown) {
+            });
+        }
+    }
+    $(document).ready(function () {
+        term = new Terminal({
+            cols: 80,
+            rows: 24,
+            useStyle: true,
+            screenKeys: false
+        });
+        term.open();
+        term.on('data', function (data) {
+            term.write(data.replace('\r', '\r\n'))
+        });
+        calWinSize();
+        resize();
+        $('.terminal').detach().appendTo('#term');
+        term.write('\x1b[31mWelcome to term.js!\x1b[m\r\n');
+        setInterval(function () {
+            requestAndWrite()
+        }, 100)
+    });
+</script>
+</html>

apps/ops/urls/api_urls.py

@@ -15,6 +15,7 @@ router.register(r'v1/history', api.AdHocRunHistorySet, 'history')
 
 urlpatterns = [
     url(r'^v1/tasks/(?P<pk>[0-9a-zA-Z\-]{36})/run/$', api.TaskRun.as_view(), name='task-run'),
+    url(r'^v1/history/(?P<pk>[0-9a-zA-Z\-]{36})/output/$', api.AdHocHistoryOutputAPI.as_view(), name='history-output'),
 ]
 
 urlpatterns += router.urls

apps/ops/urls/view_urls.py

@@ -18,4 +18,5 @@ urlpatterns = [
     url(r'^adhoc/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.AdHocDetailView.as_view(), name='adhoc-detail'),
     url(r'^adhoc/(?P<pk>[0-9a-zA-Z\-]{36})/history/$', views.AdHocHistoryView.as_view(), name='adhoc-history'),
     url(r'^adhoc/history/(?P<pk>[0-9a-zA-Z\-]{36})/$', views.AdHocHistoryDetailView.as_view(), name='adhoc-history-detail'),
+    url(r'^adhoc/history/(?P<pk>[0-9a-zA-Z\-]{36})/output/$', views.AdHocHistoryOutputView.as_view(), name='adhoc-history-output'),
 ]

apps/ops/views.py

@@ -112,6 +112,19 @@ class AdHocHistoryDetailView(AdminUserRequiredMixin, DetailView):
     model = AdHocRunHistory
     template_name = 'ops/adhoc_history_detail.html'
 
+    def get_context_data(self, **kwargs):
+        context = {
+            'app': _('Ops'),
+            'action': _('Run history detail'),
+        }
+        kwargs.update(context)
+        return super().get_context_data(**kwargs)
+
+
+class AdHocHistoryOutputView(AdminUserRequiredMixin, DetailView):
+    model = AdHocRunHistory
+    template_name = 'ops/adhoc_history_output.html'
+
     def get_context_data(self, **kwargs):
         context = {
             'app': _('Ops'),