From 09fc2776df4dfda5aee42b6ffb86f709397bdbd7 Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 30 Mar 2018 22:03:43 +0800 Subject: [PATCH] [Update] Support history view --- apps/ops/ansible/callback.py | 8 +- apps/ops/ansible/display.py | 19 ++++ apps/ops/ansible/inventory.py | 5 + apps/ops/ansible/runner.py | 37 +++++--- apps/ops/api.py | 28 +++++- apps/ops/models.py | 38 +++++--- .../templates/ops/adhoc_history_output.html | 93 +++++++++++++++++++ apps/ops/urls/api_urls.py | 1 + apps/ops/urls/view_urls.py | 1 + apps/ops/views.py | 13 +++ 10 files changed, 219 insertions(+), 24 deletions(-) create mode 100644 apps/ops/ansible/display.py create mode 100644 apps/ops/templates/ops/adhoc_history_output.html diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index 810b14c51..0af872b25 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -1,14 +1,18 @@ # ~*~ coding: utf-8 ~*~ +import sys + from ansible.plugins.callback import CallbackBase from ansible.plugins.callback.default import CallbackModule +from .display import TeeObj + class AdHocResultCallback(CallbackModule): """ Task result Callback """ - def __init__(self, display=None, options=None): + def __init__(self, display=None, options=None, file_obj=None): # result_raw example: { # "ok": {"hostname": {"task_name": {}，...},..}, # "failed": {"hostname": {"task_name": {}..}, ..}, @@ -22,6 +26,8 @@ class AdHocResultCallback(CallbackModule): self.results_raw = dict(ok={}, failed={}, unreachable={}, skipped={}) self.results_summary = dict(contacted=[], dark={}) super().__init__() + if file_obj is not None: + sys.stdout = TeeObj(file_obj) def gather_result(self, t, res): self._clean_results(res._result, res._task.action) diff --git a/apps/ops/ansible/display.py b/apps/ops/ansible/display.py new file mode 100644 index 000000000..1494eb5ef --- /dev/null +++ b/apps/ops/ansible/display.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# + +import sys + + +class TeeObj: + origin_stdout = sys.stdout + + def __init__(self, file_obj): + self.file_obj = file_obj + + def write(self, msg): + self.origin_stdout.write(msg) + self.file_obj.write(msg.replace('*', '')) + + def flush(self): + self.origin_stdout.flush() + self.file_obj.flush() diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index e2f318f71..707855cc3 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -132,6 +132,8 @@ class BaseInventory(InventoryManager): parent.add_child_group(child) def parse_hosts(self): + group_all = self.get_or_create_group('all') + ungrouped = self.get_or_create_group('ungrouped') for host_data in self.host_list: host = self.host_manager_class(host_data=host_data) self.hosts[host_data['hostname']] = host @@ -140,6 +142,9 @@ class BaseInventory(InventoryManager): for group_name in groups_data: group = self.get_or_create_group(group_name) group.add_host(host) + else: + ungrouped.add_host(host) + group_all.add_host(host) def parse_sources(self, cache=False): self.parse_groups() diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index d9a8c7e6e..3e168e987 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -9,6 +9,7 @@ from ansible.parsing.dataloader import DataLoader from ansible.executor.playbook_executor import PlaybookExecutor from ansible.playbook.play import Play import ansible.constants as C +from ansible.utils.display import Display from .callback import AdHocResultCallback, PlaybookResultCallBack, \ CommandResultCallback @@ -21,6 +22,13 @@ C.HOST_KEY_CHECKING = False logger = get_logger(__name__) +class CustomDisplay(Display): + def display(self, msg, color=None, stderr=False, screen_only=False, log_only=False): + pass + +display = CustomDisplay() + + Options = namedtuple('Options', [ 'listtags', 'listtasks', 'listhosts', 'syntax', 'connection', 'module_path', 'forks', 'remote_user', 'private_key_file', 'timeout', @@ -123,20 +131,22 @@ class AdHocRunner: ADHoc Runner接口 """ results_callback_class = AdHocResultCallback + results_callback = None loader_class = DataLoader variable_manager_class = VariableManager - options = get_default_options() default_options = get_default_options() def __init__(self, inventory, options=None): - if options: - self.options = options + self.options = self.update_options(options) self.inventory = inventory self.loader = DataLoader() self.variable_manager = VariableManager( loader=self.loader, inventory=self.inventory ) + def get_result_callback(self, file_obj=None): + return self.__class__.results_callback_class(file_obj=file_obj) + @staticmethod def check_module_args(module_name, module_args=''): if module_name in C.MODULE_REQUIRE_ARGS and not module_args: @@ -160,19 +170,24 @@ class AdHocRunner: cleaned_tasks.append(task) return cleaned_tasks - def set_option(self, k, v): - kwargs = {k: v} - self.options = self.options._replace(**kwargs) + def update_options(self, options): + if options and isinstance(options, dict): + options = self.__class__.default_options._replace(**options) + else: + options = self.__class__.default_options + return options - def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no'): + def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no', file_obj=None): """ :param tasks: [{'action': {'module': 'shell', 'args': 'ls'}, ...}, ] :param pattern: all, *, or others :param play_name: The play name + :param gather_facts: + :param file_obj: logging to file_obj :return: """ self.check_pattern(pattern) - results_callback = self.results_callback_class() + self.results_callback = self.get_result_callback(file_obj) cleaned_tasks = self.clean_tasks(tasks) play_source = dict( @@ -193,16 +208,16 @@ class AdHocRunner: variable_manager=self.variable_manager, loader=self.loader, options=self.options, - stdout_callback=results_callback, + stdout_callback=self.results_callback, passwords=self.options.passwords, ) - logger.debug("Get inventory matched hosts: {}".format( + print("Get matched hosts: {}".format( self.inventory.get_matched_hosts(pattern) )) try: tqm.run(play) - return results_callback + return self.results_callback except Exception as e: raise AnsibleError(e) finally: diff --git a/apps/ops/api.py b/apps/ops/api.py index 09f5e67e7..92933098c 100644 --- a/apps/ops/api.py +++ b/apps/ops/api.py @@ -1,8 +1,11 @@ # ~*~ coding: utf-8 ~*~ +import uuid +import re - +from django.core.cache import cache from django.shortcuts import get_object_or_404 from rest_framework import viewsets, generics +from rest_framework.generics import RetrieveAPIView from rest_framework.views import Response from .hands import IsSuperUser @@ -58,3 +61,26 @@ class AdHocRunHistorySet(viewsets.ModelViewSet): adhoc = get_object_or_404(AdHoc, id=adhoc_id) self.queryset = self.queryset.filter(adhoc=adhoc) return self.queryset + + +class AdHocHistoryOutputAPI(RetrieveAPIView): + queryset = AdHocRunHistory.objects.all() + permission_classes = (IsSuperUser,) + buff_size = 1024 * 10 + end = False + + def retrieve(self, request, *args, **kwargs): + history = self.get_object() + mark = request.query_params.get("mark") or str(uuid.uuid4()) + + with open(history.log_path, 'r') as f: + offset = cache.get(mark, 0) + f.seek(offset) + data = f.read(self.buff_size).replace('\n', '\r\n') + print(repr(data)) + mark = str(uuid.uuid4()) + cache.set(mark, f.tell(), 5) + + if history.is_finished and data == '': + self.end = True + return Response({"data": data, 'end': self.end, 'mark': mark}) diff --git a/apps/ops/models.py b/apps/ops/models.py index 1120bb206..8a34e125a 100644 --- a/apps/ops/models.py +++ b/apps/ops/models.py @@ -2,16 +2,21 @@ import json import uuid - +import os import time +import datetime + from django.db import models +from django.conf import settings from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask +from django_celery_beat.models import CrontabSchedule, IntervalSchedule, \ + PeriodicTask from common.utils import get_signer, get_logger -from common.celery import delete_celery_periodic_task, create_or_update_celery_periodic_tasks, \ - disable_celery_periodic_task +from common.celery import delete_celery_periodic_task, \ + create_or_update_celery_periodic_tasks, \ + disable_celery_periodic_task from .ansible import AdHocRunner, AnsibleError from .inventory import JMSInventory @@ -209,7 +214,8 @@ class AdHoc(models.Model): history = AdHocRunHistory(adhoc=self, task=self.task) time_start = time.time() try: - raw, summary = self._run_only() + with open(history.log_path, 'w') as f: + raw, summary = self._run_only(file_obj=f) history.is_finished = True if summary.get('dark'): history.is_success = False @@ -225,13 +231,15 @@ class AdHoc(models.Model): history.timedelta = time.time() - time_start history.save() - def _run_only(self): - runner = AdHocRunner(self.inventory) - for k, v in self.options.items(): - runner.set_option(k, v) - + def _run_only(self, file_obj=None): + runner = AdHocRunner(self.inventory, options=self.options) try: - result = runner.run(self.tasks, self.pattern, self.task.name) + result = runner.run( + self.tasks, + self.pattern, + self.task.name, + file_obj=file_obj, + ) return result.results_raw, result.results_summary except AnsibleError as e: logger.warn("Failed run adhoc {}, {}".format(self.task.name, e)) @@ -316,6 +324,14 @@ class AdHocRunHistory(models.Model): def short_id(self): return str(self.id).split('-')[-1] + @property + def log_path(self): + dt = datetime.datetime.now().strftime('%Y-%m-%d') + log_dir = os.path.join(settings.PROJECT_DIR, 'data', 'ansible', dt) + if not os.path.exists(log_dir): + os.makedirs(log_dir) + return os.path.join(log_dir, str(self.id) + '.log') + @property def result(self): if self._result: diff --git a/apps/ops/templates/ops/adhoc_history_output.html b/apps/ops/templates/ops/adhoc_history_output.html new file mode 100644 index 000000000..34f8078bf --- /dev/null +++ b/apps/ops/templates/ops/adhoc_history_output.html @@ -0,0 +1,93 @@ +{% load static %} + + + + term.js + + + + +
+
+
+
+ + + + + + diff --git a/apps/ops/urls/api_urls.py b/apps/ops/urls/api_urls.py index ab007c383..26c1c89a1 100644 --- a/apps/ops/urls/api_urls.py +++ b/apps/ops/urls/api_urls.py @@ -15,6 +15,7 @@ router.register(r'v1/history', api.AdHocRunHistorySet, 'history') urlpatterns = [ url(r'^v1/tasks/(?P[0-9a-zA-Z\-]{36})/run/$', api.TaskRun.as_view(), name='task-run'), + url(r'^v1/history/(?P[0-9a-zA-Z\-]{36})/output/$', api.AdHocHistoryOutputAPI.as_view(), name='history-output'), ] urlpatterns += router.urls diff --git a/apps/ops/urls/view_urls.py b/apps/ops/urls/view_urls.py index 080bdd06c..980f9d070 100644 --- a/apps/ops/urls/view_urls.py +++ b/apps/ops/urls/view_urls.py @@ -18,4 +18,5 @@ urlpatterns = [ url(r'^adhoc/(?P[0-9a-zA-Z\-]{36})/$', views.AdHocDetailView.as_view(), name='adhoc-detail'), url(r'^adhoc/(?P[0-9a-zA-Z\-]{36})/history/$', views.AdHocHistoryView.as_view(), name='adhoc-history'), url(r'^adhoc/history/(?P[0-9a-zA-Z\-]{36})/$', views.AdHocHistoryDetailView.as_view(), name='adhoc-history-detail'), + url(r'^adhoc/history/(?P[0-9a-zA-Z\-]{36})/output/$', views.AdHocHistoryOutputView.as_view(), name='adhoc-history-output'), ] diff --git a/apps/ops/views.py b/apps/ops/views.py index ba9e2cfeb..fbc942b56 100644 --- a/apps/ops/views.py +++ b/apps/ops/views.py @@ -112,6 +112,19 @@ class AdHocHistoryDetailView(AdminUserRequiredMixin, DetailView): model = AdHocRunHistory template_name = 'ops/adhoc_history_detail.html' + def get_context_data(self, **kwargs): + context = { + 'app': _('Ops'), + 'action': _('Run history detail'), + } + kwargs.update(context) + return super().get_context_data(**kwargs) + + +class AdHocHistoryOutputView(AdminUserRequiredMixin, DetailView): + model = AdHocRunHistory + template_name = 'ops/adhoc_history_output.html' + def get_context_data(self, **kwargs): context = { 'app': _('Ops'),