From a9d1e281bb355d083e9d9e42236907e6fb0805b0 Mon Sep 17 00:00:00 2001 From: vapao Date: Tue, 21 Jan 2020 23:29:44 +0800 Subject: [PATCH 1/3] =?UTF-8?q?F=20=E4=BF=AE=E5=A4=8D=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=AE=A1=E5=88=92=E5=92=8C=E7=9B=91=E6=8E=A7=E4=B8=AD=E5=BF=83?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=97=B6=E9=97=B4=E4=BC=9A=E8=A2=AB=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=9A=84=E5=88=B7=E6=96=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spug_web/src/pages/monitor/Table.js | 1 - spug_web/src/pages/monitor/store.js | 9 ++++++++- spug_web/src/pages/schedule/Table.js | 3 +-- spug_web/src/pages/schedule/store.js | 5 +++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/spug_web/src/pages/monitor/Table.js b/spug_web/src/pages/monitor/Table.js index ff0be36..0b27409 100644 --- a/spug_web/src/pages/monitor/Table.js +++ b/spug_web/src/pages/monitor/Table.js @@ -85,7 +85,6 @@ class ComTable extends React.Component { }, { title: '更新于', dataIndex: 'latest_run_time', - render: value => value ? moment(value).fromNow() : null }, { title: '操作', render: info => ( diff --git a/spug_web/src/pages/monitor/store.js b/spug_web/src/pages/monitor/store.js index 6ea9cfc..cf6901f 100644 --- a/spug_web/src/pages/monitor/store.js +++ b/spug_web/src/pages/monitor/store.js @@ -5,6 +5,7 @@ */ import { observable } from "mobx"; import http from 'libs/http'; +import moment from "moment"; class Store { @observable records = []; @@ -18,7 +19,13 @@ class Store { fetchRecords = () => { this.isFetching = true; http.get('/api/monitor/') - .then(res => this.records = res) + .then(res => { + res.map(item => { + const value = item['latest_run_time']; + item['latest_run_time'] = value ? moment(value).fromNow() : null + }); + this.records = res + }) .finally(() => this.isFetching = false) }; diff --git a/spug_web/src/pages/schedule/Table.js b/spug_web/src/pages/schedule/Table.js index e27dd3d..92a058e 100644 --- a/spug_web/src/pages/schedule/Table.js +++ b/spug_web/src/pages/schedule/Table.js @@ -58,9 +58,8 @@ class ComTable extends React.Component { } }, }, { - title: '最近时间', + title: '更新于', dataIndex: 'latest_run_time', - render: value => value ? moment(value).fromNow() : 'N/A' }, { title: '描述信息', dataIndex: 'desc', diff --git a/spug_web/src/pages/schedule/store.js b/spug_web/src/pages/schedule/store.js index 4a45879..5f6558d 100644 --- a/spug_web/src/pages/schedule/store.js +++ b/spug_web/src/pages/schedule/store.js @@ -5,6 +5,7 @@ */ import { observable } from "mobx"; import http from 'libs/http'; +import moment from "moment"; class Store { @observable records = []; @@ -23,6 +24,10 @@ class Store { this.isFetching = true; http.get('/api/schedule/') .then(({types, tasks}) => { + tasks.map(item => { + const value = item['latest_run_time']; + item['latest_run_time'] = value ? moment(value).fromNow() : null + }); this.records = tasks; this.types = types }) From 4e5fe4046f3c79ec9ee32c599d1b355857ad6f7e Mon Sep 17 00:00:00 2001 From: vapao Date: Tue, 21 Jan 2020 23:57:09 +0800 Subject: [PATCH 2/3] =?UTF-8?q?F=20=E4=BF=AE=E5=A4=8D=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E8=BF=9E=E6=8E=A5=E5=8F=AF=E8=83=BD?= =?UTF-8?q?=E6=84=8F=E5=A4=96=E4=B8=A2=E5=A4=B1=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spug_api/apps/monitor/scheduler.py | 16 ++++++++++------ spug_api/apps/schedule/executors.py | 2 +- spug_api/apps/schedule/scheduler.py | 16 ++++++++++------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/spug_api/apps/monitor/scheduler.py b/spug_api/apps/monitor/scheduler.py index e09b416..d902300 100644 --- a/spug_api/apps/monitor/scheduler.py +++ b/spug_api/apps/monitor/scheduler.py @@ -3,9 +3,10 @@ # Released under the MIT License. from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger -from apscheduler import events +from apscheduler.events import EVENT_SCHEDULER_SHUTDOWN, EVENT_JOB_MAX_INSTANCES, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED from django_redis import get_redis_connection from django.utils.functional import SimpleLazyObject +from django.db import close_old_connections from apps.monitor.models import Detection from apps.alarm.models import Alarm from apps.monitor.executors import dispatch @@ -25,7 +26,9 @@ class Scheduler: def __init__(self): self.scheduler = BackgroundScheduler(timezone=self.timezone) - self.scheduler.add_listener(self._handle_event, ) + self.scheduler.add_listener( + self._handle_event, + EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES | EVENT_JOB_EXECUTED) def _record_alarm(self, obj, status): duration = seconds_to_human(time.time() - obj.latest_fault_time) @@ -63,17 +66,18 @@ class Scheduler: self._do_notify('1', obj) def _handle_event(self, event): + close_old_connections() obj = SimpleLazyObject(lambda: Detection.objects.filter(pk=event.job_id).first()) - if event.code == events.EVENT_SCHEDULER_SHUTDOWN: + if event.code == EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('monitor', '1', '调度器已关闭', '调度器意外关闭，你可以在github上提交issue', False) - elif event.code == events.EVENT_JOB_MAX_INSTANCES: + elif event.code == EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('monitor', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束，请增加调度间隔或减少任务执行耗时') - elif event.code == events.EVENT_JOB_ERROR: + elif event.code == EVENT_JOB_ERROR: logger.info(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}') Notify.make_notify('monitor', '1', f'{obj.name} - 执行异常', f'{event.exception}') - elif event.code == events.EVENT_JOB_EXECUTED: + elif event.code == EVENT_JOB_EXECUTED: obj = Detection.objects.filter(pk=event.job_id).first() old_status = obj.latest_status obj.latest_status = 0 if event.retval else 1 diff --git a/spug_api/apps/schedule/executors.py b/spug_api/apps/schedule/executors.py index 3cb7dbb..1d08ad9 100644 --- a/spug_api/apps/schedule/executors.py +++ b/spug_api/apps/schedule/executors.py @@ -26,7 +26,7 @@ def host_executor(q, host, pkey, command): cli = SSH(host.hostname, host.port, host.username, pkey=pkey) exit_code, out = cli.exec_command(command) finally: - q.put((host.id, exit_code, round(time.time() - now, 3), out.decode())) + q.put((host.id, exit_code, round(time.time() - now, 3), out.decode() if out else None)) def dispatch(command, targets): diff --git a/spug_api/apps/schedule/scheduler.py b/spug_api/apps/schedule/scheduler.py index c55bee4..6e94487 100644 --- a/spug_api/apps/schedule/scheduler.py +++ b/spug_api/apps/schedule/scheduler.py @@ -4,9 +4,10 @@ from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger from apscheduler.triggers.date import DateTrigger -from apscheduler import events +from apscheduler.events import EVENT_SCHEDULER_SHUTDOWN, EVENT_JOB_MAX_INSTANCES, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED from django_redis import get_redis_connection from django.utils.functional import SimpleLazyObject +from django.db import close_old_connections from apps.schedule.models import Task from apps.notify.models import Notify from apps.schedule.executors import dispatch @@ -26,7 +27,9 @@ class Scheduler: def __init__(self): self.scheduler = BackgroundScheduler(timezone=self.timezone) - self.scheduler.add_listener(self._handle_event, ) + self.scheduler.add_listener( + self._handle_event, + EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES | EVENT_JOB_EXECUTED) @classmethod def parse_trigger(cls, trigger, trigger_args): @@ -38,17 +41,18 @@ class Scheduler: raise TypeError(f'unknown schedule policy: {trigger!r}') def _handle_event(self, event): + close_old_connections() obj = SimpleLazyObject(lambda: Task.objects.filter(pk=event.job_id).first()) - if event.code == events.EVENT_SCHEDULER_SHUTDOWN: + if event.code == EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') Notify.make_notify('schedule', '1', '调度器已关闭', '调度器意外关闭，你可以在github上提交issue') - elif event.code == events.EVENT_JOB_MAX_INSTANCES: + elif event.code == EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') Notify.make_notify('schedule', '1', f'{obj.name} - 达到调度实例上限', '一般为上个周期的执行任务还未结束，请增加调度间隔或减少任务执行耗时') - elif event.code == events.EVENT_JOB_ERROR: + elif event.code == EVENT_JOB_ERROR: logger.info(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}') Notify.make_notify('schedule', '1', f'{obj.name} - 执行异常', f'{event.exception}') - elif event.code == events.EVENT_JOB_EXECUTED: + elif event.code == EVENT_JOB_EXECUTED: if event.retval: score = 0 for item in event.retval: From a7cef081761fe95ecc3611710d35734b20208fba Mon Sep 17 00:00:00 2001 From: vapao Date: Wed, 22 Jan 2020 14:00:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?F=20=E4=BF=AE=E5=A4=8D=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E8=B0=83=E5=BA=A6=E5=8F=AF=E8=83=BD=E4=B8=A2=E5=A4=B1=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E8=BF=9E=E6=8E=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spug_api/apps/schedule/executors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spug_api/apps/schedule/executors.py b/spug_api/apps/schedule/executors.py index 1d08ad9..6e4efaa 100644 --- a/spug_api/apps/schedule/executors.py +++ b/spug_api/apps/schedule/executors.py @@ -6,6 +6,7 @@ from threading import Thread from libs.ssh import SSH from apps.host.models import Host from apps.setting.utils import AppSetting +from django.db import close_old_connections import subprocess import time @@ -30,6 +31,7 @@ def host_executor(q, host, pkey, command): def dispatch(command, targets): + close_old_connections() threads, pkey, q = [], AppSetting.get('private_key'), Queue() for t in targets: if t == 'local':