diff --git a/spug_api/apps/monitor/scheduler.py b/spug_api/apps/monitor/scheduler.py index 5f02bf2..cb66780 100644 --- a/spug_api/apps/monitor/scheduler.py +++ b/spug_api/apps/monitor/scheduler.py @@ -2,10 +2,12 @@ from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger from apscheduler import events from django_redis import get_redis_connection +from django.utils.functional import SimpleLazyObject from apps.monitor.models import Detection from apps.alarm.models import Alarm from apps.monitor.executors import dispatch from apps.monitor.utils import seconds_to_human +from apps.notify.models import Notify from django.conf import settings from libs import AttrDict, human_datetime import logging @@ -13,6 +15,7 @@ import json import time logger = logging.getLogger("django.apps.monitor") +counter = dict() class Scheduler: @@ -46,16 +49,36 @@ class Scheduler: logger.info(f'{human_datetime()} notify job_id: {obj.id}') def _handle_event(self, event): - # TODO: notify to user + obj = SimpleLazyObject(lambda: Detection.objects.filter(pk=event.job_id).first()) if event.code == events.EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') - if event.code == events.EVENT_JOB_MAX_INSTANCES: + Notify.objects.create( + title='调度器已关闭', + source='monitor', + content='调度器意外关闭,你可以在github上提交issue', + type='1', + ) + elif event.code == events.EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') - if event.code == events.EVENT_JOB_ERROR: + if time.time() - counter.get(event.job_id, time.time()) > 3600: + counter[event.job_id] = time.time() + Notify.objects.create( + title=f'{obj.name} - 达到调度实例上限', + source='monitor', + content='一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时', + type='1', + ) + elif event.code == events.EVENT_JOB_ERROR: logger.info(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}') - if event.code == events.EVENT_JOB_MISSED: - logger.info(f'EVENT_JOB_MISSED: job_id {event.job_id}') - if event.code == events.EVENT_JOB_EXECUTED: + if time.time() - counter.get(event.job_id, time.time()) > 3600: + counter[event.job_id] = time.time() + Notify.objects.create( + title=f'{obj.name} - 执行异常', + source='monitor', + content=f'{event.exception}', + type='1', + ) + elif event.code == events.EVENT_JOB_EXECUTED: obj = Detection.objects.filter(pk=event.job_id).first() old_status = obj.latest_status obj.latest_status = 0 if event.retval else 1 diff --git a/spug_api/apps/notify/__init__.py b/spug_api/apps/notify/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spug_api/apps/notify/models.py b/spug_api/apps/notify/models.py new file mode 100644 index 0000000..54d22f8 --- /dev/null +++ b/spug_api/apps/notify/models.py @@ -0,0 +1,28 @@ +from django.db import models +from libs import ModelMixin, human_datetime + + +class Notify(models.Model, ModelMixin): + TYPES = ( + ('1', '通知'), + ('2', '待办'), + ) + SOURCES = ( + ('monitor', '监控中心'), + ('schedule', '任务计划'), + ) + title = models.CharField(max_length=255) + source = models.CharField(max_length=10, choices=SOURCES) + type = models.CharField(max_length=2, choices=TYPES) + content = models.CharField(max_length=255, null=True) + unread = models.BooleanField(default=True) + link = models.CharField(max_length=255, null=True) + + created_at = models.CharField(max_length=20, default=human_datetime) + + def __repr__(self): + return '' % self.title + + class Meta: + db_table = 'notifies' + ordering = ('-id',) diff --git a/spug_api/apps/notify/urls.py b/spug_api/apps/notify/urls.py new file mode 100644 index 0000000..2174598 --- /dev/null +++ b/spug_api/apps/notify/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from .views import * + +urlpatterns = [ + path('', NotifyView.as_view()), +] diff --git a/spug_api/apps/notify/views.py b/spug_api/apps/notify/views.py new file mode 100644 index 0000000..c8da9b0 --- /dev/null +++ b/spug_api/apps/notify/views.py @@ -0,0 +1,17 @@ +from django.views.generic import View +from apps.notify.models import Notify +from libs import json_response, JsonParser, Argument + + +class NotifyView(View): + def get(self, request): + notifies = Notify.objects.filter(unread=True) + return json_response(notifies) + + def patch(self, request): + form, error = JsonParser( + Argument('id', type=int, help='参数错误') + ).parse(request.body) + if error is None: + Notify.objects.filter(pk=form.id).update(unread=False) + return json_response(error=error) diff --git a/spug_api/apps/schedule/scheduler.py b/spug_api/apps/schedule/scheduler.py index 629acf4..b764791 100644 --- a/spug_api/apps/schedule/scheduler.py +++ b/spug_api/apps/schedule/scheduler.py @@ -3,15 +3,19 @@ from apscheduler.triggers.interval import IntervalTrigger from apscheduler.triggers.date import DateTrigger from apscheduler import events from django_redis import get_redis_connection +from django.utils.functional import SimpleLazyObject from apps.schedule.models import Task +from apps.notify.models import Notify from apps.schedule.executors import dispatch from apps.alarm.utils import auto_clean_records from django.conf import settings from libs import AttrDict, human_datetime import logging import json +import time logger = logging.getLogger("django.apps.scheduler") +counter = dict() class Scheduler: @@ -31,16 +35,36 @@ class Scheduler: raise TypeError(f'unknown schedule policy: {trigger!r}') def _handle_event(self, event): - # TODO: notify to user + obj = SimpleLazyObject(lambda: Task.objects.filter(pk=event.job_id).first()) if event.code == events.EVENT_SCHEDULER_SHUTDOWN: logger.info(f'EVENT_SCHEDULER_SHUTDOWN: {event}') - if event.code == events.EVENT_JOB_MAX_INSTANCES: + Notify.objects.create( + title='调度器已关闭', + source='schedule', + content='调度器意外关闭,你可以在github上提交issue', + type='1', + ) + elif event.code == events.EVENT_JOB_MAX_INSTANCES: logger.info(f'EVENT_JOB_MAX_INSTANCES: {event}') - if event.code == events.EVENT_JOB_ERROR: + if time.time() - counter.get(event.job_id, 0) > 3600: + counter[event.job_id] = time.time() + Notify.objects.create( + title=f'{obj.name} - 达到调度实例上限', + source='schedule', + content='一般为上个周期的执行任务还未结束,请增加调度间隔或减少任务执行耗时', + type='1', + ) + elif event.code == events.EVENT_JOB_ERROR: logger.info(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}') - if event.code == events.EVENT_JOB_MISSED: - logger.info(f'EVENT_JOB_MISSED: job_id {event.job_id}') - if event.code == events.EVENT_JOB_EXECUTED: + if time.time() - counter.get(event.job_id, 0) > 3600: + counter[event.job_id] = time.time() + Notify.objects.create( + title=f'{obj.name} - 执行异常', + source='schedule', + content=f'{event.exception}', + type='1', + ) + elif event.code == events.EVENT_JOB_EXECUTED: if event.retval: score = 0 for item in event.retval: @@ -50,6 +74,14 @@ class Scheduler: latest_run_time=human_datetime(event.scheduled_run_time), latest_output=json.dumps(event.retval) ) + if score != 0 and time.time() - counter.get(event.job_id, 0) > 3600: + counter[event.job_id] = time.time() + Notify.objects.create( + title=f'{obj.name} - 执行失败', + source='schedule', + content='请在任务计划中查看失败详情', + type='1', + ) def _init_builtin_jobs(self): self.scheduler.add_job(auto_clean_records, 'cron', hour=0, minute=0) diff --git a/spug_api/spug/settings.py b/spug_api/spug/settings.py index 038ec47..8621c0a 100644 --- a/spug_api/spug/settings.py +++ b/spug_api/spug/settings.py @@ -41,6 +41,7 @@ INSTALLED_APPS = [ 'apps.config', 'apps.app', 'apps.deploy', + 'apps.notify', ] MIDDLEWARE = [ diff --git a/spug_api/spug/urls.py b/spug_api/spug/urls.py index cae2353..57e5907 100644 --- a/spug_api/spug/urls.py +++ b/spug_api/spug/urls.py @@ -27,5 +27,6 @@ urlpatterns = [ path('app/', include('apps.app.urls')), path('deploy/', include('apps.deploy.urls')), path('home/', include('apps.home.urls')), + path('notify/', include('apps.notify.urls')), path('apis/', include('apps.apis.urls')), ]