mirror of https://github.com/openspug/spug
upgrade monitor module
parent
b4bcb2606a
commit
3553036862
|
@ -4,20 +4,18 @@
|
||||||
from apscheduler.schedulers.background import BackgroundScheduler
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
from apscheduler.executors.pool import ThreadPoolExecutor
|
from apscheduler.executors.pool import ThreadPoolExecutor
|
||||||
from apscheduler.triggers.interval import IntervalTrigger
|
from apscheduler.triggers.interval import IntervalTrigger
|
||||||
from apscheduler.events import EVENT_SCHEDULER_SHUTDOWN, EVENT_JOB_MAX_INSTANCES, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
|
from apscheduler.events import EVENT_SCHEDULER_SHUTDOWN, EVENT_JOB_MAX_INSTANCES, EVENT_JOB_ERROR
|
||||||
from django_redis import get_redis_connection
|
from django_redis import get_redis_connection
|
||||||
from django.utils.functional import SimpleLazyObject
|
from django.utils.functional import SimpleLazyObject
|
||||||
from django.db import close_old_connections
|
from django.db import close_old_connections
|
||||||
from apps.monitor.models import Detection
|
from apps.monitor.models import Detection
|
||||||
from apps.alarm.models import Alarm
|
|
||||||
from apps.monitor.utils import seconds_to_human
|
|
||||||
from apps.notify.models import Notify
|
from apps.notify.models import Notify
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from libs import spug, AttrDict, human_datetime, human_diff_time
|
from libs import AttrDict, human_datetime
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
|
from random import randint
|
||||||
import logging
|
import logging
|
||||||
import json
|
import json
|
||||||
import time
|
|
||||||
|
|
||||||
MONITOR_WORKER_KEY = settings.MONITOR_WORKER_KEY
|
MONITOR_WORKER_KEY = settings.MONITOR_WORKER_KEY
|
||||||
|
|
||||||
|
@ -32,46 +30,6 @@ class Scheduler:
|
||||||
EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES
|
EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES
|
||||||
)
|
)
|
||||||
|
|
||||||
def _record_alarm(self, obj, status):
|
|
||||||
duration = seconds_to_human(time.time() - obj.latest_fault_time)
|
|
||||||
Alarm.objects.create(
|
|
||||||
name=obj.name,
|
|
||||||
type=obj.get_type_display(),
|
|
||||||
status=status,
|
|
||||||
duration=duration,
|
|
||||||
notify_grp=obj.notify_grp,
|
|
||||||
notify_mode=obj.notify_mode)
|
|
||||||
|
|
||||||
def _do_notify(self, event, obj, out):
|
|
||||||
obj.out = out
|
|
||||||
obj.grp = json.loads(obj.notify_grp)
|
|
||||||
if event == '2':
|
|
||||||
obj.duration = human_diff_time(datetime.now(), datetime.fromtimestamp(obj.latest_fault_time))
|
|
||||||
for mode in json.loads(obj.notify_mode):
|
|
||||||
if mode == '1':
|
|
||||||
spug.notify_by_wx(event, obj)
|
|
||||||
elif mode == '3':
|
|
||||||
spug.notify_by_dd(event, obj)
|
|
||||||
elif mode == '4':
|
|
||||||
spug.notify_by_email(event, obj)
|
|
||||||
elif mode == '5':
|
|
||||||
spug.notify_by_qy_wx(event, obj)
|
|
||||||
|
|
||||||
def _handle_notify(self, obj, is_notified, out):
|
|
||||||
if obj.latest_status == 0:
|
|
||||||
if is_notified:
|
|
||||||
self._record_alarm(obj, '2')
|
|
||||||
logging.warning(f'{human_datetime()} recover job_id: {obj.id}, job_name: {obj.name}')
|
|
||||||
self._do_notify('2', obj, out)
|
|
||||||
else:
|
|
||||||
if obj.fault_times >= obj.threshold:
|
|
||||||
if time.time() - obj.latest_notify_time >= obj.quiet * 60:
|
|
||||||
obj.latest_notify_time = int(time.time())
|
|
||||||
obj.save()
|
|
||||||
self._record_alarm(obj, '1')
|
|
||||||
logging.warning(f'{human_datetime()} notify job_id: {obj.id}, job_name: {obj.name}')
|
|
||||||
self._do_notify('1', obj, out)
|
|
||||||
|
|
||||||
def _handle_event(self, event):
|
def _handle_event(self, event):
|
||||||
close_old_connections()
|
close_old_connections()
|
||||||
obj = SimpleLazyObject(lambda: Detection.objects.filter(pk=event.job_id).first())
|
obj = SimpleLazyObject(lambda: Detection.objects.filter(pk=event.job_id).first())
|
||||||
|
@ -84,20 +42,6 @@ class Scheduler:
|
||||||
elif event.code == EVENT_JOB_ERROR:
|
elif event.code == EVENT_JOB_ERROR:
|
||||||
logging.warning(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}')
|
logging.warning(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}')
|
||||||
Notify.make_notify('monitor', '1', f'{obj.name} - 执行异常', f'{event.exception}')
|
Notify.make_notify('monitor', '1', f'{obj.name} - 执行异常', f'{event.exception}')
|
||||||
elif event.code == EVENT_JOB_EXECUTED:
|
|
||||||
is_ok, out = event.retval
|
|
||||||
is_notified = True if obj.latest_notify_time else False
|
|
||||||
if obj.latest_status in [0, None] and is_ok is False:
|
|
||||||
obj.latest_fault_time = int(time.time())
|
|
||||||
if is_ok:
|
|
||||||
obj.latest_notify_time = 0
|
|
||||||
obj.fault_times = 0
|
|
||||||
else:
|
|
||||||
obj.fault_times += 1
|
|
||||||
obj.latest_status = 0 if is_ok else 1
|
|
||||||
obj.latest_run_time = human_datetime(event.scheduled_run_time)
|
|
||||||
obj.save()
|
|
||||||
self._handle_notify(obj, is_notified, out)
|
|
||||||
|
|
||||||
def _dispatch(self, task_id, tp, targets, extra, threshold, quiet):
|
def _dispatch(self, task_id, tp, targets, extra, threshold, quiet):
|
||||||
close_old_connections()
|
close_old_connections()
|
||||||
|
@ -109,12 +53,14 @@ class Scheduler:
|
||||||
def _init(self):
|
def _init(self):
|
||||||
self.scheduler.start()
|
self.scheduler.start()
|
||||||
for item in Detection.objects.filter(is_active=True):
|
for item in Detection.objects.filter(is_active=True):
|
||||||
|
now = datetime.now()
|
||||||
trigger = IntervalTrigger(minutes=int(item.rate), timezone=self.timezone)
|
trigger = IntervalTrigger(minutes=int(item.rate), timezone=self.timezone)
|
||||||
self.scheduler.add_job(
|
self.scheduler.add_job(
|
||||||
self._dispatch,
|
self._dispatch,
|
||||||
trigger,
|
trigger,
|
||||||
id=str(item.id),
|
id=str(item.id),
|
||||||
args=(item.id, item.type, item.targets, item.extra, item.threshold, item.quiet),
|
args=(item.id, item.type, item.targets, item.extra, item.threshold, item.quiet),
|
||||||
|
next_run_time=now + timedelta(seconds=randint(0, 60))
|
||||||
)
|
)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
|
|
|
@ -135,7 +135,8 @@ export default observer(function () {
|
||||||
<Button type="link" onClick={() => setShowSelector(true)}>选择主机</Button>
|
<Button type="link" onClick={() => setShowSelector(true)}>选择主机</Button>
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
<Form.Item label="响应时间" style={getStyle(['1'])}>
|
<Form.Item label="响应时间" style={getStyle(['1'])}>
|
||||||
<Input suffix="ms" placeholder="最长响应时间(毫秒),不设置则默认10秒超时" onChange={e => store.record.extra = e.target.value}/>
|
<Input suffix="ms" value={extra} placeholder="最长响应时间(毫秒),不设置则默认10秒超时"
|
||||||
|
onChange={e => store.record.extra = e.target.value}/>
|
||||||
</Form.Item>
|
</Form.Item>
|
||||||
<Form.Item required label="检测端口" style={getStyle(['2'])}>
|
<Form.Item required label="检测端口" style={getStyle(['2'])}>
|
||||||
<Input value={extra} placeholder="请输入端口号" onChange={e => store.record.extra = e.target.value}/>
|
<Input value={extra} placeholder="请输入端口号" onChange={e => store.record.extra = e.target.value}/>
|
||||||
|
|
|
@ -12,7 +12,6 @@ import { http, hasPermission } from 'libs';
|
||||||
import groupStore from '../alarm/group/store';
|
import groupStore from '../alarm/group/store';
|
||||||
import hostStore from '../host/store';
|
import hostStore from '../host/store';
|
||||||
import store from './store';
|
import store from './store';
|
||||||
import lds from 'lodash';
|
|
||||||
|
|
||||||
@observer
|
@observer
|
||||||
class ComTable extends React.Component {
|
class ComTable extends React.Component {
|
||||||
|
@ -102,9 +101,9 @@ class ComTable extends React.Component {
|
||||||
<Table.Column title="频率" dataIndex="rate" render={value => `${value}分钟`}/>
|
<Table.Column title="频率" dataIndex="rate" render={value => `${value}分钟`}/>
|
||||||
<Table.Column title="状态" render={info => {
|
<Table.Column title="状态" render={info => {
|
||||||
if (info.is_active) {
|
if (info.is_active) {
|
||||||
return <Tag color="green">监控中</Tag>
|
return <Tag color="blue">已启用</Tag>
|
||||||
} else {
|
} else {
|
||||||
return <Tag color="red">未启用</Tag>
|
return <Tag color="red">已禁用</Tag>
|
||||||
}
|
}
|
||||||
}}/>
|
}}/>
|
||||||
<Table.Column title="更新于" dataIndex="latest_run_time_alias"
|
<Table.Column title="更新于" dataIndex="latest_run_time_alias"
|
||||||
|
|
|
@ -34,13 +34,6 @@ export default observer(function () {
|
||||||
{store.types.map(item => <Select.Option key={item} value={item}>{item}</Select.Option>)}
|
{store.types.map(item => <Select.Option key={item} value={item}>{item}</Select.Option>)}
|
||||||
</Select>
|
</Select>
|
||||||
</SearchForm.Item>
|
</SearchForm.Item>
|
||||||
<SearchForm.Item span={7} title="任务状态">
|
|
||||||
<Select allowClear value={store.f_status} onChange={v => store.f_status = v} placeholder="请选择">
|
|
||||||
<Select.Option value={-1}>待检测</Select.Option>
|
|
||||||
<Select.Option value={0}>正常</Select.Option>
|
|
||||||
<Select.Option value={1}>异常</Select.Option>
|
|
||||||
</Select>
|
|
||||||
</SearchForm.Item>
|
|
||||||
</SearchForm>
|
</SearchForm>
|
||||||
<ComTable/>
|
<ComTable/>
|
||||||
{store.formVisible && <ComForm/>}
|
{store.formVisible && <ComForm/>}
|
||||||
|
|
|
@ -19,7 +19,6 @@ class Store {
|
||||||
|
|
||||||
@observable f_name;
|
@observable f_name;
|
||||||
@observable f_type;
|
@observable f_type;
|
||||||
@observable f_status;
|
|
||||||
@observable f_active = '';
|
@observable f_active = '';
|
||||||
@observable f_group;
|
@observable f_group;
|
||||||
|
|
||||||
|
@ -29,13 +28,6 @@ class Store {
|
||||||
if (this.f_name) records = records.filter(x => x.name.toLowerCase().includes(this.f_name.toLowerCase()));
|
if (this.f_name) records = records.filter(x => x.name.toLowerCase().includes(this.f_name.toLowerCase()));
|
||||||
if (this.f_type) records = records.filter(x => x.type_alias === this.f_type);
|
if (this.f_type) records = records.filter(x => x.type_alias === this.f_type);
|
||||||
if (this.f_group) records = records.filter(x => x.group === this.f_group);
|
if (this.f_group) records = records.filter(x => x.group === this.f_group);
|
||||||
if (this.f_status !== undefined) {
|
|
||||||
if (this.f_status === -1) {
|
|
||||||
records = records.filter(x => x.is_active && !x.latest_status_alias);
|
|
||||||
} else {
|
|
||||||
records = records.filter(x => x.latest_status === this.f_status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return records
|
return records
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue