2021-10-22 12:06:16 +00:00
|
|
|
from django.template.loader import render_to_string
|
2024-04-09 09:07:48 +00:00
|
|
|
from django.utils.translation import gettext
|
|
|
|
from django.utils.translation import gettext_lazy as _
|
2021-05-31 09:20:38 +00:00
|
|
|
|
2021-06-28 07:33:58 +00:00
|
|
|
from notifications.backends import BACKEND
|
2024-04-09 09:07:48 +00:00
|
|
|
from notifications.models import SystemMsgSubscription
|
|
|
|
from notifications.notifications import SystemMessage
|
|
|
|
from terminal.const import TerminalType
|
2023-04-08 16:45:11 +00:00
|
|
|
from terminal.models import Terminal
|
2024-04-09 09:07:48 +00:00
|
|
|
from terminal.models.component.status import Status
|
|
|
|
from users.models import User
|
2021-05-31 09:20:38 +00:00
|
|
|
|
2021-07-30 07:42:06 +00:00
|
|
|
__all__ = ('ServerPerformanceMessage', 'ServerPerformanceCheckUtil')
|
2021-05-31 09:20:38 +00:00
|
|
|
|
|
|
|
|
|
|
|
class ServerPerformanceMessage(SystemMessage):
|
|
|
|
category = 'Operations'
|
2022-03-02 12:48:43 +00:00
|
|
|
category_label = _('App ops')
|
2021-05-31 09:20:38 +00:00
|
|
|
message_type_label = _('Server performance')
|
|
|
|
|
2021-10-22 12:06:16 +00:00
|
|
|
def __init__(self, terms_with_errors):
|
|
|
|
self.terms_with_errors = terms_with_errors
|
2021-05-31 09:20:38 +00:00
|
|
|
|
2021-09-16 06:28:39 +00:00
|
|
|
def get_html_msg(self) -> dict:
|
2022-02-16 12:07:35 +00:00
|
|
|
subject = gettext("Terminal health check warning")
|
2021-10-22 12:06:16 +00:00
|
|
|
context = {
|
|
|
|
'terms_with_errors': self.terms_with_errors
|
|
|
|
}
|
|
|
|
message = render_to_string('ops/_msg_terminal_performance.html', context)
|
2021-08-24 06:20:54 +00:00
|
|
|
return {
|
2021-10-22 12:06:16 +00:00
|
|
|
'subject': subject,
|
|
|
|
'message': message
|
2021-08-24 06:20:54 +00:00
|
|
|
}
|
2021-05-31 09:20:38 +00:00
|
|
|
|
|
|
|
@classmethod
|
|
|
|
def post_insert_to_db(cls, subscription: SystemMsgSubscription):
|
2022-02-17 12:13:31 +00:00
|
|
|
from rbac.models import Role, RoleBinding
|
|
|
|
# Todo: 需要更改这里
|
|
|
|
admin_role = Role.BuiltinRole.system_admin.get_role()
|
|
|
|
admins_ids = RoleBinding.objects.filter(role=admin_role).values_list('user_id', flat=True)
|
|
|
|
admins = User.objects.filter(id__in=admins_ids)
|
2021-05-31 09:20:38 +00:00
|
|
|
subscription.users.add(*admins)
|
2021-06-28 07:33:58 +00:00
|
|
|
subscription.receive_backends = [BACKEND.EMAIL]
|
|
|
|
subscription.save()
|
2021-07-30 07:42:06 +00:00
|
|
|
|
2021-10-20 11:45:37 +00:00
|
|
|
@classmethod
|
|
|
|
def gen_test_msg(cls):
|
2021-10-22 12:06:16 +00:00
|
|
|
from terminal.models import Terminal
|
2021-10-20 11:45:37 +00:00
|
|
|
items_mapper = ServerPerformanceCheckUtil.items_mapper
|
2021-10-22 12:06:16 +00:00
|
|
|
terms_with_errors = []
|
|
|
|
terms = Terminal.objects.all()[:5]
|
2021-10-20 11:45:37 +00:00
|
|
|
|
2021-10-22 12:06:16 +00:00
|
|
|
for i, term in enumerate(terms, 1):
|
|
|
|
errors = []
|
|
|
|
for item, data in items_mapper.items():
|
|
|
|
msg = data['alarm_msg_format']
|
|
|
|
max_threshold = data['max_threshold']
|
|
|
|
value = 123 // i+1
|
|
|
|
msg = msg.format(max_threshold=max_threshold, value=value, name=term.name)
|
|
|
|
errors.append(msg)
|
|
|
|
terms_with_errors.append([term, errors])
|
|
|
|
return cls(terms_with_errors)
|
2021-10-20 11:45:37 +00:00
|
|
|
|
2021-07-30 07:42:06 +00:00
|
|
|
|
|
|
|
class ServerPerformanceCheckUtil(object):
|
2021-08-06 11:16:18 +00:00
|
|
|
items_mapper = {
|
|
|
|
'is_alive': {
|
|
|
|
'default': False,
|
|
|
|
'max_threshold': False,
|
2021-08-18 11:11:10 +00:00
|
|
|
'alarm_msg_format': _('The terminal is offline: {name}')
|
2021-08-06 11:16:18 +00:00
|
|
|
},
|
2021-09-16 06:33:05 +00:00
|
|
|
'disk_used': {
|
2021-08-06 11:16:18 +00:00
|
|
|
'default': 0,
|
|
|
|
'max_threshold': 80,
|
2021-10-22 12:06:16 +00:00
|
|
|
'alarm_msg_format': _('Disk used more than {max_threshold}%: => {value}')
|
2021-08-06 11:16:18 +00:00
|
|
|
},
|
2021-09-16 06:33:05 +00:00
|
|
|
'memory_used': {
|
2021-08-06 11:16:18 +00:00
|
|
|
'default': 0,
|
|
|
|
'max_threshold': 85,
|
2021-10-22 12:06:16 +00:00
|
|
|
'alarm_msg_format': _('Memory used more than {max_threshold}%: => {value}'),
|
2021-08-06 11:16:18 +00:00
|
|
|
},
|
|
|
|
'cpu_load': {
|
|
|
|
'default': 0,
|
|
|
|
'max_threshold': 5,
|
2021-10-22 12:06:16 +00:00
|
|
|
'alarm_msg_format': _('CPU load more than {max_threshold}: => {value}'),
|
2021-08-06 11:16:18 +00:00
|
|
|
},
|
|
|
|
}
|
2021-07-30 07:42:06 +00:00
|
|
|
|
|
|
|
def __init__(self):
|
2021-10-22 12:06:16 +00:00
|
|
|
self.terms_with_errors = []
|
2021-08-06 11:16:18 +00:00
|
|
|
self._terminals = []
|
2021-07-30 07:42:06 +00:00
|
|
|
|
|
|
|
def check_and_publish(self):
|
|
|
|
self.check()
|
|
|
|
self.publish()
|
|
|
|
|
2021-08-06 11:16:18 +00:00
|
|
|
def check(self):
|
2021-10-22 12:06:16 +00:00
|
|
|
self.terms_with_errors = []
|
2021-08-06 11:16:18 +00:00
|
|
|
self.initial_terminals()
|
2021-10-22 12:06:16 +00:00
|
|
|
|
|
|
|
for term in self._terminals:
|
|
|
|
errors = self.check_terminal(term)
|
|
|
|
if not errors:
|
|
|
|
continue
|
|
|
|
self.terms_with_errors.append((term, errors))
|
|
|
|
|
|
|
|
def check_terminal(self, term):
|
|
|
|
errors = []
|
2021-08-06 11:16:18 +00:00
|
|
|
for item, data in self.items_mapper.items():
|
2021-10-22 12:06:16 +00:00
|
|
|
error = self.check_item(term, item, data)
|
|
|
|
if not error:
|
|
|
|
continue
|
|
|
|
errors.append(error)
|
|
|
|
return errors
|
2021-08-06 11:16:18 +00:00
|
|
|
|
2021-10-22 12:06:16 +00:00
|
|
|
@staticmethod
|
|
|
|
def check_item(term, item, data):
|
2021-08-06 11:16:18 +00:00
|
|
|
default = data['default']
|
|
|
|
max_threshold = data['max_threshold']
|
2021-10-22 12:06:16 +00:00
|
|
|
value = getattr(term.stat, item, default)
|
|
|
|
|
2021-08-06 11:16:18 +00:00
|
|
|
if isinstance(value, bool) and value != max_threshold:
|
|
|
|
return
|
|
|
|
elif isinstance(value, (int, float)) and value < max_threshold:
|
|
|
|
return
|
|
|
|
msg = data['alarm_msg_format']
|
2021-10-22 12:06:16 +00:00
|
|
|
error = msg.format(max_threshold=max_threshold, value=value, name=term.name)
|
|
|
|
return error
|
2021-08-06 11:16:18 +00:00
|
|
|
|
2021-07-30 07:42:06 +00:00
|
|
|
def publish(self):
|
2021-10-22 12:06:16 +00:00
|
|
|
if not self.terms_with_errors:
|
2021-07-30 07:42:06 +00:00
|
|
|
return
|
2021-10-22 12:06:16 +00:00
|
|
|
ServerPerformanceMessage(self.terms_with_errors).publish()
|
2021-07-30 07:42:06 +00:00
|
|
|
|
2021-08-06 11:16:18 +00:00
|
|
|
def initial_terminals(self):
|
2021-07-30 07:42:06 +00:00
|
|
|
terminals = []
|
2024-04-09 09:07:48 +00:00
|
|
|
for terminal in Terminal.objects.filter(is_deleted=False).exclude(
|
|
|
|
type__in=[TerminalType.core, TerminalType.celery]
|
|
|
|
):
|
2021-07-30 07:42:06 +00:00
|
|
|
if not terminal.is_active:
|
|
|
|
continue
|
2021-08-06 11:16:18 +00:00
|
|
|
terminal.stat = Status.get_terminal_latest_stat(terminal)
|
2021-07-30 07:42:06 +00:00
|
|
|
terminals.append(terminal)
|
2021-08-06 11:16:18 +00:00
|
|
|
self._terminals = terminals
|