From 9cd5675209549b82968e44f0150765b7e2411526 Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 26 Mar 2021 19:09:34 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9terminal=20statuts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: 优化status api perf: 优化 status api perf: 修改sesion参数 perf: 修改migrations perf: 优化数据结构 perf: 修改保留日志 perf: 优化之前的一个写法 --- apps/terminal/api/__init__.py | 2 +- apps/terminal/api/component.py | 34 --- apps/terminal/api/status.py | 74 +++++++ apps/terminal/api/terminal.py | 48 +---- apps/terminal/const.py | 1 + .../migrations/0033_auto_20210329_1711.py | 43 ++++ apps/terminal/models/session.py | 3 +- apps/terminal/models/status.py | 52 ++++- apps/terminal/models/terminal.py | 199 ++++++------------ apps/terminal/serializers/__init__.py | 1 - apps/terminal/serializers/components.py | 25 --- apps/terminal/serializers/terminal.py | 35 ++- apps/terminal/tasks.py | 2 +- apps/terminal/urls/api_urls.py | 1 - apps/terminal/utils.py | 166 +++++++++++---- 15 files changed, 380 insertions(+), 306 deletions(-) delete mode 100644 apps/terminal/api/component.py create mode 100644 apps/terminal/api/status.py create mode 100644 apps/terminal/migrations/0033_auto_20210329_1711.py delete mode 100644 apps/terminal/serializers/components.py diff --git a/apps/terminal/api/__init__.py b/apps/terminal/api/__init__.py index c0c6b8197..e6a3b3885 100644 --- a/apps/terminal/api/__init__.py +++ b/apps/terminal/api/__init__.py @@ -5,4 +5,4 @@ from .session import * from .command import * from .task import * from .storage import * -from .component import * +from .status import * diff --git a/apps/terminal/api/component.py b/apps/terminal/api/component.py deleted file mode 100644 index f881b5e98..000000000 --- a/apps/terminal/api/component.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# - -import logging -from rest_framework import generics, status -from rest_framework.views import Response - -from .. import serializers -from ..utils import ComponentsMetricsUtil -from common.permissions import IsAppUser, IsSuperUser - -logger = logging.getLogger(__file__) - - -__all__ = [ - 'ComponentsStateAPIView', 'ComponentsMetricsAPIView', -] - - -class ComponentsStateAPIView(generics.CreateAPIView): - """ koko, guacamole, omnidb 上报状态 """ - permission_classes = (IsAppUser,) - serializer_class = serializers.ComponentsStateSerializer - - -class ComponentsMetricsAPIView(generics.GenericAPIView): - """ 返回汇总组件指标数据 """ - permission_classes = (IsSuperUser,) - - def get(self, request, *args, **kwargs): - tp = request.query_params.get('type') - util = ComponentsMetricsUtil() - metrics = util.get_metrics(tp) - return Response(metrics, status=status.HTTP_200_OK) diff --git a/apps/terminal/api/status.py b/apps/terminal/api/status.py new file mode 100644 index 000000000..b39e13ba7 --- /dev/null +++ b/apps/terminal/api/status.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +# + +import logging +from django.shortcuts import get_object_or_404 +from rest_framework import viewsets, generics +from rest_framework.views import Response +from rest_framework import status + +from common.permissions import IsAppUser, IsOrgAdminOrAppUser, IsSuperUser +from ..models import Terminal, Status, Session +from .. import serializers +from ..utils import TypedComponentsStatusMetricsUtil + +logger = logging.getLogger(__file__) + + +__all__ = [ + 'StatusViewSet', + 'ComponentsMetricsAPIView', +] + + +class StatusViewSet(viewsets.ModelViewSet): + queryset = Status.objects.all() + serializer_class = serializers.StatusSerializer + permission_classes = (IsOrgAdminOrAppUser,) + session_serializer_class = serializers.SessionSerializer + task_serializer_class = serializers.TaskSerializer + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.handle_sessions() + self.perform_create(serializer) + tasks = self.request.user.terminal.task_set.filter(is_finished=False) + serializer = self.task_serializer_class(tasks, many=True) + return Response(serializer.data, status=201) + + def handle_sessions(self): + session_ids = self.request.data.get('sessions', []) + # guacamole 上报的 session 是字符串 + # "[53cd3e47-210f-41d8-b3c6-a184f3, 53cd3e47-210f-41d8-b3c6-a184f4]" + if isinstance(session_ids, str): + session_ids = session_ids[1:-1].split(',') + session_ids = [sid.strip() for sid in session_ids if sid.strip()] + Session.set_sessions_active(session_ids) + + def get_queryset(self): + terminal_id = self.kwargs.get("terminal", None) + if terminal_id: + terminal = get_object_or_404(Terminal, id=terminal_id) + return terminal.status_set.all() + return super().get_queryset() + + def perform_create(self, serializer): + serializer.validated_data.pop('sessions', None) + serializer.validated_data["terminal"] = self.request.user.terminal + return super().perform_create(serializer) + + def get_permissions(self): + if self.action == "create": + self.permission_classes = (IsAppUser,) + return super().get_permissions() + + +class ComponentsMetricsAPIView(generics.GenericAPIView): + """ 返回汇总组件指标数据 """ + permission_classes = (IsSuperUser,) + + def get(self, request, *args, **kwargs): + util = TypedComponentsStatusMetricsUtil() + metrics = util.get_metrics() + return Response(metrics, status=status.HTTP_200_OK) diff --git a/apps/terminal/api/terminal.py b/apps/terminal/api/terminal.py index 91e9d4d07..5ee19b3e2 100644 --- a/apps/terminal/api/terminal.py +++ b/apps/terminal/api/terminal.py @@ -4,8 +4,7 @@ import logging import uuid from django.core.cache import cache -from django.shortcuts import get_object_or_404 -from rest_framework import viewsets, generics +from rest_framework import generics from rest_framework.views import APIView, Response from rest_framework import status from django.conf import settings @@ -13,13 +12,13 @@ from django.conf import settings from common.drf.api import JMSBulkModelViewSet from common.utils import get_object_or_none -from common.permissions import IsAppUser, IsOrgAdminOrAppUser, IsSuperUser, WithBootstrapToken -from ..models import Terminal, Status, Session +from common.permissions import IsAppUser, IsSuperUser, WithBootstrapToken +from ..models import Terminal from .. import serializers from .. import exceptions __all__ = [ - 'TerminalViewSet', 'StatusViewSet', 'TerminalConfig', + 'TerminalViewSet', 'TerminalConfig', 'TerminalRegistrationApi', ] logger = logging.getLogger(__file__) @@ -72,45 +71,6 @@ class TerminalViewSet(JMSBulkModelViewSet): return queryset -class StatusViewSet(viewsets.ModelViewSet): - queryset = Status.objects.all() - serializer_class = serializers.StatusSerializer - permission_classes = (IsOrgAdminOrAppUser,) - session_serializer_class = serializers.SessionSerializer - task_serializer_class = serializers.TaskSerializer - - def create(self, request, *args, **kwargs): - self.handle_sessions() - tasks = self.request.user.terminal.task_set.filter(is_finished=False) - serializer = self.task_serializer_class(tasks, many=True) - return Response(serializer.data, status=201) - - def handle_sessions(self): - session_ids = self.request.data.get('sessions', []) - # guacamole 上报的 session 是字符串 - # "[53cd3e47-210f-41d8-b3c6-a184f3, 53cd3e47-210f-41d8-b3c6-a184f4]" - if isinstance(session_ids, str): - session_ids = session_ids[1:-1].split(',') - session_ids = [sid.strip() for sid in session_ids if sid.strip()] - Session.set_sessions_active(session_ids) - - def get_queryset(self): - terminal_id = self.kwargs.get("terminal", None) - if terminal_id: - terminal = get_object_or_404(Terminal, id=terminal_id) - self.queryset = terminal.status_set.all() - return self.queryset - - def perform_create(self, serializer): - serializer.validated_data["terminal"] = self.request.user.terminal - return super().perform_create(serializer) - - def get_permissions(self): - if self.action == "create": - self.permission_classes = (IsAppUser,) - return super().get_permissions() - - class TerminalConfig(APIView): permission_classes = (IsAppUser,) diff --git a/apps/terminal/const.py b/apps/terminal/const.py index 6e3a38027..830913e28 100644 --- a/apps/terminal/const.py +++ b/apps/terminal/const.py @@ -31,6 +31,7 @@ class ComponentStatusChoices(TextChoices): critical = 'critical', _('Critical') high = 'high', _('High') normal = 'normal', _('Normal') + offline = 'offline', _('Offline') @classmethod def status(cls): diff --git a/apps/terminal/migrations/0033_auto_20210329_1711.py b/apps/terminal/migrations/0033_auto_20210329_1711.py new file mode 100644 index 000000000..bbc45a8c7 --- /dev/null +++ b/apps/terminal/migrations/0033_auto_20210329_1711.py @@ -0,0 +1,43 @@ +# Generated by Django 3.1 on 2021-03-29 09:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('terminal', '0032_auto_20210302_1853'), + ] + + operations = [ + migrations.RenameField( + model_name='status', + old_name='cpu_used', + new_name='cpu_load', + ), + migrations.AlterField( + model_name='status', + name='cpu_load', + field=models.FloatField(default=0, verbose_name='CPU Load'), + ), + migrations.AddField( + model_name='status', + name='disk_used', + field=models.FloatField(default=0, verbose_name='Disk Used'), + ), + migrations.AlterField( + model_name='status', + name='boot_time', + field=models.FloatField(default=0, verbose_name='Boot Time'), + ), + migrations.AlterField( + model_name='status', + name='connections', + field=models.IntegerField(default=0, verbose_name='Connections'), + ), + migrations.AlterField( + model_name='status', + name='threads', + field=models.IntegerField(default=0, verbose_name='Threads'), + ), + ] diff --git a/apps/terminal/models/session.py b/apps/terminal/models/session.py index ee7e07a4d..89e338143 100644 --- a/apps/terminal/models/session.py +++ b/apps/terminal/models/session.py @@ -14,7 +14,6 @@ from assets.models import Asset from orgs.mixins.models import OrgModelMixin from common.db.models import ChoiceSet from ..backends import get_multi_command_storage -from .terminal import Terminal class Session(OrgModelMixin): @@ -47,7 +46,7 @@ class Session(OrgModelMixin): is_finished = models.BooleanField(default=False, db_index=True) has_replay = models.BooleanField(default=False, verbose_name=_("Replay")) has_command = models.BooleanField(default=False, verbose_name=_("Command")) - terminal = models.ForeignKey(Terminal, null=True, on_delete=models.DO_NOTHING, db_constraint=False) + terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.DO_NOTHING, db_constraint=False) protocol = models.CharField(choices=PROTOCOL.choices, default='ssh', max_length=16, db_index=True) date_start = models.DateTimeField(verbose_name=_("Date start"), db_index=True, default=timezone.now) date_end = models.DateTimeField(verbose_name=_("Date end"), null=True) diff --git a/apps/terminal/models/status.py b/apps/terminal/models/status.py index a0607e5dc..dddf8f350 100644 --- a/apps/terminal/models/status.py +++ b/apps/terminal/models/status.py @@ -3,26 +3,62 @@ from __future__ import unicode_literals import uuid from django.db import models +from django.forms.models import model_to_dict +from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ -from .terminal import Terminal +from common.utils import get_logger + + +logger = get_logger(__name__) class Status(models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True) session_online = models.IntegerField(verbose_name=_("Session Online"), default=0) - cpu_used = models.FloatField(verbose_name=_("CPU Usage")) + cpu_load = models.FloatField(verbose_name=_("CPU Load"), default=0) memory_used = models.FloatField(verbose_name=_("Memory Used")) - connections = models.IntegerField(verbose_name=_("Connections")) - threads = models.IntegerField(verbose_name=_("Threads")) - boot_time = models.FloatField(verbose_name=_("Boot Time")) - terminal = models.ForeignKey(Terminal, null=True, on_delete=models.CASCADE) + disk_used = models.FloatField(verbose_name=_("Disk Used"), default=0) + connections = models.IntegerField(verbose_name=_("Connections"), default=0) + threads = models.IntegerField(verbose_name=_("Threads"), default=0) + boot_time = models.FloatField(verbose_name=_("Boot Time"), default=0) + terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE) date_created = models.DateTimeField(auto_now_add=True) + CACHE_KEY = 'TERMINAL_STATUS_{}' + class Meta: db_table = 'terminal_status' get_latest_by = 'date_created' - def __str__(self): - return self.date_created.strftime("%Y-%m-%d %H:%M:%S") + def save_to_cache(self): + if not self.terminal: + return + key = self.CACHE_KEY.format(self.terminal.id) + data = model_to_dict(self) + cache.set(key, data, 60*3) + return data + + @classmethod + def get_terminal_latest_status(cls, terminal): + from ..utils import ComputeStatUtil + stat = cls.get_terminal_latest_stat(terminal) + return ComputeStatUtil.compute_component_status(stat) + + @classmethod + def get_terminal_latest_stat(cls, terminal): + key = cls.CACHE_KEY.format(terminal.id) + data = cache.get(key) + if not data: + return None + data.pop('terminal', None) + stat = cls(**data) + stat.terminal = terminal + return stat + + def save(self, force_insert=False, force_update=False, using=None, + update_fields=None): + self.terminal.set_alive(ttl=120) + return self.save_to_cache() + # return super().save() diff --git a/apps/terminal/models/terminal.py b/apps/terminal/models/terminal.py index 48e225cfd..e13902251 100644 --- a/apps/terminal/models/terminal.py +++ b/apps/terminal/models/terminal.py @@ -1,168 +1,63 @@ -from __future__ import unicode_literals import uuid from django.db import models +from django.core.cache import cache from django.utils.translation import ugettext_lazy as _ from django.conf import settings -from django.core.cache import cache from common.utils import get_logger from users.models import User +from .status import Status from .. import const +from ..const import ComponentStatusChoices as StatusChoice +from .session import Session logger = get_logger(__file__) -class ComputeStatusMixin: - - # system status - @staticmethod - def _common_compute_system_status(value, thresholds): - if thresholds[0] <= value <= thresholds[1]: - return const.ComponentStatusChoices.normal.value - elif thresholds[1] < value <= thresholds[2]: - return const.ComponentStatusChoices.high.value - else: - return const.ComponentStatusChoices.critical.value - - def _compute_system_cpu_load_1_status(self, value): - thresholds = [0, 5, 20] - return self._common_compute_system_status(value, thresholds) - - def _compute_system_memory_used_percent_status(self, value): - thresholds = [0, 85, 95] - return self._common_compute_system_status(value, thresholds) - - def _compute_system_disk_used_percent_status(self, value): - thresholds = [0, 80, 99] - return self._common_compute_system_status(value, thresholds) - - def _compute_system_status(self, state): - system_status_keys = [ - 'system_cpu_load_1', 'system_memory_used_percent', 'system_disk_used_percent' - ] - system_status = [] - for system_status_key in system_status_keys: - state_value = state.get(system_status_key) - if state_value is None: - msg = 'state: {}, state_key: {}, state_value: {}' - logger.debug(msg.format(state, system_status_key, state_value)) - state_value = 0 - status = getattr(self, f'_compute_{system_status_key}_status')(state_value) - system_status.append(status) - return system_status - - def _compute_component_status(self, state): - system_status = self._compute_system_status(state) - if const.ComponentStatusChoices.critical in system_status: - return const.ComponentStatusChoices.critical - elif const.ComponentStatusChoices.high in system_status: - return const.ComponentStatusChoices.high - else: - return const.ComponentStatusChoices.normal - - @staticmethod - def _compute_component_status_display(status): - return getattr(const.ComponentStatusChoices, status).label - - -class TerminalStateMixin(ComputeStatusMixin): - CACHE_KEY_COMPONENT_STATE = 'CACHE_KEY_COMPONENT_STATE_TERMINAL_{}' - CACHE_TIMEOUT = 120 +class TerminalStatusMixin: + ALIVE_KEY = 'TERMINAL_ALIVE_{}' + id: str @property - def cache_key(self): - return self.CACHE_KEY_COMPONENT_STATE.format(str(self.id)) - - # get - def _get_from_cache(self): - return cache.get(self.cache_key) - - def _set_to_cache(self, state): - cache.set(self.cache_key, state, self.CACHE_TIMEOUT) - - # set - def _add_status(self, state): - status = self._compute_component_status(state) - status_display = self._compute_component_status_display(status) - state.update({ - 'status': status, - 'status_display': status_display - }) + def latest_status(self): + return Status.get_terminal_latest_status(self) @property - def state(self): - state = self._get_from_cache() - return state or {} - - @state.setter - def state(self, state): - self._add_status(state) - self._set_to_cache(state) - - -class TerminalStatusMixin(TerminalStateMixin): - - # alive - @property - def is_alive(self): - return bool(self.state) - - # status - @property - def status(self): - if self.is_alive: - return self.state['status'] - else: - return const.ComponentStatusChoices.critical.value + def latest_status_display(self): + return self.latest_status.label @property - def status_display(self): - return self._compute_component_status_display(self.status) + def latest_stat(self): + return Status.get_terminal_latest_stat(self) @property def is_normal(self): - return self.status == const.ComponentStatusChoices.normal.value + return self.latest_status == StatusChoice.normal @property def is_high(self): - return self.status == const.ComponentStatusChoices.high.value + return self.latest_status == StatusChoice.high @property def is_critical(self): - return self.status == const.ComponentStatusChoices.critical.value - - -class Terminal(TerminalStatusMixin, models.Model): - id = models.UUIDField(default=uuid.uuid4, primary_key=True) - name = models.CharField(max_length=128, verbose_name=_('Name')) - type = models.CharField( - choices=const.TerminalTypeChoices.choices, default=const.TerminalTypeChoices.koko.value, - max_length=64, verbose_name=_('type') - ) - remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address')) - ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222) - http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000) - command_storage = models.CharField(max_length=128, verbose_name=_("Command storage"), default='default') - replay_storage = models.CharField(max_length=128, verbose_name=_("Replay storage"), default='default') - user = models.OneToOneField(User, related_name='terminal', verbose_name='Application User', null=True, on_delete=models.CASCADE) - is_accepted = models.BooleanField(default=False, verbose_name='Is Accepted') - is_deleted = models.BooleanField(default=False) - date_created = models.DateTimeField(auto_now_add=True) - comment = models.TextField(blank=True, verbose_name=_('Comment')) + return self.latest_status == StatusChoice.critical @property - def is_active(self): - if self.user and self.user.is_active: - return True - return False + def is_alive(self): + key = self.ALIVE_KEY.format(self.id) + # return self.latest_status != StatusChoice.offline + return cache.get(key, False) - @is_active.setter - def is_active(self, active): - if self.user: - self.user.is_active = active - self.user.save() + def set_alive(self, ttl=120): + key = self.ALIVE_KEY.format(self.id) + cache.set(key, True, ttl) + + +class StorageMixin: + command_storage: str + replay_storage: str def get_command_storage(self): from .storage import CommandStorage @@ -198,6 +93,44 @@ class Terminal(TerminalStatusMixin, models.Model): config = self.get_replay_storage_config() return {"TERMINAL_REPLAY_STORAGE": config} + +class Terminal(StorageMixin, TerminalStatusMixin, models.Model): + id = models.UUIDField(default=uuid.uuid4, primary_key=True) + name = models.CharField(max_length=128, verbose_name=_('Name')) + type = models.CharField( + choices=const.TerminalTypeChoices.choices, default=const.TerminalTypeChoices.koko.value, + max_length=64, verbose_name=_('type') + ) + remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address')) + ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222) + http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000) + command_storage = models.CharField(max_length=128, verbose_name=_("Command storage"), default='default') + replay_storage = models.CharField(max_length=128, verbose_name=_("Replay storage"), default='default') + user = models.OneToOneField(User, related_name='terminal', verbose_name='Application User', null=True, on_delete=models.CASCADE) + is_accepted = models.BooleanField(default=False, verbose_name='Is Accepted') + is_deleted = models.BooleanField(default=False) + date_created = models.DateTimeField(auto_now_add=True) + comment = models.TextField(blank=True, verbose_name=_('Comment')) + + + @property + def is_active(self): + if self.user and self.user.is_active: + return True + return False + + @is_active.setter + def is_active(self, active): + if self.user: + self.user.is_active = active + self.user.save() + + def get_online_sessions(self): + return Session.objects.filter(terminal=self, is_finished=False) + + def get_online_session_count(self): + return self.get_online_sessions().count() + @staticmethod def get_login_title_setting(): login_title = None diff --git a/apps/terminal/serializers/__init__.py b/apps/terminal/serializers/__init__.py index e958d7955..f1714dc21 100644 --- a/apps/terminal/serializers/__init__.py +++ b/apps/terminal/serializers/__init__.py @@ -4,4 +4,3 @@ from .terminal import * from .session import * from .storage import * from .command import * -from .components import * diff --git a/apps/terminal/serializers/components.py b/apps/terminal/serializers/components.py deleted file mode 100644 index d6e6d7f56..000000000 --- a/apps/terminal/serializers/components.py +++ /dev/null @@ -1,25 +0,0 @@ - -from rest_framework import serializers -from django.utils.translation import ugettext_lazy as _ - - -class ComponentsStateSerializer(serializers.Serializer): - # system - system_cpu_load_1 = serializers.FloatField( - required=False, label=_("System cpu load (1 minutes)") - ) - system_memory_used_percent = serializers.FloatField( - required=False, label=_('System memory used percent') - ) - system_disk_used_percent = serializers.FloatField( - required=False, label=_('System disk used percent') - ) - # sessions - session_active_count = serializers.IntegerField( - required=False, label=_("Session active count") - ) - - def save(self, **kwargs): - request = self.context['request'] - terminal = request.user.terminal - terminal.state = self.validated_data diff --git a/apps/terminal/serializers/terminal.py b/apps/terminal/serializers/terminal.py index caffba522..765634393 100644 --- a/apps/terminal/serializers/terminal.py +++ b/apps/terminal/serializers/terminal.py @@ -9,15 +9,34 @@ from common.utils import get_request_ip from ..models import ( Terminal, Status, Session, Task, CommandStorage, ReplayStorage ) -from .components import ComponentsStateSerializer + + +class StatusSerializer(serializers.ModelSerializer): + sessions = serializers.ListSerializer( + child=serializers.CharField(max_length=35), write_only=True + ) + + class Meta: + fields = [ + 'id', + 'cpu_load', 'memory_used', 'disk_used', + 'session_online', 'sessions', + 'terminal', 'date_created', + ] + extra_kwargs = { + "cpu_load": {'default': 0}, + "memory_used": {'default': 0}, + "disk_used": {'default': 0}, + } + model = Status class TerminalSerializer(BulkModelSerializer): session_online = serializers.SerializerMethodField() is_alive = serializers.BooleanField(read_only=True) - status = serializers.CharField(read_only=True) - status_display = serializers.CharField(read_only=True) - state = ComponentsStateSerializer(read_only=True) + status = serializers.CharField(read_only=True, source='latest_status') + status_display = serializers.CharField(read_only=True, source='latest_status_display') + stat = StatusSerializer(read_only=True, source='latest_stat') class Meta: model = Terminal @@ -25,7 +44,7 @@ class TerminalSerializer(BulkModelSerializer): 'id', 'name', 'type', 'remote_addr', 'http_port', 'ssh_port', 'comment', 'is_accepted', "is_active", 'session_online', 'is_alive', 'date_created', 'command_storage', 'replay_storage', - 'status', 'status_display', 'state' + 'status', 'status_display', 'stat' ] read_only_fields = ['type', 'date_created'] @@ -59,12 +78,6 @@ class TerminalSerializer(BulkModelSerializer): return Session.objects.filter(terminal=obj, is_finished=False).count() -class StatusSerializer(serializers.ModelSerializer): - class Meta: - fields = ['id', 'terminal'] - model = Status - - class TaskSerializer(BulkModelSerializer): class Meta: fields = '__all__' diff --git a/apps/terminal/tasks.py b/apps/terminal/tasks.py index b743f10f9..701aeac96 100644 --- a/apps/terminal/tasks.py +++ b/apps/terminal/tasks.py @@ -29,7 +29,7 @@ logger = get_task_logger(__name__) @after_app_ready_start @after_app_shutdown_clean_periodic def delete_terminal_status_period(): - yesterday = timezone.now() - datetime.timedelta(days=1) + yesterday = timezone.now() - datetime.timedelta(days=7) Status.objects.filter(date_created__lt=yesterday).delete() diff --git a/apps/terminal/urls/api_urls.py b/apps/terminal/urls/api_urls.py index 38b6df976..57fb6eb73 100644 --- a/apps/terminal/urls/api_urls.py +++ b/apps/terminal/urls/api_urls.py @@ -35,7 +35,6 @@ urlpatterns = [ path('command-storages//test-connective/', api.CommandStorageTestConnectiveApi.as_view(), name='command-storage-test-connective'), # components path('components/metrics/', api.ComponentsMetricsAPIView.as_view(), name='components-metrics'), - path('components/state/', api.ComponentsStateAPIView.as_view(), name='components-state'), # v2: get session's replay # path('v2/sessions//replay/', # api.SessionReplayV2ViewSet.as_view({'get': 'retrieve'}), diff --git a/apps/terminal/utils.py b/apps/terminal/utils.py index 8ceff0166..b13383fba 100644 --- a/apps/terminal/utils.py +++ b/apps/terminal/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # import os +from itertools import groupby from django.conf import settings from django.core.files.storage import default_storage @@ -10,9 +11,7 @@ import jms_storage from common.tasks import send_mail_async from common.utils import get_logger, reverse -from settings.models import Setting from . import const - from .models import ReplayStorage, Session, Command logger = get_logger(__name__) @@ -141,23 +140,73 @@ def send_command_execution_alert_mail(command): send_mail_async.delay(subject, message, recipient_list, html_message=message) -class ComponentsMetricsUtil(object): - +class ComputeStatUtil: + # system status @staticmethod - def get_components(tp=None): + def _common_compute_system_status(value, thresholds): + if thresholds[0] <= value <= thresholds[1]: + return const.ComponentStatusChoices.normal.value + elif thresholds[1] < value <= thresholds[2]: + return const.ComponentStatusChoices.high.value + else: + return const.ComponentStatusChoices.critical.value + + @classmethod + def _compute_system_stat_status(cls, stat): + system_stat_thresholds_mapper = { + 'cpu_load': [0, 5, 20], + 'memory_used': [0, 85, 95], + 'disk_used': [0, 80, 99] + } + system_status = {} + for stat_key, thresholds in system_stat_thresholds_mapper.items(): + stat_value = getattr(stat, stat_key) + if stat_value is None: + msg = 'stat: {}, stat_key: {}, stat_value: {}' + logger.debug(msg.format(stat, stat_key, stat_value)) + stat_value = 0 + status = cls._common_compute_system_status(stat_value, thresholds) + system_status[stat_key] = status + return system_status + + @classmethod + def compute_component_status(cls, stat): + if not stat: + return const.ComponentStatusChoices.offline + system_status_values = cls._compute_system_stat_status(stat).values() + if const.ComponentStatusChoices.critical in system_status_values: + return const.ComponentStatusChoices.critical + elif const.ComponentStatusChoices.high in system_status_values: + return const.ComponentStatusChoices.high + else: + return const.ComponentStatusChoices.normal + + +class TypedComponentsStatusMetricsUtil(object): + def __init__(self): + self.components = [] + self.grouped_components = [] + self.get_components() + + def get_components(self): from .models import Terminal components = Terminal.objects.filter(is_deleted=False).order_by('type') - if tp: - components = components.filter(type=tp) - return components + grouped_components = groupby(components, lambda c: c.type) + grouped_components = [(i[0], list(i[1])) for i in grouped_components] + self.grouped_components = grouped_components + self.components = components - def get_metrics(self, tp=None): - components = self.get_components(tp) - total_count = normal_count = high_count = critical_count = offline_count = \ - session_active_total = 0 - for component in components: - total_count += 1 - if component.is_alive: + def get_metrics(self): + metrics = [] + for _tp, components in self.grouped_components: + normal_count = high_count = critical_count = 0 + total_count = offline_count = session_online_total = 0 + + for component in components: + total_count += 1 + if not component.is_alive: + offline_count += 1 + continue if component.is_normal: normal_count += 1 elif component.is_high: @@ -165,20 +214,23 @@ class ComponentsMetricsUtil(object): else: # critical critical_count += 1 - session_active_total += component.state.get('session_active_count', 0) - else: - offline_count += 1 - return { - 'total': total_count, - 'normal': normal_count, - 'high': high_count, - 'critical': critical_count, - 'offline': offline_count, - 'session_active': session_active_total - } + session_online_total += component.get_online_session_count() + metrics.append({ + 'total': total_count, + 'normal': normal_count, + 'high': high_count, + 'critical': critical_count, + 'offline': offline_count, + 'session_active': session_online_total, + 'type': _tp, + }) + return metrics -class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil): +class ComponentsPrometheusMetricsUtil(TypedComponentsStatusMetricsUtil): + def __init__(self): + super().__init__() + self.metrics = self.get_metrics() @staticmethod def convert_status_metrics(metrics): @@ -190,50 +242,74 @@ class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil): 'offline': metrics['offline'] } - def get_prometheus_metrics_text(self): + def get_component_status_metrics(self): prometheus_metrics = list() - # 各组件状态个数汇总 prometheus_metrics.append('# JumpServer 各组件状态个数汇总') status_metric_text = 'jumpserver_components_status_total{component_type="%s", status="%s"} %s' - for tp in const.TerminalTypeChoices.types(): + for metric in self.metrics: + tp = metric['type'] prometheus_metrics.append(f'## 组件: {tp}') - metrics_tp = self.get_metrics(tp) - status_metrics = self.convert_status_metrics(metrics_tp) + status_metrics = self.convert_status_metrics(metric) for status, value in status_metrics.items(): metric_text = status_metric_text % (tp, status, value) prometheus_metrics.append(metric_text) + return prometheus_metrics - prometheus_metrics.append('\n') - + def get_component_session_metrics(self): + prometheus_metrics = list() # 各组件在线会话数汇总 prometheus_metrics.append('# JumpServer 各组件在线会话数汇总') session_active_metric_text = 'jumpserver_components_session_active_total{component_type="%s"} %s' - for tp in const.TerminalTypeChoices.types(): + + for metric in self.metrics: + tp = metric['type'] prometheus_metrics.append(f'## 组件: {tp}') - metrics_tp = self.get_metrics(tp) - metric_text = session_active_metric_text % (tp, metrics_tp['session_active']) + metric_text = session_active_metric_text % (tp, metric['session_active']) prometheus_metrics.append(metric_text) + return prometheus_metrics - prometheus_metrics.append('\n') - + def get_component_stat_metrics(self): + prometheus_metrics = list() # 各组件节点指标 prometheus_metrics.append('# JumpServer 各组件一些指标') state_metric_text = 'jumpserver_components_%s{component_type="%s", component="%s"} %s' - states = [ + stats_key = [ + 'cpu_load', 'memory_used', 'disk_used', 'session_online' + ] + old_stats_key = [ 'system_cpu_load_1', 'system_memory_used_percent', 'system_disk_used_percent', 'session_active_count' ] - for state in states: - prometheus_metrics.append(f'## 指标: {state}') - components = self.get_components() - for component in components: + old_stats_key_mapper = dict(zip(stats_key, old_stats_key)) + + for stat_key in stats_key: + prometheus_metrics.append(f'## 指标: {stat_key}') + for component in self.components: if not component.is_alive: continue + component_stat = component.latest_stat + if not component_stat: + continue metric_text = state_metric_text % ( - state, component.type, component.name, component.state.get(state) + stat_key, component.type, component.name, getattr(component_stat, stat_key) ) prometheus_metrics.append(metric_text) + old_stat_key = old_stats_key_mapper.get(stat_key) + old_metric_text = state_metric_text % ( + old_stat_key, component.type, component.name, getattr(component_stat, stat_key) + ) + prometheus_metrics.append(old_metric_text) + return prometheus_metrics + def get_prometheus_metrics_text(self): + prometheus_metrics = list() + for method in [ + self.get_component_status_metrics, + self.get_component_session_metrics, + self.get_component_stat_metrics + ]: + prometheus_metrics.extend(method()) + prometheus_metrics.append('\n') prometheus_metrics_text = '\n'.join(prometheus_metrics) return prometheus_metrics_text