perf: 修改组件状态

pull/9019/head
ibuler 2022-11-04 11:40:16 +08:00
parent b0ae9b47ca
commit 30106bdbbb
11 changed files with 93 additions and 175 deletions

View File

@ -21,7 +21,7 @@ __all__ = ['StatusViewSet', 'ComponentsMetricsAPIView']
class StatusViewSet(viewsets.ModelViewSet):
queryset = Status.objects.all()
serializer_class = serializers.StatusSerializer
serializer_class = serializers.StatSerializer
session_serializer_class = serializers.SessionSerializer
task_serializer_class = serializers.TaskSerializer
@ -52,7 +52,7 @@ class StatusViewSet(viewsets.ModelViewSet):
terminal_id = self.kwargs.get("terminal", None)
if terminal_id:
terminal = get_object_or_404(Terminal, id=terminal_id)
return terminal.status.all()
return terminal.status_set.all()
return super().get_queryset()

View File

@ -61,7 +61,7 @@ class CommandStorageViewSet(BaseStorageViewSetMixin, viewsets.ModelViewSet):
if not filterset.is_valid():
raise utils.translate_validation(filterset.errors)
command_qs = filterset.qs
if storage.type == const.CommandStorageTypeChoices.es:
if storage.type == const.CommandStorageType.es:
command_count = command_qs.count(limit_to_max_result_window=False)
else:
command_count = command_qs.count()

View File

@ -47,7 +47,7 @@ class TerminalViewSet(JMSBulkModelViewSet):
s = self.request.query_params.get('status')
if not s:
return queryset
filtered_queryset_id = [str(q.id) for q in queryset if q.latest_status == s]
filtered_queryset_id = [str(q.id) for q in queryset if q.load == s]
queryset = queryset.filter(id__in=filtered_queryset_id)
return queryset

View File

@ -8,7 +8,7 @@ from django.utils.translation import ugettext_lazy as _
# --------------------------------
class ReplayStorageTypeChoices(TextChoices):
class ReplayStorageType(TextChoices):
null = 'null', 'Null',
server = 'server', 'Server'
s3 = 's3', 'S3'
@ -20,7 +20,7 @@ class ReplayStorageTypeChoices(TextChoices):
cos = 'cos', 'COS'
class CommandStorageTypeChoices(TextChoices):
class CommandStorageType(TextChoices):
null = 'null', 'Null',
server = 'server', 'Server'
es = 'es', 'Elasticsearch'
@ -29,7 +29,7 @@ class CommandStorageTypeChoices(TextChoices):
# Component Status Choices
# ------------------------
class ComponentStatusChoices(TextChoices):
class ComponentLoad(TextChoices):
critical = 'critical', _('Critical')
high = 'high', _('High')
normal = 'normal', _('Normal')
@ -40,7 +40,7 @@ class ComponentStatusChoices(TextChoices):
return set(dict(cls.choices).keys())
class TerminalTypeChoices(TextChoices):
class TerminalType(TextChoices):
koko = 'koko', 'KoKo'
guacamole = 'guacamole', 'Guacamole'
omnidb = 'omnidb', 'OmniDB'

View File

@ -1,10 +1,6 @@
from __future__ import unicode_literals
import uuid
from django.db import models
from django.forms.models import model_to_dict
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger
@ -22,56 +18,12 @@ class Status(models.Model):
connections = models.IntegerField(verbose_name=_("Connections"), default=0)
threads = models.IntegerField(verbose_name=_("Threads"), default=0)
boot_time = models.FloatField(verbose_name=_("Boot Time"), default=0)
terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE, related_name='status')
terminal = models.ForeignKey('terminal.Terminal', null=True, on_delete=models.CASCADE)
date_created = models.DateTimeField(auto_now_add=True)
CACHE_KEY = 'TERMINAL_STATUS_{}'
class Meta:
db_table = 'terminal_status'
get_latest_by = 'date_created'
verbose_name = _("Status")
def save_to_cache(self):
if not self.terminal:
return
key = self.CACHE_KEY.format(self.terminal.id)
data = model_to_dict(self)
cache.set(key, data, 60*3)
return data
@classmethod
def get_terminal_latest_status(cls, terminal):
from ...utils import ComputeStatUtil
stat = cls.get_terminal_latest_stat(terminal)
return ComputeStatUtil.compute_component_status(stat)
@classmethod
def get_terminal_latest_stat(cls, terminal):
key = cls.CACHE_KEY.format(terminal.id)
data = cache.get(key)
if not data:
return None
data.pop('terminal', None)
stat = cls(**data)
stat.terminal = terminal
stat.is_alive = terminal.is_alive
stat.keep_one_decimal_place()
return stat
def keep_one_decimal_place(self):
keys = ['cpu_load', 'memory_used', 'disk_used']
for key in keys:
value = getattr(self, key, 0)
if not isinstance(value, (int, float)):
continue
value = '%.1f' % value
setattr(self, key, float(value))
def save(self, force_insert=False, force_update=False, using=None,
update_fields=None):
self.terminal.set_alive(ttl=120)
return self.save_to_cache()
# return super().save()

View File

@ -53,21 +53,21 @@ class CommonStorageModelMixin(models.Model):
class CommandStorage(CommonStorageModelMixin, CommonModelMixin):
type = models.CharField(
max_length=16, choices=const.CommandStorageTypeChoices.choices,
default=const.CommandStorageTypeChoices.server.value, verbose_name=_('Type'),
max_length=16, choices=const.CommandStorageType.choices,
default=const.CommandStorageType.server.value, verbose_name=_('Type'),
)
@property
def type_null(self):
return self.type == const.CommandStorageTypeChoices.null.value
return self.type == const.CommandStorageType.null.value
@property
def type_server(self):
return self.type == const.CommandStorageTypeChoices.server.value
return self.type == const.CommandStorageType.server.value
@property
def type_es(self):
return self.type == const.CommandStorageTypeChoices.es.value
return self.type == const.CommandStorageType.es.value
@property
def type_null_or_server(self):
@ -138,17 +138,17 @@ class CommandStorage(CommonStorageModelMixin, CommonModelMixin):
class ReplayStorage(CommonStorageModelMixin, CommonModelMixin):
type = models.CharField(
max_length=16, choices=const.ReplayStorageTypeChoices.choices,
default=const.ReplayStorageTypeChoices.server.value, verbose_name=_('Type')
max_length=16, choices=const.ReplayStorageType.choices,
default=const.ReplayStorageType.server.value, verbose_name=_('Type')
)
@property
def type_null(self):
return self.type == const.ReplayStorageTypeChoices.null.value
return self.type == const.ReplayStorageType.null.value
@property
def type_server(self):
return self.type == const.ReplayStorageTypeChoices.server.value
return self.type == const.ReplayStorageType.server.value
@property
def type_null_or_server(self):
@ -156,11 +156,11 @@ class ReplayStorage(CommonStorageModelMixin, CommonModelMixin):
@property
def type_swift(self):
return self.type == const.ReplayStorageTypeChoices.swift.value
return self.type == const.ReplayStorageType.swift.value
@property
def type_ceph(self):
return self.type == const.ReplayStorageTypeChoices.ceph.value
return self.type == const.ReplayStorageType.ceph.value
@property
def config(self):
@ -168,7 +168,7 @@ class ReplayStorage(CommonStorageModelMixin, CommonModelMixin):
# add type config
if self.type_ceph:
_type = const.ReplayStorageTypeChoices.s3.value
_type = const.ReplayStorageType.s3.value
else:
_type = self.type
_config.update({'TYPE': _type})

View File

@ -1,16 +1,15 @@
import uuid
from django.utils import timezone
from django.db import models
from django.core.cache import cache
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from common.utils import get_logger
from common.utils import get_logger, lazyproperty
from users.models import User
from orgs.utils import tmp_to_root_org
from .status import Status
from terminal.const import TerminalTypeChoices as TypeChoices
from terminal.const import ComponentStatusChoices as StatusChoice
from terminal.const import TerminalType as TypeChoices, ComponentLoad as StatusChoice
from ..session import Session
@ -18,42 +17,24 @@ logger = get_logger(__file__)
class TerminalStatusMixin:
ALIVE_KEY = 'TERMINAL_ALIVE_{}'
id: str
ALIVE_KEY = 'TERMINAL_ALIVE_{}'
status_set: models.Manager
@property
def latest_status(self):
return Status.get_terminal_latest_status(self)
@lazyproperty
def last_stat(self):
return self.status_set.order_by('date_created').last()
@property
def latest_status_display(self):
return self.latest_status.label
@property
def latest_stat(self):
return Status.get_terminal_latest_stat(self)
@property
def is_normal(self):
return self.latest_status == StatusChoice.normal
@property
def is_high(self):
return self.latest_status == StatusChoice.high
@property
def is_critical(self):
return self.latest_status == StatusChoice.critical
@lazyproperty
def load(self):
from ...utils import ComputeLoadUtil
return ComputeLoadUtil.compute_load(self.last_stat)
@property
def is_alive(self):
key = self.ALIVE_KEY.format(self.id)
# return self.latest_status != StatusChoice.offline
return cache.get(key, False)
def set_alive(self, ttl=120):
key = self.ALIVE_KEY.format(self.id)
cache.set(key, True, ttl)
if not self.last_stat:
return False
return self.last_stat.date_created > timezone.now() - timezone.timedelta(seconds=120)
class StorageMixin:

View File

@ -118,13 +118,13 @@ class ReplayStorageTypeAzureSerializer(serializers.Serializer):
# mapping
replay_storage_type_serializer_classes_mapping = {
const.ReplayStorageTypeChoices.s3.value: ReplayStorageTypeS3Serializer,
const.ReplayStorageTypeChoices.ceph.value: ReplayStorageTypeCephSerializer,
const.ReplayStorageTypeChoices.swift.value: ReplayStorageTypeSwiftSerializer,
const.ReplayStorageTypeChoices.oss.value: ReplayStorageTypeOSSSerializer,
const.ReplayStorageTypeChoices.azure.value: ReplayStorageTypeAzureSerializer,
const.ReplayStorageTypeChoices.obs.value: ReplayStorageTypeOBSSerializer,
const.ReplayStorageTypeChoices.cos.value: ReplayStorageTypeCOSSerializer
const.ReplayStorageType.s3.value: ReplayStorageTypeS3Serializer,
const.ReplayStorageType.ceph.value: ReplayStorageTypeCephSerializer,
const.ReplayStorageType.swift.value: ReplayStorageTypeSwiftSerializer,
const.ReplayStorageType.oss.value: ReplayStorageTypeOSSSerializer,
const.ReplayStorageType.azure.value: ReplayStorageTypeAzureSerializer,
const.ReplayStorageType.obs.value: ReplayStorageTypeOBSSerializer,
const.ReplayStorageType.cos.value: ReplayStorageTypeCOSSerializer
}
@ -172,7 +172,7 @@ class CommandStorageTypeESSerializer(serializers.Serializer):
# mapping
command_storage_type_serializer_classes_mapping = {
const.CommandStorageTypeChoices.es.value: CommandStorageTypeESSerializer
const.CommandStorageType.es.value: CommandStorageTypeESSerializer
}

View File

@ -2,28 +2,26 @@ from rest_framework import serializers
from django.utils.translation import ugettext_lazy as _
from common.drf.serializers import BulkModelSerializer
from common.utils import is_uuid
from common.utils import get_request_ip, pretty_string
from common.drf.fields import LabeledChoiceField
from common.utils import get_request_ip, pretty_string, is_uuid
from users.serializers import ServiceAccountSerializer
from .. import const
from ..models import (
Terminal, Status, Task, CommandStorage, ReplayStorage
)
from ..models import Terminal, Status, Task, CommandStorage, ReplayStorage
class StatusSerializer(serializers.ModelSerializer):
class StatSerializer(serializers.ModelSerializer):
sessions = serializers.ListSerializer(
child=serializers.CharField(max_length=36), write_only=True
child=serializers.CharField(max_length=36),
write_only=True
)
class Meta:
model = Status
fields_mini = ['id']
fields_write_only = ['sessions', ]
fields_small = fields_mini + fields_write_only + [
'cpu_load', 'memory_used', 'disk_used',
'session_online',
'date_created'
'session_online', 'date_created'
]
fields_fk = ['terminal']
fields = fields_small + fields_fk
@ -32,30 +30,28 @@ class StatusSerializer(serializers.ModelSerializer):
"memory_used": {'default': 0},
"disk_used": {'default': 0},
}
model = Status
class TerminalSerializer(BulkModelSerializer):
session_online = serializers.ReadOnlyField(source='get_online_session_count')
is_alive = serializers.BooleanField(read_only=True)
is_active = serializers.BooleanField(read_only=True, label='Is active')
status = serializers.ChoiceField(
read_only=True, choices=const.ComponentStatusChoices.choices,
source='latest_status', label=_('Load status')
load = LabeledChoiceField(
read_only=True, choices=const.ComponentLoad.choices,
label=_('Load status')
)
status_display = serializers.CharField(read_only=True, source='latest_status_display')
stat = StatusSerializer(read_only=True, source='latest_stat')
stat = StatSerializer(read_only=True, source='last_stat')
class Meta:
model = Terminal
fields_mini = ['id', 'name']
fields_small = fields_mini + [
'type', 'remote_addr', 'http_port', 'ssh_port',
'session_online', 'command_storage', 'replay_storage',
'is_accepted', "is_active", 'is_alive',
'type', 'remote_addr', 'session_online',
'command_storage', 'replay_storage',
'is_active', 'is_alive',
'date_created', 'comment',
]
fields_fk = ['status', 'status_display', 'stat']
fields_fk = ['load', 'stat']
fields = fields_small + fields_fk
read_only_fields = ['type', 'date_created']
extra_kwargs = {

View File

@ -9,8 +9,8 @@ from common.db.utils import close_old_connections
from common.decorator import Singleton
from common.utils import get_disk_usage, get_cpu_load, get_memory_usage, get_logger
from .serializers.terminal import TerminalRegistrationSerializer, StatusSerializer
from .const import TerminalTypeChoices
from .serializers.terminal import TerminalRegistrationSerializer, StatSerializer
from .const import TerminalType
from .models import Terminal
__all__ = ['CoreTerminal', 'CeleryTerminal']
@ -51,16 +51,18 @@ class BaseTerminal(object):
'disk_used': get_disk_usage(path=settings.BASE_DIR),
'sessions': [],
}
status_serializer = StatusSerializer(data=heartbeat_data)
status_serializer = StatSerializer(data=heartbeat_data)
status_serializer.is_valid()
status_serializer.validated_data.pop('sessions', None)
terminal = self.get_or_register_terminal()
status_serializer.validated_data['terminal'] = terminal
try:
status_serializer.save()
status = status_serializer.save()
print("Save status ok: ", status)
time.sleep(self.interval)
except OperationalError:
print("Save status error, close old connections")
close_old_connections()
def get_or_register_terminal(self):
@ -90,8 +92,8 @@ class CoreTerminal(BaseTerminal):
def __init__(self):
super().__init__(
suffix_name=TerminalTypeChoices.core.label,
_type=TerminalTypeChoices.core.value
suffix_name=TerminalType.core.label,
_type=TerminalType.core.value
)
@ -99,6 +101,6 @@ class CoreTerminal(BaseTerminal):
class CeleryTerminal(BaseTerminal):
def __init__(self):
super().__init__(
suffix_name=TerminalTypeChoices.celery.label,
_type=TerminalTypeChoices.celery.value
suffix_name=TerminalType.celery.label,
_type=TerminalType.celery.value
)

View File

@ -1,8 +1,11 @@
# -*- coding: utf-8 -*-
#
import os
import time
from itertools import groupby, chain
from collections import defaultdict
from django.utils import timezone
from django.conf import settings
from django.core.files.storage import default_storage
import jms_storage
@ -75,16 +78,16 @@ def get_session_replay_url(session):
return local_path, url
class ComputeStatUtil:
class ComputeLoadUtil:
# system status
@staticmethod
def _common_compute_system_status(value, thresholds):
if thresholds[0] <= value <= thresholds[1]:
return const.ComponentStatusChoices.normal.value
return const.ComponentLoad.normal.value
elif thresholds[1] < value <= thresholds[2]:
return const.ComponentStatusChoices.high.value
return const.ComponentLoad.high.value
else:
return const.ComponentStatusChoices.critical.value
return const.ComponentLoad.critical.value
@classmethod
def _compute_system_stat_status(cls, stat):
@ -105,16 +108,16 @@ class ComputeStatUtil:
return system_status
@classmethod
def compute_component_status(cls, stat):
if not stat:
return const.ComponentStatusChoices.offline
def compute_load(cls, stat):
if not stat or time.time() - stat.date_created.timestamp() > 150:
return const.ComponentLoad.offline
system_status_values = cls._compute_system_stat_status(stat).values()
if const.ComponentStatusChoices.critical in system_status_values:
return const.ComponentStatusChoices.critical
elif const.ComponentStatusChoices.high in system_status_values:
return const.ComponentStatusChoices.high
if const.ComponentLoad.critical in system_status_values:
return const.ComponentLoad.critical
elif const.ComponentLoad.high in system_status_values:
return const.ComponentLoad.high
else:
return const.ComponentStatusChoices.normal
return const.ComponentLoad.normal
class TypedComponentsStatusMetricsUtil(object):
@ -134,31 +137,15 @@ class TypedComponentsStatusMetricsUtil(object):
def get_metrics(self):
metrics = []
for _tp, components in self.grouped_components:
normal_count = high_count = critical_count = 0
total_count = offline_count = session_online_total = 0
metric = {
'normal': 0, 'high': 0, 'critical': 0, 'offline': 0,
'total': 0, 'session_active': 0, 'type': _tp
}
for component in components:
total_count += 1
if not component.is_alive:
offline_count += 1
continue
if component.is_normal:
normal_count += 1
elif component.is_high:
high_count += 1
else:
# critical
critical_count += 1
session_online_total += component.get_online_session_count()
metrics.append({
'total': total_count,
'normal': normal_count,
'high': high_count,
'critical': critical_count,
'offline': offline_count,
'session_active': session_online_total,
'type': _tp,
})
metric[component.load] += 1
metric['total'] += 1
metric['session_active'] += component.get_online_session_count()
metrics.append(metric)
return metrics