stash (#5178)

* Dev (#4791)

* fix(xpack): 修复last login太长的问题 (#4786)

Co-authored-by: ibuler <ibuler@qq.com>

* perf: 更新密码中也发送邮件 (#4789)

Co-authored-by: ibuler <ibuler@qq.com>

* fix(terminal): 修复获取螺旋的异步api

* fix(terminal): 修复有的录像存储有问题的导致下载录像的bug

* fix(orgs): 修复组织添加用户bug

* perf(requirements): 修改jms-storage==0.0.34 (#4797)

Co-authored-by: Bai <bugatti_it@163.com>

Co-authored-by: fit2bot <68588906+fit2bot@users.noreply.github.com>
Co-authored-by: ibuler <ibuler@qq.com>
Co-authored-by: Bai <bugatti_it@163.com>

* stash

* feat(system): 添加系统app

* stash

* fix: 修复一些bug

Co-authored-by: xinwen <coderWen@126.com>
Co-authored-by: ibuler <ibuler@qq.com>
Co-authored-by: Bai <bugatti_it@163.com>
Co-authored-by: Jiangjie.Bai <32935519+BaiJiangJie@users.noreply.github.com>

apps/jumpserver/settings/base.py

@@ -48,6 +48,7 @@ INSTALLED_APPS = [
     'authentication.apps.AuthenticationConfig',  # authentication
     'applications.apps.ApplicationsConfig',
     'tickets.apps.TicketsConfig',
+    'system.apps.SystemConfig',
     'jms_oidc_rp',
     'rest_framework',
     'rest_framework_swagger',

apps/jumpserver/urls.py

@@ -23,6 +23,7 @@ api_v1 = [
     path('common/', include('common.urls.api_urls', namespace='api-common')),
     path('applications/', include('applications.urls.api_urls', namespace='api-applications')),
     path('tickets/', include('tickets.urls.api_urls', namespace='api-tickets')),
+    path('system/', include('system.urls', namespace='api-system')),
 ]
 
 api_v2 = [
@@ -63,7 +64,7 @@ urlpatterns = [
     # External apps url
     path('core/auth/captcha/', include('captcha.urls')),
     path('core/', include(app_view_patterns)),
-    path('ui/', views.UIView.as_view())
+    path('ui/', views.UIView.as_view()),
 ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \

apps/system/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

apps/system/api.py

@@ -0,0 +1,17 @@
+# ~*~ coding: utf-8 ~*~
+from common.permissions import IsOrgAdminOrAppUser
+from common.drf.api import JMSBulkModelViewSet
+from common.utils import get_logger
+from . import serializers
+from .models import Stat
+
+logger = get_logger(__name__)
+__all__ = ['StatViewSet']
+
+
+class StatViewSet(JMSBulkModelViewSet):
+    queryset = Stat.objects.all()
+    filter_fields = ('id', 'key', 'value', 'component')
+    search_fields = filter_fields
+    permission_classes = (IsOrgAdminOrAppUser,)
+    serializer_class = serializers.StatSerializer

apps/system/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class SystemConfig(AppConfig):
+    name = 'system'

apps/system/migrations/0001_initial.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.1 on 2020-11-25 06:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Stat',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('node', models.CharField(max_length=128)),
+                ('ip', models.GenericIPAddressField()),
+                ('component', models.CharField(choices=[('core', 'Core'), ('koko', 'KoKo'), ('guacamole', 'Guacamole'), ('omnidb', 'OmniDB')], max_length=16)),
+                ('key', models.CharField(db_index=True, max_length=16, verbose_name='Item key')),
+                ('value', models.FloatField()),
+                ('datetime', models.DateTimeField()),
+            ],
+        ),
+    ]

apps/system/models.py

@@ -0,0 +1,68 @@
+import time
+
+from django.db import models
+import psutil
+from django.utils import timezone
+
+from django.utils.translation import ugettext_lazy as _
+
+
+class Stat(models.Model):
+    class Components(models.TextChoices):
+        core = 'core', 'Core'
+        koko = 'koko', 'KoKo'
+        guacamole = 'guacamole', 'Guacamole'
+        omnidb = 'omnidb', 'OmniDB'
+
+    class Keys(models.TextChoices):
+        cpu_load_1 = 'cpu_load', 'CPU load'
+        memory_used_percent = 'memory_used_percent', _('Memory used percent')
+        disk_used_percent = 'disk_used_percent', _('Disk used percent')
+        session_active = 'session_active', _('Session active')
+        session_processed = 'session_processed', _('Session processed')
+
+    node = models.CharField(max_length=128)
+    ip = models.GenericIPAddressField()
+    component = models.CharField(choices=Components.choices, max_length=16)
+    key = models.CharField(db_index=True, max_length=16, verbose_name=_('Item key'))
+    value = models.FloatField()
+    datetime = models.DateTimeField()
+
+    def __str__(self):
+        return f'{self.key}:{self.value}'
+
+    @staticmethod
+    def collect_local_stats():
+        memory_percent = psutil.virtual_memory().percent
+        cpu_load = psutil.getloadavg()
+        cpu_load_1 = round(cpu_load[0], 2)
+        cpu_load_5 = round(cpu_load[1], 2)
+        cpu_load_15 = round(cpu_load[2], 2)
+        cpu_percent = psutil.cpu_percent()
+        stats = dict(
+            memory_percent=memory_percent,
+            cpu_load_1=cpu_load_1,
+            cpu_load_5=cpu_load_5,
+            cpu_load_15=cpu_load_15,
+            cpu_load=cpu_load_1,
+            cpu_percent=cpu_percent
+        )
+        return stats
+
+    @classmethod
+    def keep_collect_local_stats(cls):
+        data = {
+            'node': 'core-01',
+            'ip': '192.168.1.1',
+            'component': 'core'
+        }
+        while True:
+            stats = cls.collect_local_stats()
+            data['datetime'] = timezone.now()
+            items = []
+            for k, v in stats.items():
+                data['key'] = k
+                data['value'] = v
+                items.append(cls(**data))
+            cls.objects.bulk_create(items, ignore_conflicts=True)
+            time.sleep(60)

apps/system/serializers.py

@@ -0,0 +1,12 @@
+from common.drf.serializers import BulkModelSerializer
+
+from .models import Stat
+
+
+class StatSerializer(BulkModelSerializer):
+    class Meta:
+        model = Stat
+        fields = (
+            'id', 'node', 'ip', 'component',
+            'key', 'value', 'datetime',
+        )

apps/system/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

apps/system/urls.py

@@ -0,0 +1,14 @@
+# coding:utf-8
+from rest_framework_bulk.routes import BulkRouter
+
+from . import api
+
+app_name = 'system'
+
+router = BulkRouter()
+router.register(r'stats', api.StatViewSet, 'stat')
+
+urlpatterns = [
+]
+
+urlpatterns += router.urls

apps/system/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

utils/generate_fake_data/generate.py

@@ -15,6 +15,7 @@ django.setup()
 from resources.assets import AssetsGenerator, NodesGenerator, SystemUsersGenerator, AdminUsersGenerator
 from resources.users import UserGroupGenerator, UserGenerator
 from resources.perms import AssetPermissionGenerator
+from resources.system import StatGenerator
 
 
 resource_generator_mapper = {
@@ -24,7 +25,8 @@ resource_generator_mapper = {
     'admin_user': AdminUsersGenerator,
     'user': UserGenerator,
     'user_group': UserGroupGenerator,
-    'asset_permission': AssetPermissionGenerator
+    'asset_permission': AssetPermissionGenerator,
+    'stat': StatGenerator
 }
 
 

utils/generate_fake_data/resources/system.py

@@ -0,0 +1,69 @@
+import random
+
+from .base import FakeDataGenerator
+from system.models import *
+
+
+class StatGenerator(FakeDataGenerator):
+    resource = 'stat'
+
+    nodes = [
+        {
+            'node': 'guacamole-01',
+            'ip': '192.168.1.1',
+            'component': 'guacamole'
+        },
+        {
+            'node': 'koko-01',
+            'ip': '192.168.1.2',
+            'component': 'koko'
+        },
+        {
+            'node': 'omnidb-01',
+            'ip': '192.168.1.3',
+            'component': 'omnidb'
+        },
+        {
+            'node': 'core-01',
+            'ip': '192.168.1.4',
+            'component': 'core'
+        }
+    ]
+    items_value_range = {
+        'cpu_load': (0, 3.0),
+        'memory_used_percent': (20, 10.0),
+        'disk_used_percent': (30, 80.0),
+        'thread': (100, 100),
+        'goroutine': (200, 500),
+        'replay_upload_health': (0, [0, 1]),
+        'command_upload_health': (0, [0, 1]),
+        'session_active': (100, 50),
+        'session_processed': (400, 400),
+        'session_failed': (50, 100),
+        'session_succeeded': (500, 300)
+    }
+
+    def do_generate(self, batch, batch_size):
+        datetime = timezone.now()
+        for i in batch:
+            datetime = datetime - timezone.timedelta(minutes=1)
+            items = []
+            for node in self.nodes:
+                for key, values in self.items_value_range.items():
+                    base, r = values
+                    if isinstance(r, int):
+                        value = int(random.random() * r)
+                    elif isinstance(r, float):
+                        value = round(random.random() * r, 2)
+                    elif isinstance(r, list):
+                        value = random.choice(r)
+                    else:
+                        continue
+                    value += base
+                    node.update({
+                        'key': key,
+                        'value': value,
+                        'datetime': datetime
+                    })
+                    items.append(Stat(**node))
+            Stat.objects.bulk_create(items, ignore_conflicts=True)

utils/generate_fake_data/resources/users.py

@@ -47,7 +47,7 @@ class UserGenerator(FakeDataGenerator):
     def do_generate(self, batch, batch_size):
         users = []
         for i in batch:
-            username = forgery_py.internet.user_name(True)
+            username = forgery_py.internet.user_name(True) + '-' + str(i)
             email = forgery_py.internet.email_address()
             u = User(
                 username=username,