mirror of https://github.com/jumpserver/jumpserver
perf: navigation page
parent
8a3bc51faa
commit
0b802b1782
|
@ -1,22 +1,21 @@
|
|||
import datetime
|
||||
|
||||
import pytz
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from django.utils import timezone as dj_timezone
|
||||
from rest_framework.fields import DateTimeField
|
||||
|
||||
max = datetime.datetime.max.replace(tzinfo=datetime.timezone.utc)
|
||||
max = datetime.max.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def astimezone(dt: datetime.datetime, tzinfo: pytz.tzinfo.DstTzInfo):
|
||||
def astimezone(dt: datetime, tzinfo: pytz.tzinfo.DstTzInfo):
|
||||
assert dj_timezone.is_aware(dt)
|
||||
return tzinfo.normalize(dt.astimezone(tzinfo))
|
||||
|
||||
|
||||
def as_china_cst(dt: datetime.datetime):
|
||||
def as_china_cst(dt: datetime):
|
||||
return astimezone(dt, pytz.timezone('Asia/Shanghai'))
|
||||
|
||||
|
||||
def as_current_tz(dt: datetime.datetime):
|
||||
def as_current_tz(dt: datetime):
|
||||
return astimezone(dt, dj_timezone.get_current_timezone())
|
||||
|
||||
|
||||
|
@ -36,6 +35,15 @@ def local_now_date_display(fmt='%Y-%m-%d'):
|
|||
return local_now().strftime(fmt)
|
||||
|
||||
|
||||
def local_zero_hour(fmt='%Y-%m-%d'):
|
||||
return datetime.strptime(local_now().strftime(fmt), fmt)
|
||||
|
||||
|
||||
def local_monday():
|
||||
zero_hour_time = local_zero_hour()
|
||||
return zero_hour_time - timedelta(zero_hour_time.weekday())
|
||||
|
||||
|
||||
_rest_dt_field = DateTimeField()
|
||||
dt_parser = _rest_dt_field.to_internal_value
|
||||
dt_formatter = _rest_dt_field.to_representation
|
||||
|
|
|
@ -3,45 +3,51 @@ import time
|
|||
from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
from django.utils.timesince import timesince
|
||||
from django.db.models import Count, Max
|
||||
from django.db.models import Count, Max, F
|
||||
from django.http.response import JsonResponse, HttpResponse
|
||||
from rest_framework.views import APIView
|
||||
from rest_framework.permissions import AllowAny
|
||||
from collections import Counter
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
|
||||
from users.models import User
|
||||
from assets.models import Asset
|
||||
from assets.const import AllTypes
|
||||
from terminal.models import Session
|
||||
from terminal.utils import ComponentsPrometheusMetricsUtil
|
||||
from orgs.utils import current_org
|
||||
from common.utils import lazyproperty
|
||||
from common.utils.timezone import local_now, local_zero_hour
|
||||
from orgs.caches import OrgResourceStatisticsCache
|
||||
|
||||
|
||||
__all__ = ['IndexApi']
|
||||
|
||||
|
||||
class DatesLoginMetricMixin:
|
||||
request: Request
|
||||
|
||||
@lazyproperty
|
||||
def days(self):
|
||||
query_params = self.request.query_params
|
||||
if query_params.get('monthly'):
|
||||
return 30
|
||||
return 7
|
||||
# monthly
|
||||
count = query_params.get('days')
|
||||
return count if count else 0
|
||||
|
||||
@lazyproperty
|
||||
def sessions_queryset(self):
|
||||
days = timezone.now() - timezone.timedelta(days=self.days)
|
||||
sessions_queryset = Session.objects.filter(date_start__gt=days)
|
||||
days = self.days
|
||||
if days == 0:
|
||||
t = local_zero_hour()
|
||||
else:
|
||||
t = local_now() - timezone.timedelta(days=days)
|
||||
sessions_queryset = Session.objects.filter(date_start__gte=t)
|
||||
return sessions_queryset
|
||||
|
||||
@lazyproperty
|
||||
def session_dates_list(self):
|
||||
now = timezone.now()
|
||||
now = local_now()
|
||||
dates = [(now - timezone.timedelta(days=i)).date() for i in range(self.days)]
|
||||
dates.reverse()
|
||||
# dates = self.sessions_queryset.dates('date_start', 'day')
|
||||
return dates
|
||||
|
||||
def get_dates_metrics_date(self):
|
||||
|
@ -63,7 +69,7 @@ class DatesLoginMetricMixin:
|
|||
|
||||
def __set_data_to_cache(self, date, tp, count):
|
||||
cache_key = self.get_cache_key(date, tp)
|
||||
cache.set(cache_key, count, 3600*24*7)
|
||||
cache.set(cache_key, count, 3600 * 24 * 7)
|
||||
|
||||
@staticmethod
|
||||
def get_date_start_2_end(d):
|
||||
|
@ -162,40 +168,45 @@ class DatesLoginMetricMixin:
|
|||
def dates_total_count_disabled_assets(self):
|
||||
return Asset.objects.filter(is_active=False).count()
|
||||
|
||||
# 以下是从week中而来
|
||||
def get_dates_login_times_top5_users(self):
|
||||
users = self.sessions_queryset.values_list('user_id', flat=True)
|
||||
users = [
|
||||
{'user': user, 'total': total}
|
||||
for user, total in Counter(users).most_common(5)
|
||||
]
|
||||
return users
|
||||
|
||||
def get_dates_total_count_login_users(self):
|
||||
return len(set(self.sessions_queryset.values_list('user_id', flat=True)))
|
||||
|
||||
def get_dates_total_count_login_times(self):
|
||||
return self.sessions_queryset.count()
|
||||
|
||||
def get_dates_login_times_top10_assets(self):
|
||||
@lazyproperty
|
||||
def get_type_to_assets(self):
|
||||
result = Asset.objects.annotate(type=F('platform__type')). \
|
||||
values('type').order_by('type').annotate(total=Count(1))
|
||||
all_types_dict = dict(AllTypes.choices())
|
||||
result = list(result)
|
||||
for i in result:
|
||||
tp = i['type']
|
||||
i['label'] = all_types_dict[tp]
|
||||
return result
|
||||
|
||||
def get_dates_login_times_assets(self):
|
||||
assets = self.sessions_queryset.values("asset") \
|
||||
.annotate(total=Count("asset")) \
|
||||
.annotate(last=Max("date_start")).order_by("-total")[:10]
|
||||
.annotate(total=Count("asset")) \
|
||||
.annotate(last=Max("date_start")).order_by("-total")
|
||||
assets = assets[:10]
|
||||
for asset in assets:
|
||||
asset['last'] = str(asset['last'])
|
||||
return list(assets)
|
||||
|
||||
def get_dates_login_times_top10_users(self):
|
||||
def get_dates_login_times_users(self):
|
||||
users = self.sessions_queryset.values("user_id") \
|
||||
.annotate(total=Count("user_id")) \
|
||||
.annotate(user=Max('user')) \
|
||||
.annotate(last=Max("date_start")).order_by("-total")[:10]
|
||||
.annotate(total=Count("user_id")) \
|
||||
.annotate(user=Max('user')) \
|
||||
.annotate(last=Max("date_start")).order_by("-total")
|
||||
users = users[:10]
|
||||
for user in users:
|
||||
user['last'] = str(user['last'])
|
||||
return list(users)
|
||||
|
||||
def get_dates_login_record_top10_sessions(self):
|
||||
sessions = self.sessions_queryset.order_by('-date_start')[:10]
|
||||
def get_dates_login_record_sessions(self):
|
||||
sessions = self.sessions_queryset.order_by('-date_start')
|
||||
sessions = sessions[:10]
|
||||
for session in sessions:
|
||||
session.avatar_url = User.get_avatar_url("")
|
||||
sessions = [
|
||||
|
@ -229,11 +240,13 @@ class IndexApi(DatesLoginMetricMixin, APIView):
|
|||
if _all or query_params.get('total_count') or query_params.get('total_count_users'):
|
||||
data.update({
|
||||
'total_count_users': caches.users_amount,
|
||||
'total_count_users_this_week': caches.new_users_amount_this_week,
|
||||
})
|
||||
|
||||
if _all or query_params.get('total_count') or query_params.get('total_count_assets'):
|
||||
data.update({
|
||||
'total_count_assets': caches.assets_amount,
|
||||
'total_count_assets_this_week': caches.new_assets_amount_this_week,
|
||||
})
|
||||
|
||||
if _all or query_params.get('total_count') or query_params.get('total_count_online_users'):
|
||||
|
@ -246,6 +259,23 @@ class IndexApi(DatesLoginMetricMixin, APIView):
|
|||
'total_count_online_sessions': caches.total_count_online_sessions,
|
||||
})
|
||||
|
||||
if _all or query_params.get('total_count') or query_params.get('total_count_today_failed_sessions'):
|
||||
data.update({
|
||||
'total_count_today_failed_sessions': caches.total_count_today_failed_sessions,
|
||||
})
|
||||
if _all or query_params.get('total_count') or query_params.get('total_count_today_login_users'):
|
||||
data.update({
|
||||
'total_count_today_login_users': caches.total_count_today_login_users,
|
||||
})
|
||||
if _all or query_params.get('total_count') or query_params.get('total_count_today_active_assets'):
|
||||
data.update({
|
||||
'total_count_today_active_assets': caches.total_count_today_active_assets,
|
||||
})
|
||||
if _all or query_params.get('total_count') or query_params.get('total_count_type_to_assets_amount'):
|
||||
data.update({
|
||||
'total_count_type_to_assets_amount': self.get_type_to_assets,
|
||||
})
|
||||
|
||||
if _all or query_params.get('dates_metrics'):
|
||||
data.update({
|
||||
'dates_metrics_date': self.get_dates_metrics_date(),
|
||||
|
@ -274,24 +304,19 @@ class IndexApi(DatesLoginMetricMixin, APIView):
|
|||
'dates_total_count_login_times': self.get_dates_total_count_login_times(),
|
||||
})
|
||||
|
||||
if _all or query_params.get('dates_login_times_top5_users'):
|
||||
data.update({
|
||||
'dates_login_times_top5_users': self.get_dates_login_times_top5_users(),
|
||||
})
|
||||
|
||||
if _all or query_params.get('dates_login_times_top10_assets'):
|
||||
data.update({
|
||||
'dates_login_times_top10_assets': self.get_dates_login_times_top10_assets(),
|
||||
'dates_login_times_top10_assets': self.get_dates_login_times_assets(),
|
||||
})
|
||||
|
||||
if _all or query_params.get('dates_login_times_top10_users'):
|
||||
data.update({
|
||||
'dates_login_times_top10_users': self.get_dates_login_times_top10_users(),
|
||||
'dates_login_times_top10_users': self.get_dates_login_times_users(),
|
||||
})
|
||||
|
||||
if _all or query_params.get('dates_login_record_top10_sessions'):
|
||||
data.update({
|
||||
'dates_login_record_top10_sessions': self.get_dates_login_record_top10_sessions()
|
||||
'dates_login_record_top10_sessions': self.get_dates_login_record_sessions()
|
||||
})
|
||||
|
||||
return JsonResponse(data, status=200)
|
||||
|
@ -353,4 +378,3 @@ class PrometheusMetricsApi(HealthApiMixin):
|
|||
util = ComponentsPrometheusMetricsUtil()
|
||||
metrics_text = util.get_prometheus_metrics_text()
|
||||
return HttpResponse(metrics_text, content_type='text/plain; version=0.0.4; charset=utf-8')
|
||||
|
||||
|
|
|
@ -1,11 +1,14 @@
|
|||
from django.db.transaction import on_commit
|
||||
|
||||
from orgs.models import Organization
|
||||
from orgs.tasks import refresh_org_cache_task
|
||||
from orgs.utils import current_org, tmp_to_org
|
||||
|
||||
from common.cache import Cache, IntegerField
|
||||
from common.utils import get_logger
|
||||
from common.utils.timezone import local_zero_hour, local_monday
|
||||
from users.models import UserGroup, User
|
||||
from audits.models import UserLoginLog
|
||||
from audits.const import LoginStatusChoices
|
||||
from assets.models import Node, Domain, Asset, Account
|
||||
from terminal.models import Session
|
||||
from perms.models import AssetPermission
|
||||
|
@ -35,30 +38,35 @@ class OrgRelatedCache(Cache):
|
|||
"""
|
||||
在事务提交之后再发送信号,防止因事务的隔离性导致未获得最新的数据
|
||||
"""
|
||||
|
||||
def func():
|
||||
logger.debug(f'CACHE: Send refresh task {self}.{fields}')
|
||||
refresh_org_cache_task.delay(self, *fields)
|
||||
|
||||
on_commit(func)
|
||||
|
||||
def expire(self, *fields):
|
||||
def func():
|
||||
super(OrgRelatedCache, self).expire(*fields)
|
||||
|
||||
on_commit(func)
|
||||
|
||||
|
||||
class OrgResourceStatisticsCache(OrgRelatedCache):
|
||||
users_amount = IntegerField()
|
||||
groups_amount = IntegerField(queryset=UserGroup.objects)
|
||||
|
||||
assets_amount = IntegerField()
|
||||
new_users_amount_this_week = IntegerField()
|
||||
new_assets_amount_this_week = IntegerField()
|
||||
nodes_amount = IntegerField(queryset=Node.objects)
|
||||
accounts_amount = IntegerField(queryset=Account.objects)
|
||||
domains_amount = IntegerField(queryset=Domain.objects)
|
||||
# gateways_amount = IntegerField(queryset=Gateway.objects)
|
||||
groups_amount = IntegerField(queryset=UserGroup.objects)
|
||||
accounts_amount = IntegerField(queryset=Account.objects)
|
||||
asset_perms_amount = IntegerField(queryset=AssetPermission.objects)
|
||||
|
||||
total_count_online_users = IntegerField()
|
||||
total_count_online_sessions = IntegerField()
|
||||
total_count_today_login_users = IntegerField()
|
||||
total_count_today_active_assets = IntegerField()
|
||||
total_count_today_failed_sessions = IntegerField()
|
||||
|
||||
def __init__(self, org):
|
||||
super().__init__()
|
||||
|
@ -70,18 +78,56 @@ class OrgResourceStatisticsCache(OrgRelatedCache):
|
|||
def get_current_org(self):
|
||||
return self.org
|
||||
|
||||
def get_users(self):
|
||||
return User.get_org_users(self.org)
|
||||
|
||||
@staticmethod
|
||||
def get_assets():
|
||||
return Asset.objects.all()
|
||||
|
||||
def compute_users_amount(self):
|
||||
amount = User.get_org_users(self.org).count()
|
||||
return amount
|
||||
users = self.get_users()
|
||||
return users.count()
|
||||
|
||||
def compute_new_users_amount_this_week(self):
|
||||
monday_time = local_monday()
|
||||
users = self.get_users().filter(date_joined__gte=monday_time)
|
||||
return users.count()
|
||||
|
||||
def compute_assets_amount(self):
|
||||
if self.org.is_root():
|
||||
return Asset.objects.all().count()
|
||||
node = Node.org_root()
|
||||
return node.assets_amount
|
||||
assets = self.get_assets()
|
||||
return assets.count()
|
||||
|
||||
def compute_total_count_online_users(self):
|
||||
return Session.objects.filter(is_finished=False).values_list('user_id').distinct().count()
|
||||
def compute_new_assets_amount_this_week(self):
|
||||
monday_time = local_monday()
|
||||
assets = self.get_assets().filter(date_created__gte=monday_time)
|
||||
return assets.count()
|
||||
|
||||
def compute_total_count_online_sessions(self):
|
||||
@staticmethod
|
||||
def compute_total_count_online_users():
|
||||
return Session.objects.filter(
|
||||
is_finished=False
|
||||
).values_list('user_id').distinct().count()
|
||||
|
||||
@staticmethod
|
||||
def compute_total_count_online_sessions():
|
||||
return Session.objects.filter(is_finished=False).count()
|
||||
|
||||
@staticmethod
|
||||
def compute_total_count_today_login_users():
|
||||
t = local_zero_hour()
|
||||
return UserLoginLog.objects.filter(
|
||||
datetime__gte=t, status=LoginStatusChoices.success
|
||||
).values('username').distinct().count()
|
||||
|
||||
@staticmethod
|
||||
def compute_total_count_today_active_assets():
|
||||
t = local_zero_hour()
|
||||
return Session.objects.filter(
|
||||
date_start__gte=t, is_success=False
|
||||
).values('asset_id').distinct().count()
|
||||
|
||||
@staticmethod
|
||||
def compute_total_count_today_failed_sessions():
|
||||
t = local_zero_hour()
|
||||
return Session.objects.filter(date_start__gte=t, is_success=False).count()
|
||||
|
|
|
@ -2,8 +2,9 @@ from django.db.models.signals import post_save, pre_delete, pre_save, post_delet
|
|||
from django.dispatch import receiver
|
||||
|
||||
from orgs.models import Organization
|
||||
from assets.models import Node
|
||||
from assets.models import Node, Account
|
||||
from perms.models import AssetPermission
|
||||
from audits.models import UserLoginLog
|
||||
from users.models import UserGroup, User
|
||||
from users.signals import pre_user_leave_org
|
||||
from terminal.models import Session
|
||||
|
@ -74,12 +75,15 @@ def on_user_delete_refresh_cache(sender, instance, **kwargs):
|
|||
|
||||
class OrgResourceStatisticsRefreshUtil:
|
||||
model_cache_field_mapper = {
|
||||
AssetPermission: ['asset_perms_amount'],
|
||||
Domain: ['domains_amount'],
|
||||
Node: ['nodes_amount'],
|
||||
Asset: ['assets_amount'],
|
||||
Domain: ['domains_amount'],
|
||||
UserGroup: ['groups_amount'],
|
||||
RoleBinding: ['users_amount']
|
||||
Account: ['accounts_amount'],
|
||||
UserLoginLog: ['total_count_today_login_users'],
|
||||
RoleBinding: ['users_amount', 'new_users_amount_this_week'],
|
||||
Asset: ['assets_amount', 'new_assets_amount_this_week'],
|
||||
AssetPermission: ['asset_perms_amount'],
|
||||
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
|
Loading…
Reference in New Issue