github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: 优化 dashboard api (#11478) 

Co-authored-by: feng <1304903146@qq.com>
pull/11464/head^2
fit2bot 2023-08-31 15:05:05 +08:00 committed by GitHub
parent 40a460870a
commit db0b43ee84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 62 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
apps/jumpserver/api.py
View File

@ -2,7 +2,8 @@ import time
from collections import defaultdict from collections import defaultdict
from django.core.cache import cache from django.core.cache import cache
from django.db.models import Count, Max, F from django.db.models import Count, Max, F, CharField
from django.db.models.functions import Cast
from django.http.response import JsonResponse, HttpResponse from django.http.response import JsonResponse, HttpResponse
from django.utils import timezone from django.utils import timezone
from django.utils.timesince import timesince from django.utils.timesince import timesince
@ -13,6 +14,7 @@ from rest_framework.views import APIView
from assets.const import AllTypes from assets.const import AllTypes
from assets.models import Asset from assets.models import Asset
from audits.api import OperateLogViewSet
from audits.const import LoginStatusChoices from audits.const import LoginStatusChoices
from audits.models import UserLoginLog, PasswordChangeLog, OperateLog, FTPLog, JobLog from audits.models import UserLoginLog, PasswordChangeLog, OperateLog, FTPLog, JobLog
from common.utils import lazyproperty from common.utils import lazyproperty
@ -57,83 +59,78 @@ class DateTimeMixin:
@lazyproperty @lazyproperty
def dates_list(self): def dates_list(self):
now = local_now() return [
dates = [(now - timezone.timedelta(days=i)).date() for i in range(self.days)] (local_now() - timezone.timedelta(days=i)).date()
dates.reverse() for i in range(self.days - 1, -1, -1)
return dates ]
def get_dates_metrics_date(self): def get_dates_metrics_date(self):
dates_metrics_date = [d.strftime('%m-%d') for d in self.dates_list] or ['0'] return [d.strftime('%m-%d') for d in self.dates_list] or ['0']
return dates_metrics_date
@lazyproperty def get_logs_queryset_filter(self, qs, query_field, is_timestamp=False):
def users(self): dt = self.days_to_datetime
return self.org.get_members() t = dt.timestamp() if is_timestamp else dt
query = {f'{query_field}__gte': t}
@lazyproperty return qs.filter(**query)
def sessions_queryset(self):
t = self.days_to_datetime
sessions_queryset = Session.objects.filter(date_start__gte=t)
return sessions_queryset
def get_logs_queryset(self, queryset, query_params): def get_logs_queryset(self, queryset, query_params):
query = {} query = {}
users = self.org.get_members()
if not self.org.is_root(): if not self.org.is_root():
if query_params == 'username': if query_params == 'username':
query = { query = {
f'{query_params}__in': self.users.values_list('username', flat=True) f'{query_params}__in': users.values_list('username', flat=True)
} }
else: else:
query = { query = {
f'{query_params}__in': [str(user) for user in self.users] f'{query_params}__in': [str(user) for user in users]
} }
queryset = queryset.filter(**query) queryset = queryset.filter(**query)
return queryset return queryset
@lazyproperty
def sessions_queryset(self):
return self.get_logs_queryset_filter(Session.objects, 'date_start')
@lazyproperty @lazyproperty
def login_logs_queryset(self): def login_logs_queryset(self):
t = self.days_to_datetime qs = UserLoginLog.objects.all()
queryset = UserLoginLog.objects.filter(datetime__gte=t) qs = self.get_logs_queryset_filter(qs, 'datetime')
queryset = self.get_logs_queryset(queryset, 'username') queryset = self.get_logs_queryset(qs, 'username')
return queryset return queryset
@lazyproperty @lazyproperty
def password_change_logs_queryset(self): def password_change_logs_queryset(self):
t = self.days_to_datetime qs = PasswordChangeLog.objects.all()
queryset = PasswordChangeLog.objects.filter(datetime__gte=t) qs = self.get_logs_queryset_filter(qs, 'datetime')
queryset = self.get_logs_queryset(queryset, 'user') queryset = self.get_logs_queryset(qs, 'user')
return queryset return queryset
@lazyproperty @lazyproperty
def operate_logs_queryset(self): def operate_logs_queryset(self):
from audits.api import OperateLogViewSet qs = OperateLogViewSet().get_queryset()
t = self.days_to_datetime return self.get_logs_queryset_filter(qs, 'datetime')
queryset = OperateLogViewSet().get_queryset().filter(datetime__gte=t)
return queryset
@lazyproperty @lazyproperty
def ftp_logs_queryset(self): def ftp_logs_queryset(self):
t = self.days_to_datetime qs = FTPLog.objects.all()
queryset = FTPLog.objects.filter(date_start__gte=t) qs = self.get_logs_queryset_filter(qs, 'date_start')
queryset = self.get_logs_queryset(queryset, 'user') return self.get_logs_queryset(qs, 'user')
return queryset
@lazyproperty @lazyproperty
def command_queryset(self): def command_queryset(self):
t = self.days_to_datetime qs = Command.objects.all()
t = t.timestamp() return self.get_logs_queryset_filter(qs, 'timestamp', is_timestamp=True)
queryset = Command.objects.filter(timestamp__gte=t)
return queryset
@lazyproperty @lazyproperty
def job_logs_queryset(self): def job_logs_queryset(self):
t = self.days_to_datetime qs = JobLog.objects.all()
queryset = JobLog.objects.filter(date_created__gte=t) return self.get_logs_queryset_filter(qs, 'date_start')
return queryset
class DatesLoginMetricMixin: class DatesLoginMetricMixin:
dates_list: list dates_list: list
date_start_end: tuple
command_queryset: Command.objects command_queryset: Command.objects
sessions_queryset: Session.objects sessions_queryset: Session.objects
ftp_logs_queryset: OperateLog.objects ftp_logs_queryset: OperateLog.objects
@ -142,6 +139,17 @@ class DatesLoginMetricMixin:
operate_logs_queryset: OperateLog.objects operate_logs_queryset: OperateLog.objects
password_change_logs_queryset: PasswordChangeLog.objects password_change_logs_queryset: PasswordChangeLog.objects
@lazyproperty
def get_type_to_assets(self):
result = Asset.objects.annotate(type=F('platform__type')). \
values('type').order_by('type').annotate(total=Count(1))
all_types_dict = dict(AllTypes.choices())
result = list(result)
for i in result:
tp = i['type']
i['label'] = all_types_dict.get(tp, tp)
return result
def filter_date_start_end(self, queryset, field_name): def filter_date_start_end(self, queryset, field_name):
query = {f'{field_name}__range': self.date_start_end} query = {f'{field_name}__range': self.date_start_end}
return queryset.filter(**query) return queryset.filter(**query)
@ -172,41 +180,23 @@ class DatesLoginMetricMixin:
def get_dates_metrics_total_count_sessions(self): def get_dates_metrics_total_count_sessions(self):
return self.get_date_metrics(Session.objects, 'date_start', 'id') return self.get_date_metrics(Session.objects, 'date_start', 'id')
@lazyproperty
def get_type_to_assets(self):
result = Asset.objects.annotate(type=F('platform__type')). \
values('type').order_by('type').annotate(total=Count(1))
all_types_dict = dict(AllTypes.choices())
result = list(result)
for i in result:
tp = i['type']
i['label'] = all_types_dict.get(tp, tp)
return result
def get_dates_login_times_assets(self): def get_dates_login_times_assets(self):
assets = self.sessions_queryset.values("asset") \ assets = self.sessions_queryset.values("asset") \
.annotate(total=Count("asset")) \ .annotate(total=Count("asset")) \
.annotate(last=Max("date_start")).order_by("-total") .annotate(last=Cast(Max("date_start"), output_field=CharField())) \
assets = assets[:10] .order_by("-total")
for asset in assets: return list(assets[:10])
asset['last'] = str(asset['last'])
return list(assets)
def get_dates_login_times_users(self): def get_dates_login_times_users(self):
users = self.sessions_queryset.values("user_id") \ users = self.sessions_queryset.values("user_id") \
.annotate(total=Count("user_id")) \ .annotate(total=Count("user_id")) \
.annotate(user=Max('user')) \ .annotate(user=Max('user')) \
.annotate(last=Max("date_start")).order_by("-total") .annotate(last=Cast(Max("date_start"), output_field=CharField())) \
users = users[:10] .order_by("-total")
for user in users: return list(users[:10])
user['last'] = str(user['last'])
return list(users)
def get_dates_login_record_sessions(self): def get_dates_login_record_sessions(self):
sessions = self.sessions_queryset.order_by('-date_start') sessions = self.sessions_queryset.order_by('-date_start')
sessions = sessions[:10]
for session in sessions:
session.avatar_url = User.get_avatar_url("")
sessions = [ sessions = [
{ {
'user': session.user, 'user': session.user,
@ -215,7 +205,7 @@ class DatesLoginMetricMixin:
'date_start': str(session.date_start), 'date_start': str(session.date_start),
'timesince': timesince(session.date_start) 'timesince': timesince(session.date_start)
} }
for session in sessions for session in sessions[:10]
] ]
return sessions return sessions
@ -249,12 +239,13 @@ class DatesLoginMetricMixin:
@lazyproperty @lazyproperty
def job_logs_running_amount(self): def job_logs_running_amount(self):
return self.job_logs_queryset.filter(status__in=[JobStatus.running]).count() return self.job_logs_queryset.filter(status=JobStatus.running).count()
@lazyproperty @lazyproperty
def job_logs_failed_amount(self): def job_logs_failed_amount(self):
return self.job_logs_queryset.filter( return self.job_logs_queryset.filter(
status__in=[JobStatus.failed, JobStatus.timeout]).count() status__in=[JobStatus.failed, JobStatus.timeout]
).count()
@lazyproperty @lazyproperty
def job_logs_amount(self): def job_logs_amount(self):
@ -264,6 +255,10 @@ class DatesLoginMetricMixin:
def sessions_amount(self): def sessions_amount(self):
return self.sessions_queryset.count() return self.sessions_queryset.count()
@lazyproperty
def online_sessions_amount(self):
return self.sessions_queryset.filter(is_finished=False).count()
@lazyproperty @lazyproperty
def ftp_logs_amount(self): def ftp_logs_amount(self):
return self.ftp_logs_queryset.count() return self.ftp_logs_queryset.count()
@ -360,9 +355,8 @@ class IndexApi(DateTimeMixin, DatesLoginMetricMixin, APIView):
}) })
if _all or query_params.get('total_count') or query_params.get('total_count_history_sessions'): if _all or query_params.get('total_count') or query_params.get('total_count_history_sessions'):
count = self.sessions_amount - caches.total_count_online_sessions
data.update({ data.update({
'total_count_history_sessions': count if count > 0 else 0, 'total_count_history_sessions': self.sessions_amount - self.online_sessions_amount,
}) })
if _all or query_params.get('total_count') or query_params.get('total_count_ftp_logs'): if _all or query_params.get('total_count') or query_params.get('total_count_ftp_logs'):