diff --git a/apps/accounts/api/automations/__init__.py b/apps/accounts/api/automations/__init__.py index 61035a089..a81361e35 100644 --- a/apps/accounts/api/automations/__init__.py +++ b/apps/accounts/api/automations/__init__.py @@ -1,6 +1,7 @@ from .backup import * from .base import * from .change_secret import * +from .change_secret_dashboard import * from .check_account import * from .gather_account import * from .push_account import * diff --git a/apps/accounts/api/automations/change_secret.py b/apps/accounts/api/automations/change_secret.py index 05ee515dc..186a49e4e 100644 --- a/apps/accounts/api/automations/change_secret.py +++ b/apps/accounts/api/automations/change_secret.py @@ -54,7 +54,10 @@ class ChangeSecretRecordViewSet(mixins.ListModelMixin, OrgGenericViewSet): return super().get_permissions() def get_queryset(self): - return ChangeSecretRecord.objects.all() + qs = ChangeSecretRecord.get_valid_records() + return qs.objects.filter( + execution__automation__type=self.tp + ) @action(methods=['post'], detail=False, url_path='execute') def execute(self, request, *args, **kwargs): diff --git a/apps/accounts/api/automations/change_secret_dashboard.py b/apps/accounts/api/automations/change_secret_dashboard.py new file mode 100644 index 000000000..c3393f36c --- /dev/null +++ b/apps/accounts/api/automations/change_secret_dashboard.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +# +from collections import defaultdict + +from django.http.response import JsonResponse +from django.utils import timezone +from rest_framework.views import APIView + +from accounts.const import AutomationTypes, ChangeSecretRecordStatusChoice +from accounts.models import ChangeSecretAutomation, AutomationExecution, ChangeSecretRecord +from assets.models import Node, Asset +from common.utils import lazyproperty +from common.utils.timezone import local_zero_hour, local_now +from ops.celery import app + +__all__ = ['ChangeSecretDashboardApi'] + + +class ChangeSecretDashboardApi(APIView): + http_method_names = ['get'] + rbac_perms = { + 'GET': 'accounts.view_changesecretautomation', + } + + tp = AutomationTypes.change_secret + task_name = 'accounts.tasks.automation.execute_account_automation_task' + + @lazyproperty + def days(self): + count = self.request.query_params.get('days', 1) + return int(count) + + @property + def days_to_datetime(self): + if self.days == 1: + return local_zero_hour() + return local_now() - timezone.timedelta(days=self.days) + + def get_queryset_date_filter(self, qs, query_field='date_updated'): + return qs.filter(**{f'{query_field}__gte': self.days_to_datetime}) + + @lazyproperty + def date_range_list(self): + return [ + (local_now() - timezone.timedelta(days=i)).date() + for i in range(self.days - 1, -1, -1) + ] + + def filter_by_date_range(self, queryset, field_name): + date_range_bounds = self.days_to_datetime.date(), (local_now() + timezone.timedelta(days=1)).date() + return queryset.filter(**{f'{field_name}__range': date_range_bounds}) + + def calculate_daily_metrics(self, queryset, date_field): + filtered_queryset = self.filter_by_date_range(queryset, date_field) + results = filtered_queryset.values_list(date_field, 'status') + + status_counts = defaultdict(lambda: defaultdict(int)) + + for date_finished, status in results: + date_str = str(date_finished.date()) + if status == ChangeSecretRecordStatusChoice.failed: + status_counts[date_str]['failed'] += 1 + elif status == ChangeSecretRecordStatusChoice.success: + status_counts[date_str]['success'] += 1 + + metrics = defaultdict(list) + for date in self.date_range_list: + date_str = str(date) + for status in ['success', 'failed']: + metrics[status].append(status_counts[date_str].get(status, 0)) + + return metrics + + def get_daily_success_and_failure_metrics(self): + metrics = self.calculate_daily_metrics(self.change_secret_records_queryset, 'date_finished') + return metrics.get('success', []), metrics.get('failed', []) + + @lazyproperty + def change_secrets_queryset(self): + return ChangeSecretAutomation.objects.all() + + @lazyproperty + def change_secret_executions_queryset(self): + return AutomationExecution.objects.filter(automation__type=self.tp) + + @lazyproperty + def change_secret_records_queryset(self): + return ChangeSecretRecord.get_valid_records().filter(execution__automation__type=self.tp) + + def get_change_secret_asset_queryset(self): + qs = self.get_queryset_date_filter(self.change_secrets_queryset) + node_ids = qs.filter(nodes__isnull=False).values_list('nodes', flat=True).distinct() + nodes = Node.objects.filter(id__in=node_ids) + node_asset_ids = Node.get_nodes_all_assets(*nodes).values_list('id', flat=True) + direct_asset_ids = qs.filter(assets__isnull=False).values_list('assets', flat=True).distinct() + asset_ids = set(list(direct_asset_ids) + list(node_asset_ids)) + return Asset.objects.filter(id__in=asset_ids) + + def get_filtered_counts(self, qs, field): + return self.get_queryset_date_filter(qs, field).count() + + @staticmethod + def get_status_counts(records): + pending = ChangeSecretRecordStatusChoice.pending + failed = ChangeSecretRecordStatusChoice.failed + total_ids = {str(i) for i in records.exclude(status=pending).values('execution_id').distinct()} + failed_ids = {str(i) for i in records.filter(status=failed).values('execution_id').distinct()} + total = len(total_ids) + failed = len(total_ids & failed_ids) + return { + 'total_count_change_secret_executions': total, + 'total_count_success_change_secret_executions': total - failed, + 'total_count_failed_change_secret_executions': failed, + } + + def get(self, request, *args, **kwargs): + query_params = self.request.query_params + data = {} + + if query_params.get('total_count_change_secrets'): + data['total_count_change_secrets'] = self.get_filtered_counts( + self.change_secrets_queryset, 'date_updated' + ) + + if query_params.get('total_count_periodic_change_secrets'): + data['total_count_periodic_change_secrets'] = self.get_filtered_counts( + self.change_secrets_queryset.filter(is_periodic=True), 'date_updated' + ) + + if query_params.get('total_count_change_secret_assets'): + data['total_count_change_secret_assets'] = self.get_change_secret_asset_queryset().count() + + if query_params.get('total_count_change_secret_status'): + records = self.get_queryset_date_filter(self.change_secret_records_queryset, 'date_finished') + data.update(self.get_status_counts(records)) + + if query_params.get('total_count_change_secret_status'): + records = self.get_queryset_date_filter(self.change_secret_records_queryset, 'date_finished') + data.update(self.get_status_counts(records)) + + if query_params.get('daily_success_and_failure_metrics'): + success, failed = self.get_daily_success_and_failure_metrics() + data.update({ + 'dates_metrics_date': [date.strftime('%m-%d') for date in self.date_range_list] or ['0'], + 'dates_metrics_total_count_success': success, + 'dates_metrics_total_count_failed': failed, + }) + + if query_params.get('total_count_ongoing_change_secret'): + execution_ids = [] + inspect = app.control.inspect() + active_tasks = inspect.active() + for tasks in active_tasks.values(): + for task in tasks: + _id = task.get('id') + name = task.get('name') + tp = task.kwargs.get('tp') + if name == self.task_name and tp == self.tp: + execution_ids.append(_id) + + snapshots = self.change_secret_executions_queryset.filter( + id__in=execution_ids).values_list('id', 'snapshot') + + asset_ids = {asset for i in snapshots for asset in i.get('assets', [])} + account_ids = {account for i in snapshots for account in i.get('accounts', [])} + data['total_count_ongoing_change_secret'] = len(execution_ids) + data['total_count_ongoing_change_secret_assets'] = len(asset_ids) + data['total_count_ongoing_change_secret_accounts'] = len(account_ids) + + return JsonResponse(data, status=200) diff --git a/apps/accounts/api/automations/push_account.py b/apps/accounts/api/automations/push_account.py index 1fa5c1219..7b27cc7ca 100644 --- a/apps/accounts/api/automations/push_account.py +++ b/apps/accounts/api/automations/push_account.py @@ -45,8 +45,9 @@ class PushAccountRecordViewSet(ChangeSecretRecordViewSet): tp = AutomationTypes.push_account def get_queryset(self): - return ChangeSecretRecord.objects.filter( - execution__automation__type=AutomationTypes.push_account + qs = ChangeSecretRecord.get_valid_records() + return qs.objects.filter( + execution__automation__type=self.tp ) diff --git a/apps/accounts/models/automations/change_secret.py b/apps/accounts/models/automations/change_secret.py index dff4d5cc4..4e42af544 100644 --- a/apps/accounts/models/automations/change_secret.py +++ b/apps/accounts/models/automations/change_secret.py @@ -1,4 +1,5 @@ from django.db import models +from django.db.models import Q from django.utils.translation import gettext_lazy as _ from accounts.const import ( @@ -48,3 +49,9 @@ class ChangeSecretRecord(JMSBaseModel): def __str__(self): return f'{self.account.username}@{self.asset}' + + @staticmethod + def get_valid_records(): + return ChangeSecretRecord.objects.exclude( + Q(execution__isnull=True) | Q(asset__isnull=True) | Q(account__isnull=True) + ) diff --git a/apps/accounts/urls.py b/apps/accounts/urls.py index 352dcfaae..bc901816b 100644 --- a/apps/accounts/urls.py +++ b/apps/accounts/urls.py @@ -49,6 +49,7 @@ urlpatterns = [ path('push-account//nodes/', api.PushAccountNodeAddRemoveApi.as_view(), name='push-account-add-or-remove-node'), path('push-account//assets/', api.PushAccountAssetsListApi.as_view(), name='push-account-assets'), + path('change-secret-dashboard/', api.ChangeSecretDashboardApi.as_view(), name='change-secret-dashboard'), ] urlpatterns += router.urls