perf: update status

apps/accounts/filters.py

@@ -11,31 +11,39 @@ from .models import Account, GatheredAccount, ChangeSecretRecord
 
 
 class AccountFilterSet(BaseFilterSet):
-    ip = drf_filters.CharFilter(field_name='address', lookup_expr='exact')
-    hostname = drf_filters.CharFilter(field_name='name', lookup_expr='exact')
-    username = drf_filters.CharFilter(field_name="username", lookup_expr='exact')
-    address = drf_filters.CharFilter(field_name="asset__address", lookup_expr='exact')
-    asset_id = drf_filters.CharFilter(field_name="asset", lookup_expr='exact')
-    asset = drf_filters.CharFilter(field_name='asset', lookup_expr='exact')
-    assets = drf_filters.CharFilter(field_name='asset_id', lookup_expr='exact')
-    nodes = drf_filters.CharFilter(method='filter_nodes')
-    node_id = drf_filters.CharFilter(method='filter_nodes')
-    has_secret = drf_filters.BooleanFilter(method='filter_has_secret')
-    platform = drf_filters.CharFilter(field_name='asset__platform_id', lookup_expr='exact')
-    category = drf_filters.CharFilter(field_name='asset__platform__category', lookup_expr='exact')
-    type = drf_filters.CharFilter(field_name='asset__platform__type', lookup_expr='exact')
-    latest_discovery = drf_filters.BooleanFilter(method='filter_latest')
-    latest_accessed = drf_filters.BooleanFilter(method='filter_latest')
-    latest_updated = drf_filters.BooleanFilter(method='filter_latest')
-    latest_secret_changed = drf_filters.BooleanFilter(method='filter_latest')
-    latest_secret_change_failed = drf_filters.BooleanFilter(method='filter_latest')
-    risk = drf_filters.CharFilter(method='filter_risk', )
-    long_time_no_change_secret = drf_filters.BooleanFilter(method='filter_long_time')
-    long_time_no_verified = drf_filters.BooleanFilter(method='filter_long_time')
+    ip = drf_filters.CharFilter(field_name="address", lookup_expr="exact")
+    hostname = drf_filters.CharFilter(field_name="name", lookup_expr="exact")
+    username = drf_filters.CharFilter(field_name="username", lookup_expr="exact")
+    address = drf_filters.CharFilter(field_name="asset__address", lookup_expr="exact")
+    asset_id = drf_filters.CharFilter(field_name="asset", lookup_expr="exact")
+    asset = drf_filters.CharFilter(field_name="asset", lookup_expr="exact")
+    assets = drf_filters.CharFilter(field_name="asset_id", lookup_expr="exact")
+    nodes = drf_filters.CharFilter(method="filter_nodes")
+    node_id = drf_filters.CharFilter(method="filter_nodes")
+    has_secret = drf_filters.BooleanFilter(method="filter_has_secret")
+    platform = drf_filters.CharFilter(
+        field_name="asset__platform_id", lookup_expr="exact"
+    )
+    category = drf_filters.CharFilter(
+        field_name="asset__platform__category", lookup_expr="exact"
+    )
+    type = drf_filters.CharFilter(
+        field_name="asset__platform__type", lookup_expr="exact"
+    )
+    latest_discovery = drf_filters.BooleanFilter(method="filter_latest")
+    latest_accessed = drf_filters.BooleanFilter(method="filter_latest")
+    latest_updated = drf_filters.BooleanFilter(method="filter_latest")
+    latest_secret_changed = drf_filters.BooleanFilter(method="filter_latest")
+    latest_secret_change_failed = drf_filters.BooleanFilter(method="filter_latest")
+    risk = drf_filters.CharFilter(
+        method="filter_risk",
+    )
+    long_time_no_change_secret = drf_filters.BooleanFilter(method="filter_long_time")
+    long_time_no_verified = drf_filters.BooleanFilter(method="filter_long_time")
 
     @staticmethod
     def filter_has_secret(queryset, name, has_secret):
-        q = Q(_secret__isnull=True) | Q(_secret='')
+        q = Q(_secret__isnull=True) | Q(_secret="")
         if has_secret:
             return queryset.exclude(q)
         else:
@@ -45,15 +53,15 @@ class AccountFilterSet(BaseFilterSet):
     def filter_long_time(queryset, name, value):
         date = timezone.now() - timezone.timedelta(days=30)
 
-        if name == 'long_time_no_change_secret':
-            field = 'date_change_secret'
-            confirm_field = 'change_secret_status'
+        if name == "long_time_no_change_secret":
+            field = "date_change_secret"
+            confirm_field = "change_secret_status"
         else:
-            field = 'date_verified'
-            confirm_field = 'connectivity'
+            field = "date_verified"
+            confirm_field = "connectivity"
 
-        q = Q(**{f'{field}__lt': date}) | Q(**{f'{field}__isnull': True})
-        confirm_q = {f'{confirm_field}': 'na'}
+        q = Q(**{f"{field}__lt": date}) | Q(**{f"{field}__isnull": True})
+        confirm_q = {f"{confirm_field}": "na"}
         queryset = queryset.exclude(**confirm_q).filter(q)
         return queryset
 
@@ -62,9 +70,11 @@ class AccountFilterSet(BaseFilterSet):
         if not value:
             return queryset
 
-        queryset = queryset.prefetch_related('risks') \
-            .annotate(risk=F('risks__risk'), confirmed=F('risks__confirmed')) \
+        queryset = (
+            queryset.prefetch_related("risks")
+            .annotate(risk=F("risks__risk"), confirmed=F("risks__confirmed"))
             .filter(risk=value, confirmed=False)
+        )
         return queryset
 
     @staticmethod
@@ -75,17 +85,19 @@ class AccountFilterSet(BaseFilterSet):
         date = timezone.now() - timezone.timedelta(days=7)
         kwargs = {}
 
-        if name == 'latest_discovery':
-            kwargs.update({'date_created__gte': date, 'source': 'collected'})
-        elif name == 'latest_accessed':
-            kwargs.update({'date_last_login__gte': date})
-        elif name == 'latest_updated':
-            kwargs.update({'date_updated__gte': date})
-        elif name == 'latest_secret_changed':
-            kwargs.update({'date_change_secret__gt': date})
+        if name == "latest_discovery":
+            kwargs.update({"date_created__gte": date, "source": "collected"})
+        elif name == "latest_accessed":
+            kwargs.update({"date_last_login__gte": date})
+        elif name == "latest_updated":
+            kwargs.update({"date_updated__gte": date})
+        elif name == "latest_secret_changed":
+            kwargs.update({"date_change_secret__gt": date})
 
-        if name == 'latest_secret_change_failed':
-            queryset = queryset.filter(date_change_secret__gt=date).exclude(change_secret_status='ok')
+        if name == "latest_secret_change_failed":
+            queryset = queryset.filter(date_change_secret__gt=date).exclude(
+                change_secret_status="ok"
+            )
 
         if kwargs:
             queryset = queryset.filter(date_last_login__gte=date)
@@ -100,19 +112,22 @@ class AccountFilterSet(BaseFilterSet):
         node_qs = Node.objects.none()
         for node in nodes:
             node_qs |= node.get_all_children(with_self=True)
-        node_ids = list(node_qs.values_list('id', flat=True))
+        node_ids = list(node_qs.values_list("id", flat=True))
         queryset = queryset.filter(asset__nodes__in=node_ids)
         return queryset
 
     class Meta:
         model = Account
-        fields = ['id', 'asset', 'source_id', 'secret_type', 'category', 'type']
+        fields = ["id", "asset", "source_id", "secret_type", "category", "type"]
 
 
 class GatheredAccountFilterSet(BaseFilterSet):
-    node_id = drf_filters.CharFilter(method='filter_nodes')
-    asset_id = drf_filters.CharFilter(field_name='asset_id', lookup_expr='exact')
-    asset_name = drf_filters.CharFilter(field_name='asset__name', lookup_expr='icontains')
+    node_id = drf_filters.CharFilter(method="filter_nodes")
+    asset_id = drf_filters.CharFilter(field_name="asset_id", lookup_expr="exact")
+    asset_name = drf_filters.CharFilter(
+        field_name="asset__name", lookup_expr="icontains"
+    )
+    status = drf_filters.CharFilter(field_name="status", lookup_expr="exact")
 
     @staticmethod
     def filter_nodes(queryset, name, value):
@@ -120,15 +135,20 @@ class GatheredAccountFilterSet(BaseFilterSet):
 
     class Meta:
         model = GatheredAccount
-        fields = ['id', 'username']
+        fields = ["id", "username"]
 
 
 class ChangeSecretRecordFilterSet(BaseFilterSet):
-    asset_name = drf_filters.CharFilter(field_name='asset__name', lookup_expr='icontains')
-    account_username = drf_filters.CharFilter(field_name='account__username', lookup_expr='icontains')
-    execution_id = drf_filters.CharFilter(field_name='execution_id', lookup_expr='exact')
-
-    days = drf_filters.NumberFilter(method='filter_days')
+    asset_name = drf_filters.CharFilter(
+        field_name="asset__name", lookup_expr="icontains"
+    )
+    account_username = drf_filters.CharFilter(
+        field_name="account__username", lookup_expr="icontains"
+    )
+    execution_id = drf_filters.CharFilter(
+        field_name="execution_id", lookup_expr="exact"
+    )
+    days = drf_filters.NumberFilter(method="filter_days")
 
     @staticmethod
     def filter_days(queryset, name, value):
@@ -141,4 +161,4 @@ class ChangeSecretRecordFilterSet(BaseFilterSet):
 
     class Meta:
         model = ChangeSecretRecord
-        fields = ['id', 'status', 'asset_id', 'execution']
+        fields = ["id", "status", "asset_id", "execution"]

apps/common/drf/filters.py

@@ -4,6 +4,7 @@ import base64
 import json
 import logging
 from collections import defaultdict
+from django.utils import timezone
 
 from django.core.cache import cache
 from django.core.exceptions import ImproperlyConfigured
@@ -18,18 +19,26 @@ from rest_framework.filters import OrderingFilter
 from common import const
 from common.db.fields import RelatedManager
 
-logger = logging.getLogger('jumpserver.common')
+logger = logging.getLogger("jumpserver.common")
 
 __all__ = [
-    "DatetimeRangeFilterBackend", "IDSpmFilterBackend",
-    'IDInFilterBackend', "CustomFilterBackend",
-    "BaseFilterSet", 'IDNotFilterBackend',
-    'NotOrRelFilterBackend', 'LabelFilterBackend',
-    'RewriteOrderingFilter', 'AttrRulesFilterBackend'
+    "DatetimeRangeFilterBackend",
+    "IDSpmFilterBackend",
+    "IDInFilterBackend",
+    "CustomFilterBackend",
+    "BaseFilterSet",
+    "IDNotFilterBackend",
+    "NotOrRelFilterBackend",
+    "LabelFilterBackend",
+    "RewriteOrderingFilter",
+    "AttrRulesFilterBackend",
 ]
 
 
 class BaseFilterSet(drf_filters.FilterSet):
+    days = drf_filters.NumberFilter(method="filter_days")
+    days__lt = drf_filters.NumberFilter(method="filter_days")
+
     def do_nothing(self, queryset, name, value):
         return queryset
 
@@ -38,6 +47,22 @@ class BaseFilterSet(drf_filters.FilterSet):
             return self.form.cleaned_data[k]
         return default
 
+    @staticmethod
+    def filter_days(queryset, name, value):
+        try:
+            value = int(value)
+        except ValueError:
+            return queryset.none()
+
+        if name == 'days':
+            arg = 'date_created__gte'
+        else:
+            arg = 'date_created__lt'
+
+        date = timezone.now() - timezone.timedelta(days=value)
+        kwargs = {arg: date}
+        return queryset.filter(**kwargs)
+
 
 class DatetimeRangeFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
@@ -50,18 +75,20 @@ class DatetimeRangeFilterBackend(filters.BaseFilterBackend):
             for v in date_range_keyword:
                 ret.append(
                     coreapi.Field(
-                        name=v, location='query', required=False, type='string',
+                        name=v,
+                        location="query",
+                        required=False,
+                        type="string",
                         schema=coreschema.String(
-                            title=v,
-                            description='%s %s' % (attr, v)
-                        )
+                            title=v, description="%s %s" % (attr, v)
+                        ),
                     )
                 )
 
         return ret
 
     def _get_date_range_filter_fields(self, view):
-        if not hasattr(view, 'date_range_filter_fields'):
+        if not hasattr(view, "date_range_filter_fields"):
             return {}
         try:
             return dict(view.date_range_filter_fields)
@@ -75,7 +102,9 @@ class DatetimeRangeFilterBackend(filters.BaseFilterBackend):
                             ('db column', ('query param date from', 'query param date to'))
                         ]
                 ```
-            """.format(view.name)
+            """.format(
+                view.name
+            )
             logger.error(msg)
             raise ImproperlyConfigured(msg)
 
@@ -110,14 +139,17 @@ class IDSpmFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         return [
             coreapi.Field(
-                name='spm', location='query', required=False,
-                type='string', example='',
-                description='Pre post objects id get spm id, then using filter'
+                name="spm",
+                location="query",
+                required=False,
+                type="string",
+                example="",
+                description="Pre post objects id get spm id, then using filter",
             )
         ]
 
     def filter_queryset(self, request, queryset, view):
-        spm = request.query_params.get('spm')
+        spm = request.query_params.get("spm")
         if not spm:
             return queryset
         cache_key = const.KEY_CACHE_RESOURCE_IDS.format(spm)
@@ -127,7 +159,7 @@ class IDSpmFilterBackend(filters.BaseFilterBackend):
             return queryset.none()
         if isinstance(resource_ids, str):
             resource_ids = [resource_ids]
-        if hasattr(view, 'filter_spm_queryset'):
+        if hasattr(view, "filter_spm_queryset"):
             queryset = view.filter_spm_queryset(resource_ids, queryset)
         else:
             queryset = queryset.filter(id__in=resource_ids)
@@ -138,17 +170,20 @@ class IDInFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         return [
             coreapi.Field(
-                name='ids', location='query', required=False,
-                type='string', example='/api/v1/users/users?ids=1,2,3',
-                description='Filter by id set'
+                name="ids",
+                location="query",
+                required=False,
+                type="string",
+                example="/api/v1/users/users?ids=1,2,3",
+                description="Filter by id set",
             )
         ]
 
     def filter_queryset(self, request, queryset, view):
-        ids = request.query_params.get('ids')
+        ids = request.query_params.get("ids")
         if not ids:
             return queryset
-        id_list = [i.strip() for i in ids.split(',')]
+        id_list = [i.strip() for i in ids.split(",")]
         queryset = queryset.filter(id__in=id_list)
         return queryset
 
@@ -157,17 +192,20 @@ class IDNotFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         return [
             coreapi.Field(
-                name='id!', location='query', required=False,
-                type='string', example='/api/v1/users/users?id!=1,2,3',
-                description='Exclude by id set'
+                name="id!",
+                location="query",
+                required=False,
+                type="string",
+                example="/api/v1/users/users?id!=1,2,3",
+                description="Exclude by id set",
             )
         ]
 
     def filter_queryset(self, request, queryset, view):
-        ids = request.query_params.get('id!')
+        ids = request.query_params.get("id!")
         if not ids:
             return queryset
-        id_list = [i.strip() for i in ids.split(',')]
+        id_list = [i.strip() for i in ids.split(",")]
         queryset = queryset.exclude(id__in=id_list)
         return queryset
 
@@ -176,26 +214,30 @@ class LabelFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         return [
             coreapi.Field(
-                name='label', location='query', required=False,
-                type='string', example='/api/v1/users/users?label=abc',
-                description='Filter by label'
+                name="label",
+                location="query",
+                required=False,
+                type="string",
+                example="/api/v1/users/users?label=abc",
+                description="Filter by label",
             )
         ]
 
     @staticmethod
     def parse_labels(labels_id):
         from labels.models import Label
-        label_ids = [i.strip() for i in labels_id.split(',')]
+
+        label_ids = [i.strip() for i in labels_id.split(",")]
         cleaned = []
 
         args = []
         for label_id in label_ids:
             kwargs = {}
-            if ':' in label_id:
-                k, v = label_id.split(':', 1)
-                kwargs['name'] = k.strip()
-                if v != '*':
-                    kwargs['value'] = v.strip()
+            if ":" in label_id:
+                k, v = label_id.split(":", 1)
+                kwargs["name"] = k.strip()
+                if v != "*":
+                    kwargs["value"] = v.strip()
                 args.append(kwargs)
             else:
                 cleaned.append(label_id)
@@ -209,14 +251,14 @@ class LabelFilterBackend(filters.BaseFilterBackend):
         return cleaned
 
     def filter_queryset(self, request, queryset, view):
-        labels_id = request.query_params.get('labels')
+        labels_id = request.query_params.get("labels")
         if not labels_id:
             return queryset
 
-        if not hasattr(queryset, 'model'):
+        if not hasattr(queryset, "model"):
             return queryset
 
-        if not hasattr(queryset.model, 'label_model'):
+        if not hasattr(queryset.model, "label_model"):
             return queryset
 
         model = queryset.model.label_model()
@@ -225,7 +267,8 @@ class LabelFilterBackend(filters.BaseFilterBackend):
         model_name = model._meta.model_name
 
         full_resources = labeled_resource_cls.objects.filter(
-            res_type__app_label=app_label, res_type__model=model_name,
+            res_type__app_label=app_label,
+            res_type__model=model_name,
         )
         labels = self.parse_labels(labels_id)
         grouped = defaultdict(set)
@@ -234,8 +277,10 @@ class LabelFilterBackend(filters.BaseFilterBackend):
 
         matched_ids = set()
         for name, label_ids in grouped.items():
-            resources = model.filter_resources_by_labels(full_resources, label_ids, rel='any')
-            res_ids = resources.values_list('res_id', flat=True)
+            resources = model.filter_resources_by_labels(
+                full_resources, label_ids, rel="any"
+            )
+            res_ids = resources.values_list("res_id", flat=True)
             if not matched_ids:
                 matched_ids = set(res_ids)
             else:
@@ -249,16 +294,14 @@ class CustomFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         fields = []
         defaults = dict(
-            location='query', required=False,
-            type='string', example='',
-            description=''
+            location="query", required=False, type="string", example="", description=""
         )
-        if not hasattr(view, 'custom_filter_fields'):
+        if not hasattr(view, "custom_filter_fields"):
             return []
 
         for field in view.custom_filter_fields:
             if isinstance(field, str):
-                defaults['name'] = field
+                defaults["name"] = field
             elif isinstance(field, dict):
                 defaults.update(field)
             else:
@@ -270,7 +313,7 @@ class CustomFilterBackend(filters.BaseFilterBackend):
         return queryset
 
 
-def current_user_filter(user_field='user'):
+def current_user_filter(user_field="user"):
     class CurrentUserFilter(filters.BaseFilterBackend):
         def filter_queryset(self, request, queryset, view):
             return queryset.filter(**{user_field: request.user})
@@ -290,27 +333,30 @@ class AttrRulesFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         return [
             coreapi.Field(
-                name='attr_rules', location='query', required=False,
-                type='string', example='/api/v1/users/users?attr_rules=jsonbase64',
-                description='Filter by json like {"type": "attrs", "attrs": []} to base64'
+                name="attr_rules",
+                location="query",
+                required=False,
+                type="string",
+                example="/api/v1/users/users?attr_rules=jsonbase64",
+                description='Filter by json like {"type": "attrs", "attrs": []} to base64',
             )
         ]
 
     def filter_queryset(self, request, queryset, view):
-        attr_rules = request.query_params.get('attr_rules')
+        attr_rules = request.query_params.get("attr_rules")
         if not attr_rules:
             return queryset
 
         try:
-            attr_rules = base64.b64decode(attr_rules.encode('utf-8'))
+            attr_rules = base64.b64decode(attr_rules.encode("utf-8"))
         except Exception:
-            raise ValidationError({'attr_rules': 'attr_rules should be base64'})
+            raise ValidationError({"attr_rules": "attr_rules should be base64"})
         try:
             attr_rules = json.loads(attr_rules)
         except Exception:
-            raise ValidationError({'attr_rules': 'attr_rules should be json'})
+            raise ValidationError({"attr_rules": "attr_rules should be json"})
 
-        logger.debug('attr_rules: %s', attr_rules)
+        logger.debug("attr_rules: %s", attr_rules)
         qs = RelatedManager.get_to_filter_qs(attr_rules, queryset.model)
         for q in qs:
             queryset = queryset.filter(q)
@@ -321,33 +367,38 @@ class NotOrRelFilterBackend(filters.BaseFilterBackend):
     def get_schema_fields(self, view):
         return [
             coreapi.Field(
-                name='_rel', location='query', required=False,
-                type='string', example='/api/v1/users/users?name=abc&username=def&_rel=union',
-                description='Filter by rel, or not, default is and'
+                name="_rel",
+                location="query",
+                required=False,
+                type="string",
+                example="/api/v1/users/users?name=abc&username=def&_rel=union",
+                description="Filter by rel, or not, default is and",
             )
         ]
 
     def filter_queryset(self, request, queryset, view):
-        _rel = request.query_params.get('_rel')
-        if not _rel or _rel not in ('or', 'not'):
+        _rel = request.query_params.get("_rel")
+        if not _rel or _rel not in ("or", "not"):
             return queryset
-        if _rel == 'not':
+        if _rel == "not":
             queryset.query.where.negated = True
-        elif _rel == 'or':
-            queryset.query.where.connector = 'OR'
+        elif _rel == "or":
+            queryset.query.where.connector = "OR"
         queryset._result_cache = None
         return queryset
 
 
 class RewriteOrderingFilter(OrderingFilter):
-    default_ordering_if_has = ('name', )
+    default_ordering_if_has = ("name",)
 
     def get_default_ordering(self, view):
         ordering = super().get_default_ordering(view)
         # 如果 view.ordering = [] 表示不排序, 这样可以节约性能 (比如: 用户授权的资产)
         if ordering is not None:
             return ordering
-        ordering_fields = getattr(view, 'ordering_fields', self.ordering_fields)
+        ordering_fields = getattr(view, "ordering_fields", self.ordering_fields)
         if ordering_fields:
-            ordering = tuple([f for f in ordering_fields if f in self.default_ordering_if_has])
+            ordering = tuple(
+                [f for f in ordering_fields if f in self.default_ordering_if_has]
+            )
         return ordering