Merge branch 'api_perf' into lina

apps/assets/api/admin_user.py

@@ -36,7 +36,7 @@ class AdminUserViewSet(OrgBulkModelViewSet):
 
     def get_queryset(self):
         queryset = super().get_queryset()
-        queryset = queryset.annotate(_assets_amount=Count('assets'))
+        queryset = queryset.annotate(assets_amount=Count('assets'))
         return queryset
 
     def destroy(self, request, *args, **kwargs):

apps/assets/serializers/asset.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 from rest_framework import serializers
-from django.db.models import Prefetch, F
+from django.db.models import Prefetch, F, Count
 
 from django.utils.translation import ugettext_lazy as _
 
@@ -73,21 +73,35 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
     class Meta:
         model = Asset
         list_serializer_class = AdaptedBulkListSerializer
-        fields = [
+        fields_mini = ['id', 'hostname', 'ip']
-            'id', 'ip', 'hostname', 'protocol', 'port',
+        fields_small = fields_mini + [
-            'protocols', 'platform', 'is_active', 'public_ip', 'domain',
+            'protocol', 'port', 'protocols', 'is_active', 'public_ip',
-            'admin_user', 'nodes', 'labels', 'number', 'vendor', 'model', 'sn',
+            'number', 'vendor', 'model', 'sn', 'cpu_model', 'cpu_count',
-            'cpu_model', 'cpu_count', 'cpu_cores', 'cpu_vcpus', 'memory',
+            'cpu_cores', 'cpu_vcpus', 'memory', 'disk_total', 'disk_info',
-            'disk_total', 'disk_info', 'os', 'os_version', 'os_arch',
+            'os', 'os_version', 'os_arch', 'hostname_raw', 'comment',
-            'hostname_raw', 'comment', 'created_by', 'date_created',
+            'created_by', 'date_created', 'hardware_info',
-            'hardware_info',
         ]
-        read_only_fields = (
+        fields_fk = [
+            'admin_user', 'domain', 'platform'
+        ]
+        fk_only_fields = {
+            'platform': ['name']
+        }
+        fields_m2m = [
+            'nodes', 'labels',
+        ]
+        annotates_fields = {
+            # 'admin_user_display': 'admin_user__name'
+        }
+        fields_as = list(annotates_fields.keys())
+        fields = fields_small + fields_fk + fields_m2m + fields_as
+        read_only_fields = [
             'vendor', 'model', 'sn', 'cpu_model', 'cpu_count',
             'cpu_cores', 'cpu_vcpus', 'memory', 'disk_total', 'disk_info',
             'os', 'os_version', 'os_arch', 'hostname_raw',
             'created_by', 'date_created',
-        )
+        ] + fields_as
+
         extra_kwargs = {
             'protocol': {'write_only': True},
             'port': {'write_only': True},
@@ -98,11 +112,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer):
     @classmethod
     def setup_eager_loading(cls, queryset):
         """ Perform necessary eager loading of data. """
-        queryset = queryset.prefetch_related(
+        queryset = queryset.select_related('admin_user', 'domain', 'platform')
-            Prefetch('nodes', queryset=Node.objects.all().only('id')),
-            Prefetch('labels', queryset=Label.objects.all().only('id')),
-        ).select_related('admin_user', 'domain', 'platform') \
-         .annotate(platform_base=F('platform__base'))
         return queryset
 
     def compatible_with_old_protocol(self, validated_data):

apps/common/mixins/serializers.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
 #
+from collections import Iterable
 
+from django.db.models import Prefetch, F
 from django.core.exceptions import ObjectDoesNotExist
 from rest_framework.utils import html
 from rest_framework.settings import api_settings
 from rest_framework.exceptions import ValidationError
 from rest_framework.fields import SkipField, empty
 
-__all__ = ['BulkSerializerMixin', 'BulkListSerializerMixin']
+__all__ = ['BulkSerializerMixin', 'BulkListSerializerMixin', 'CommonSerializerMixin']
 
 
 class BulkSerializerMixin(object):
@@ -113,3 +115,121 @@ class BulkListSerializerMixin(object):
             raise ValidationError(errors)
 
         return ret
+
+
+class BaseDynamicFieldsPlugin:
+    def __init__(self, serializer):
+        self.serializer = serializer
+
+    def can_dynamic(self):
+        try:
+            request = self.serializer.context['request']
+            method = request.method
+        except (AttributeError, TypeError, KeyError):
+            # The serializer was not initialized with request context.
+            return False
+
+        if method != 'GET':
+            return False
+        return True
+
+    def get_request(self):
+        return self.serializer.context['request']
+
+    def get_query_params(self):
+        request = self.get_request()
+        try:
+            query_params = request.query_params
+        except AttributeError:
+            # DRF 2
+            query_params = getattr(request, 'QUERY_PARAMS', request.GET)
+        return query_params
+
+    def get_exclude_field_names(self):
+        return set()
+
+
+class QueryFieldsMixin(BaseDynamicFieldsPlugin):
+    # https://github.com/wimglenn/djangorestframework-queryfields/
+
+    # If using Django filters in the API, these labels mustn't conflict with any model field names.
+    include_arg_name = 'fields'
+    exclude_arg_name = 'fields!'
+
+    # Split field names by this string.  It doesn't necessarily have to be a single character.
+    # Avoid RFC 1738 reserved characters i.e. ';', '/', '?', ':', '@', '=' and '&'
+    delimiter = ','
+
+    def get_exclude_field_names(self):
+        query_params = self.get_query_params()
+        includes = query_params.getlist(self.include_arg_name)
+        include_field_names = {name for names in includes for name in names.split(self.delimiter) if name}
+
+        excludes = query_params.getlist(self.exclude_arg_name)
+        exclude_field_names = {name for names in excludes for name in names.split(self.delimiter) if name}
+
+        if not include_field_names and not exclude_field_names:
+            # No user fields filtering was requested, we have nothing to do here.
+            return []
+
+        serializer_field_names = set(self.serializer.fields)
+        fields_to_drop = serializer_field_names & exclude_field_names
+
+        if include_field_names:
+            fields_to_drop |= serializer_field_names - include_field_names
+        return fields_to_drop
+
+
+class SizedModelFieldsMixin(BaseDynamicFieldsPlugin):
+    arg_name = 'fields_size'
+
+    def can_dynamic(self):
+        if not hasattr(self.serializer, 'Meta'):
+            return False
+        can = super().can_dynamic()
+        return can
+
+    def get_exclude_field_names(self):
+        query_params = self.get_query_params()
+        size = query_params.get(self.arg_name)
+        if not size:
+            return []
+        size_fields = getattr(self.serializer.Meta, 'fields_{}'.format(size), None)
+        if not size_fields or not isinstance(size_fields, Iterable):
+            return []
+        serializer_field_names = set(self.serializer.fields)
+        fields_to_drop = serializer_field_names - set(size_fields)
+        return fields_to_drop
+
+
+class DynamicFieldsMixin:
+    dynamic_fields_plugins = [QueryFieldsMixin, SizedModelFieldsMixin]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        exclude_field_names = set()
+        for cls in self.dynamic_fields_plugins:
+            plugin = cls(self)
+            if not plugin.can_dynamic():
+                continue
+            exclude_field_names |= set(plugin.get_exclude_field_names())
+
+        for field in exclude_field_names or []:
+            self.fields.pop(field, None)
+
+
+class EagerLoadQuerySetFields:
+    def setup_eager_loading(self, queryset):
+        """ Perform necessary eager loading of data. """
+        queryset = queryset.prefetch_related(
+            Prefetch('nodes'),
+            Prefetch('labels'),
+        ).select_related('admin_user', 'domain', 'platform') \
+         .annotate(platform_base=F('platform__base'))
+        return queryset
+
+
+class CommonSerializerMixin(DynamicFieldsMixin):
+    pass
+

apps/orgs/mixins/serializers.py

@@ -5,7 +5,7 @@ from rest_framework import serializers
 from rest_framework.validators import UniqueTogetherValidator
 
 from common.validators import ProjectUniqueValidator
-from common.mixins import BulkSerializerMixin
+from common.mixins import BulkSerializerMixin, CommonSerializerMixin
 from ..utils import get_current_org_id_for_serializer
 
 
@@ -16,7 +16,7 @@ __all__ = [
 ]
 
 
-class OrgResourceSerializerMixin(serializers.Serializer):
+class OrgResourceSerializerMixin(CommonSerializerMixin, serializers.Serializer):
     """
     通过API批量操作资源时, 自动给每个资源添加所需属性org_id的值为current_org_id
     (同时为serializer.is_valid()对Model的unique_together校验做准备)

apps/users/api/user.py

@@ -31,7 +31,6 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet):
     search_fields = filter_fields
     serializer_classes = {
         'default': serializers.UserSerializer,
-        'display': serializers.UserDisplaySerializer
     }
     permission_classes = (IsOrgAdmin, CanUpdateDeleteUser)
 

apps/users/serializers/user.py

@@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
 from common.utils import validate_ssh_public_key
-from common.mixins import BulkSerializerMixin
+from common.mixins import CommonSerializerMixin
 from common.serializers import AdaptedBulkListSerializer
 from common.permissions import CanUpdateDeleteUser
 from ..models import User
@@ -13,7 +13,7 @@ from ..models import User
 __all__ = [
     'UserSerializer', 'UserPKUpdateSerializer',
     'ChangeUserPasswordSerializer', 'ResetOTPSerializer',
-    'UserProfileSerializer', 'UserDisplaySerializer',
+    'UserProfileSerializer',
 ]
 
 
@@ -22,7 +22,7 @@ class UserOrgSerializer(serializers.Serializer):
     name = serializers.CharField()
 
 
-class UserSerializer(BulkSerializerMixin, serializers.ModelSerializer):
+class UserSerializer(CommonSerializerMixin, serializers.ModelSerializer):
     EMAIL_SET_PASSWORD = _('Reset link will be generated and sent to the user')
     CUSTOM_PASSWORD = _('Set password')
     PASSWORD_STRATEGY_CHOICES = (
@@ -33,18 +33,27 @@ class UserSerializer(BulkSerializerMixin, serializers.ModelSerializer):
         choices=PASSWORD_STRATEGY_CHOICES, required=False, initial=0,
         label=_('Password strategy'), write_only=True
     )
+    can_update = serializers.SerializerMethodField()
+    can_delete = serializers.SerializerMethodField()
 
     class Meta:
         model = User
         list_serializer_class = AdaptedBulkListSerializer
-        fields = [
+        # mini 是指能识别对象的最小单元
-            'id', 'name', 'username', 'password', 'email', 'public_key',
+        fields_mini = ['id', 'name', 'username']
-            'groups', 'role', 'wechat', 'phone', 'mfa_level',
+        # small 指的是 不需要计算的直接能从一张表中获取到的数据
+        fields_small = fields_mini + [
+            'password', 'email', 'public_key', 'wechat', 'phone', 'mfa_level',
             'comment', 'source', 'is_valid', 'is_expired',
             'is_active', 'created_by', 'is_first_login',
             'password_strategy', 'date_password_last_updated', 'date_expired',
-            'avatar_url',
+            'avatar_url', 'source_display',
         ]
+        fields = fields_small + [
+            'groups', 'role', 'groups_display', 'role_display',
+            'can_update', 'can_delete'
+        ]
+
         extra_kwargs = {
             'password': {'write_only': True, 'required': False, 'allow_null': True, 'allow_blank': True},
             'public_key': {'write_only': True},
@@ -53,6 +62,11 @@ class UserSerializer(BulkSerializerMixin, serializers.ModelSerializer):
             'is_expired': {'label': _('Is expired')},
             'avatar_url': {'label': _('Avatar url')},
             'created_by': {'read_only': True, 'allow_blank': True},
+            'can_update': {'read_only': True},
+            'can_delete': {'read_only': True},
+            'groups_display': {'label': _('Groups name')},
+            'source_display': {'label': _('Source name')},
+            'role_display': {'label': _('Role name')},
         }
 
     def __init__(self, *args, **kwargs):
@@ -60,7 +74,9 @@ class UserSerializer(BulkSerializerMixin, serializers.ModelSerializer):
         self.set_role_choices()
 
     def set_role_choices(self):
-        role = self.fields['role']
+        role = self.fields.get('role')
+        if not role:
+            return
         choices = role._choices
         choices.pop('App', None)
         role._choices = choices
@@ -114,17 +130,6 @@ class UserSerializer(BulkSerializerMixin, serializers.ModelSerializer):
         attrs.pop('password_strategy', None)
         return attrs
 
-
-class UserDisplaySerializer(UserSerializer):
-    can_update = serializers.SerializerMethodField()
-    can_delete = serializers.SerializerMethodField()
-
-    class Meta(UserSerializer.Meta):
-        fields = UserSerializer.Meta.fields + [
-            'groups_display', 'role_display', 'source_display',
-            'can_update', 'can_delete',
-        ]
-
     def get_can_update(self, obj):
         return CanUpdateDeleteUser.has_update_object_permission(
             self.context['request'], self.context['view'], obj
@@ -135,17 +140,6 @@ class UserDisplaySerializer(UserSerializer):
             self.context['request'], self.context['view'], obj
         )
 
-    def get_extra_kwargs(self):
-        kwargs = super().get_extra_kwargs()
-        kwargs.update({
-            'can_update': {'read_only': True},
-            'can_delete': {'read_only': True},
-            'groups_display': {'label': _('Groups name')},
-            'source_display': {'label': _('Source name')},
-            'role_display': {'label': _('Role name')},
-        })
-        return kwargs
-
 
 class UserPKUpdateSerializer(serializers.ModelSerializer):
     class Meta: