[Update] 优化sql, or方式改为union (#3682)

* [Update] 优化sql, or方式改为union

* [Update] 优化union操作，直接union后，queryset 的一些参数不能使用，如annoate, 如filter assets__isnull=True

apps/common/utils/django.py

@@ -2,6 +2,7 @@
 #
 import re
 from django.shortcuts import reverse as dj_reverse
+from django.db.models import Subquery, QuerySet
 from django.conf import settings
 from django.utils import timezone
 
@@ -35,3 +36,16 @@ def date_expired_default():
         years = 70
     return timezone.now() + timezone.timedelta(days=365*years)
 
+
+def union_queryset(*args, base_queryset=None):
+    if len(args) == 1:
+        return args[0]
+    elif len(args) == 0:
+        raise ValueError("args is empty")
+    args = [q.order_by() for q in args]
+    sub_query = args[0].union(*args[1:])
+    queryset_id = list(sub_query.values_list('id', flat=True))
+    if not base_queryset:
+        base_queryset = args[0].model.objects
+    queryset = base_queryset.filter(id__in=queryset_id)
+    return queryset

apps/perms/api/asset_permission.py

@@ -1,11 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 
-from django.db.models import Q
-
 from common.permissions import IsOrgAdmin
 from orgs.mixins.api import OrgModelViewSet
-from common.utils import get_object_or_none
+from common.utils import get_object_or_none, union_queryset
 from ..models import AssetPermission
 from ..hands import (
     User, UserGroup, Asset, Node, SystemUser,
@@ -111,9 +109,9 @@ class AssetPermissionViewSet(OrgModelViewSet):
                 continue
             ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True)
             inherit_all_nodes.update(ancestor_keys)
-        queryset = queryset.filter(
-            Q(assets__in=assets) | Q(nodes__key__in=inherit_all_nodes)
-        ).distinct()
+        assets_queryset = queryset.filter(assets__in=assets)
+        nodes_queryset = queryset.filter(nodes__key__in=inherit_all_nodes)
+        queryset = union_queryset(assets_queryset, nodes_queryset)
         return queryset
 
     def filter_user(self, queryset):
@@ -131,9 +129,9 @@ class AssetPermissionViewSet(OrgModelViewSet):
             queryset = queryset.filter(users=user)
             return queryset
         groups = user.groups.all()
-        queryset = queryset.filter(
-            Q(users=user) | Q(user_groups__in=groups)
-        ).distinct()
+        users_queryset = queryset.filter(users=user)
+        groups_queryset = queryset.filter(user_groups__in=groups)
+        queryset = union_queryset(users_queryset, groups_queryset)
         return queryset
 
     def filter_user_group(self, queryset):

apps/perms/models/asset_permission.py

@@ -93,7 +93,7 @@ class AssetPermission(BasePermission):
             models.Prefetch('nodes', queryset=Node.objects.all().only('key')),
             models.Prefetch('assets', queryset=Asset.objects.all().only('id')),
             models.Prefetch('system_users', queryset=SystemUser.objects.all().only('id'))
-        )
+        ).order_by()
 
     def get_all_assets(self):
         from assets.models import Node

apps/perms/models/base.py

@@ -8,7 +8,7 @@ from django.db.models import Q
 from django.utils import timezone
 from orgs.mixins.models import OrgModelMixin
 
-from common.utils import date_expired_default, set_or_append_attr_bulk
+from common.utils import date_expired_default, union_queryset
 from orgs.mixins.models import OrgManager
 
 
@@ -83,7 +83,8 @@ class BasePermission(OrgModelMixin):
         from users.models import User
         users_id = self.users.all().values_list('id', flat=True)
         groups_id = self.user_groups.all().values_list('id', flat=True)
-        users = User.objects.filter(
-            Q(id__in=users_id) | Q(groups__id__in=groups_id)
-        ).distinct()
+        users = User.objects.filter(id__in=users_id)
+        if groups_id:
+            groups_users = User.objects.filter(groups__id__in=groups_id)
+            users = union_queryset(users, groups_users)
         return users

apps/perms/utils/asset_permission.py

@@ -9,7 +9,7 @@ from django.db.models import Q
 from django.conf import settings
 
 from orgs.utils import set_to_root_org
-from common.utils import get_logger, timeit, lazyproperty
+from common.utils import get_logger, timeit, lazyproperty, union_queryset
 from common.tree import TreeNode
 from assets.utils import TreeService
 from ..models import AssetPermission
@@ -25,13 +25,16 @@ __all__ = [
 
 
 def get_user_permissions(user, include_group=True):
-    permissions = AssetPermission.get_queryset_with_prefetch().filter(users=user)
+    permissions = AssetPermission.objects.filter(users=user)
     if include_group:
         groups = user.groups.all()
-        permissions_groups = AssetPermission.get_queryset_with_prefetch().filter(
+        permissions_groups = AssetPermission.objects.filter(
             user_groups__in=groups
         )
-        permissions = permissions.union(permissions_groups)
+        base_queryset = AssetPermission.get_queryset_with_prefetch()
+        permissions = union_queryset(
+            permissions, permissions_groups, base_queryset=base_queryset
+        )
     return permissions
 
 
@@ -42,13 +45,12 @@ def get_user_group_permissions(user_group):
 
 
 def get_asset_permissions(asset, include_node=True):
-    permissions = AssetPermission.get_queryset_with_prefetch().filter(asset=asset)
+    permissions = AssetPermission.objects.filter(asset=asset)
     if include_node:
         nodes = asset.get_all_nodes(flat=True)
-        permissions_nodes = AssetPermission.get_queryset_with_prefetch().filter(
-            nodes__in=nodes
-        )
-        permissions = permissions.union(permissions_nodes)
+        base_queryset = AssetPermission.get_queryset_with_prefetch()
+        permissions_nodes = AssetPermission.objects.filter(nodes__in=nodes)
+        permissions = union_queryset(permissions, permissions_nodes, base_queryset=base_queryset)
     return permissions
 
 

apps/perms/utils/database_app_permission.py

@@ -1,11 +1,11 @@
 # coding: utf-8
 #
 
-from django.db.models import Q
 from django.utils.translation import ugettext as _
-from orgs.utils import set_to_root_org
 
+from orgs.utils import set_to_root_org
 from ..models import DatabaseAppPermission
+from common.utils import union_queryset
 from common.tree import TreeNode
 from applications.models import DatabaseApp
 from assets.models import SystemUser
@@ -17,13 +17,15 @@ __all__ = [
     'parse_database_app_to_tree_node'
 ]
 
+
 def get_user_database_app_permissions(user, include_group=True):
+    permissions = DatabaseAppPermission.objects.all().valid().filter(users=user)
     if include_group:
         groups = user.groups.all()
-        arg = Q(users=user) | Q(user_groups__in=groups)
-    else:
-        arg = Q(users=user)
-    return DatabaseAppPermission.objects.all().valid().filter(arg)
+        groups_permissions = DatabaseAppPermission.objects.all().valid()\
+            .filter(user_groups__in=groups)
+        permissions = union_queryset(permissions, groups_permissions)
+    return permissions
 
 
 def get_user_group_database_app_permission(user_group):

apps/perms/utils/remote_app_permission.py

@@ -1,10 +1,10 @@
 #  coding: utf-8
 #
 
-from django.db.models import Q
 from django.utils.translation import ugettext as _
 
 from common.tree import TreeNode
+from common.utils import union_queryset
 from orgs.utils import set_to_root_org
 
 from ..models import RemoteAppPermission
@@ -18,12 +18,14 @@ __all__ = [
 
 
 def get_user_remote_app_permissions(user, include_group=True):
+    permissions = RemoteAppPermission.objects.all().valid().filter(users=user)
     if include_group:
         groups = user.groups.all()
-        arg = Q(users=user) | Q(user_groups__in=groups)
-    else:
-        arg = Q(users=user)
-    return RemoteAppPermission.objects.all().valid().filter(arg)
+        groups_permissions = RemoteAppPermission.objects.all().valid().filter(
+            user_groups__in=groups
+        )
+        permissions = union_queryset(permissions, groups_permissions)
+    return permissions
 
 
 def get_user_group_remote_app_permissions(user_group):