From 3ddeb97ea5f76798e817ebc300093a8c9fa88fd9 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Tue, 20 Dec 2022 20:36:12 +0800 Subject: [PATCH 01/11] =?UTF-8?q?fix:=20=E8=A7=A3=E5=86=B3=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E5=91=BD=E4=BB=A4=E5=BC=95=E5=8F=B7=E9=80=A0=E6=88=90?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/models/job.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py index 6f0c7f696..f554343ab 100644 --- a/apps/ops/models/job.py +++ b/apps/ops/models/job.py @@ -163,9 +163,10 @@ class JobExecution(JMSOrgBaseModel): def compile_shell(self): if self.current_job.type != 'adhoc': return - result = "{}{}{} ".format('\'', self.current_job.args, '\'') - result += "chdir={}".format(self.current_job.chdir) - return result + result = self.current_job.args + result += " chdir={}".format(self.current_job.chdir) + return self.job.args + # return result def get_runner(self): inv = self.current_job.inventory From 560ff651c46e1bbc87c99a97b3e334bd714502c8 Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Tue, 20 Dec 2022 20:36:44 +0800 Subject: [PATCH 02/11] =?UTF-8?q?perf:=20=E5=88=A0=E9=99=A4=E6=97=A0?= =?UTF-8?q?=E7=94=A8=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/models/job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py index f554343ab..5c8885221 100644 --- a/apps/ops/models/job.py +++ b/apps/ops/models/job.py @@ -166,7 +166,6 @@ class JobExecution(JMSOrgBaseModel): result = self.current_job.args result += " chdir={}".format(self.current_job.chdir) return self.job.args - # return result def get_runner(self): inv = self.current_job.inventory From f65146cd4531374aa522369c3f82fca8327540d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=B0=8F=E7=99=BD?= <296015668@qq.com> Date: Wed, 21 Dec 2022 11:21:48 +0800 Subject: [PATCH 03/11] =?UTF-8?q?chore:=20=E6=B7=BB=E5=8A=A0=20mysql-clien?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 1 + Dockerfile.loong64 | 1 + 2 files changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6898a63c3..1dc3eacfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ ARG TOOLS=" \ ca-certificates \ curl \ default-libmysqlclient-dev \ + default-mysql-client \ locales \ openssh-client \ sshpass \ diff --git a/Dockerfile.loong64 b/Dockerfile.loong64 index 27002794f..455ee9fac 100644 --- a/Dockerfile.loong64 +++ b/Dockerfile.loong64 @@ -33,6 +33,7 @@ ARG TOOLS=" \ ca-certificates \ curl \ default-libmysqlclient-dev \ + default-mysql-client \ locales \ openssh-client \ sshpass \ From 3bef5825002d041e8f913556564343cbaccabb13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=B0=8F=E7=99=BD?= <296015668@qq.com> Date: Wed, 21 Dec 2022 11:56:39 +0800 Subject: [PATCH 04/11] =?UTF-8?q?chore:=20=E6=B7=BB=E5=8A=A0=20vim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 1 + Dockerfile.loong64 | 1 + 2 files changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1dc3eacfa..37f573be9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,7 @@ ARG TOOLS=" \ sshpass \ telnet \ unzip \ + vim \ wget" ARG APT_MIRROR=http://mirrors.ustc.edu.cn diff --git a/Dockerfile.loong64 b/Dockerfile.loong64 index 455ee9fac..5e942d5a3 100644 --- a/Dockerfile.loong64 +++ b/Dockerfile.loong64 @@ -39,6 +39,7 @@ ARG TOOLS=" \ sshpass \ telnet \ unzip \ + vim \ wget" RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \ From 5198ac1cc0a2e20b73c635071844ae23cee2f5b1 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 21 Dec 2022 15:16:54 +0800 Subject: [PATCH 05/11] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=20UserAssetGra?= =?UTF-8?q?ntedTreeNodeRelation=20id=20=E4=B8=BA=20AutoField?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ops/migrations/0032_auto_20221221_1513.py | 23 +++++++++++++++++++ .../migrations/0034_auto_20221220_1956.py | 5 ---- apps/perms/models/perm_node.py | 1 + 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 apps/ops/migrations/0032_auto_20221221_1513.py diff --git a/apps/ops/migrations/0032_auto_20221221_1513.py b/apps/ops/migrations/0032_auto_20221221_1513.py new file mode 100644 index 000000000..3a706f4b7 --- /dev/null +++ b/apps/ops/migrations/0032_auto_20221221_1513.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.14 on 2022-12-21 07:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ops', '0031_auto_20221220_1956'), + ] + + operations = [ + migrations.AlterField( + model_name='historicaljob', + name='created_by', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by'), + ), + migrations.AlterField( + model_name='historicaljob', + name='updated_by', + field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by'), + ), + ] diff --git a/apps/perms/migrations/0034_auto_20221220_1956.py b/apps/perms/migrations/0034_auto_20221220_1956.py index 01b7f0f5f..41f6528d1 100644 --- a/apps/perms/migrations/0034_auto_20221220_1956.py +++ b/apps/perms/migrations/0034_auto_20221220_1956.py @@ -41,11 +41,6 @@ class Migration(migrations.Migration): name='created_by', field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by'), ), - migrations.AlterField( - model_name='userassetgrantedtreenoderelation', - name='id', - field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False), - ), migrations.AlterField( model_name='userassetgrantedtreenoderelation', name='updated_by', diff --git a/apps/perms/models/perm_node.py b/apps/perms/models/perm_node.py index 675e36e9c..89354a06b 100644 --- a/apps/perms/models/perm_node.py +++ b/apps/perms/models/perm_node.py @@ -16,6 +16,7 @@ class NodeFrom(TextChoices): class UserAssetGrantedTreeNodeRelation(FamilyMixin, JMSOrgBaseModel): NodeFrom = NodeFrom + id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name=_('ID')) user = models.ForeignKey('users.User', db_constraint=False, on_delete=models.CASCADE) node = models.ForeignKey('assets.Node', default=None, on_delete=models.CASCADE, db_constraint=False, null=False, related_name='granted_node_rels') From 0e534f32514f5bb373b6a072947572234619d27f Mon Sep 17 00:00:00 2001 From: Aaron3S Date: Wed, 21 Dec 2022 17:14:07 +0800 Subject: [PATCH 06/11] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/api/adhoc.py | 2 ++ apps/ops/api/job.py | 13 +++++++++---- apps/ops/api/playbook.py | 3 +++ apps/ops/serializers/job.py | 7 +++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/apps/ops/api/adhoc.py b/apps/ops/api/adhoc.py index 8caa59beb..aa7b890c5 100644 --- a/apps/ops/api/adhoc.py +++ b/apps/ops/api/adhoc.py @@ -15,6 +15,8 @@ class AdHocViewSet(OrgBulkModelViewSet): permission_classes = () model = AdHoc + def allow_bulk_destroy(self, qs, filtered): + return True def get_queryset(self): queryset = super().get_queryset() return queryset.filter(creator=self.request.user) diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index 9b36a5aba..0f37732bd 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -23,6 +23,9 @@ class JobViewSet(OrgBulkModelViewSet): permission_classes = () model = Job + def allow_bulk_destroy(self, qs, filtered): + return True + def get_queryset(self): queryset = super().get_queryset() queryset = queryset.filter(creator=self.request.user) @@ -31,20 +34,21 @@ class JobViewSet(OrgBulkModelViewSet): return queryset def perform_create(self, serializer): + run_after_save = serializer.validated_data.pop('run_after_save', False) instance = serializer.save() - run_after_save = serializer.validated_data.get('run_after_save', False) if instance.instant or run_after_save: self.run_job(instance, serializer) def perform_update(self, serializer): + run_after_save = serializer.validated_data.pop('run_after_save', False) instance = serializer.save() - run_after_save = serializer.validated_data.get('run_after_save', False) if run_after_save: self.run_job(instance, serializer) - @staticmethod - def run_job(job, serializer): + def run_job(self, job, serializer): execution = job.create_execution() + execution.creator = self.request.user + execution.save() task = run_ops_job_execution.delay(execution.id) set_task_to_serializer_data(serializer, task) @@ -58,6 +62,7 @@ class JobExecutionViewSet(OrgBulkModelViewSet): def perform_create(self, serializer): instance = serializer.save() instance.job_version = instance.job.version + instance.creator = self.request.user instance.save() task = run_ops_job_execution.delay(instance.id) set_task_to_serializer_data(serializer, task) diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index 009bfc2b8..aaafacd58 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -21,6 +21,9 @@ class PlaybookViewSet(OrgBulkModelViewSet): permission_classes = () model = Playbook + def allow_bulk_destroy(self, qs, filtered): + return True + def get_queryset(self): queryset = super().get_queryset() queryset = queryset.filter(creator=self.request.user) diff --git a/apps/ops/serializers/job.py b/apps/ops/serializers/job.py index 005b88cdc..386c4e92f 100644 --- a/apps/ops/serializers/job.py +++ b/apps/ops/serializers/job.py @@ -9,12 +9,11 @@ from orgs.mixins.serializers import BulkOrgResourceModelSerializer class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin): creator = ReadableHiddenField(default=serializers.CurrentUserDefault()) - run_after_save = serializers.BooleanField(label=_("Run after save"), read_only=True, default=False, required=False) + run_after_save = serializers.BooleanField(label=_("Run after save"), default=False, required=False) class Meta: model = Job - read_only_fields = ["id", "date_last_run", "date_created", "date_updated", "average_time_cost", - "run_after_save"] + read_only_fields = ["id", "date_last_run", "date_created", "date_updated", "average_time_cost"] fields = read_only_fields + [ "name", "instant", "type", "module", "args", "playbook", "assets", "runas_policy", "runas", "creator", "use_parameter_define", @@ -23,7 +22,7 @@ class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin): "chdir", "comment", "summary", - "is_periodic", "interval", "crontab" + "is_periodic", "interval", "crontab", "run_after_save" ] From c304a58c059fc2876e82dd019795676576c0f23e Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 21 Dec 2022 17:17:54 +0800 Subject: [PATCH 07/11] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9k8s=20=E6=A0=91?= =?UTF-8?q?=20(#9228)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: feng <1304903146@qq.com> --- apps/assets/utils/k8s.py | 88 +++++++------------ .../serializers/connection_token.py | 4 + .../user_permission/tree/node_with_asset.py | 74 +++++++++------- 3 files changed, 79 insertions(+), 87 deletions(-) diff --git a/apps/assets/utils/k8s.py b/apps/assets/utils/k8s.py index 8e703d4b6..5ffd1612d 100644 --- a/apps/assets/utils/k8s.py +++ b/apps/assets/utils/k8s.py @@ -1,16 +1,13 @@ # -*- coding: utf-8 -*- from urllib3.exceptions import MaxRetryError -from urllib.parse import urlencode, parse_qsl +from urllib.parse import urlencode from kubernetes import client from kubernetes.client import api_client from kubernetes.client.api import core_v1_api from kubernetes.client.exceptions import ApiException -from rest_framework.generics import get_object_or_404 - from common.utils import get_logger -from assets.models import Account, Asset from ..const import CloudTypes, Category @@ -94,57 +91,45 @@ class KubernetesClient: return f'{gateway.address}:{gateway.port}' @classmethod - def get_kubernetes_data(cls, app_id, username): - asset = get_object_or_404(Asset, id=app_id) - account = get_object_or_404(Account, asset=asset, username=username) + def get_kubernetes_data(cls, asset, secret): k8s_url = f'{asset.address}:{asset.port}' proxy_url = cls.get_proxy_url(asset) - k8s = cls(k8s_url, account.secret, proxy=proxy_url) + k8s = cls(k8s_url, secret, proxy=proxy_url) return k8s.get_pods() class KubernetesTree: - def __init__(self, tree_id): - self.tree_id = str(tree_id) + def __init__(self, asset, secret): + self.asset = asset + self.secret = secret - @staticmethod - def create_tree_id(pid, tp, v): - i = dict(parse_qsl(pid)) - i[tp] = v - tree_id = urlencode(i) - return tree_id - - def as_tree_node(self, app): - pid = app.create_app_tree_pid(self.tree_id) - app_id = str(app.id) + def as_asset_tree_node(self): + i = str(self.asset.id) + name = str(self.asset) node = self.create_tree_node( - app_id, pid, app.name, 'k8s' + i, i, name, 'asset', is_open=True, ) return node - def as_asset_tree_node(self, asset): - i = urlencode({'asset_id': self.tree_id}) - node = self.create_tree_node( - i, str(asset.id), str(asset), 'asset', is_open=True, - ) + def as_namespace_node(self, name, tp, counts=0): + i = urlencode({'namespace': name}) + pid = str(self.asset.id) + name = f'{name}({counts})' + node = self.create_tree_node(i, pid, name, tp, icon='cloud') return node - def as_account_tree_node(self, account, parent_info): - username = account.username - name = str(account) - pid = urlencode({'asset_id': self.tree_id}) - i = self.create_tree_id(pid, 'account', username) - parent_info.update({'account': username}) - node = self.create_tree_node( - i, pid, name, 'account', icon='user-tie' - ) + def as_pod_tree_node(self, namespace, name, tp, counts=0): + pid = urlencode({'namespace': namespace}) + i = urlencode({'namespace': namespace, 'pod': name}) + name = f'{name}({counts})' + node = self.create_tree_node(i, pid, name, tp, icon='cloud') return node - def as_namespace_pod_tree_node(self, name, tp, counts=0, is_container=False): - i = self.create_tree_id(self.tree_id, tp, name) - name = name if is_container else f'{name}({counts})' + def as_container_tree_node(self, namespace, pod, name, tp): + pid = urlencode({'namespace': namespace, 'pod': pod}) + i = urlencode({'namespace': namespace, 'pod': pod, 'container': name}) node = self.create_tree_node( - i, self.tree_id, name, tp, icon='cloud', is_container=is_container + i, pid, name, tp, icon='cloud', is_container=True ) return node @@ -169,36 +154,31 @@ class KubernetesTree: } return node - def async_tree_node(self, parent_info): - pod_name = parent_info.get('pod') - asset_id = parent_info.get('asset_id') - namespace = parent_info.get('namespace') - account_username = parent_info.get('account') - + def async_tree_node(self, namespace, pod): tree = [] - data = KubernetesClient.get_kubernetes_data(asset_id, account_username) + data = KubernetesClient.get_kubernetes_data(self.asset, self.secret) if not data: return tree - if pod_name: + if pod: for container in next( filter( - lambda x: x['pod_name'] == pod_name, data[namespace] + lambda x: x['pod_name'] == pod, data[namespace] ) )['containers']: - container_node = self.as_namespace_pod_tree_node( - container, 'container', is_container=True + container_node = self.as_container_tree_node( + namespace, pod, container, 'container' ) tree.append(container_node) elif namespace: for pod in data[namespace]: - pod_nodes = self.as_namespace_pod_tree_node( - pod['pod_name'], 'pod', len(pod['containers']) + pod_nodes = self.as_pod_tree_node( + namespace, pod['pod_name'], 'pod', len(pod['containers']) ) tree.append(pod_nodes) - elif account_username: + else: for namespace, pods in data.items(): - namespace_node = self.as_namespace_pod_tree_node( + namespace_node = self.as_namespace_node( namespace, 'namespace', len(pods) ) tree.append(namespace_node) diff --git a/apps/authentication/serializers/connection_token.py b/apps/authentication/serializers/connection_token.py index e45037853..2fd9ae16e 100644 --- a/apps/authentication/serializers/connection_token.py +++ b/apps/authentication/serializers/connection_token.py @@ -1,6 +1,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers +from common.drf.fields import EncryptedField from orgs.mixins.serializers import OrgResourceModelSerializerMixin from ..models import ConnectionToken @@ -11,6 +12,9 @@ __all__ = [ class ConnectionTokenSerializer(OrgResourceModelSerializerMixin): expire_time = serializers.IntegerField(read_only=True, label=_('Expired time')) + input_secret = EncryptedField( + label=_("Input secret"), max_length=40960, required=False, allow_blank=True + ) class Meta: model = ConnectionToken diff --git a/apps/perms/api/user_permission/tree/node_with_asset.py b/apps/perms/api/user_permission/tree/node_with_asset.py index adec9c079..fd8c8c233 100644 --- a/apps/perms/api/user_permission/tree/node_with_asset.py +++ b/apps/perms/api/user_permission/tree/node_with_asset.py @@ -3,14 +3,16 @@ from urllib.parse import parse_qsl from django.conf import settings from django.db.models import F, Value, CharField -from rest_framework.generics import ListAPIView -from rest_framework.generics import get_object_or_404 from rest_framework.request import Request from rest_framework.response import Response +from rest_framework.generics import ListAPIView +from rest_framework.generics import get_object_or_404 +from rest_framework.exceptions import PermissionDenied, NotFound -from assets.api import SerializeToTreeNodeMixin -from assets.models import Asset, Account from assets.utils import KubernetesTree +from assets.models import Asset, Account +from assets.api import SerializeToTreeNodeMixin +from authentication.models import ConnectionToken from common.utils import get_object_or_none, lazyproperty from common.utils.common import timeit from perms.hands import Node @@ -133,41 +135,47 @@ class UserPermedNodeChildrenWithAssetsAsTreeApi(BaseUserNodeWithAssetAsTreeApi): class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView): """ 用户授权的K8s树 """ - @staticmethod - def asset(asset_id): - kwargs = {'id': asset_id, 'is_active': True} - asset = get_object_or_404(Asset, **kwargs) - return asset + def get_token(self): + token_id = self.request.query_params.get('token') + token = get_object_or_404(ConnectionToken, pk=token_id) + if token.is_expired: + raise PermissionDenied('Token is expired') + token.renewal() + return token - def get_accounts(self, asset): + def get_account_secret(self, token: ConnectionToken): util = PermAccountUtil() - accounts = util.get_permed_accounts_for_user(self.user, asset) - ignore_username = [Account.AliasAccount.INPUT, Account.AliasAccount.USER] - accounts = filter(lambda x: x.username not in ignore_username, accounts) + accounts = util.get_permed_accounts_for_user(self.user, token.asset) + account_username = token.account + accounts = filter(lambda x: x.username == account_username, accounts) accounts = list(accounts) - return accounts + if not accounts: + raise NotFound('Account is not found') + account = accounts[0] + if account.username in [ + Account.AliasAccount.INPUT, Account.AliasAccount.USER + ]: + return token.input_secret + else: + return account.secret + + def get_namespace_and_pod(self): + key = self.request.query_params.get('key') + namespace_and_pod = dict(parse_qsl(key)) + pod = namespace_and_pod.get('pod') + namespace = namespace_and_pod.get('namespace') + return namespace, pod def list(self, request: Request, *args, **kwargs): - tree_id = request.query_params.get('tree_id') - key = request.query_params.get('key', {}) + token = self.get_token() + asset = token.asset + secret = self.get_account_secret(token) + namespace, pod = self.get_namespace_and_pod() tree = [] - parent_info = dict(parse_qsl(key)) - account_username = parent_info.get('account') - - asset_id = parent_info.get('asset_id') - asset_id = tree_id if not asset_id else asset_id - - if tree_id and not key and not account_username: - asset = self.asset(asset_id) - accounts = self.get_accounts(asset) - asset_node = KubernetesTree(tree_id).as_asset_tree_node(asset) + k8s_tree_instance = KubernetesTree(asset, secret) + if not any([namespace, pod]): + asset_node = k8s_tree_instance.as_asset_tree_node() tree.append(asset_node) - for account in accounts: - account_node = KubernetesTree(tree_id).as_account_tree_node( - account, parent_info, - ) - tree.append(account_node) - elif key and account_username: - tree = KubernetesTree(key).async_tree_node(parent_info) + tree.extend(k8s_tree_instance.async_tree_node(namespace, pod)) return Response(data=tree) From 510ca9a5b855dacbdbcce7fb5a68bfd6977e12fd Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Wed, 21 Dec 2022 17:32:55 +0800 Subject: [PATCH 08/11] perf: k8s tree --- apps/perms/api/user_permission/tree/node_with_asset.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/perms/api/user_permission/tree/node_with_asset.py b/apps/perms/api/user_permission/tree/node_with_asset.py index fd8c8c233..f65cfc0b1 100644 --- a/apps/perms/api/user_permission/tree/node_with_asset.py +++ b/apps/perms/api/user_permission/tree/node_with_asset.py @@ -159,8 +159,8 @@ class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView): else: return account.secret - def get_namespace_and_pod(self): - key = self.request.query_params.get('key') + @staticmethod + def get_namespace_and_pod(key): namespace_and_pod = dict(parse_qsl(key)) pod = namespace_and_pod.get('pod') namespace = namespace_and_pod.get('namespace') @@ -170,11 +170,12 @@ class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView): token = self.get_token() asset = token.asset secret = self.get_account_secret(token) - namespace, pod = self.get_namespace_and_pod() + key = self.request.query_params.get('key') + namespace, pod = self.get_namespace_and_pod(key) tree = [] k8s_tree_instance = KubernetesTree(asset, secret) - if not any([namespace, pod]): + if not any([namespace, pod]) and not key: asset_node = k8s_tree_instance.as_asset_tree_node() tree.append(asset_node) tree.extend(k8s_tree_instance.async_tree_node(namespace, pod)) From 34cc3b233d5bb94b31b943ef918be4778f2f1508 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 21 Dec 2022 17:36:44 +0800 Subject: [PATCH 09/11] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E6=8E=88=E6=9D=83=E8=B5=84=E4=BA=A7=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=B7=A5=E5=85=B7(=E9=87=8D=E6=9E=84=E4=B8=AD..)=20(#9225)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: 重构用户授权资产查询工具(重构中..) * perf: 修改 get_perm_nodes_assets 名称 * refactor: 优化用户授权节点查询工具; 删除UnionQuerySet工具 Co-authored-by: Bai --- apps/common/db/models.py | 84 +----- apps/perms/api/asset_permission_relation.py | 5 +- apps/perms/api/user_permission/assets.py | 10 +- apps/perms/api/user_permission/nodes.py | 4 +- .../user_permission/tree/node_with_asset.py | 26 +- apps/perms/filters.py | 9 +- apps/perms/models/asset_permission.py | 15 +- apps/perms/models/perm_node.py | 30 +- apps/perms/utils/__init__.py | 2 +- apps/perms/utils/user_perm.py | 219 ++++++++++++++ apps/perms/utils/user_permission.py | 280 ------------------ 11 files changed, 269 insertions(+), 415 deletions(-) create mode 100644 apps/perms/utils/user_perm.py delete mode 100644 apps/perms/utils/user_permission.py diff --git a/apps/common/db/models.py b/apps/common/db/models.py index 7be495538..ae726b85d 100644 --- a/apps/common/db/models.py +++ b/apps/common/db/models.py @@ -9,9 +9,8 @@ - 此文件中添加代码的时候，注意不要跟 `django.db.models` 中的命名冲突 """ -import inspect import uuid -from functools import reduce, partial +from functools import reduce from django.db import models from django.db import transaction @@ -96,87 +95,6 @@ def output_as_string(field_name): return ExpressionWrapper(F(field_name), output_field=models.CharField()) -class UnionQuerySet(QuerySet): - after_union = ['order_by'] - not_return_qs = [ - 'query', 'get', 'create', 'get_or_create', - 'update_or_create', 'bulk_create', 'count', - 'latest', 'earliest', 'first', 'last', 'aggregate', - 'exists', 'update', 'delete', 'as_manager', 'explain', - ] - - def __init__(self, *queryset_list): - self.queryset_list = queryset_list - self.after_union_items = [] - self.before_union_items = [] - - def __execute(self): - queryset_list = [] - for qs in self.queryset_list: - for attr, args, kwargs in self.before_union_items: - qs = getattr(qs, attr)(*args, **kwargs) - queryset_list.append(qs) - union_qs = reduce(lambda x, y: x.union(y), queryset_list) - for attr, args, kwargs in self.after_union_items: - union_qs = getattr(union_qs, attr)(*args, **kwargs) - return union_qs - - def __before_union_perform(self, item, *args, **kwargs): - self.before_union_items.append((item, args, kwargs)) - return self.__clone(*self.queryset_list) - - def __after_union_perform(self, item, *args, **kwargs): - self.after_union_items.append((item, args, kwargs)) - return self.__clone(*self.queryset_list) - - def __clone(self, *queryset_list): - uqs = UnionQuerySet(*queryset_list) - uqs.after_union_items = self.after_union_items - uqs.before_union_items = self.before_union_items - return uqs - - def __getattribute__(self, item): - if item.startswith('__') or item in UnionQuerySet.__dict__ or item in [ - 'queryset_list', 'after_union_items', 'before_union_items' - ]: - return object.__getattribute__(self, item) - - if item in UnionQuerySet.not_return_qs: - return getattr(self.__execute(), item) - - origin_item = object.__getattribute__(self, 'queryset_list')[0] - origin_attr = getattr(origin_item, item, None) - if not inspect.ismethod(origin_attr): - return getattr(self.__execute(), item) - - if item in UnionQuerySet.after_union: - attr = partial(self.__after_union_perform, item) - else: - attr = partial(self.__before_union_perform, item) - return attr - - def __getitem__(self, item): - return self.__execute()[item] - - def __iter__(self): - return iter(self.__execute()) - - def __str__(self): - return str(self.__execute()) - - def __repr__(self): - return repr(self.__execute()) - - @classmethod - def test_it(cls): - from assets.models import Asset - assets1 = Asset.objects.filter(hostname__startswith='a') - assets2 = Asset.objects.filter(hostname__startswith='b') - - qs = cls(assets1, assets2) - return qs - - class MultiTableChildQueryset(QuerySet): def bulk_create(self, objs, batch_size=None): diff --git a/apps/perms/api/asset_permission_relation.py b/apps/perms/api/asset_permission_relation.py index c2c116248..5b85cb971 100644 --- a/apps/perms/api/asset_permission_relation.py +++ b/apps/perms/api/asset_permission_relation.py @@ -9,7 +9,7 @@ from orgs.mixins.api import OrgBulkModelViewSet from orgs.utils import current_org from perms import serializers from perms import models -from perms.utils.user_permission import UserGrantedAssetsQueryUtils +from perms.utils import AssetPermissionPermAssetUtil from assets.serializers import AccountSerializer __all__ = [ @@ -95,8 +95,7 @@ class AssetPermissionAllAssetListApi(generics.ListAPIView): def get_queryset(self): pk = self.kwargs.get("pk") - query_utils = UserGrantedAssetsQueryUtils(None, asset_perm_ids=[pk]) - assets = query_utils.get_all_granted_assets() + assets = AssetPermissionPermAssetUtil(perm_ids=[pk]).get_all_assets() return assets diff --git a/apps/perms/api/user_permission/assets.py b/apps/perms/api/user_permission/assets.py index e494b7a4a..e499a3127 100644 --- a/apps/perms/api/user_permission/assets.py +++ b/apps/perms/api/user_permission/assets.py @@ -6,7 +6,7 @@ from assets.api.asset.asset import AssetFilterSet from perms import serializers from perms.pagination import AllPermedAssetPagination from perms.pagination import NodePermedAssetPagination -from perms.utils.user_permission import UserGrantedAssetsQueryUtils +from perms.utils import UserPermAssetUtil from common.utils import get_logger, lazyproperty from .mixin import ( @@ -43,21 +43,23 @@ class BaseUserPermedAssetsApi(SelfOrPKUserMixin, ListAPIView): def get_assets(self): return Asset.objects.none() + query_asset_util: UserPermAssetUtil + @lazyproperty def query_asset_util(self): - return UserGrantedAssetsQueryUtils(self.user) + return UserPermAssetUtil(self.user) class UserAllPermedAssetsApi(BaseUserPermedAssetsApi): pagination_class = AllPermedAssetPagination def get_assets(self): - return self.query_asset_util.get_all_granted_assets() + return self.query_asset_util.get_all_assets() class UserDirectPermedAssetsApi(BaseUserPermedAssetsApi): def get_assets(self): - return self.query_asset_util.get_direct_granted_assets() + return self.query_asset_util.get_direct_assets() class UserFavoriteAssetsApi(BaseUserPermedAssetsApi): diff --git a/apps/perms/api/user_permission/nodes.py b/apps/perms/api/user_permission/nodes.py index 1972909e9..793ad3858 100644 --- a/apps/perms/api/user_permission/nodes.py +++ b/apps/perms/api/user_permission/nodes.py @@ -7,7 +7,7 @@ from rest_framework.generics import ListAPIView from assets.models import Node from common.utils import get_logger, lazyproperty from perms import serializers -from perms.utils.user_permission import UserGrantedNodesQueryUtils +from perms.utils import UserPermNodeUtil from .mixin import SelfOrPKUserMixin logger = get_logger(__name__) @@ -32,7 +32,7 @@ class BaseUserPermedNodesApi(SelfOrPKUserMixin, ListAPIView): @lazyproperty def query_node_util(self): - return UserGrantedNodesQueryUtils(self.user) + return UserPermNodeUtil(self.user) class UserAllPermedNodesApi(BaseUserPermedNodesApi): diff --git a/apps/perms/api/user_permission/tree/node_with_asset.py b/apps/perms/api/user_permission/tree/node_with_asset.py index f65cfc0b1..204859f36 100644 --- a/apps/perms/api/user_permission/tree/node_with_asset.py +++ b/apps/perms/api/user_permission/tree/node_with_asset.py @@ -17,11 +17,8 @@ from common.utils import get_object_or_none, lazyproperty from common.utils.common import timeit from perms.hands import Node from perms.models import PermNode -from perms.utils import PermAccountUtil -from perms.utils.permission import AssetPermissionUtil -from perms.utils.user_permission import ( - UserGrantedNodesQueryUtils, UserGrantedAssetsQueryUtils, -) +from perms.utils import PermAccountUtil, UserPermNodeUtil, AssetPermissionUtil +from perms.utils import UserPermAssetUtil from .mixin import RebuildTreeMixin from ..mixin import SelfOrPKUserMixin @@ -54,13 +51,12 @@ class BaseUserNodeWithAssetAsTreeApi( class UserPermedNodesWithAssetsAsTreeApi(BaseUserNodeWithAssetAsTreeApi): - query_node_util: UserGrantedNodesQueryUtils - query_asset_util: UserGrantedAssetsQueryUtils + query_node_util: UserPermNodeUtil + query_asset_util: UserPermAssetUtil def get_nodes_assets(self): - perm_ids = AssetPermissionUtil().get_permissions_for_user(self.request.user, flat=True) - self.query_node_util = UserGrantedNodesQueryUtils(self.request.user, perm_ids) - self.query_asset_util = UserGrantedAssetsQueryUtils(self.request.user, perm_ids) + self.query_node_util = UserPermNodeUtil(self.request.user) + self.query_asset_util = UserPermAssetUtil(self.request.user) ung_nodes, ung_assets = self._get_nodes_assets_for_ungrouped() fav_nodes, fav_assets = self._get_nodes_assets_for_favorite() all_nodes, all_assets = self._get_nodes_assets_for_all() @@ -89,9 +85,9 @@ class UserPermedNodesWithAssetsAsTreeApi(BaseUserNodeWithAssetAsTreeApi): def _get_nodes_assets_for_all(self): nodes = self.query_node_util.get_whole_tree_nodes(with_special=False) if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - assets = self.query_asset_util.get_direct_granted_nodes_assets() + assets = self.query_asset_util.get_perm_nodes_assets() else: - assets = self.query_asset_util.get_all_granted_assets() + assets = self.query_asset_util.get_all_assets() assets = assets.annotate(parent_key=F('nodes__key')).prefetch_related('platform') return nodes, assets @@ -102,8 +98,8 @@ class UserPermedNodeChildrenWithAssetsAsTreeApi(BaseUserNodeWithAssetAsTreeApi): def get_nodes_assets(self): nodes = PermNode.objects.none() assets = Asset.objects.none() - query_node_util = UserGrantedNodesQueryUtils(self.user) - query_asset_util = UserGrantedAssetsQueryUtils(self.user) + query_node_util = UserPermNodeUtil(self.user) + query_asset_util = UserPermAssetUtil(self.user) node_key = self.query_node_key if not node_key: nodes = query_node_util.get_top_level_nodes() @@ -113,7 +109,7 @@ class UserPermedNodeChildrenWithAssetsAsTreeApi(BaseUserNodeWithAssetAsTreeApi): assets = query_asset_util.get_favorite_assets() else: nodes = query_node_util.get_node_children(node_key) - assets = query_asset_util.get_node_assets(node_key) + assets = query_asset_util.get_node_assets(key=node_key) assets = assets.prefetch_related('platform') return nodes, assets diff --git a/apps/perms/filters.py b/apps/perms/filters.py index 05f202655..8743c38be 100644 --- a/apps/perms/filters.py +++ b/apps/perms/filters.py @@ -1,7 +1,6 @@ from django_filters import rest_framework as filters from django.db.models import QuerySet, Q -from common.db.models import UnionQuerySet from common.drf.filters import BaseFilterSet from common.utils import get_object_or_none from users.models import User, UserGroup @@ -169,10 +168,10 @@ class AssetPermissionFilter(PermissionBaseFilter): inherit_all_node_ids = Node.objects.filter(key__in=inherit_all_node_keys).values_list('id', flat=True) inherit_all_node_ids = list(inherit_all_node_ids) - qs1 = queryset.filter(assets__in=asset_ids).distinct() - qs2 = queryset.filter(nodes__in=inherit_all_node_ids).distinct() - - qs = UnionQuerySet(qs1, qs2) + qs1_ids = queryset.filter(assets__in=asset_ids).distinct().values_list('id', flat=True) + qs2_ids = queryset.filter(nodes__in=inherit_all_node_ids).distinct().values_list('id', flat=True) + qs_ids = list(qs1_ids) + list(qs2_ids) + qs = queryset.filter(id__in=qs_ids) return qs def filter_effective(self, queryset): diff --git a/apps/perms/models/asset_permission.py b/apps/perms/models/asset_permission.py index ac6fecdb4..989f10cbc 100644 --- a/apps/perms/models/asset_permission.py +++ b/apps/perms/models/asset_permission.py @@ -5,14 +5,14 @@ from django.db.models import Q from django.utils import timezone from django.utils.translation import ugettext_lazy as _ +from users.models import User from assets.models import Asset, Account -from common.db.models import UnionQuerySet -from common.utils import date_expired_default -from common.utils.timezone import local_now from orgs.mixins.models import JMSOrgBaseModel from orgs.mixins.models import OrgManager +from common.utils import date_expired_default +from common.utils.timezone import local_now + from perms.const import ActionChoices -from users.models import User __all__ = ['AssetPermission', 'ActionChoices'] @@ -104,9 +104,10 @@ class AssetPermission(JMSOrgBaseModel): group_ids = self.user_groups.all().values_list('id', flat=True) user_ids = list(user_ids) group_ids = list(group_ids) - qs1 = User.objects.filter(id__in=user_ids).distinct() - qs2 = User.objects.filter(groups__id__in=group_ids).distinct() - qs = UnionQuerySet(qs1, qs2) + qs1_ids = User.objects.filter(id__in=user_ids).distinct().values_list('id', flat=True) + qs2_ids = User.objects.filter(groups__id__in=group_ids).distinct().values_list('id', flat=True) + qs_ids = list(qs1_ids) + list(qs2_ids) + qs = User.objects.filter(id__in=qs_ids) return qs def get_all_assets(self, flat=False): diff --git a/apps/perms/models/perm_node.py b/apps/perms/models/perm_node.py index 89354a06b..ed4b4b213 100644 --- a/apps/perms/models/perm_node.py +++ b/apps/perms/models/perm_node.py @@ -36,15 +36,14 @@ class UserAssetGrantedTreeNodeRelation(FamilyMixin, JMSOrgBaseModel): return self.node_parent_key @classmethod - def get_node_granted_status(cls, user, key): + def get_node_from_with_node(cls, user, key): ancestor_keys = set(cls.get_node_ancestor_keys(key, with_self=True)) - ancestor_rel_nodes = cls.objects.filter(user=user, node_key__in=ancestor_keys) - - for rel_node in ancestor_rel_nodes: - if rel_node.key == key: - return rel_node.node_from, rel_node - if rel_node.node_from == cls.NodeFrom.granted: - return cls.NodeFrom.granted, None + ancestor_nodes = cls.objects.filter(user=user, node_key__in=ancestor_keys) + for node in ancestor_nodes: + if node.key == key: + return node.node_from, node + if node.node_from == cls.NodeFrom.granted: + return node.node_from, None return '', None @@ -91,15 +90,16 @@ class PermNode(Node): node.assets_amount = assets_amount return node - def get_granted_status(self, user): - status, rel_node = UserAssetGrantedTreeNodeRelation.get_node_granted_status(user, self.key) - self.node_from = status - if rel_node: - self.granted_assets_amount = rel_node.node_assets_amount - return status + def compute_node_from_and_assets_amount(self, user): + node_from, node = UserAssetGrantedTreeNodeRelation.get_node_from_with_node( + user, self.key + ) + self.node_from = node_from + if node: + self.granted_assets_amount = node.node_assets_amount def save(self): - # 这是个只读 Model + """ 这是个只读 Model """ raise NotImplementedError diff --git a/apps/perms/utils/__init__.py b/apps/perms/utils/__init__.py index 8563b21e9..808bdd1b8 100644 --- a/apps/perms/utils/__init__.py +++ b/apps/perms/utils/__init__.py @@ -1,4 +1,4 @@ from .permission import * -from .user_permission import * from .account import * from .user_perm_tree import * +from .user_perm import * diff --git a/apps/perms/utils/user_perm.py b/apps/perms/utils/user_perm.py new file mode 100644 index 000000000..a5f956de8 --- /dev/null +++ b/apps/perms/utils/user_perm.py @@ -0,0 +1,219 @@ +from assets.models import FavoriteAsset, Asset + +from django.conf import settings +from django.db.models import Q + +from common.utils.common import timeit + +from perms.models import AssetPermission, PermNode, UserAssetGrantedTreeNodeRelation + +from .permission import AssetPermissionUtil + + +__all__ = ['AssetPermissionPermAssetUtil', 'UserPermAssetUtil', 'UserPermNodeUtil'] + + +class AssetPermissionPermAssetUtil: + + def __init__(self, perm_ids): + self.perm_ids = perm_ids + + def get_all_assets(self): + """ 获取所有授权的资产 """ + node_asset_ids = self.get_perm_nodes_assets(flat=True) + direct_asset_ids = self.get_direct_assets(flat=True) + asset_ids = list(node_asset_ids) + list(direct_asset_ids) + assets = Asset.objects.filter(id__in=asset_ids) + return assets + + def get_perm_nodes_assets(self, flat=False): + """ 获取所有授权节点下的资产 """ + node_ids = AssetPermission.nodes.through.objects \ + .filter(assetpermission_id__in=self.perm_ids) \ + .values_list('node_id', flat=True) \ + .distinct() + node_ids = list(node_ids) + nodes = PermNode.objects.filter(id__in=node_ids).only('id', 'key') + assets = PermNode.get_nodes_all_assets(*nodes) + if flat: + return assets.values_list('id', flat=True) + return assets + + def get_direct_assets(self, flat=False): + """ 获取直接授权的资产 """ + assets = Asset.objects.order_by() \ + .filter(granted_by_permissions__id__in=self.perm_ids) \ + .distinct() + if flat: + return assets.values_list('id', flat=True) + return assets + + +class UserPermAssetUtil(AssetPermissionPermAssetUtil): + + def __init__(self, user): + self.user = user + perm_ids = AssetPermissionUtil().get_permissions_for_user(self.user, flat=True) + super().__init__(perm_ids) + + def get_ungroup_assets(self): + return self.get_direct_assets() + + def get_favorite_assets(self): + assets = self.get_all_assets() + asset_ids = FavoriteAsset.objects.filter(user=self.user).values_list('asset_id', flat=True) + assets = assets.filter(id__in=list(asset_ids)) + return assets + + def get_node_assets(self, key): + node = PermNode.objects.get(key=key) + node.compute_node_from_and_assets_amount(self.user) + if node.node_from == node.NodeFrom.granted: + assets = Asset.objects.filter(nodes__id=node.id).order_by() + elif node.node_from == node.NodeFrom.asset: + assets = self._get_indirect_perm_node_assets(node) + else: + assets = Asset.objects.none() + assets = assets.order_by('name') + return assets + + def get_node_all_assets(self, node_id): + """ 获取节点下的所有资产 """ + node = PermNode.objects.get(id=node_id) + node.compute_node_from_and_assets_amount(self.user) + if node.node_from == node.NodeFrom.granted: + assets = PermNode.get_nodes_all_assets() + elif node.node_from in (node.NodeFrom.asset, node.NodeFrom.child): + node.assets_amount = node.granted_assets_amount + assets = self._get_indirect_perm_node_all_assets(node) + else: + node.assets_amount = 0 + assets = Asset.objects.none() + return node, assets + + def _get_indirect_perm_node_assets(self, node): + """ 获取间接授权节点下的直接资产 """ + assets = self.get_direct_assets() + assets = assets.filter(nodes__id=node.id).order_by().distinct() + return assets + + def _get_indirect_perm_node_all_assets(self, node): + """ 获取间接授权节点下的所有资产 + 此算法依据 `UserAssetGrantedTreeNodeRelation` 的数据查询 + 1. 查询该节点下的直接授权节点 + 2. 查询该节点下授权资产关联的节点 + """ + # 查询节点下直接授权的子节点 + asset_ids = set() + children_from_granted = UserAssetGrantedTreeNodeRelation.objects \ + .filter(user=self.user) \ + .filter(node_key__startwith=f'{node.key}:', node_from=node.NodeFrom.granted) \ + .only('node_id', 'node_key') + for n in children_from_granted: + n.id = n.node_id + _assets = PermNode.get_nodes_all_assets(*children_from_granted) + _asset_ids = _assets.values_list('id', flat=True) + asset_ids.update(list(_asset_ids)) + + # 查询节点下资产授权的节点 + children_from_assets = UserAssetGrantedTreeNodeRelation.objects \ + .filter(user=self.user) \ + .filter(node_key__startwith=f'{node.key}:', node_from=node.NodeFrom.asset) \ + .values_list('node_id', flat=True) + children_from_assets = set(children_from_assets) + if node.node_from == node.NodeFrom.asset: + children_from_assets.add(node.id) + _asset_ids = Asset.objects \ + .filter(node__id__in=children_from_assets) \ + .filter(granted_by_permissions__id__in=self.perm_ids) \ + .distinct() \ + .order_by() \ + .values_list('id', flat=True) + asset_ids.update(list(_asset_ids)) + + return Asset.objects.filter(id__in=asset_ids) + + +class UserPermNodeUtil: + + def __init__(self, user): + self.user = user + self.perm_ids = AssetPermissionUtil().get_permissions_for_user(self.user, flat=True) + + def get_favorite_node(self): + assets_amount = UserPermAssetUtil(self.user).get_favorite_assets().count() + return PermNode.get_favorite_node(assets_amount) + + def get_ungrouped_node(self): + assets_amount = UserPermAssetUtil(self.user).get_direct_assets().count() + return PermNode.get_favorite_node(assets_amount) + + def get_top_level_nodes(self): + nodes = self.get_special_nodes() + real_nodes = self._get_indirect_perm_node_children(key='') + if len(real_nodes) == 1: + children = self.get_node_children(real_nodes[0].key) + nodes.extend(children) + return nodes + + def get_special_nodes(self): + nodes = [] + if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + ung_node = self.get_ungrouped_node() + nodes.append(ung_node) + fav_node = self.get_favorite_node() + nodes.append(fav_node) + return nodes + + def get_node_children(self, key): + if not key: + return self.get_top_level_nodes() + + if key in [PermNode.FAVORITE_NODE_KEY, PermNode.UNGROUPED_NODE_KEY]: + return PermNode.objects.none() + + node = PermNode.objects.get(key=key) + node.compute_node_from_and_assets_amount(self.user) + if node.node_from == node.NodeFrom.granted: + children = PermNode.objects.filter(parent_key=key) + elif node.node_from in (node.NodeFrom.asset, node.NodeFrom.child): + children = self._get_indirect_perm_node_children(key) + else: + children = PermNode.objects.none() + children = sorted(children, key=lambda x: x.value) + return children + + def _get_indirect_perm_node_children(self, key): + """ 获取未直接授权节点的子节点 """ + children = PermNode.objects.filter(granted_node_rels__user=self.user, parent_key=key) + children = children.annotate(**PermNode.annotate_granted_node_rel_fields).distinct() + for node in children: + node.assets_amount = node.granted_assets_amount + return children + + @timeit + def get_whole_tree_nodes(self, with_special=True): + user_nodes = PermNode.objects.filter(granted_node_rels__user=self.user) + user_nodes = user_nodes.annotate(**PermNode.annotate_granted_node_rel_fields).distinct() + + key_node_mapper = {} + q_nodes_descendant = Q() + for node in user_nodes: + node.assets_amount = node.granted_assets_amount + key_node_mapper[node.key] = node + if node.node_from == node.NodeFrom.granted: + """ 直接授权的节点, 增加后代节点的过滤条件 """ + q_nodes_descendant |= Q(key__startswith=f'{node.key}:') + if q_nodes_descendant: + descendant_nodes = PermNode.objects.filter(q_nodes_descendant) + for node in descendant_nodes: + key_node_mapper[node.key] = node + + nodes = [] + if with_special: + special_nodes = self.get_special_nodes() + nodes.extend(special_nodes) + nodes.extend(list(key_node_mapper.values())) + + return nodes + diff --git a/apps/perms/utils/user_permission.py b/apps/perms/utils/user_permission.py deleted file mode 100644 index 8ee5992dd..000000000 --- a/apps/perms/utils/user_permission.py +++ /dev/null @@ -1,280 +0,0 @@ -from collections import defaultdict -from typing import List, Tuple - -from django.conf import settings -from django.db.models import Q, QuerySet -from django.utils.translation import gettext as _ - -from users.models import User -from assets.utils import NodeAssetsUtil -from assets.models import ( - Asset, - FavoriteAsset, - AssetQuerySet, - NodeQuerySet -) -from orgs.utils import ( - tmp_to_org, - current_org, - ensure_in_real_or_default_org, -) -from common.db.models import output_as_string, UnionQuerySet -from common.utils import get_logger -from common.utils.common import lazyproperty, timeit - -from perms.models import ( - AssetPermission, - PermNode, - UserAssetGrantedTreeNodeRelation -) -from .permission import AssetPermissionUtil - -NodeFrom = UserAssetGrantedTreeNodeRelation.NodeFrom -NODE_ONLY_FIELDS = ('id', 'key', 'parent_key', 'org_id') - -logger = get_logger(__name__) - - -class UserGrantedUtilsBase: - user: User - - def __init__(self, user, asset_perm_ids=None): - self.user = user - self._asset_perm_ids = asset_perm_ids and set(asset_perm_ids) - - @lazyproperty - def asset_perm_ids(self) -> set: - if self._asset_perm_ids: - return self._asset_perm_ids - - asset_perm_ids = AssetPermissionUtil().get_permissions_for_user(self.user, flat=True) - return asset_perm_ids - - -class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): - - def get_favorite_assets(self) -> QuerySet: - assets = self.get_all_granted_assets() - asset_ids = FavoriteAsset.objects.filter(user=self.user).values_list('asset_id', flat=True) - assets = assets.filter(id__in=list(asset_ids)) - return assets - - def get_ungroup_assets(self) -> AssetQuerySet: - return self.get_direct_granted_assets() - - def get_direct_granted_assets(self) -> AssetQuerySet: - queryset = Asset.objects.order_by().filter( - granted_by_permissions__id__in=self.asset_perm_ids - ).distinct() - return queryset - - def get_direct_granted_nodes_assets(self) -> AssetQuerySet: - granted_node_ids = AssetPermission.nodes.through.objects.filter( - assetpermission_id__in=self.asset_perm_ids - ).values_list('node_id', flat=True).distinct() - granted_node_ids = list(granted_node_ids) - granted_nodes = PermNode.objects.filter(id__in=granted_node_ids).only('id', 'key') - queryset = PermNode.get_nodes_all_assets(*granted_nodes) - return queryset - - def get_all_granted_assets(self) -> QuerySet: - nodes_assets = self.get_direct_granted_nodes_assets() - assets = self.get_direct_granted_assets() - # queryset = UnionQuerySet(nodes_assets, assets) - # return queryset - node_asset_ids = nodes_assets.values_list('id', flat=True) - direct_asset_ids = assets.values_list('id', flat=True) - asset_ids = list(node_asset_ids) + list(direct_asset_ids) - asset = Asset.objects.filter(id__in=asset_ids) - return asset - - def get_node_all_assets(self, id) -> Tuple[PermNode, QuerySet]: - node = PermNode.objects.get(id=id) - granted_status = node.get_granted_status(self.user) - if granted_status == NodeFrom.granted: - assets = PermNode.get_nodes_all_assets(node) - return node, assets - elif granted_status in (NodeFrom.asset, NodeFrom.child): - node.use_granted_assets_amount() - assets = self._get_indirect_granted_node_all_assets(node) - return node, assets - else: - node.assets_amount = 0 - return node, Asset.objects.none() - - def get_node_assets(self, key) -> AssetQuerySet: - node = PermNode.objects.get(key=key) - granted_status = node.get_granted_status(self.user) - - if granted_status == NodeFrom.granted: - assets = Asset.objects.order_by().filter(nodes__id=node.id) - elif granted_status == NodeFrom.asset: - assets = self._get_indirect_granted_node_assets(node.id) - else: - assets = Asset.objects.none() - assets = assets.order_by('name') - return assets - - def _get_indirect_granted_node_assets(self, id) -> AssetQuerySet: - assets = Asset.objects.order_by().filter(nodes__id=id).distinct() & self.get_direct_granted_assets() - return assets - - def _get_indirect_granted_node_all_assets(self, node) -> QuerySet: - """ - 此算法依据 `UserAssetGrantedTreeNodeRelation` 的数据查询 - 1. 查询该节点下的直接授权节点 - 2. 查询该节点下授权资产关联的节点 - """ - user = self.user - - # 查询该节点下的授权节点 - granted_nodes = UserAssetGrantedTreeNodeRelation.objects.filter( - user=user, node_from=NodeFrom.granted - ).filter( - Q(node_key__startswith=f'{node.key}:') - ).only('node_id', 'node_key') - - for n in granted_nodes: - n.id = n.node_id - - node_assets = PermNode.get_nodes_all_assets(*granted_nodes) - - # 查询该节点下的资产授权节点 - only_asset_granted_node_ids = UserAssetGrantedTreeNodeRelation.objects.filter( - user=user, node_from=NodeFrom.asset - ).filter( - Q(node_key__startswith=f'{node.key}:') - ).values_list('node_id', flat=True) - - only_asset_granted_node_ids = list(only_asset_granted_node_ids) - if node.node_from == NodeFrom.asset: - only_asset_granted_node_ids.append(node.id) - - assets = Asset.objects.filter( - nodes__id__in=only_asset_granted_node_ids, - granted_by_permissions__id__in=self.asset_perm_ids - ).distinct().order_by() - granted_assets = UnionQuerySet(node_assets, assets) - return granted_assets - - -class UserGrantedNodesQueryUtils(UserGrantedUtilsBase): - def sort(self, nodes): - nodes = sorted(nodes, key=lambda x: x.value) - return nodes - - def get_node_children(self, key): - if not key: - return self.get_top_level_nodes() - - nodes = PermNode.objects.none() - if key in [PermNode.FAVORITE_NODE_KEY, PermNode.UNGROUPED_NODE_KEY]: - return nodes - - node = PermNode.objects.get(key=key) - granted_status = node.get_granted_status(self.user) - if granted_status == NodeFrom.granted: - nodes = PermNode.objects.filter(parent_key=key) - elif granted_status in (NodeFrom.asset, NodeFrom.child): - nodes = self.get_indirect_granted_node_children(key) - nodes = self.sort(nodes) - return nodes - - def get_indirect_granted_node_children(self, key): - """ - 获取用户授权树中未授权节点的子节点 - 只匹配在 `UserAssetGrantedTreeNodeRelation` 中存在的节点 - """ - user = self.user - nodes = PermNode.objects.filter( - granted_node_rels__user=user, - parent_key=key - ).annotate( - **PermNode.annotate_granted_node_rel_fields - ).distinct() - - # 设置节点授权资产数量 - for node in nodes: - node.use_granted_assets_amount() - return nodes - - def get_top_level_nodes(self): - nodes = self.get_special_nodes() - real_nodes = self.get_indirect_granted_node_children('') - nodes.extend(real_nodes) - if len(real_nodes) == 1: - children = self.get_node_children(real_nodes[0].key) - nodes.extend(children) - return nodes - - def get_ungrouped_node(self): - assets_util = UserGrantedAssetsQueryUtils(self.user, self.asset_perm_ids) - assets_amount = assets_util.get_direct_granted_assets().count() - return PermNode.get_ungrouped_node(assets_amount) - - def get_favorite_node(self): - assets_query_utils = UserGrantedAssetsQueryUtils(self.user, self.asset_perm_ids) - assets_amount = assets_query_utils.get_favorite_assets().values_list('id').count() - return PermNode.get_favorite_node(assets_amount) - - @staticmethod - def get_root_node(): - name = _('My assets') - node = { - 'id': '', - 'name': name, - 'title': name, - 'pId': '', - 'open': True, - 'isParent': True, - 'meta': { - 'type': 'root' - } - } - return node - - def get_special_nodes(self): - nodes = [] - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - ungrouped_node = self.get_ungrouped_node() - nodes.append(ungrouped_node) - favorite_node = self.get_favorite_node() - nodes.append(favorite_node) - return nodes - - @timeit - def get_whole_tree_nodes(self, with_special=True): - """ - 这里的 granted nodes, 是整棵树需要的node，推算出来的也算 - :param with_special: - :return: - """ - nodes = PermNode.objects.filter(granted_node_rels__user=self.user) \ - .annotate(**PermNode.annotate_granted_node_rel_fields) \ - .distinct() - - key_to_node_mapper = {} - nodes_descendant_q = Q() - - for node in nodes: - node.use_granted_assets_amount() - key_to_node_mapper[node.key] = node - - if node.node_from == NodeFrom.granted: - # 直接授权的节点 - # 增加查询后代节点的过滤条件 - nodes_descendant_q |= Q(key__startswith=f'{node.key}:') - - if nodes_descendant_q: - descendant_nodes = PermNode.objects.filter( - nodes_descendant_q - ) - for node in descendant_nodes: - key_to_node_mapper[node.key] = node - - all_nodes = [] - if with_special: - special_nodes = self.get_special_nodes() - all_nodes.extend(special_nodes) - all_nodes.extend(key_to_node_mapper.values()) - return all_nodes From d4e215aeaa71605c87a5555b35cc8161cb00bd9d Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 21 Dec 2022 18:00:50 +0800 Subject: [PATCH 10/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dluna=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E8=B5=84=E4=BA=A7=E6=A0=91=E5=8A=A0=E8=BD=BD=E4=B8=8D?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../user_permission/tree/node_with_asset.py | 5 +++-- apps/perms/models/perm_node.py | 18 +++++++++++++----- apps/perms/utils/user_perm.py | 1 + 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/apps/perms/api/user_permission/tree/node_with_asset.py b/apps/perms/api/user_permission/tree/node_with_asset.py index 204859f36..e67020962 100644 --- a/apps/perms/api/user_permission/tree/node_with_asset.py +++ b/apps/perms/api/user_permission/tree/node_with_asset.py @@ -96,16 +96,17 @@ class UserPermedNodeChildrenWithAssetsAsTreeApi(BaseUserNodeWithAssetAsTreeApi): """ 用户授权的节点的子节点与资产树 """ def get_nodes_assets(self): - nodes = PermNode.objects.none() - assets = Asset.objects.none() query_node_util = UserPermNodeUtil(self.user) query_asset_util = UserPermAssetUtil(self.user) node_key = self.query_node_key if not node_key: nodes = query_node_util.get_top_level_nodes() + assets = Asset.objects.none() elif node_key == PermNode.UNGROUPED_NODE_KEY: + nodes = PermNode.objects.none() assets = query_asset_util.get_ungroup_assets() elif node_key == PermNode.FAVORITE_NODE_KEY: + nodes = PermNode.objects.none() assets = query_asset_util.get_favorite_assets() else: nodes = query_node_util.get_node_children(node_key) diff --git a/apps/perms/models/perm_node.py b/apps/perms/models/perm_node.py index ed4b4b213..438cd5b2d 100644 --- a/apps/perms/models/perm_node.py +++ b/apps/perms/models/perm_node.py @@ -16,13 +16,18 @@ class NodeFrom(TextChoices): class UserAssetGrantedTreeNodeRelation(FamilyMixin, JMSOrgBaseModel): NodeFrom = NodeFrom - id = models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name=_('ID')) + id = models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name=_('ID') + ) user = models.ForeignKey('users.User', db_constraint=False, on_delete=models.CASCADE) - node = models.ForeignKey('assets.Node', default=None, on_delete=models.CASCADE, - db_constraint=False, null=False, related_name='granted_node_rels') + node = models.ForeignKey( + 'assets.Node', default=None, on_delete=models.CASCADE, db_constraint=False, null=False, + related_name='granted_node_rels' + ) node_key = models.CharField(max_length=64, verbose_name=_("Key"), db_index=True) - node_parent_key = models.CharField(max_length=64, default='', verbose_name=_('Parent key'), - db_index=True) + node_parent_key = models.CharField( + max_length=64, default='', verbose_name=_('Parent key'), db_index=True + ) node_from = models.CharField(choices=NodeFrom.choices, max_length=16, db_index=True) node_assets_amount = models.IntegerField(default=0) comment = '' @@ -68,6 +73,9 @@ class PermNode(Node): 'node_from': F('granted_node_rels__node_from') } + def __str__(self): + return f'{self.name}' + def use_granted_assets_amount(self): self.assets_amount = self.granted_assets_amount diff --git a/apps/perms/utils/user_perm.py b/apps/perms/utils/user_perm.py index a5f956de8..ba7f37855 100644 --- a/apps/perms/utils/user_perm.py +++ b/apps/perms/utils/user_perm.py @@ -151,6 +151,7 @@ class UserPermNodeUtil: def get_top_level_nodes(self): nodes = self.get_special_nodes() real_nodes = self._get_indirect_perm_node_children(key='') + nodes.extend(real_nodes) if len(real_nodes) == 1: children = self.get_node_children(real_nodes[0].key) nodes.extend(children) From e347e05210ea8dc12226271514447fa69f7d8787 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Wed, 21 Dec 2022 18:26:06 +0800 Subject: [PATCH 11/11] perf: terminal type remove ssh --- apps/terminal/connect_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/terminal/connect_methods.py b/apps/terminal/connect_methods.py index 10bf30c81..454abb522 100644 --- a/apps/terminal/connect_methods.py +++ b/apps/terminal/connect_methods.py @@ -153,7 +153,7 @@ class ConnectMethodUtil: protocols = { TerminalType.koko: { 'web_methods': [WebMethod.web_cli, WebMethod.web_sftp], - 'listen': [Protocol.ssh, Protocol.http], + 'listen': [Protocol.http], 'support': [ Protocol.ssh, Protocol.telnet, Protocol.mysql, Protocol.postgresql,