From 8291a81efd530691a0655222dc084f50bd3f16f0 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 5 Dec 2023 11:16:34 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=94=AF=E6=8C=81=E5=85=A8=E5=B1=80?= =?UTF-8?q?=E7=9A=84=20labels=20(#12043)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 支持全局的 labels * perf: stash * stash * stash * stash * stash * perf: 优化 labels * stash * perf: add debug sql * perf: 修改 labels * perf: 优化提交 * perf: 优化提交 labels * perf: 基本完成 * perf: 完成 labels 搜索 * perf: 优化 labels * perf: 去掉不用 debug --------- Co-authored-by: ibuler --- apps/accounts/models/account.py | 5 +- apps/accounts/models/template.py | 3 +- apps/accounts/serializers/account/account.py | 5 +- apps/accounts/serializers/account/base.py | 6 +- apps/assets/api/__init__.py | 1 - apps/assets/api/asset/asset.py | 18 +- apps/assets/api/label.py | 43 - apps/assets/filters.py | 52 -- .../migrations/0126_remove_asset_labels.py | 18 + apps/assets/models/asset/common.py | 12 +- apps/assets/models/domain.py | 3 +- apps/assets/models/platform.py | 3 +- apps/assets/serializers/__init__.py | 9 +- apps/assets/serializers/asset/common.py | 11 +- apps/assets/serializers/domain.py | 9 +- apps/assets/serializers/label.py | 47 - apps/assets/serializers/platform.py | 10 +- apps/assets/urls/api_urls.py | 1 - apps/audits/utils.py | 15 +- apps/common/api/mixin.py | 4 +- apps/common/drf/filters.py | 44 +- apps/common/serializers/fields.py | 30 +- apps/common/serializers/mixin.py | 28 +- apps/common/signal_handlers.py | 4 + apps/jumpserver/conf.py | 5 +- apps/jumpserver/rewriting/pagination.py | 7 + apps/jumpserver/settings/base.py | 4 + apps/jumpserver/settings/custom.py | 1 + apps/jumpserver/settings/libs.py | 1 + apps/jumpserver/urls.py | 1 + apps/labels/__init__.py | 0 apps/labels/admin.py | 3 + apps/labels/api.py | 141 +++ apps/labels/apps.py | 6 + apps/labels/const.py | 0 apps/labels/migrations/0001_initial.py | 54 ++ .../migrations/0002_auto_20231103_1659.py | 52 ++ ..._alter_labeledresource_options_and_more.py | 28 + apps/labels/migrations/__init__.py | 0 apps/labels/mixins.py | 13 + apps/labels/models.py | 38 + apps/labels/serializers.py | 54 ++ apps/labels/tests.py | 3 + apps/labels/urls.py | 21 + apps/labels/views.py | 3 + apps/locale/ja/LC_MESSAGES/django.mo | 4 +- apps/locale/ja/LC_MESSAGES/django.po | 803 ++++++++++-------- apps/locale/zh/LC_MESSAGES/django.mo | 4 +- apps/locale/zh/LC_MESSAGES/django.po | 792 +++++++++-------- apps/ops/models/job.py | 3 +- apps/ops/models/playbook.py | 3 +- apps/ops/serializers/job.py | 15 +- apps/ops/serializers/playbook.py | 7 +- apps/perms/models/asset_permission.py | 3 +- apps/perms/serializers/permission.py | 7 +- apps/rbac/api/__init__.py | 2 +- apps/rbac/api/content_type.py | 11 + apps/rbac/const.py | 2 +- apps/rbac/models/permission.py | 51 ++ apps/rbac/serializers/__init__.py | 2 +- apps/rbac/serializers/content_type.py | 13 + apps/rbac/tree.py | 2 + apps/rbac/urls/api_urls.py | 2 +- apps/users/api/user.py | 4 - apps/users/models/group.py | 4 +- apps/users/models/user.py | 3 +- apps/users/serializers/group.py | 10 +- apps/users/serializers/profile.py | 2 +- apps/users/serializers/user.py | 11 +- poetry.lock | 19 + pyproject.toml | 1 + 71 files changed, 1618 insertions(+), 978 deletions(-) delete mode 100644 apps/assets/api/label.py create mode 100644 apps/assets/migrations/0126_remove_asset_labels.py delete mode 100644 apps/assets/serializers/label.py create mode 100644 apps/labels/__init__.py create mode 100644 apps/labels/admin.py create mode 100644 apps/labels/api.py create mode 100644 apps/labels/apps.py create mode 100644 apps/labels/const.py create mode 100644 apps/labels/migrations/0001_initial.py create mode 100644 apps/labels/migrations/0002_auto_20231103_1659.py create mode 100644 apps/labels/migrations/0003_alter_labeledresource_options_and_more.py create mode 100644 apps/labels/migrations/__init__.py create mode 100644 apps/labels/mixins.py create mode 100644 apps/labels/models.py create mode 100644 apps/labels/serializers.py create mode 100644 apps/labels/tests.py create mode 100644 apps/labels/urls.py create mode 100644 apps/labels/views.py create mode 100644 apps/rbac/api/content_type.py create mode 100644 apps/rbac/serializers/content_type.py diff --git a/apps/accounts/models/account.py b/apps/accounts/models/account.py index 8a9187190..c3c052b1b 100644 --- a/apps/accounts/models/account.py +++ b/apps/accounts/models/account.py @@ -4,6 +4,7 @@ from simple_history.models import HistoricalRecords from assets.models.base import AbsConnectivity from common.utils import lazyproperty +from labels.mixins import LabeledMixin from .base import BaseAccount from .mixins import VaultModelMixin from ..const import Source @@ -42,7 +43,7 @@ class AccountHistoricalRecords(HistoricalRecords): return super().create_history_model(model, inherited) -class Account(AbsConnectivity, BaseAccount): +class Account(AbsConnectivity, LabeledMixin, BaseAccount): asset = models.ForeignKey( 'assets.Asset', related_name='accounts', on_delete=models.CASCADE, verbose_name=_('Asset') @@ -71,7 +72,7 @@ class Account(AbsConnectivity, BaseAccount): ] def __str__(self): - return '{}'.format(self.username) + return '{}({})'.format(self.name, self.asset.name) @lazyproperty def platform(self): diff --git a/apps/accounts/models/template.py b/apps/accounts/models/template.py index c56be1464..63ed1b20d 100644 --- a/apps/accounts/models/template.py +++ b/apps/accounts/models/template.py @@ -3,13 +3,14 @@ from django.db.models import Count, Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from labels.mixins import LabeledMixin from .account import Account from .base import BaseAccount, SecretWithRandomMixin __all__ = ['AccountTemplate', ] -class AccountTemplate(BaseAccount, SecretWithRandomMixin): +class AccountTemplate(LabeledMixin, BaseAccount, SecretWithRandomMixin): su_from = models.ForeignKey( 'self', related_name='su_to', null=True, on_delete=models.SET_NULL, verbose_name=_("Su from") diff --git a/apps/accounts/serializers/account/account.py b/apps/accounts/serializers/account/account.py index 8b8d9be36..c1098a6d1 100644 --- a/apps/accounts/serializers/account/account.py +++ b/apps/accounts/serializers/account/account.py @@ -66,6 +66,9 @@ class AccountCreateUpdateSerializerMixin(serializers.Serializer): name = initial_data.get('name') if name is not None: return + request = self.context.get('request') + if request and request.method == 'PATCH': + return if not name: name = initial_data.get('username') if self.instance and self.instance.name == name: @@ -238,7 +241,7 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize queryset = queryset.prefetch_related( 'asset', 'asset__platform', 'asset__platform__automation' - ) + ).prefetch_related('labels', 'labels__label') return queryset diff --git a/apps/accounts/serializers/account/base.py b/apps/accounts/serializers/account/base.py index 5289ea25b..23dec0d3e 100644 --- a/apps/accounts/serializers/account/base.py +++ b/apps/accounts/serializers/account/base.py @@ -5,6 +5,7 @@ from rest_framework import serializers from accounts.const import SecretType from accounts.models import BaseAccount from accounts.utils import validate_password_for_ansible, validate_ssh_key +from common.serializers import ResourceLabelsMixin from common.serializers.fields import EncryptedField, LabeledChoiceField from orgs.mixins.serializers import BulkOrgResourceModelSerializer @@ -60,8 +61,7 @@ class AuthValidateMixin(serializers.Serializer): return super().update(instance, validated_data) -class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer): - +class BaseAccountSerializer(AuthValidateMixin, ResourceLabelsMixin, BulkOrgResourceModelSerializer): class Meta: model = BaseAccount fields_mini = ['id', 'name', 'username'] @@ -70,7 +70,7 @@ class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer): 'privileged', 'is_active', 'spec_info', ] fields_other = ['created_by', 'date_created', 'date_updated', 'comment'] - fields = fields_small + fields_other + fields = fields_small + fields_other + ['labels'] read_only_fields = [ 'spec_info', 'date_verified', 'created_by', 'date_created', ] diff --git a/apps/assets/api/__init__.py b/apps/assets/api/__init__.py index 8ae0cd9bd..9eccd4977 100644 --- a/apps/assets/api/__init__.py +++ b/apps/assets/api/__init__.py @@ -2,7 +2,6 @@ from .asset import * from .category import * from .domain import * from .favorite_asset import * -from .label import * from .mixin import * from .node import * from .platform import * diff --git a/apps/assets/api/asset/asset.py b/apps/assets/api/asset/asset.py index 4b36b4bbe..729aa41bf 100644 --- a/apps/assets/api/asset/asset.py +++ b/apps/assets/api/asset/asset.py @@ -3,7 +3,6 @@ from collections import defaultdict import django_filters -from django.db.models import Q from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ from rest_framework import status @@ -14,7 +13,7 @@ from rest_framework.status import HTTP_200_OK from accounts.tasks import push_accounts_to_assets_task, verify_accounts_connectivity_task from assets import serializers from assets.exceptions import NotSupportedTemporarilyError -from assets.filters import IpInFilterBackend, LabelFilterBackend, NodeFilterBackend +from assets.filters import IpInFilterBackend, NodeFilterBackend from assets.models import Asset, Gateway, Platform, Protocol from assets.tasks import test_assets_connectivity_manual, update_assets_hardware_info_manual from common.api import SuggestionMixin @@ -33,7 +32,6 @@ __all__ = [ class AssetFilterSet(BaseFilterSet): - labels = django_filters.CharFilter(method='filter_labels') platform = django_filters.CharFilter(method='filter_platform') domain = django_filters.CharFilter(method='filter_domain') type = django_filters.CharFilter(field_name="platform__type", lookup_expr="exact") @@ -64,7 +62,7 @@ class AssetFilterSet(BaseFilterSet): class Meta: model = Asset fields = [ - "id", "name", "address", "is_active", "labels", + "id", "name", "address", "is_active", "type", "category", "platform", ] @@ -87,16 +85,6 @@ class AssetFilterSet(BaseFilterSet): value = value.split(',') return queryset.filter(protocols__name__in=value).distinct() - @staticmethod - def filter_labels(queryset, name, value): - if ':' in value: - n, v = value.split(':', 1) - queryset = queryset.filter(labels__name=n, labels__value=v) - else: - q = Q(labels__name__contains=value) | Q(labels__value__contains=value) - queryset = queryset.filter(q).distinct() - return queryset - class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet): """ @@ -121,7 +109,7 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet): ("sync_platform_protocols", "assets.change_asset"), ) extra_filter_backends = [ - LabelFilterBackend, IpInFilterBackend, + IpInFilterBackend, NodeFilterBackend, AttrRulesFilterBackend ] diff --git a/apps/assets/api/label.py b/apps/assets/api/label.py deleted file mode 100644 index d970d2180..000000000 --- a/apps/assets/api/label.py +++ /dev/null @@ -1,43 +0,0 @@ -# ~*~ coding: utf-8 ~*~ -# Copyright (C) 2014-2018 Beijing DuiZhan Technology Co.,Ltd. All Rights Reserved. -# -# Licensed under the GNU General Public License v2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.gnu.org/licenses/gpl-2.0.html -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from django.db.models import Count - -from common.utils import get_logger -from orgs.mixins.api import OrgBulkModelViewSet -from ..models import Label -from .. import serializers - - -logger = get_logger(__file__) -__all__ = ['LabelViewSet'] - - -class LabelViewSet(OrgBulkModelViewSet): - model = Label - filterset_fields = ("name", "value") - search_fields = filterset_fields - serializer_class = serializers.LabelSerializer - - def list(self, request, *args, **kwargs): - if request.query_params.get("distinct"): - self.serializer_class = serializers.LabelDistinctSerializer - self.queryset = self.queryset.values("name").distinct() - return super().list(request, *args, **kwargs) - - def get_queryset(self): - self.queryset = Label.objects.prefetch_related( - 'assets').annotate(asset_count=Count("assets")) - return self.queryset diff --git a/apps/assets/filters.py b/apps/assets/filters.py index 17e8414bf..f1fe6d666 100644 --- a/apps/assets/filters.py +++ b/apps/assets/filters.py @@ -5,7 +5,6 @@ from rest_framework import filters from rest_framework.compat import coreapi, coreschema from assets.utils import get_node_from_request, is_query_node_all_assets -from .models import Label class AssetByNodeFilterBackend(filters.BaseFilterBackend): @@ -72,57 +71,6 @@ class NodeFilterBackend(filters.BaseFilterBackend): return queryset.filter(nodes__key=node.key).distinct() -class LabelFilterBackend(filters.BaseFilterBackend): - sep = ':' - query_arg = 'label' - - def get_schema_fields(self, view): - example = self.sep.join(['os', 'linux']) - return [ - coreapi.Field( - name=self.query_arg, location='query', required=False, - type='string', example=example, description='' - ) - ] - - def get_query_labels(self, request): - labels_query = request.query_params.getlist(self.query_arg) - if not labels_query: - return None - - q = None - for kv in labels_query: - if '#' in kv: - self.sep = '#' - break - - for kv in labels_query: - if self.sep not in kv: - continue - key, value = kv.strip().split(self.sep)[:2] - if not all([key, value]): - continue - if q: - q |= Q(name=key, value=value) - else: - q = Q(name=key, value=value) - if not q: - return [] - labels = Label.objects.filter(q, is_active=True) \ - .values_list('id', flat=True) - return labels - - def filter_queryset(self, request, queryset, view): - labels = self.get_query_labels(request) - if labels is None: - return queryset - if len(labels) == 0: - return queryset.none() - for label in labels: - queryset = queryset.filter(labels=label) - return queryset - - class IpInFilterBackend(filters.BaseFilterBackend): def filter_queryset(self, request, queryset, view): ips = request.query_params.get('ips') diff --git a/apps/assets/migrations/0126_remove_asset_labels.py b/apps/assets/migrations/0126_remove_asset_labels.py new file mode 100644 index 000000000..44590dc4c --- /dev/null +++ b/apps/assets/migrations/0126_remove_asset_labels.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.10 on 2023-11-22 07:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0125_auto_20231011_1053'), + ('labels', '0002_auto_20231103_1659'), + ] + + operations = [ + migrations.RemoveField( + model_name='asset', + name='labels', + ), + ] diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index a22e552e9..93294fd5b 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -13,7 +13,9 @@ from django.utils.translation import gettext_lazy as _ from assets import const from common.db.fields import EncryptMixin from common.utils import lazyproperty +from labels.mixins import LabeledMixin from orgs.mixins.models import OrgManager, JMSOrgBaseModel +from rbac.models import ContentType from ..base import AbsConnectivity from ..platform import Platform @@ -150,7 +152,7 @@ class JSONFilterMixin: return None -class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel): +class Asset(NodesRelationMixin, LabeledMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel): Category = const.Category Type = const.AllTypes @@ -162,7 +164,6 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets', verbose_name=_("Nodes")) is_active = models.BooleanField(default=True, verbose_name=_('Is active')) - labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels")) gathered_info = models.JSONField(verbose_name=_('Gathered info'), default=dict, blank=True) # 资产的一些信息，如 硬件信息 custom_info = models.JSONField(verbose_name=_('Custom info'), default=dict) @@ -171,6 +172,13 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode def __str__(self): return '{0.name}({0.address})'.format(self) + def get_labels(self): + from labels.models import Label, LabeledResource + res_type = ContentType.objects.get_for_model(self.__class__) + label_ids = LabeledResource.objects.filter(res_type=res_type, res_id=self.id) \ + .values_list('label_id', flat=True) + return Label.objects.filter(id__in=label_ids) + @staticmethod def get_spec_values(instance, fields): info = {} diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index 587f41d7c..e424a2d46 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -6,6 +6,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from common.utils import get_logger +from labels.mixins import LabeledMixin from orgs.mixins.models import JMSOrgBaseModel from .gateway import Gateway @@ -14,7 +15,7 @@ logger = get_logger(__file__) __all__ = ['Domain'] -class Domain(JMSOrgBaseModel): +class Domain(LabeledMixin, JMSOrgBaseModel): name = models.CharField(max_length=128, verbose_name=_('Name')) class Meta: diff --git a/apps/assets/models/platform.py b/apps/assets/models/platform.py index 8fed01acf..74fcb9cfb 100644 --- a/apps/assets/models/platform.py +++ b/apps/assets/models/platform.py @@ -9,6 +9,7 @@ from common.db.models import JMSBaseModel __all__ = ['Platform', 'PlatformProtocol', 'PlatformAutomation'] from common.utils import lazyproperty +from labels.mixins import LabeledMixin class PlatformProtocol(models.Model): @@ -74,7 +75,7 @@ class PlatformAutomation(models.Model): platform = models.OneToOneField('Platform', on_delete=models.CASCADE, related_name='automation', null=True) -class Platform(JMSBaseModel): +class Platform(LabeledMixin, JMSBaseModel): """ 对资产提供 约束和默认值 对资产进行抽象 diff --git a/apps/assets/serializers/__init__.py b/apps/assets/serializers/__init__.py index cbf21454d..e071e24c0 100644 --- a/apps/assets/serializers/__init__.py +++ b/apps/assets/serializers/__init__.py @@ -2,11 +2,10 @@ # from .asset import * -from .label import * -from .node import * -from .gateway import * +from .automations import * +from .cagegory import * from .domain import * from .favorite_asset import * +from .gateway import * +from .node import * from .platform import * -from .cagegory import * -from .automations import * diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index 75d8c4c19..6c45944de 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -11,13 +11,14 @@ from accounts.serializers import AccountSerializer from common.const import UUID_PATTERN from common.serializers import ( WritableNestedModelSerializer, SecretReadableMixin, - CommonModelSerializer, MethodSerializer + CommonModelSerializer, MethodSerializer, ResourceLabelsMixin ) from common.serializers.common import DictSerializer from common.serializers.fields import LabeledChoiceField +from labels.models import Label from orgs.mixins.serializers import BulkOrgResourceModelSerializer from ...const import Category, AllTypes -from ...models import Asset, Node, Platform, Label, Protocol +from ...models import Asset, Node, Platform, Protocol __all__ = [ 'AssetSerializer', 'AssetSimpleSerializer', 'MiniAssetSerializer', @@ -117,10 +118,9 @@ class AccountSecretSerializer(SecretReadableMixin, CommonModelSerializer): } -class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSerializer): +class AssetSerializer(BulkOrgResourceModelSerializer, ResourceLabelsMixin, WritableNestedModelSerializer): category = LabeledChoiceField(choices=Category.choices, read_only=True, label=_('Category')) type = LabeledChoiceField(choices=AllTypes.choices(), read_only=True, label=_('Type')) - labels = AssetLabelSerializer(many=True, required=False, label=_('Label')) protocols = AssetProtocolsSerializer(many=True, required=False, label=_('Protocols'), default=()) accounts = AssetAccountSerializer(many=True, required=False, allow_null=True, write_only=True, label=_('Account')) nodes_display = serializers.ListField(read_only=False, required=False, label=_("Node path")) @@ -201,8 +201,9 @@ class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSeriali @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('domain', 'nodes', 'labels', 'protocols') \ + queryset = queryset.prefetch_related('domain', 'nodes', 'protocols', ) \ .prefetch_related('platform', 'platform__automation') \ + .prefetch_related('labels', 'labels__label') \ .annotate(category=F("platform__category")) \ .annotate(type=F("platform__type")) return queryset diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index a22d1a9ad..d2b3e3550 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -3,6 +3,7 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers +from common.serializers import ResourceLabelsMixin from common.serializers.fields import ObjectRelatedField from orgs.mixins.serializers import BulkOrgResourceModelSerializer from .gateway import GatewayWithAccountSecretSerializer @@ -11,7 +12,7 @@ from ..models import Domain, Asset __all__ = ['DomainSerializer', 'DomainWithGatewaySerializer'] -class DomainSerializer(BulkOrgResourceModelSerializer): +class DomainSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer): gateways = ObjectRelatedField( many=True, required=False, label=_('Gateway'), read_only=True, ) @@ -41,6 +42,12 @@ class DomainSerializer(BulkOrgResourceModelSerializer): instance = super().update(instance, validated_data) return instance + @classmethod + def setup_eager_loading(cls, queryset): + queryset = queryset \ + .prefetch_related('labels', 'labels__label') + return queryset + class DomainWithGatewaySerializer(serializers.ModelSerializer): gateways = GatewayWithAccountSecretSerializer(many=True, read_only=True) diff --git a/apps/assets/serializers/label.py b/apps/assets/serializers/label.py deleted file mode 100644 index 3d913aeea..000000000 --- a/apps/assets/serializers/label.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# -from django.db.models import Count -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from orgs.mixins.serializers import BulkOrgResourceModelSerializer -from ..models import Label - - -class LabelSerializer(BulkOrgResourceModelSerializer): - asset_count = serializers.ReadOnlyField(label=_("Assets amount")) - - class Meta: - model = Label - fields_mini = ['id', 'name'] - fields_small = fields_mini + [ - 'value', 'category', 'is_active', - 'date_created', 'comment', - ] - fields_m2m = ['asset_count', 'assets'] - fields = fields_small + fields_m2m - read_only_fields = ( - 'category', 'date_created', 'asset_count', - ) - extra_kwargs = { - 'assets': {'required': False, 'label': _('Asset')} - } - - @classmethod - def setup_eager_loading(cls, queryset): - queryset = queryset.prefetch_related('assets') \ - .annotate(asset_count=Count('assets')) - return queryset - - -class LabelDistinctSerializer(BulkOrgResourceModelSerializer): - value = serializers.SerializerMethodField() - - class Meta: - model = Label - fields = ("name", "value") - - @staticmethod - def get_value(obj): - labels = Label.objects.filter(name=obj["name"]) - return ', '.join([label.value for label in labels]) diff --git a/apps/assets/serializers/platform.py b/apps/assets/serializers/platform.py index 2a0634a43..f67df9906 100644 --- a/apps/assets/serializers/platform.py +++ b/apps/assets/serializers/platform.py @@ -5,7 +5,7 @@ from rest_framework.validators import UniqueValidator from common.serializers import ( WritableNestedModelSerializer, type_field_map, MethodSerializer, - DictSerializer, create_serializer_class + DictSerializer, create_serializer_class, ResourceLabelsMixin ) from common.serializers.fields import LabeledChoiceField from common.utils import lazyproperty @@ -123,7 +123,7 @@ class PlatformCustomField(serializers.Serializer): choices = serializers.ListField(default=list, label=_("Choices"), required=False) -class PlatformSerializer(WritableNestedModelSerializer): +class PlatformSerializer(ResourceLabelsMixin, WritableNestedModelSerializer): SU_METHOD_CHOICES = [ ("sudo", "sudo su -"), ("su", "su - "), @@ -160,6 +160,7 @@ class PlatformSerializer(WritableNestedModelSerializer): fields = fields_small + [ "protocols", "domain_enabled", "su_enabled", "su_method", "automation", "comment", "custom_fields", + "labels" ] + read_only_fields extra_kwargs = { "su_enabled": {"label": _('Su enabled')}, @@ -201,9 +202,8 @@ class PlatformSerializer(WritableNestedModelSerializer): @classmethod def setup_eager_loading(cls, queryset): - queryset = queryset.prefetch_related( - 'protocols', 'automation' - ) + queryset = queryset.prefetch_related('protocols', 'automation') \ + .prefetch_related('labels', 'labels__label') return queryset def validate_protocols(self, protocols): diff --git a/apps/assets/urls/api_urls.py b/apps/assets/urls/api_urls.py index 5644d7dc0..408550ece 100644 --- a/apps/assets/urls/api_urls.py +++ b/apps/assets/urls/api_urls.py @@ -17,7 +17,6 @@ router.register(r'clouds', api.CloudViewSet, 'cloud') router.register(r'gpts', api.GPTViewSet, 'gpt') router.register(r'customs', api.CustomViewSet, 'custom') router.register(r'platforms', api.AssetPlatformViewSet, 'platform') -router.register(r'labels', api.LabelViewSet, 'label') router.register(r'nodes', api.NodeViewSet, 'node') router.register(r'domains', api.DomainViewSet, 'domain') router.register(r'gateways', api.GatewayViewSet, 'gateway') diff --git a/apps/audits/utils.py b/apps/audits/utils.py index 44e858098..244df5d22 100644 --- a/apps/audits/utils.py +++ b/apps/audits/utils.py @@ -2,6 +2,7 @@ import copy from datetime import datetime from itertools import chain +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.db import models from common.db.fields import RelatedManager @@ -65,13 +66,19 @@ def _get_instance_field_value( continue data.setdefault(k, v) continue - data.setdefault(str(f.verbose_name), value) + elif isinstance(f, GenericRelation): + value = [str(v) for v in value.all()] + elif isinstance(f, GenericForeignKey): + continue + try: + data.setdefault(str(f.verbose_name), value) + except Exception as e: + print(f.__dict__) + raise e return data -def model_to_dict_for_operate_log( - instance, include_model_fields=True, include_related_fields=False -): +def model_to_dict_for_operate_log(instance, include_model_fields=True, include_related_fields=False): model_need_continue_fields = ['date_updated'] m2m_need_continue_fields = ['history_passwords'] diff --git a/apps/common/api/mixin.py b/apps/common/api/mixin.py index d93459104..c87d5f22a 100644 --- a/apps/common/api/mixin.py +++ b/apps/common/api/mixin.py @@ -11,7 +11,7 @@ from rest_framework.settings import api_settings from common.drf.filters import ( IDSpmFilterBackend, CustomFilterBackend, IDInFilterBackend, - IDNotFilterBackend, NotOrRelFilterBackend + IDNotFilterBackend, NotOrRelFilterBackend, LabelFilterBackend ) from common.utils import get_logger, lazyproperty from .action import RenderToJsonMixin @@ -111,7 +111,7 @@ class ExtraFilterFieldsMixin: """ default_added_filters = ( CustomFilterBackend, IDSpmFilterBackend, IDInFilterBackend, - IDNotFilterBackend, + IDNotFilterBackend, LabelFilterBackend ) filter_backends = api_settings.DEFAULT_FILTER_BACKENDS extra_filter_fields = [] diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index eb6878019..c58c289ab 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -21,7 +21,7 @@ __all__ = [ "DatetimeRangeFilterBackend", "IDSpmFilterBackend", 'IDInFilterBackend', "CustomFilterBackend", "BaseFilterSet", 'IDNotFilterBackend', - 'NotOrRelFilterBackend', + 'NotOrRelFilterBackend', 'LabelFilterBackend', ] @@ -168,6 +168,48 @@ class IDNotFilterBackend(filters.BaseFilterBackend): return queryset +class LabelFilterBackend(filters.BaseFilterBackend): + def get_schema_fields(self, view): + return [ + coreapi.Field( + name='label', location='query', required=False, + type='string', example='/api/v1/users/users?label=abc', + description='Filter by label' + ) + ] + + def filter_queryset(self, request, queryset, view): + label_id = request.query_params.get('label') + if not label_id: + return queryset + + if not hasattr(queryset, 'model'): + return queryset + + if not hasattr(queryset.model, 'labels'): + return queryset + + kwargs = {} + if ':' in label_id: + k, v = label_id.split(':', 1) + kwargs['label__name'] = k + if v != '*': + kwargs['label__value'] = v + else: + kwargs['label_id'] = label_id + + model = queryset.model + labeled_resource_cls = model.labels.field.related_model + app_label = model._meta.app_label + model_name = model._meta.model_name + + res_ids = labeled_resource_cls.objects.filter( + res_type__app_label=app_label, res_type__model=model_name, + ).filter(**kwargs).values_list('res_id', flat=True) + queryset = queryset.filter(id__in=set(res_ids)) + return queryset + + class CustomFilterBackend(filters.BaseFilterBackend): def get_schema_fields(self, view): diff --git a/apps/common/serializers/fields.py b/apps/common/serializers/fields.py index 5fe019c65..3054a5af4 100644 --- a/apps/common/serializers/fields.py +++ b/apps/common/serializers/fields.py @@ -20,7 +20,8 @@ __all__ = [ "TreeChoicesField", "LabeledMultipleChoiceField", "PhoneField", - "JSONManyToManyField" + "JSONManyToManyField", + "LabelRelatedField", ] @@ -99,6 +100,33 @@ class LabeledMultipleChoiceField(serializers.MultipleChoiceField): return data +class LabelRelatedField(serializers.RelatedField): + def __init__(self, **kwargs): + queryset = kwargs.pop("queryset", None) + if queryset is None: + from labels.models import LabeledResource + queryset = LabeledResource.objects.all() + + kwargs = {**kwargs} + read_only = kwargs.get("read_only", False) + if not read_only: + kwargs["queryset"] = queryset + super().__init__(**kwargs) + + def to_representation(self, value): + if value is None: + return value + return str(value.label) + + def to_internal_value(self, data): + from labels.models import LabeledResource, Label + if data is None: + return data + k, v = data.split(":", 1) + label, __ = Label.objects.get_or_create(name=k, value=v, defaults={'name': k, 'value': v}) + return LabeledResource(label=label) + + class ObjectRelatedField(serializers.RelatedField): default_error_messages = { "required": _("This field is required."), diff --git a/apps/common/serializers/mixin.py b/apps/common/serializers/mixin.py index d665d6346..1a749f6d3 100644 --- a/apps/common/serializers/mixin.py +++ b/apps/common/serializers/mixin.py @@ -7,6 +7,7 @@ else: from collections import Iterable from django.core.exceptions import ObjectDoesNotExist from django.db.models import NOT_PROVIDED +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.exceptions import ValidationError from rest_framework.fields import SkipField, empty @@ -14,14 +15,13 @@ from rest_framework.settings import api_settings from rest_framework.utils import html from common.db.fields import EncryptMixin -from common.serializers.fields import EncryptedField, LabeledChoiceField, ObjectRelatedField +from common.serializers.fields import EncryptedField, LabeledChoiceField, ObjectRelatedField, LabelRelatedField __all__ = [ 'BulkSerializerMixin', 'BulkListSerializerMixin', 'CommonSerializerMixin', 'CommonBulkSerializerMixin', 'SecretReadableMixin', 'CommonModelSerializer', - 'CommonBulkModelSerializer', - + 'CommonBulkModelSerializer', 'ResourceLabelsMixin', ] @@ -391,3 +391,25 @@ class CommonBulkSerializerMixin(BulkSerializerMixin, CommonSerializerMixin): class CommonBulkModelSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer): pass + + +class ResourceLabelsMixin(serializers.Serializer): + labels = LabelRelatedField(many=True, label=_('Labels'), ) + + def update(self, instance, validated_data): + labels = validated_data.pop('labels', None) + res = super().update(instance, validated_data) + if labels is not None: + instance.labels.set(labels, bulk=False) + return res + + def create(self, validated_data): + labels = validated_data.pop('labels', None) + instance = super().create(validated_data) + if labels is not None: + instance.labels.set(labels, bulk=False) + return instance + + @classmethod + def setup_eager_loading(cls, queryset): + return queryset.prefetch_related('labels') diff --git a/apps/common/signal_handlers.py b/apps/common/signal_handlers.py index df2019b60..588a63632 100644 --- a/apps/common/signal_handlers.py +++ b/apps/common/signal_handlers.py @@ -66,6 +66,10 @@ def digest_sql_query(): for table_name, queries in table_queries.items(): if table_name.startswith('rbac_') or table_name.startswith('auth_permission'): continue + + for query in queries: + sql = query['sql'] + print(" # {}: {}".format(query['time'], sql)) if len(queries) < 3: continue print("- Table: {}".format(table_name)) diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index ec224d7c8..45d21aeab 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -586,8 +586,9 @@ class Config(dict): # FTP 文件上传下载备份阈值，单位(M)，当值小于等于0时，不备份 'FTP_FILE_MAX_STORE': 100, - # API 请求次数限制 - 'MAX_LIMIT_PER_PAGE': 100, + # API 分页 + 'MAX_LIMIT_PER_PAGE': 10000, + 'DEFAULT_PAGE_SIZE': None, 'LIMIT_SUPER_PRIV': False, diff --git a/apps/jumpserver/rewriting/pagination.py b/apps/jumpserver/rewriting/pagination.py index cd38fd0ba..9a5fd263d 100644 --- a/apps/jumpserver/rewriting/pagination.py +++ b/apps/jumpserver/rewriting/pagination.py @@ -11,3 +11,10 @@ class MaxLimitOffsetPagination(LimitOffsetPagination): return queryset.values_list('id').order_by().count() except (AttributeError, TypeError, FieldError): return len(queryset) + + def paginate_queryset(self, queryset, request, view=None): + if view and hasattr(view, 'page_max_limit'): + self.max_limit = view.page_max_limit + if view and hasattr(view, 'page_default_limit'): + self.default_limit = view.page_default_limit + return super().paginate_queryset(queryset, request, view) diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index 4da1dfc7e..e2bfbc5c1 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -109,6 +109,7 @@ for host_port in ALLOWED_DOMAINS: continue CSRF_TRUSTED_ORIGINS.append('{}://*.{}'.format(schema, origin)) +CORS_ALLOWED_ORIGINS = [o.replace('*.', '') for o in CSRF_TRUSTED_ORIGINS] CSRF_FAILURE_VIEW = 'jumpserver.views.other.csrf_failure' # print("CSRF_TRUSTED_ORIGINS: ") # for origin in CSRF_TRUSTED_ORIGINS: @@ -134,6 +135,7 @@ INSTALLED_APPS = [ 'acls.apps.AclsConfig', 'notifications.apps.NotificationsConfig', 'rbac.apps.RBACConfig', + 'labels.apps.LabelsConfig', 'rest_framework', 'rest_framework_swagger', 'drf_yasg', @@ -142,6 +144,7 @@ INSTALLED_APPS = [ 'django_filters', 'bootstrap3', 'captcha', + 'corsheaders', 'private_storage', 'django_celery_beat', 'django.contrib.auth', @@ -160,6 +163,7 @@ MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.locale.LocaleMiddleware', + 'corsheaders.middleware.CorsMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index 3c0cfed58..fa83fa9fd 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -207,6 +207,7 @@ SESSION_RSA_PUBLIC_KEY_NAME = 'jms_public_key' OPERATE_LOG_ELASTICSEARCH_CONFIG = CONFIG.OPERATE_LOG_ELASTICSEARCH_CONFIG MAX_LIMIT_PER_PAGE = CONFIG.MAX_LIMIT_PER_PAGE +DEFAULT_PAGE_SIZE = CONFIG.DEFAULT_PAGE_SIZE # Magnus DB Port MAGNUS_ORACLE_PORTS = CONFIG.MAGNUS_ORACLE_PORTS diff --git a/apps/jumpserver/settings/libs.py b/apps/jumpserver/settings/libs.py index 817db8861..1f2c333ec 100644 --- a/apps/jumpserver/settings/libs.py +++ b/apps/jumpserver/settings/libs.py @@ -46,6 +46,7 @@ REST_FRAMEWORK = { 'DATETIME_FORMAT': '%Y/%m/%d %H:%M:%S %z', 'DATETIME_INPUT_FORMATS': ['%Y/%m/%d %H:%M:%S %z', 'iso-8601', '%Y-%m-%d %H:%M:%S %z'], 'DEFAULT_PAGINATION_CLASS': 'jumpserver.rewriting.pagination.MaxLimitOffsetPagination', + 'PAGE_SIZE': CONFIG.DEFAULT_PAGE_SIZE, 'EXCEPTION_HANDLER': 'common.drf.exc_handlers.common_exception_handler', } diff --git a/apps/jumpserver/urls.py b/apps/jumpserver/urls.py index 8fbea643c..881d5c57a 100644 --- a/apps/jumpserver/urls.py +++ b/apps/jumpserver/urls.py @@ -28,6 +28,7 @@ api_v1 = [ path('acls/', include('acls.urls.api_urls', namespace='api-acls')), path('notifications/', include('notifications.urls.api_urls', namespace='api-notifications')), path('rbac/', include('rbac.urls.api_urls', namespace='api-rbac')), + path('labels/', include('labels.urls', namespace='api-label')), path('prometheus/metrics/', api.PrometheusMetricsApi.as_view()), ] diff --git a/apps/labels/__init__.py b/apps/labels/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/labels/admin.py b/apps/labels/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/apps/labels/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/apps/labels/api.py b/apps/labels/api.py new file mode 100644 index 000000000..bd2685036 --- /dev/null +++ b/apps/labels/api.py @@ -0,0 +1,141 @@ +from django.shortcuts import get_object_or_404 +from rest_framework.decorators import action +from rest_framework.response import Response + +from common.api.generic import JMSModelViewSet +from common.utils import is_true +from orgs.mixins.api import OrgBulkModelViewSet +from orgs.mixins.models import OrgModelMixin +from orgs.utils import current_org +from rbac.models import ContentType +from rbac.serializers import ContentTypeSerializer +from . import serializers +from .models import Label, LabeledResource + +__all__ = ['LabelViewSet'] + + +class ContentTypeViewSet(JMSModelViewSet): + serializer_class = ContentTypeSerializer + http_method_names = ['get', 'head', 'options'] + rbac_perms = { + 'default': 'labels.view_contenttype', + 'resources': 'labels.view_contenttype', + } + page_default_limit = None + can_labeled_content_type = [] + model = ContentType + + @classmethod + def get_can_labeled_content_type_ids(cls): + if cls.can_labeled_content_type: + return cls.can_labeled_content_type + content_types = ContentType.objects.all() + for ct in content_types: + model_cls = ct.model_class() + if not model_cls: + continue + if model_cls._meta.parents: + continue + if 'labels' in model_cls._meta._forward_fields_map.keys(): + # if issubclass(model_cls, LabeledMixin): + cls.can_labeled_content_type.append(ct.id) + return cls.can_labeled_content_type + + def get_queryset(self): + ids = self.get_can_labeled_content_type_ids() + queryset = ContentType.objects.filter(id__in=ids) + return queryset + + @action(methods=['GET'], detail=True, serializer_class=serializers.ContentTypeResourceSerializer) + def resources(self, request, *args, **kwargs): + self.page_default_limit = 100 + content_type = self.get_object() + model = content_type.model_class() + + if issubclass(model, OrgModelMixin): + queryset = model.objects.filter(org_id=current_org.id) + else: + queryset = model.objects.all() + + keyword = request.query_params.get('search') + if keyword: + queryset = content_type.filter_queryset(queryset, keyword) + return self.get_paginated_response_from_queryset(queryset) + + +class LabelContentTypeResourceViewSet(JMSModelViewSet): + serializer_class = serializers.ContentTypeResourceSerializer + rbac_perms = { + 'default': 'labels.view_labeledresource', + 'update': 'labels.change_labeledresource', + } + ordering_fields = ('res_type', 'date_created') + + def get_queryset(self): + label_pk = self.kwargs.get('label') + res_type = self.kwargs.get('res_type') + label = get_object_or_404(Label, pk=label_pk) + content_type = get_object_or_404(ContentType, id=res_type) + bound = self.request.query_params.get('bound', '1') + res_ids = LabeledResource.objects.filter(res_type=content_type, label=label) \ + .values_list('res_id', flat=True) + res_ids = set(res_ids) + model = content_type.model_class() + if is_true(bound): + queryset = model.objects.filter(id__in=list(res_ids)) + else: + queryset = model.objects.exclude(id__in=list(res_ids)) + keyword = self.request.query_params.get('search') + if keyword: + queryset = content_type.filter_queryset(queryset, keyword) + return queryset + + def put(self, request, *args, **kwargs): + label_pk = self.kwargs.get('label') + res_type = self.kwargs.get('res_type') + content_type = get_object_or_404(ContentType, id=res_type) + label = get_object_or_404(Label, pk=label_pk) + res_ids = request.data.get('res_ids', []) + + LabeledResource.objects \ + .filter(res_type=content_type, label=label) \ + .exclude(res_id__in=res_ids).delete() + resources = [] + for res_id in res_ids: + resources.append(LabeledResource(res_type=content_type, res_id=res_id, label=label, org_id=current_org.id)) + LabeledResource.objects.bulk_create(resources, ignore_conflicts=True) + return Response({"total": len(res_ids)}) + + +class LabelViewSet(OrgBulkModelViewSet): + model = Label + filterset_fields = ("name", "value") + search_fields = filterset_fields + serializer_classes = { + 'default': serializers.LabelSerializer, + 'resource_types': ContentTypeSerializer, + } + rbac_perms = { + 'resource_types': 'labels.view_label', + 'keys': 'labels.view_label', + } + + @action(methods=['GET'], detail=False) + def keys(self, request, *args, **kwargs): + queryset = Label.objects.all() + keyword = request.query_params.get('search') + if keyword: + queryset = queryset.filter(name__icontains=keyword) + keys = queryset.values_list('name', flat=True).distinct() + return Response(keys) + + +class LabeledResourceViewSet(OrgBulkModelViewSet): + model = LabeledResource + filterset_fields = ("label__name", "label__value", "res_type", "res_id", "label") + search_fields = filterset_fields + serializer_classes = { + 'default': serializers.LabeledResourceSerializer, + } + ordering_fields = ('res_type', 'date_created') diff --git a/apps/labels/apps.py b/apps/labels/apps.py new file mode 100644 index 000000000..434a5c6df --- /dev/null +++ b/apps/labels/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class LabelsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'labels' diff --git a/apps/labels/const.py b/apps/labels/const.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/labels/migrations/0001_initial.py b/apps/labels/migrations/0001_initial.py new file mode 100644 index 000000000..cefe74e5c --- /dev/null +++ b/apps/labels/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 4.1.10 on 2023-11-06 10:38 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.CreateModel( + name='Label', + fields=[ + ('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('comment', models.TextField(blank=True, default='', verbose_name='Comment')), + ('internal', models.BooleanField(default=False, verbose_name='Internal')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('name', models.CharField(db_index=True, max_length=64, verbose_name='Name')), + ('value', models.CharField(max_length=64, verbose_name='Value')), + ], + options={ + 'verbose_name': 'Label', + 'unique_together': {('name', 'value', 'org_id')}, + }, + ), + migrations.CreateModel( + name='LabeledResource', + fields=[ + ('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('comment', models.TextField(blank=True, default='', verbose_name='Comment')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('res_id', models.CharField(db_index=True, max_length=36, verbose_name='Resource ID')), + ('label', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='labels.label')), + ('res_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/apps/labels/migrations/0002_auto_20231103_1659.py b/apps/labels/migrations/0002_auto_20231103_1659.py new file mode 100644 index 000000000..49386ed39 --- /dev/null +++ b/apps/labels/migrations/0002_auto_20231103_1659.py @@ -0,0 +1,52 @@ +# Generated by Django 4.1.10 on 2023-11-03 08:59 + +from django.db import migrations + + +def migrate_assets_labels(apps, schema_editor): + old_label_model = apps.get_model('assets', 'Label') + new_label_model = apps.get_model('labels', 'Label') + asset_model = apps.get_model('assets', 'Asset') + labeled_item_model = apps.get_model('labels', 'LabeledResource') + + old_labels = old_label_model.objects.all() + new_labels = [] + old_new_label_map = {} + for label in old_labels: + new_label = new_label_model(name=label.name, value=label.value, org_id=label.org_id) + old_new_label_map[label.id] = new_label + new_labels.append(new_label) + new_label_model.objects.bulk_create(new_labels, ignore_conflicts=True) + + label_relations = asset_model.labels.through.objects.all() + bulk_size = 1000 + count = 0 + content_type = apps.get_model('contenttypes', 'contenttype').objects.get_for_model(asset_model) + + while True: + relations = label_relations[count:count + bulk_size] + if not relations: + break + count += bulk_size + + tagged_items = [] + for relation in relations: + new_label = old_new_label_map[relation.label_id] + tagged_item = labeled_item_model( + label_id=new_label.id, res_type=content_type, + res_id=relation.asset_id, org_id=new_label.org_id + ) + tagged_items.append(tagged_item) + labeled_item_model.objects.bulk_create(tagged_items, ignore_conflicts=True) + + +class Migration(migrations.Migration): + + dependencies = [ + ('labels', '0001_initial'), + ('assets', '0125_auto_20231011_1053') + ] + + operations = [ + migrations.RunPython(migrate_assets_labels), + ] diff --git a/apps/labels/migrations/0003_alter_labeledresource_options_and_more.py b/apps/labels/migrations/0003_alter_labeledresource_options_and_more.py new file mode 100644 index 000000000..852376180 --- /dev/null +++ b/apps/labels/migrations/0003_alter_labeledresource_options_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.1.10 on 2023-11-15 10:58 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('labels', '0002_auto_20231103_1659'), + ] + + operations = [ + migrations.AlterModelOptions( + name='labeledresource', + options={'verbose_name': 'Labeled resource'}, + ), + migrations.AlterField( + model_name='labeledresource', + name='label', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='labeled_resources', to='labels.label'), + ), + migrations.AlterUniqueTogether( + name='labeledresource', + unique_together={('label', 'res_type', 'res_id', 'org_id')}, + ), + ] diff --git a/apps/labels/migrations/__init__.py b/apps/labels/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/apps/labels/mixins.py b/apps/labels/mixins.py new file mode 100644 index 000000000..23fa90386 --- /dev/null +++ b/apps/labels/mixins.py @@ -0,0 +1,13 @@ +from django.contrib.contenttypes.fields import GenericRelation +from django.db import models + +from .models import LabeledResource + +__all__ = ['LabeledMixin'] + + +class LabeledMixin(models.Model): + labels = GenericRelation(LabeledResource, object_id_field='res_id', content_type_field='res_type') + + class Meta: + abstract = True diff --git a/apps/labels/models.py b/apps/labels/models.py new file mode 100644 index 000000000..3e38da48e --- /dev/null +++ b/apps/labels/models.py @@ -0,0 +1,38 @@ +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from common.utils import lazyproperty +from orgs.mixins.models import JMSOrgBaseModel + + +class Label(JMSOrgBaseModel): + name = models.CharField(max_length=64, verbose_name=_("Name"), db_index=True) + value = models.CharField(max_length=64, unique=False, verbose_name=_("Value")) + internal = models.BooleanField(default=False, verbose_name=_("Internal")) + + class Meta: + unique_together = [('name', 'value', 'org_id')] + verbose_name = _('Label') + + @lazyproperty + def res_count(self): + return self.labeled_resources.count() + + def __str__(self): + return '{}:{}'.format(self.name, self.value) + + +class LabeledResource(JMSOrgBaseModel): + label = models.ForeignKey(Label, on_delete=models.CASCADE, related_name='labeled_resources') + res_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + res_id = models.CharField(max_length=36, verbose_name=_("Resource ID"), db_index=True) + resource = GenericForeignKey('res_type', 'res_id') + + class Meta: + unique_together = [('label', 'res_type', 'res_id', 'org_id')] + verbose_name = _('Labeled resource') + + def __str__(self): + return '{} => {}'.format(self.label, self.resource) diff --git a/apps/labels/serializers.py b/apps/labels/serializers.py new file mode 100644 index 000000000..476b3c06f --- /dev/null +++ b/apps/labels/serializers.py @@ -0,0 +1,54 @@ +from django.contrib.contenttypes.models import ContentType +from django.db.models import Count +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from common.serializers.fields import ObjectRelatedField +from orgs.mixins.serializers import BulkOrgResourceModelSerializer +from .models import Label, LabeledResource + +__all__ = ['LabelSerializer', 'LabeledResourceSerializer', 'ContentTypeResourceSerializer'] + + +class LabelSerializer(BulkOrgResourceModelSerializer): + class Meta: + model = Label + fields = ['id', 'name', 'value', 'res_count', 'date_created', 'date_updated'] + read_only_fields = ('date_created', 'date_updated', 'res_count') + extra_kwargs = { + 'res_count': {'label': _('Resource count')}, + } + + @classmethod + def setup_eager_loading(cls, queryset): + """ Perform necessary eager loading of data. """ + queryset = queryset.annotate(res_count=Count('labeled_resources')) + return queryset + + +class LabeledResourceSerializer(serializers.ModelSerializer): + res_type = ObjectRelatedField( + queryset=ContentType.objects, attrs=('app_label', 'model', 'name'), label=_("Resource type") + ) + label = ObjectRelatedField(queryset=Label.objects, attrs=('name', 'value')) + resource = serializers.CharField(label=_("Resource")) + + class Meta: + model = LabeledResource + fields = ('id', 'label', 'res_type', 'res_id', 'date_created', 'resource', 'date_updated') + read_only_fields = ('date_created', 'date_updated', 'resource') + + @classmethod + def setup_eager_loading(cls, queryset): + """ Perform necessary eager loading of data. """ + queryset = queryset.select_related('label', 'res_type') + return queryset + + +class ContentTypeResourceSerializer(serializers.Serializer): + id = serializers.CharField() + name = serializers.SerializerMethodField() + + @staticmethod + def get_name(obj): + return str(obj) diff --git a/apps/labels/tests.py b/apps/labels/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/apps/labels/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/apps/labels/urls.py b/apps/labels/urls.py new file mode 100644 index 000000000..a3971917f --- /dev/null +++ b/apps/labels/urls.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# + +from rest_framework_bulk.routes import BulkRouter + +from . import api + +app_name = 'labels' + +router = BulkRouter() +router.register(r'labels', api.LabelViewSet, 'label') +router.register(r'labels/(?P