perf: 支持全局的 labels (#12043)

* perf: 支持全局的 labels

* perf: stash

* stash

* stash

* stash

* stash

* perf: 优化 labels

* stash

* perf: add debug sql

* perf: 修改 labels

* perf: 优化提交

* perf: 优化提交 labels

* perf: 基本完成

* perf: 完成 labels 搜索

* perf: 优化 labels

* perf: 去掉不用 debug

---------

Co-authored-by: ibuler <ibuler@qq.com>
pull/12253/head
fit2bot 12 months ago committed by GitHub
parent a91cb1afd5
commit 8291a81efd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -4,6 +4,7 @@ from simple_history.models import HistoricalRecords
from assets.models.base import AbsConnectivity
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
from .base import BaseAccount
from .mixins import VaultModelMixin
from ..const import Source
@ -42,7 +43,7 @@ class AccountHistoricalRecords(HistoricalRecords):
return super().create_history_model(model, inherited)
class Account(AbsConnectivity, BaseAccount):
class Account(AbsConnectivity, LabeledMixin, BaseAccount):
asset = models.ForeignKey(
'assets.Asset', related_name='accounts',
on_delete=models.CASCADE, verbose_name=_('Asset')
@ -71,7 +72,7 @@ class Account(AbsConnectivity, BaseAccount):
]
def __str__(self):
return '{}'.format(self.username)
return '{}({})'.format(self.name, self.asset.name)
@lazyproperty
def platform(self):

@ -3,13 +3,14 @@ from django.db.models import Count, Q
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from labels.mixins import LabeledMixin
from .account import Account
from .base import BaseAccount, SecretWithRandomMixin
__all__ = ['AccountTemplate', ]
class AccountTemplate(BaseAccount, SecretWithRandomMixin):
class AccountTemplate(LabeledMixin, BaseAccount, SecretWithRandomMixin):
su_from = models.ForeignKey(
'self', related_name='su_to', null=True,
on_delete=models.SET_NULL, verbose_name=_("Su from")

@ -66,6 +66,9 @@ class AccountCreateUpdateSerializerMixin(serializers.Serializer):
name = initial_data.get('name')
if name is not None:
return
request = self.context.get('request')
if request and request.method == 'PATCH':
return
if not name:
name = initial_data.get('username')
if self.instance and self.instance.name == name:
@ -238,7 +241,7 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize
queryset = queryset.prefetch_related(
'asset', 'asset__platform',
'asset__platform__automation'
)
).prefetch_related('labels', 'labels__label')
return queryset

@ -5,6 +5,7 @@ from rest_framework import serializers
from accounts.const import SecretType
from accounts.models import BaseAccount
from accounts.utils import validate_password_for_ansible, validate_ssh_key
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import EncryptedField, LabeledChoiceField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
@ -60,8 +61,7 @@ class AuthValidateMixin(serializers.Serializer):
return super().update(instance, validated_data)
class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
class BaseAccountSerializer(AuthValidateMixin, ResourceLabelsMixin, BulkOrgResourceModelSerializer):
class Meta:
model = BaseAccount
fields_mini = ['id', 'name', 'username']
@ -70,7 +70,7 @@ class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
'privileged', 'is_active', 'spec_info',
]
fields_other = ['created_by', 'date_created', 'date_updated', 'comment']
fields = fields_small + fields_other
fields = fields_small + fields_other + ['labels']
read_only_fields = [
'spec_info', 'date_verified', 'created_by', 'date_created',
]

@ -2,7 +2,6 @@ from .asset import *
from .category import *
from .domain import *
from .favorite_asset import *
from .label import *
from .mixin import *
from .node import *
from .platform import *

@ -3,7 +3,6 @@
from collections import defaultdict
import django_filters
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.utils.translation import gettext as _
from rest_framework import status
@ -14,7 +13,7 @@ from rest_framework.status import HTTP_200_OK
from accounts.tasks import push_accounts_to_assets_task, verify_accounts_connectivity_task
from assets import serializers
from assets.exceptions import NotSupportedTemporarilyError
from assets.filters import IpInFilterBackend, LabelFilterBackend, NodeFilterBackend
from assets.filters import IpInFilterBackend, NodeFilterBackend
from assets.models import Asset, Gateway, Platform, Protocol
from assets.tasks import test_assets_connectivity_manual, update_assets_hardware_info_manual
from common.api import SuggestionMixin
@ -33,7 +32,6 @@ __all__ = [
class AssetFilterSet(BaseFilterSet):
labels = django_filters.CharFilter(method='filter_labels')
platform = django_filters.CharFilter(method='filter_platform')
domain = django_filters.CharFilter(method='filter_domain')
type = django_filters.CharFilter(field_name="platform__type", lookup_expr="exact")
@ -64,7 +62,7 @@ class AssetFilterSet(BaseFilterSet):
class Meta:
model = Asset
fields = [
"id", "name", "address", "is_active", "labels",
"id", "name", "address", "is_active",
"type", "category", "platform",
]
@ -87,16 +85,6 @@ class AssetFilterSet(BaseFilterSet):
value = value.split(',')
return queryset.filter(protocols__name__in=value).distinct()
@staticmethod
def filter_labels(queryset, name, value):
if ':' in value:
n, v = value.split(':', 1)
queryset = queryset.filter(labels__name=n, labels__value=v)
else:
q = Q(labels__name__contains=value) | Q(labels__value__contains=value)
queryset = queryset.filter(q).distinct()
return queryset
class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet):
"""
@ -121,7 +109,7 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet):
("sync_platform_protocols", "assets.change_asset"),
)
extra_filter_backends = [
LabelFilterBackend, IpInFilterBackend,
IpInFilterBackend,
NodeFilterBackend, AttrRulesFilterBackend
]

@ -1,43 +0,0 @@
# ~*~ coding: utf-8 ~*~
# Copyright (C) 2014-2018 Beijing DuiZhan Technology Co.,Ltd. All Rights Reserved.
#
# Licensed under the GNU General Public License v2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.gnu.org/licenses/gpl-2.0.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from django.db.models import Count
from common.utils import get_logger
from orgs.mixins.api import OrgBulkModelViewSet
from ..models import Label
from .. import serializers
logger = get_logger(__file__)
__all__ = ['LabelViewSet']
class LabelViewSet(OrgBulkModelViewSet):
model = Label
filterset_fields = ("name", "value")
search_fields = filterset_fields
serializer_class = serializers.LabelSerializer
def list(self, request, *args, **kwargs):
if request.query_params.get("distinct"):
self.serializer_class = serializers.LabelDistinctSerializer
self.queryset = self.queryset.values("name").distinct()
return super().list(request, *args, **kwargs)
def get_queryset(self):
self.queryset = Label.objects.prefetch_related(
'assets').annotate(asset_count=Count("assets"))
return self.queryset

@ -5,7 +5,6 @@ from rest_framework import filters
from rest_framework.compat import coreapi, coreschema
from assets.utils import get_node_from_request, is_query_node_all_assets
from .models import Label
class AssetByNodeFilterBackend(filters.BaseFilterBackend):
@ -72,57 +71,6 @@ class NodeFilterBackend(filters.BaseFilterBackend):
return queryset.filter(nodes__key=node.key).distinct()
class LabelFilterBackend(filters.BaseFilterBackend):
sep = ':'
query_arg = 'label'
def get_schema_fields(self, view):
example = self.sep.join(['os', 'linux'])
return [
coreapi.Field(
name=self.query_arg, location='query', required=False,
type='string', example=example, description=''
)
]
def get_query_labels(self, request):
labels_query = request.query_params.getlist(self.query_arg)
if not labels_query:
return None
q = None
for kv in labels_query:
if '#' in kv:
self.sep = '#'
break
for kv in labels_query:
if self.sep not in kv:
continue
key, value = kv.strip().split(self.sep)[:2]
if not all([key, value]):
continue
if q:
q |= Q(name=key, value=value)
else:
q = Q(name=key, value=value)
if not q:
return []
labels = Label.objects.filter(q, is_active=True) \
.values_list('id', flat=True)
return labels
def filter_queryset(self, request, queryset, view):
labels = self.get_query_labels(request)
if labels is None:
return queryset
if len(labels) == 0:
return queryset.none()
for label in labels:
queryset = queryset.filter(labels=label)
return queryset
class IpInFilterBackend(filters.BaseFilterBackend):
def filter_queryset(self, request, queryset, view):
ips = request.query_params.get('ips')

@ -0,0 +1,18 @@
# Generated by Django 4.1.10 on 2023-11-22 07:33
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('assets', '0125_auto_20231011_1053'),
('labels', '0002_auto_20231103_1659'),
]
operations = [
migrations.RemoveField(
model_name='asset',
name='labels',
),
]

@ -13,7 +13,9 @@ from django.utils.translation import gettext_lazy as _
from assets import const
from common.db.fields import EncryptMixin
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
from orgs.mixins.models import OrgManager, JMSOrgBaseModel
from rbac.models import ContentType
from ..base import AbsConnectivity
from ..platform import Platform
@ -150,7 +152,7 @@ class JSONFilterMixin:
return None
class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel):
class Asset(NodesRelationMixin, LabeledMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel):
Category = const.Category
Type = const.AllTypes
@ -162,7 +164,6 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode
nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets',
verbose_name=_("Nodes"))
is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels"))
gathered_info = models.JSONField(verbose_name=_('Gathered info'), default=dict, blank=True) # 资产的一些信息,如 硬件信息
custom_info = models.JSONField(verbose_name=_('Custom info'), default=dict)
@ -171,6 +172,13 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode
def __str__(self):
return '{0.name}({0.address})'.format(self)
def get_labels(self):
from labels.models import Label, LabeledResource
res_type = ContentType.objects.get_for_model(self.__class__)
label_ids = LabeledResource.objects.filter(res_type=res_type, res_id=self.id) \
.values_list('label_id', flat=True)
return Label.objects.filter(id__in=label_ids)
@staticmethod
def get_spec_values(instance, fields):
info = {}

@ -6,6 +6,7 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from common.utils import get_logger
from labels.mixins import LabeledMixin
from orgs.mixins.models import JMSOrgBaseModel
from .gateway import Gateway
@ -14,7 +15,7 @@ logger = get_logger(__file__)
__all__ = ['Domain']
class Domain(JMSOrgBaseModel):
class Domain(LabeledMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'))
class Meta:

@ -9,6 +9,7 @@ from common.db.models import JMSBaseModel
__all__ = ['Platform', 'PlatformProtocol', 'PlatformAutomation']
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
class PlatformProtocol(models.Model):
@ -74,7 +75,7 @@ class PlatformAutomation(models.Model):
platform = models.OneToOneField('Platform', on_delete=models.CASCADE, related_name='automation', null=True)
class Platform(JMSBaseModel):
class Platform(LabeledMixin, JMSBaseModel):
"""
对资产提供 约束和默认值
对资产进行抽象

@ -2,11 +2,10 @@
#
from .asset import *
from .label import *
from .node import *
from .gateway import *
from .automations import *
from .cagegory import *
from .domain import *
from .favorite_asset import *
from .gateway import *
from .node import *
from .platform import *
from .cagegory import *
from .automations import *

@ -11,13 +11,14 @@ from accounts.serializers import AccountSerializer
from common.const import UUID_PATTERN
from common.serializers import (
WritableNestedModelSerializer, SecretReadableMixin,
CommonModelSerializer, MethodSerializer
CommonModelSerializer, MethodSerializer, ResourceLabelsMixin
)
from common.serializers.common import DictSerializer
from common.serializers.fields import LabeledChoiceField
from labels.models import Label
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from ...const import Category, AllTypes
from ...models import Asset, Node, Platform, Label, Protocol
from ...models import Asset, Node, Platform, Protocol
__all__ = [
'AssetSerializer', 'AssetSimpleSerializer', 'MiniAssetSerializer',
@ -117,10 +118,9 @@ class AccountSecretSerializer(SecretReadableMixin, CommonModelSerializer):
}
class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSerializer):
class AssetSerializer(BulkOrgResourceModelSerializer, ResourceLabelsMixin, WritableNestedModelSerializer):
category = LabeledChoiceField(choices=Category.choices, read_only=True, label=_('Category'))
type = LabeledChoiceField(choices=AllTypes.choices(), read_only=True, label=_('Type'))
labels = AssetLabelSerializer(many=True, required=False, label=_('Label'))
protocols = AssetProtocolsSerializer(many=True, required=False, label=_('Protocols'), default=())
accounts = AssetAccountSerializer(many=True, required=False, allow_null=True, write_only=True, label=_('Account'))
nodes_display = serializers.ListField(read_only=False, required=False, label=_("Node path"))
@ -201,8 +201,9 @@ class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSeriali
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.prefetch_related('domain', 'nodes', 'labels', 'protocols') \
queryset = queryset.prefetch_related('domain', 'nodes', 'protocols', ) \
.prefetch_related('platform', 'platform__automation') \
.prefetch_related('labels', 'labels__label') \
.annotate(category=F("platform__category")) \
.annotate(type=F("platform__type"))
return queryset

@ -3,6 +3,7 @@
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from .gateway import GatewayWithAccountSecretSerializer
@ -11,7 +12,7 @@ from ..models import Domain, Asset
__all__ = ['DomainSerializer', 'DomainWithGatewaySerializer']
class DomainSerializer(BulkOrgResourceModelSerializer):
class DomainSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
gateways = ObjectRelatedField(
many=True, required=False, label=_('Gateway'), read_only=True,
)
@ -41,6 +42,12 @@ class DomainSerializer(BulkOrgResourceModelSerializer):
instance = super().update(instance, validated_data)
return instance
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset \
.prefetch_related('labels', 'labels__label')
return queryset
class DomainWithGatewaySerializer(serializers.ModelSerializer):
gateways = GatewayWithAccountSecretSerializer(many=True, read_only=True)

@ -1,47 +0,0 @@
# -*- coding: utf-8 -*-
#
from django.db.models import Count
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from ..models import Label
class LabelSerializer(BulkOrgResourceModelSerializer):
asset_count = serializers.ReadOnlyField(label=_("Assets amount"))
class Meta:
model = Label
fields_mini = ['id', 'name']
fields_small = fields_mini + [
'value', 'category', 'is_active',
'date_created', 'comment',
]
fields_m2m = ['asset_count', 'assets']
fields = fields_small + fields_m2m
read_only_fields = (
'category', 'date_created', 'asset_count',
)
extra_kwargs = {
'assets': {'required': False, 'label': _('Asset')}
}
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset.prefetch_related('assets') \
.annotate(asset_count=Count('assets'))
return queryset
class LabelDistinctSerializer(BulkOrgResourceModelSerializer):
value = serializers.SerializerMethodField()
class Meta:
model = Label
fields = ("name", "value")
@staticmethod
def get_value(obj):
labels = Label.objects.filter(name=obj["name"])
return ', '.join([label.value for label in labels])

@ -5,7 +5,7 @@ from rest_framework.validators import UniqueValidator
from common.serializers import (
WritableNestedModelSerializer, type_field_map, MethodSerializer,
DictSerializer, create_serializer_class
DictSerializer, create_serializer_class, ResourceLabelsMixin
)
from common.serializers.fields import LabeledChoiceField
from common.utils import lazyproperty
@ -123,7 +123,7 @@ class PlatformCustomField(serializers.Serializer):
choices = serializers.ListField(default=list, label=_("Choices"), required=False)
class PlatformSerializer(WritableNestedModelSerializer):
class PlatformSerializer(ResourceLabelsMixin, WritableNestedModelSerializer):
SU_METHOD_CHOICES = [
("sudo", "sudo su -"),
("su", "su - "),
@ -160,6 +160,7 @@ class PlatformSerializer(WritableNestedModelSerializer):
fields = fields_small + [
"protocols", "domain_enabled", "su_enabled",
"su_method", "automation", "comment", "custom_fields",
"labels"
] + read_only_fields
extra_kwargs = {
"su_enabled": {"label": _('Su enabled')},
@ -201,9 +202,8 @@ class PlatformSerializer(WritableNestedModelSerializer):
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset.prefetch_related(
'protocols', 'automation'
)
queryset = queryset.prefetch_related('protocols', 'automation') \
.prefetch_related('labels', 'labels__label')
return queryset
def validate_protocols(self, protocols):

@ -17,7 +17,6 @@ router.register(r'clouds', api.CloudViewSet, 'cloud')
router.register(r'gpts', api.GPTViewSet, 'gpt')
router.register(r'customs', api.CustomViewSet, 'custom')
router.register(r'platforms', api.AssetPlatformViewSet, 'platform')
router.register(r'labels', api.LabelViewSet, 'label')
router.register(r'nodes', api.NodeViewSet, 'node')
router.register(r'domains', api.DomainViewSet, 'domain')
router.register(r'gateways', api.GatewayViewSet, 'gateway')

@ -2,6 +2,7 @@ import copy
from datetime import datetime
from itertools import chain
from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
from django.db import models
from common.db.fields import RelatedManager
@ -65,13 +66,19 @@ def _get_instance_field_value(
continue
data.setdefault(k, v)
continue
data.setdefault(str(f.verbose_name), value)
elif isinstance(f, GenericRelation):
value = [str(v) for v in value.all()]
elif isinstance(f, GenericForeignKey):
continue
try:
data.setdefault(str(f.verbose_name), value)
except Exception as e:
print(f.__dict__)
raise e
return data
def model_to_dict_for_operate_log(
instance, include_model_fields=True, include_related_fields=False
):
def model_to_dict_for_operate_log(instance, include_model_fields=True, include_related_fields=False):
model_need_continue_fields = ['date_updated']
m2m_need_continue_fields = ['history_passwords']

@ -11,7 +11,7 @@ from rest_framework.settings import api_settings
from common.drf.filters import (
IDSpmFilterBackend, CustomFilterBackend, IDInFilterBackend,
IDNotFilterBackend, NotOrRelFilterBackend
IDNotFilterBackend, NotOrRelFilterBackend, LabelFilterBackend
)
from common.utils import get_logger, lazyproperty
from .action import RenderToJsonMixin
@ -111,7 +111,7 @@ class ExtraFilterFieldsMixin:
"""
default_added_filters = (
CustomFilterBackend, IDSpmFilterBackend, IDInFilterBackend,
IDNotFilterBackend,
IDNotFilterBackend, LabelFilterBackend
)
filter_backends = api_settings.DEFAULT_FILTER_BACKENDS
extra_filter_fields = []

@ -21,7 +21,7 @@ __all__ = [
"DatetimeRangeFilterBackend", "IDSpmFilterBackend",
'IDInFilterBackend', "CustomFilterBackend",
"BaseFilterSet", 'IDNotFilterBackend',
'NotOrRelFilterBackend',
'NotOrRelFilterBackend', 'LabelFilterBackend',
]
@ -168,6 +168,48 @@ class IDNotFilterBackend(filters.BaseFilterBackend):
return queryset
class LabelFilterBackend(filters.BaseFilterBackend):
def get_schema_fields(self, view):
return [
coreapi.Field(
name='label', location='query', required=False,
type='string', example='/api/v1/users/users?label=abc',
description='Filter by label'
)
]
def filter_queryset(self, request, queryset, view):
label_id = request.query_params.get('label')
if not label_id:
return queryset
if not hasattr(queryset, 'model'):
return queryset
if not hasattr(queryset.model, 'labels'):
return queryset
kwargs = {}
if ':' in label_id:
k, v = label_id.split(':', 1)
kwargs['label__name'] = k
if v != '*':
kwargs['label__value'] = v
else:
kwargs['label_id'] = label_id
model = queryset.model
labeled_resource_cls = model.labels.field.related_model
app_label = model._meta.app_label
model_name = model._meta.model_name
res_ids = labeled_resource_cls.objects.filter(
res_type__app_label=app_label, res_type__model=model_name,
).filter(**kwargs).values_list('res_id', flat=True)
queryset = queryset.filter(id__in=set(res_ids))
return queryset
class CustomFilterBackend(filters.BaseFilterBackend):
def get_schema_fields(self, view):

@ -20,7 +20,8 @@ __all__ = [
"TreeChoicesField",
"LabeledMultipleChoiceField",
"PhoneField",
"JSONManyToManyField"
"JSONManyToManyField",
"LabelRelatedField",
]
@ -99,6 +100,33 @@ class LabeledMultipleChoiceField(serializers.MultipleChoiceField):
return data
class LabelRelatedField(serializers.RelatedField):
def __init__(self, **kwargs):
queryset = kwargs.pop("queryset", None)
if queryset is None:
from labels.models import LabeledResource
queryset = LabeledResource.objects.all()
kwargs = {**kwargs}
read_only = kwargs.get("read_only", False)
if not read_only:
kwargs["queryset"] = queryset
super().__init__(**kwargs)
def to_representation(self, value):
if value is None:
return value
return str(value.label)
def to_internal_value(self, data):
from labels.models import LabeledResource, Label
if data is None:
return data
k, v = data.split(":", 1)
label, __ = Label.objects.get_or_create(name=k, value=v, defaults={'name': k, 'value': v})
return LabeledResource(label=label)
class ObjectRelatedField(serializers.RelatedField):
default_error_messages = {
"required": _("This field is required."),

@ -7,6 +7,7 @@ else:
from collections import Iterable
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import NOT_PROVIDED
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from rest_framework.fields import SkipField, empty
@ -14,14 +15,13 @@ from rest_framework.settings import api_settings
from rest_framework.utils import html
from common.db.fields import EncryptMixin
from common.serializers.fields import EncryptedField, LabeledChoiceField, ObjectRelatedField
from common.serializers.fields import EncryptedField, LabeledChoiceField, ObjectRelatedField, LabelRelatedField
__all__ = [
'BulkSerializerMixin', 'BulkListSerializerMixin',
'CommonSerializerMixin', 'CommonBulkSerializerMixin',
'SecretReadableMixin', 'CommonModelSerializer',
'CommonBulkModelSerializer',
'CommonBulkModelSerializer', 'ResourceLabelsMixin',
]
@ -391,3 +391,25 @@ class CommonBulkSerializerMixin(BulkSerializerMixin, CommonSerializerMixin):
class CommonBulkModelSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer):
pass
class ResourceLabelsMixin(serializers.Serializer):
labels = LabelRelatedField(many=True, label=_('Labels'), )
def update(self, instance, validated_data):
labels = validated_data.pop('labels', None)
res = super().update(instance, validated_data)
if labels is not None:
instance.labels.set(labels, bulk=False)
return res
def create(self, validated_data):
labels = validated_data.pop('labels', None)
instance = super().create(validated_data)
if labels is not None:
instance.labels.set(labels, bulk=False)
return instance
@classmethod
def setup_eager_loading(cls, queryset):
return queryset.prefetch_related('labels')

@ -66,6 +66,10 @@ def digest_sql_query():
for table_name, queries in table_queries.items():
if table_name.startswith('rbac_') or table_name.startswith('auth_permission'):
continue
for query in queries:
sql = query['sql']
print(" # {}: {}".format(query['time'], sql))
if len(queries) < 3:
continue
print("- Table: {}".format(table_name))

@ -586,8 +586,9 @@ class Config(dict):
# FTP 文件上传下载备份阈值,单位(M)当值小于等于0时不备份
'FTP_FILE_MAX_STORE': 100,
# API 请求次数限制
'MAX_LIMIT_PER_PAGE': 100,
# API 分页
'MAX_LIMIT_PER_PAGE': 10000,
'DEFAULT_PAGE_SIZE': None,
'LIMIT_SUPER_PRIV': False,

@ -11,3 +11,10 @@ class MaxLimitOffsetPagination(LimitOffsetPagination):
return queryset.values_list('id').order_by().count()
except (AttributeError, TypeError, FieldError):
return len(queryset)
def paginate_queryset(self, queryset, request, view=None):
if view and hasattr(view, 'page_max_limit'):
self.max_limit = view.page_max_limit
if view and hasattr(view, 'page_default_limit'):
self.default_limit = view.page_default_limit
return super().paginate_queryset(queryset, request, view)

@ -109,6 +109,7 @@ for host_port in ALLOWED_DOMAINS:
continue
CSRF_TRUSTED_ORIGINS.append('{}://*.{}'.format(schema, origin))
CORS_ALLOWED_ORIGINS = [o.replace('*.', '') for o in CSRF_TRUSTED_ORIGINS]
CSRF_FAILURE_VIEW = 'jumpserver.views.other.csrf_failure'
# print("CSRF_TRUSTED_ORIGINS: ")
# for origin in CSRF_TRUSTED_ORIGINS:
@ -134,6 +135,7 @@ INSTALLED_APPS = [
'acls.apps.AclsConfig',
'notifications.apps.NotificationsConfig',
'rbac.apps.RBACConfig',
'labels.apps.LabelsConfig',
'rest_framework',
'rest_framework_swagger',
'drf_yasg',
@ -142,6 +144,7 @@ INSTALLED_APPS = [
'django_filters',
'bootstrap3',
'captcha',
'corsheaders',
'private_storage',
'django_celery_beat',
'django.contrib.auth',
@ -160,6 +163,7 @@ MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.locale.LocaleMiddleware',
'corsheaders.middleware.CorsMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',

@ -207,6 +207,7 @@ SESSION_RSA_PUBLIC_KEY_NAME = 'jms_public_key'
OPERATE_LOG_ELASTICSEARCH_CONFIG = CONFIG.OPERATE_LOG_ELASTICSEARCH_CONFIG
MAX_LIMIT_PER_PAGE = CONFIG.MAX_LIMIT_PER_PAGE
DEFAULT_PAGE_SIZE = CONFIG.DEFAULT_PAGE_SIZE
# Magnus DB Port
MAGNUS_ORACLE_PORTS = CONFIG.MAGNUS_ORACLE_PORTS

@ -46,6 +46,7 @@ REST_FRAMEWORK = {
'DATETIME_FORMAT': '%Y/%m/%d %H:%M:%S %z',
'DATETIME_INPUT_FORMATS': ['%Y/%m/%d %H:%M:%S %z', 'iso-8601', '%Y-%m-%d %H:%M:%S %z'],
'DEFAULT_PAGINATION_CLASS': 'jumpserver.rewriting.pagination.MaxLimitOffsetPagination',
'PAGE_SIZE': CONFIG.DEFAULT_PAGE_SIZE,
'EXCEPTION_HANDLER': 'common.drf.exc_handlers.common_exception_handler',
}

@ -28,6 +28,7 @@ api_v1 = [
path('acls/', include('acls.urls.api_urls', namespace='api-acls')),
path('notifications/', include('notifications.urls.api_urls', namespace='api-notifications')),
path('rbac/', include('rbac.urls.api_urls', namespace='api-rbac')),
path('labels/', include('labels.urls', namespace='api-label')),
path('prometheus/metrics/', api.PrometheusMetricsApi.as_view()),
]

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

@ -0,0 +1,141 @@
from django.shortcuts import get_object_or_404
from rest_framework.decorators import action
from rest_framework.response import Response
from common.api.generic import JMSModelViewSet
from common.utils import is_true
from orgs.mixins.api import OrgBulkModelViewSet
from orgs.mixins.models import OrgModelMixin
from orgs.utils import current_org
from rbac.models import ContentType
from rbac.serializers import ContentTypeSerializer
from . import serializers
from .models import Label, LabeledResource
__all__ = ['LabelViewSet']
class ContentTypeViewSet(JMSModelViewSet):
serializer_class = ContentTypeSerializer
http_method_names = ['get', 'head', 'options']
rbac_perms = {
'default': 'labels.view_contenttype',
'resources': 'labels.view_contenttype',
}
page_default_limit = None
can_labeled_content_type = []
model = ContentType
@classmethod
def get_can_labeled_content_type_ids(cls):
if cls.can_labeled_content_type:
return cls.can_labeled_content_type
content_types = ContentType.objects.all()
for ct in content_types:
model_cls = ct.model_class()
if not model_cls:
continue
if model_cls._meta.parents:
continue
if 'labels' in model_cls._meta._forward_fields_map.keys():
# if issubclass(model_cls, LabeledMixin):
cls.can_labeled_content_type.append(ct.id)
return cls.can_labeled_content_type
def get_queryset(self):
ids = self.get_can_labeled_content_type_ids()
queryset = ContentType.objects.filter(id__in=ids)
return queryset
@action(methods=['GET'], detail=True, serializer_class=serializers.ContentTypeResourceSerializer)
def resources(self, request, *args, **kwargs):
self.page_default_limit = 100
content_type = self.get_object()
model = content_type.model_class()
if issubclass(model, OrgModelMixin):
queryset = model.objects.filter(org_id=current_org.id)
else:
queryset = model.objects.all()
keyword = request.query_params.get('search')
if keyword:
queryset = content_type.filter_queryset(queryset, keyword)
return self.get_paginated_response_from_queryset(queryset)
class LabelContentTypeResourceViewSet(JMSModelViewSet):
serializer_class = serializers.ContentTypeResourceSerializer
rbac_perms = {
'default': 'labels.view_labeledresource',
'update': 'labels.change_labeledresource',
}
ordering_fields = ('res_type', 'date_created')
def get_queryset(self):
label_pk = self.kwargs.get('label')
res_type = self.kwargs.get('res_type')
label = get_object_or_404(Label, pk=label_pk)
content_type = get_object_or_404(ContentType, id=res_type)
bound = self.request.query_params.get('bound', '1')
res_ids = LabeledResource.objects.filter(res_type=content_type, label=label) \
.values_list('res_id', flat=True)
res_ids = set(res_ids)
model = content_type.model_class()
if is_true(bound):
queryset = model.objects.filter(id__in=list(res_ids))
else:
queryset = model.objects.exclude(id__in=list(res_ids))
keyword = self.request.query_params.get('search')
if keyword:
queryset = content_type.filter_queryset(queryset, keyword)
return queryset
def put(self, request, *args, **kwargs):
label_pk = self.kwargs.get('label')
res_type = self.kwargs.get('res_type')
content_type = get_object_or_404(ContentType, id=res_type)
label = get_object_or_404(Label, pk=label_pk)
res_ids = request.data.get('res_ids', [])
LabeledResource.objects \
.filter(res_type=content_type, label=label) \
.exclude(res_id__in=res_ids).delete()
resources = []
for res_id in res_ids:
resources.append(LabeledResource(res_type=content_type, res_id=res_id, label=label, org_id=current_org.id))
LabeledResource.objects.bulk_create(resources, ignore_conflicts=True)
return Response({"total": len(res_ids)})
class LabelViewSet(OrgBulkModelViewSet):
model = Label
filterset_fields = ("name", "value")
search_fields = filterset_fields
serializer_classes = {
'default': serializers.LabelSerializer,
'resource_types': ContentTypeSerializer,
}
rbac_perms = {
'resource_types': 'labels.view_label',
'keys': 'labels.view_label',
}
@action(methods=['GET'], detail=False)
def keys(self, request, *args, **kwargs):
queryset = Label.objects.all()
keyword = request.query_params.get('search')
if keyword:
queryset = queryset.filter(name__icontains=keyword)
keys = queryset.values_list('name', flat=True).distinct()
return Response(keys)
class LabeledResourceViewSet(OrgBulkModelViewSet):
model = LabeledResource
filterset_fields = ("label__name", "label__value", "res_type", "res_id", "label")
search_fields = filterset_fields
serializer_classes = {
'default': serializers.LabeledResourceSerializer,
}
ordering_fields = ('res_type', 'date_created')

@ -0,0 +1,6 @@
from django.apps import AppConfig
class LabelsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'labels'

@ -0,0 +1,54 @@
# Generated by Django 4.1.10 on 2023-11-06 10:38
from django.db import migrations, models
import django.db.models.deletion
import uuid
class Migration(migrations.Migration):
initial = True
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
]
operations = [
migrations.CreateModel(
name='Label',
fields=[
('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')),
('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')),
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('internal', models.BooleanField(default=False, verbose_name='Internal')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('name', models.CharField(db_index=True, max_length=64, verbose_name='Name')),
('value', models.CharField(max_length=64, verbose_name='Value')),
],
options={
'verbose_name': 'Label',
'unique_together': {('name', 'value', 'org_id')},
},
),
migrations.CreateModel(
name='LabeledResource',
fields=[
('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')),
('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')),
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('res_id', models.CharField(db_index=True, max_length=36, verbose_name='Resource ID')),
('label', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='labels.label')),
('res_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')),
],
options={
'abstract': False,
},
),
]

@ -0,0 +1,52 @@
# Generated by Django 4.1.10 on 2023-11-03 08:59
from django.db import migrations
def migrate_assets_labels(apps, schema_editor):
old_label_model = apps.get_model('assets', 'Label')
new_label_model = apps.get_model('labels', 'Label')
asset_model = apps.get_model('assets', 'Asset')
labeled_item_model = apps.get_model('labels', 'LabeledResource')
old_labels = old_label_model.objects.all()
new_labels = []
old_new_label_map = {}
for label in old_labels:
new_label = new_label_model(name=label.name, value=label.value, org_id=label.org_id)
old_new_label_map[label.id] = new_label
new_labels.append(new_label)
new_label_model.objects.bulk_create(new_labels, ignore_conflicts=True)
label_relations = asset_model.labels.through.objects.all()
bulk_size = 1000
count = 0
content_type = apps.get_model('contenttypes', 'contenttype').objects.get_for_model(asset_model)
while True:
relations = label_relations[count:count + bulk_size]
if not relations:
break
count += bulk_size
tagged_items = []
for relation in relations:
new_label = old_new_label_map[relation.label_id]
tagged_item = labeled_item_model(
label_id=new_label.id, res_type=content_type,
res_id=relation.asset_id, org_id=new_label.org_id
)
tagged_items.append(tagged_item)
labeled_item_model.objects.bulk_create(tagged_items, ignore_conflicts=True)
class Migration(migrations.Migration):
dependencies = [
('labels', '0001_initial'),
('assets', '0125_auto_20231011_1053')
]
operations = [
migrations.RunPython(migrate_assets_labels),
]

@ -0,0 +1,28 @@
# Generated by Django 4.1.10 on 2023-11-15 10:58
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
('labels', '0002_auto_20231103_1659'),
]
operations = [
migrations.AlterModelOptions(
name='labeledresource',
options={'verbose_name': 'Labeled resource'},
),
migrations.AlterField(
model_name='labeledresource',
name='label',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='labeled_resources', to='labels.label'),
),
migrations.AlterUniqueTogether(
name='labeledresource',
unique_together={('label', 'res_type', 'res_id', 'org_id')},
),
]

@ -0,0 +1,13 @@
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from .models import LabeledResource
__all__ = ['LabeledMixin']
class LabeledMixin(models.Model):
labels = GenericRelation(LabeledResource, object_id_field='res_id', content_type_field='res_type')
class Meta:
abstract = True

@ -0,0 +1,38 @@
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.translation import gettext_lazy as _
from common.utils import lazyproperty
from orgs.mixins.models import JMSOrgBaseModel
class Label(JMSOrgBaseModel):
name = models.CharField(max_length=64, verbose_name=_("Name"), db_index=True)
value = models.CharField(max_length=64, unique=False, verbose_name=_("Value"))
internal = models.BooleanField(default=False, verbose_name=_("Internal"))
class Meta:
unique_together = [('name', 'value', 'org_id')]
verbose_name = _('Label')
@lazyproperty
def res_count(self):
return self.labeled_resources.count()
def __str__(self):
return '{}:{}'.format(self.name, self.value)
class LabeledResource(JMSOrgBaseModel):
label = models.ForeignKey(Label, on_delete=models.CASCADE, related_name='labeled_resources')
res_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
res_id = models.CharField(max_length=36, verbose_name=_("Resource ID"), db_index=True)
resource = GenericForeignKey('res_type', 'res_id')
class Meta:
unique_together = [('label', 'res_type', 'res_id', 'org_id')]
verbose_name = _('Labeled resource')
def __str__(self):
return '{} => {}'.format(self.label, self.resource)

@ -0,0 +1,54 @@
from django.contrib.contenttypes.models import ContentType
from django.db.models import Count
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers.fields import ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from .models import Label, LabeledResource
__all__ = ['LabelSerializer', 'LabeledResourceSerializer', 'ContentTypeResourceSerializer']
class LabelSerializer(BulkOrgResourceModelSerializer):
class Meta:
model = Label
fields = ['id', 'name', 'value', 'res_count', 'date_created', 'date_updated']
read_only_fields = ('date_created', 'date_updated', 'res_count')
extra_kwargs = {
'res_count': {'label': _('Resource count')},
}
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.annotate(res_count=Count('labeled_resources'))
return queryset
class LabeledResourceSerializer(serializers.ModelSerializer):
res_type = ObjectRelatedField(
queryset=ContentType.objects, attrs=('app_label', 'model', 'name'), label=_("Resource type")
)
label = ObjectRelatedField(queryset=Label.objects, attrs=('name', 'value'))
resource = serializers.CharField(label=_("Resource"))
class Meta:
model = LabeledResource
fields = ('id', 'label', 'res_type', 'res_id', 'date_created', 'resource', 'date_updated')
read_only_fields = ('date_created', 'date_updated', 'resource')
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.select_related('label', 'res_type')
return queryset
class ContentTypeResourceSerializer(serializers.Serializer):
id = serializers.CharField()
name = serializers.SerializerMethodField()
@staticmethod
def get_name(obj):
return str(obj)

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

@ -0,0 +1,21 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
from rest_framework_bulk.routes import BulkRouter
from . import api
app_name = 'labels'
router = BulkRouter()
router.register(r'labels', api.LabelViewSet, 'label')
router.register(r'labels/(?P<label>.*)/resource-types/(?P<res_type>.*)/resources',
api.LabelContentTypeResourceViewSet, 'label-content-type-resource')
router.register(r'labeled-resources', api.LabeledResourceViewSet, 'labeled-resource')
router.register(r'resource-types', api.ContentTypeViewSet, 'content-type')
urlpatterns = [
]
urlpatterns += router.urls

@ -0,0 +1,3 @@
from django.shortcuts import render
# Create your views here.

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:185acf00dbd3e3ef37e9faf300c438e120f257fc876e72135cecb1234dd5e453
size 166353
oid sha256:790917753a2bc455aaa6a74322b18f7cbdb7ba860f4c08c46263e7762fb3fbe7
size 165198

File diff suppressed because it is too large Load Diff

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e4af35b685fe0eb73e4dd9790669164bb823065d994a24f5441c9cff7a6e99e7
size 135811
oid sha256:a46c7aff5f314cbab9849e4c11671f81b849f98fad73887ced5b2012635b8a97
size 135290

File diff suppressed because it is too large Load Diff

@ -22,6 +22,7 @@ from acls.models import CommandFilterACL
from assets.models import Asset
from assets.automations.base.manager import SSHTunnelManager
from common.db.encoder import ModelJSONFieldEncoder
from labels.mixins import LabeledMixin
from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner, CommandInBlackListException
from ops.mixin import PeriodTaskModelMixin
from ops.variables import *
@ -132,7 +133,7 @@ class JMSPermedInventory(JMSInventory):
return mapper
class Job(JMSOrgBaseModel, PeriodTaskModelMixin):
class Job(LabeledMixin, JMSOrgBaseModel, PeriodTaskModelMixin):
name = models.CharField(max_length=128, null=True, verbose_name=_('Name'))
instant = models.BooleanField(default=False)

@ -6,6 +6,7 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from private_storage.fields import PrivateFileField
from labels.mixins import LabeledMixin
from ops.const import CreateMethods
from ops.exception import PlaybookNoValidEntry
from orgs.mixins.models import JMSOrgBaseModel
@ -23,7 +24,7 @@ dangerous_keywords = (
)
class Playbook(JMSOrgBaseModel):
class Playbook(LabeledMixin, JMSOrgBaseModel):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
name = models.CharField(max_length=128, verbose_name=_('Name'), null=True)
path = PrivateFileField(upload_to='playbooks/')

@ -3,23 +3,21 @@ import uuid
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from assets.models import Node, Asset
from perms.models import PermNode
from perms.utils.user_perm import UserPermAssetUtil
from assets.models import Asset
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import ReadableHiddenField
from ops.mixin import PeriodTaskSerializerMixin
from ops.models import Job, JobExecution
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
class JobSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
run_after_save = serializers.BooleanField(label=_("Run after save"), default=False, required=False)
nodes = serializers.ListField(required=False, child=serializers.CharField())
date_last_run = serializers.DateTimeField(label=_('Date last run'), read_only=True)
name = serializers.CharField(label=_('Name'), max_length=128, allow_blank=True, required=False)
assets = serializers.PrimaryKeyRelatedField(label=_('Assets'), queryset=Asset.objects, many=True,
required=False)
assets = serializers.PrimaryKeyRelatedField(label=_('Assets'), queryset=Asset.objects, many=True, required=False)
def to_internal_value(self, data):
instant = data.get('instant', False)
@ -36,11 +34,12 @@ class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
class Meta:
model = Job
read_only_fields = [
"id", "date_last_run", "date_created", "date_updated", "average_time_cost"
"id", "date_last_run", "date_created",
"date_updated", "average_time_cost"
]
fields = read_only_fields + [
"name", "instant", "type", "module",
"args", "playbook", "assets",
"args", "playbook", "assets", "labels",
"runas_policy", "runas", "creator",
"use_parameter_define", "parameters_define",
"timeout", "chdir", "comment", "summary",

@ -2,10 +2,10 @@ import os
from rest_framework import serializers
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import ReadableHiddenField
from ops.models import Playbook
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from django.utils.translation import gettext_lazy as _
def parse_playbook_name(path):
@ -13,7 +13,7 @@ def parse_playbook_name(path):
return file_name.split(".")[-2]
class PlaybookSerializer(BulkOrgResourceModelSerializer):
class PlaybookSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
path = serializers.FileField(required=False)
@ -27,5 +27,6 @@ class PlaybookSerializer(BulkOrgResourceModelSerializer):
model = Playbook
read_only_fields = ["id", "date_created", "date_updated"]
fields = read_only_fields + [
"id", 'path', "name", "comment", "creator", 'create_method', 'vcs_url',
"id", 'path', "name", "comment", "creator",
'create_method', 'vcs_url', 'labels'
]

@ -10,6 +10,7 @@ from accounts.models import Account
from assets.models import Asset
from common.utils import date_expired_default
from common.utils.timezone import local_now
from labels.mixins import LabeledMixin
from orgs.mixins.models import JMSOrgBaseModel
from orgs.mixins.models import OrgManager
from perms.const import ActionChoices
@ -56,7 +57,7 @@ def default_protocols():
return ['all']
class AssetPermission(JMSOrgBaseModel):
class AssetPermission(LabeledMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'))
users = models.ManyToManyField(
'users.User', related_name='%(class)ss', blank=True, verbose_name=_("User")

@ -7,6 +7,7 @@ from rest_framework import serializers
from accounts.models import AccountTemplate, Account
from accounts.tasks import push_accounts_to_assets_task
from assets.models import Asset, Node
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import BitChoicesField, ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from perms.models import ActionChoices, AssetPermission
@ -26,7 +27,7 @@ class ActionChoicesField(BitChoicesField):
return data
class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
class AssetPermissionSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
users = ObjectRelatedField(queryset=User.objects, many=True, required=False, label=_('User'))
user_groups = ObjectRelatedField(
queryset=UserGroup.objects, many=True, required=False, label=_('User group')
@ -50,7 +51,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
"is_valid", "comment", "from_ticket",
]
fields_small = fields_mini + fields_generic
fields_m2m = ["users", "user_groups", "assets", "nodes"]
fields_m2m = ["users", "user_groups", "assets", "nodes", "labels"]
fields = fields_mini + fields_m2m + fields_generic
read_only_fields = ["created_by", "date_created", "from_ticket"]
extra_kwargs = {
@ -130,7 +131,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
"""Perform necessary eager loading of data."""
queryset = queryset.prefetch_related(
"users", "user_groups", "assets", "nodes",
)
).prefetch_related('labels', 'labels__label')
return queryset
@staticmethod

@ -1,4 +1,4 @@
from .content_type import *
from .permission import *
from .role import *
from .rolebinding import *

@ -0,0 +1,11 @@
from rest_framework import viewsets
from .. import serializers
from ..models import ContentType
class ContentTypeViewSet(viewsets.ModelViewSet):
serializer_class = serializers.ContentTypeSerializer
filterset_fields = ("app_label", "model",)
search_fields = filterset_fields
queryset = ContentType.objects.all()

@ -73,7 +73,7 @@ exclude_permissions = (
('perms', 'rebuildusertreetask', '*', '*'),
('perms', 'permedasset', '*', 'permedasset'),
('perms', 'permedapplication', 'add,change,delete', 'permedapplication'),
('rbac', 'contenttype', '*', '*'),
('rbac', 'contenttype', 'add,change,delete', '*'),
('rbac', 'permission', 'add,delete,change', 'permission'),
('rbac', 'rolebinding', '*', '*'),
('rbac', 'systemrolebinding', 'change', 'systemrolebinding'),

@ -1,8 +1,10 @@
from django.apps import apps
from django.contrib.auth.models import ContentType as DjangoContentType
from django.contrib.auth.models import Permission as DjangoPermission
from django.db.models import Q
from django.utils.translation import gettext_lazy as _
from common.utils import lazyproperty
from .. import const
Scope = const.Scope
@ -14,10 +16,59 @@ class ContentType(DjangoContentType):
class Meta:
proxy = True
_apps_map = {}
@property
def app_model(self):
return '%s.%s' % (self.app_label, self.model)
@classmethod
def apps_map(cls):
from ..tree import app_nodes_data
if cls._apps_map:
return cls._apps_map
mapper = {}
for d in app_nodes_data:
i = d['id']
name = d.get('name')
if not name:
config = apps.get_app_config(d['id'])
if config:
name = config.verbose_name
if name:
mapper[i] = name
cls._apps_map = mapper
return mapper
@property
def app_display(self):
return self.apps_map().get(self.app_label)
@lazyproperty
def fields(self):
model = self.model_class()
return model._meta.fields
@lazyproperty
def field_names(self):
return [f.name for f in self.fields]
@lazyproperty
def filter_field_names(self):
names = []
if 'name' in self.field_names:
names.append('name')
if 'address' in self.field_names:
names.append('address')
return names
def filter_queryset(self, queryset, keyword):
q = Q()
for name in self.filter_field_names:
q |= Q(**{name + '__icontains': keyword})
return queryset.filter(q)
class Permission(DjangoPermission):
""" 权限类 """

@ -1,4 +1,4 @@
from .content_type import *
from .permission import *
from .role import *
from .rolebinding import *

@ -0,0 +1,13 @@
from rest_framework import serializers
from ..models import ContentType
__all__ = ['ContentTypeSerializer']
class ContentTypeSerializer(serializers.ModelSerializer):
app_display = serializers.CharField()
class Meta:
model = ContentType
fields = ('id', 'app_label', 'app_display', 'model', 'name')

@ -39,7 +39,9 @@ app_nodes_data = [
{'id': 'rbac', 'view': 'view_console'},
{'id': 'settings', 'view': 'view_setting'},
{'id': 'tickets', 'view': 'view_other'},
{'id': 'labels', 'view': 'view_label'},
{'id': 'authentication', 'view': 'view_other'},
{'id': 'ops', 'view': 'view_workbench'},
]
# 额外其他节点,可以在不同的层次,需要指定父节点,可以将一些 model 归类到这个节点下面

@ -6,7 +6,6 @@ from .. import api
app_name = 'rbac'
router = BulkRouter()
router.register(r'roles', api.RoleViewSet, 'role')
router.register(r'role-bindings', api.RoleBindingViewSet, 'role-binding')
@ -18,6 +17,7 @@ router.register(r'org-roles', api.OrgRoleViewSet, 'org-role')
router.register(r'org-role-bindings', api.OrgRoleBindingViewSet, 'org-role-binding')
router.register(r'permissions', api.PermissionViewSet, 'permission')
router.register(r'content-types', api.ContentTypeViewSet, 'content-type')
system_role_router = routers.NestedDefaultRouter(router, r'system-roles', lookup='system_role')
system_role_router.register(r'permissions', api.SystemRolePermissionsViewSet, 'system-role-permission')

@ -52,10 +52,6 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, SuggestionMixin, BulkModelV
'bulk_remove': 'users.remove_user',
}
def get_queryset(self):
queryset = super().get_queryset().prefetch_related('groups')
return queryset
def allow_bulk_destroy(self, qs, filtered):
is_valid = filtered.count() < qs.count()
if not is_valid:

@ -1,15 +1,15 @@
# -*- coding: utf-8 -*-
from django.db import models
from django.utils.translation import gettext_lazy as _
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
from orgs.mixins.models import JMSOrgBaseModel
__all__ = ['UserGroup']
class UserGroup(JMSOrgBaseModel):
class UserGroup(LabeledMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'))
def __str__(self):

@ -24,6 +24,7 @@ from common.utils import (
date_expired_default, get_logger, lazyproperty,
random_string, bulk_create_with_signal
)
from labels.mixins import LabeledMixin
from orgs.utils import current_org
from rbac.const import Scope
from ..signals import (
@ -734,7 +735,7 @@ class JSONFilterMixin:
return models.Q(id__in=user_id)
class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, JSONFilterMixin, AbstractUser):
class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, LabeledMixin, JSONFilterMixin, AbstractUser):
class Source(models.TextChoices):
local = 'local', _('Local')
ldap = 'ldap', 'LDAP/AD'

@ -3,7 +3,8 @@
from django.db.models import Count
from django.utils.translation import gettext_lazy as _
from common.serializers.mixin import ObjectRelatedField
from common.serializers.fields import ObjectRelatedField
from common.serializers.mixin import ResourceLabelsMixin
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from .. import utils
from ..models import User, UserGroup
@ -13,7 +14,7 @@ __all__ = [
]
class UserGroupSerializer(BulkOrgResourceModelSerializer):
class UserGroupSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
users = ObjectRelatedField(
required=False, many=True, queryset=User.objects, label=_('User'),
)
@ -24,7 +25,7 @@ class UserGroupSerializer(BulkOrgResourceModelSerializer):
fields_small = fields_mini + [
'comment', 'date_created', 'created_by'
]
fields = fields_mini + fields_small + ['users']
fields = fields_mini + fields_small + ['users', 'labels']
extra_kwargs = {
'created_by': {'label': _('Created by'), 'read_only': True},
'users_amount': {'label': _('Users amount')},
@ -43,5 +44,6 @@ class UserGroupSerializer(BulkOrgResourceModelSerializer):
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.prefetch_related('users').annotate(users_amount=Count('users'))
queryset = queryset.prefetch_related('users', 'labels', 'labels__label') \
.annotate(users_amount=Count('users'))
return queryset

@ -143,9 +143,9 @@ class UserProfileSerializer(UserSerializer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
system_roles_field = self.fields.get('system_roles')
org_roles_field = self.fields.get('org_roles')
if system_roles_field:
system_roles_field.read_only = True
org_roles_field = self.fields.get('org_roles')
if org_roles_field:
org_roles_field.read_only = True

@ -6,7 +6,7 @@ from functools import partial
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers import CommonBulkSerializerMixin
from common.serializers import CommonBulkSerializerMixin, ResourceLabelsMixin
from common.serializers.fields import (
EncryptedField, ObjectRelatedField, LabeledChoiceField, PhoneField
)
@ -81,7 +81,7 @@ class RolesSerializerMixin(serializers.Serializer):
return fields
class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, serializers.ModelSerializer):
class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, ResourceLabelsMixin, serializers.ModelSerializer):
password_strategy = LabeledChoiceField(
choices=PasswordStrategy.choices,
default=PasswordStrategy.email,
@ -143,7 +143,7 @@ class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, serializer
# 外键的字段
fields_fk = []
# 多对多字段
fields_m2m = ["groups", "system_roles", "org_roles", ]
fields_m2m = ["groups", "system_roles", "org_roles", "labels"]
# 在serializer 上定义的字段
fields_custom = ["login_blocked", "password_strategy"]
fields = fields_verbose + fields_fk + fields_m2m + fields_custom
@ -259,6 +259,11 @@ class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, serializer
)
return instance
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset.prefetch_related('groups', 'labels', 'labels__label')
return queryset
class UserRetrieveSerializer(UserSerializer):
login_confirm_settings = serializers.PrimaryKeyRelatedField(

19
poetry.lock generated

@ -1919,6 +1919,25 @@ type = "legacy"
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
reference = "tsinghua"
[[package]]
name = "django-cors-headers"
version = "4.3.0"
description = "django-cors-headers is a Django application for handling the server headers required for Cross-Origin Resource Sharing (CORS)."
optional = false
python-versions = ">=3.8"
files = [
{file = "django_cors_headers-4.3.0-py3-none-any.whl", hash = "sha256:bd36c7aea0d070e462f3383f0dc9ef717e5fdc2b10a99c98c285f16da84ffba2"},
{file = "django_cors_headers-4.3.0.tar.gz", hash = "sha256:25aabc94d4837678c1edf442c7f68a5f5fd151f6767b0e0b01c61a2179d02711"},
]
[package.dependencies]
Django = ">=3.2"
[package.source]
type = "legacy"
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
reference = "tsinghua"
[[package]]
name = "django-debug-toolbar"
version = "4.1.0"

@ -142,6 +142,7 @@ channels-redis = "4.1.0"
fido2 = "^1.1.2"
ua-parser = "^0.18.0"
user-agents = "^2.2.0"
django-cors-headers = "^4.3.0"
mistune = "0.8.4"
openai = "^1.3.7"

Loading…
Cancel
Save