github
/
jumpserver
mirror of https://github.com/jumpserver/jumpserver
1
0
Fork 0
Code Issues Releases Wiki Activity

perf: 支持全局的 labels (#12043) 

* perf: 支持全局的 labels

* perf: stash

* stash

* stash

* stash

* stash

* perf: 优化 labels

* stash

* perf: add debug sql

* perf: 修改 labels

* perf: 优化提交

* perf: 优化提交 labels

* perf: 基本完成

* perf: 完成 labels 搜索

* perf: 优化 labels

* perf: 去掉不用 debug

---------

Co-authored-by: ibuler <ibuler@qq.com>
pull/12253/head
fit2bot 2023-12-05 11:16:34 +08:00 committed by GitHub
parent a91cb1afd5
commit 8291a81efd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
71 changed files with 1618 additions and 978 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/accounts/models/account.py
View File

@ -4,6 +4,7 @@ from simple_history.models import HistoricalRecords
from assets.models.base import AbsConnectivity
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
from .base import BaseAccount
from .mixins import VaultModelMixin
from ..const import Source
@ -42,7 +43,7 @@ class AccountHistoricalRecords(HistoricalRecords):
return super().create_history_model(model, inherited)
class Account(AbsConnectivity, BaseAccount):
class Account(AbsConnectivity, LabeledMixin, BaseAccount):
asset = models.ForeignKey(
'assets.Asset', related_name='accounts',
on_delete=models.CASCADE, verbose_name=_('Asset')
@ -71,7 +72,7 @@ class Account(AbsConnectivity, BaseAccount):
]
def __str__(self):
return '{}'.format(self.username)
return '{}({})'.format(self.name, self.asset.name)
@lazyproperty
def platform(self):

3
apps/accounts/models/template.py
View File

@ -3,13 +3,14 @@ from django.db.models import Count, Q
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from labels.mixins import LabeledMixin
from .account import Account
from .base import BaseAccount, SecretWithRandomMixin
__all__ = ['AccountTemplate', ]
class AccountTemplate(BaseAccount, SecretWithRandomMixin):
class AccountTemplate(LabeledMixin, BaseAccount, SecretWithRandomMixin):
su_from = models.ForeignKey(
'self', related_name='su_to', null=True,
on_delete=models.SET_NULL, verbose_name=_("Su from")

5
apps/accounts/serializers/account/account.py
View File

@ -66,6 +66,9 @@ class AccountCreateUpdateSerializerMixin(serializers.Serializer):
name = initial_data.get('name')
if name is not None:
return
request = self.context.get('request')
if request and request.method == 'PATCH':
return
if not name:
name = initial_data.get('username')
if self.instance and self.instance.name == name:
@ -238,7 +241,7 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize
queryset = queryset.prefetch_related(
'asset', 'asset__platform',
'asset__platform__automation'
)
).prefetch_related('labels', 'labels__label')
return queryset

6
apps/accounts/serializers/account/base.py
View File

@ -5,6 +5,7 @@ from rest_framework import serializers
from accounts.const import SecretType
from accounts.models import BaseAccount
from accounts.utils import validate_password_for_ansible, validate_ssh_key
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import EncryptedField, LabeledChoiceField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
@ -60,8 +61,7 @@ class AuthValidateMixin(serializers.Serializer):
return super().update(instance, validated_data)
class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
class BaseAccountSerializer(AuthValidateMixin, ResourceLabelsMixin, BulkOrgResourceModelSerializer):
class Meta:
model = BaseAccount
fields_mini = ['id', 'name', 'username']
@ -70,7 +70,7 @@ class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
'privileged', 'is_active', 'spec_info',
]
fields_other = ['created_by', 'date_created', 'date_updated', 'comment']
fields = fields_small + fields_other
fields = fields_small + fields_other + ['labels']
read_only_fields = [
'spec_info', 'date_verified', 'created_by', 'date_created',
]

1
apps/assets/api/__init__.py
View File

@ -2,7 +2,6 @@ from .asset import *
from .category import *
from .domain import *
from .favorite_asset import *
from .label import *
from .mixin import *
from .node import *
from .platform import *

18
apps/assets/api/asset/asset.py
View File

@ -3,7 +3,6 @@
from collections import defaultdict
import django_filters
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.utils.translation import gettext as _
from rest_framework import status
@ -14,7 +13,7 @@ from rest_framework.status import HTTP_200_OK
from accounts.tasks import push_accounts_to_assets_task, verify_accounts_connectivity_task
from assets import serializers
from assets.exceptions import NotSupportedTemporarilyError
from assets.filters import IpInFilterBackend, LabelFilterBackend, NodeFilterBackend
from assets.filters import IpInFilterBackend, NodeFilterBackend
from assets.models import Asset, Gateway, Platform, Protocol
from assets.tasks import test_assets_connectivity_manual, update_assets_hardware_info_manual
from common.api import SuggestionMixin
@ -33,7 +32,6 @@ __all__ = [
class AssetFilterSet(BaseFilterSet):
labels = django_filters.CharFilter(method='filter_labels')
platform = django_filters.CharFilter(method='filter_platform')
domain = django_filters.CharFilter(method='filter_domain')
type = django_filters.CharFilter(field_name="platform__type", lookup_expr="exact")
@ -64,7 +62,7 @@ class AssetFilterSet(BaseFilterSet):
class Meta:
model = Asset
fields = [
"id", "name", "address", "is_active", "labels",
"id", "name", "address", "is_active",
"type", "category", "platform",
]
@ -87,16 +85,6 @@ class AssetFilterSet(BaseFilterSet):
value = value.split(',')
return queryset.filter(protocols__name__in=value).distinct()
@staticmethod
def filter_labels(queryset, name, value):
if ':' in value:
n, v = value.split(':', 1)
queryset = queryset.filter(labels__name=n, labels__value=v)
else:
q = Q(labels__name__contains=value) | Q(labels__value__contains=value)
queryset = queryset.filter(q).distinct()
return queryset
class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet):
"""
@ -121,7 +109,7 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet):
("sync_platform_protocols", "assets.change_asset"),
)
extra_filter_backends = [
LabelFilterBackend, IpInFilterBackend,
IpInFilterBackend,
NodeFilterBackend, AttrRulesFilterBackend
]

43
apps/assets/api/label.py
View File

@ -1,43 +0,0 @@
# ~*~ coding: utf-8 ~*~
# Copyright (C) 2014-2018 Beijing DuiZhan Technology Co.,Ltd. All Rights Reserved.
#
# Licensed under the GNU General Public License v2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.gnu.org/licenses/gpl-2.0.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from django.db.models import Count
from common.utils import get_logger
from orgs.mixins.api import OrgBulkModelViewSet
from ..models import Label
from .. import serializers
logger = get_logger(__file__)
__all__ = ['LabelViewSet']
class LabelViewSet(OrgBulkModelViewSet):
model = Label
filterset_fields = ("name", "value")
search_fields = filterset_fields
serializer_class = serializers.LabelSerializer
def list(self, request, *args, **kwargs):
if request.query_params.get("distinct"):
self.serializer_class = serializers.LabelDistinctSerializer
self.queryset = self.queryset.values("name").distinct()
return super().list(request, *args, **kwargs)
def get_queryset(self):
self.queryset = Label.objects.prefetch_related(
'assets').annotate(asset_count=Count("assets"))
return self.queryset

52
apps/assets/filters.py
View File

@ -5,7 +5,6 @@ from rest_framework import filters
from rest_framework.compat import coreapi, coreschema
from assets.utils import get_node_from_request, is_query_node_all_assets
from .models import Label
class AssetByNodeFilterBackend(filters.BaseFilterBackend):
@ -72,57 +71,6 @@ class NodeFilterBackend(filters.BaseFilterBackend):
return queryset.filter(nodes__key=node.key).distinct()
class LabelFilterBackend(filters.BaseFilterBackend):
sep = ':'
query_arg = 'label'
def get_schema_fields(self, view):
example = self.sep.join(['os', 'linux'])
return [
coreapi.Field(
name=self.query_arg, location='query', required=False,
type='string', example=example, description=''
)
]
def get_query_labels(self, request):
labels_query = request.query_params.getlist(self.query_arg)
if not labels_query:
return None
q = None
for kv in labels_query:
if '#' in kv:
self.sep = '#'
break
for kv in labels_query:
if self.sep not in kv:
continue
key, value = kv.strip().split(self.sep)[:2]
if not all([key, value]):
continue
if q:
q |= Q(name=key, value=value)
else:
q = Q(name=key, value=value)
if not q:
return []
labels = Label.objects.filter(q, is_active=True) \
.values_list('id', flat=True)
return labels
def filter_queryset(self, request, queryset, view):
labels = self.get_query_labels(request)
if labels is None:
return queryset
if len(labels) == 0:
return queryset.none()
for label in labels:
queryset = queryset.filter(labels=label)
return queryset
class IpInFilterBackend(filters.BaseFilterBackend):
def filter_queryset(self, request, queryset, view):
ips = request.query_params.get('ips')

18
apps/assets/migrations/0126_remove_asset_labels.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.10 on 2023-11-22 07:33
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('assets', '0125_auto_20231011_1053'),
('labels', '0002_auto_20231103_1659'),
]
operations = [
migrations.RemoveField(
model_name='asset',
name='labels',
),
]

12
apps/assets/models/asset/common.py
View File

@ -13,7 +13,9 @@ from django.utils.translation import gettext_lazy as _
from assets import const
from common.db.fields import EncryptMixin
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
from orgs.mixins.models import OrgManager, JMSOrgBaseModel
from rbac.models import ContentType
from ..base import AbsConnectivity
from ..platform import Platform
@ -150,7 +152,7 @@ class JSONFilterMixin:
return None
class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel):
class Asset(NodesRelationMixin, LabeledMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseModel):
Category = const.Category
Type = const.AllTypes
@ -162,7 +164,6 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode
nodes = models.ManyToManyField('assets.Node', default=default_node, related_name='assets',
verbose_name=_("Nodes"))
is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
labels = models.ManyToManyField('assets.Label', blank=True, related_name='assets', verbose_name=_("Labels"))
gathered_info = models.JSONField(verbose_name=_('Gathered info'), default=dict, blank=True) # 资产的一些信息，如 硬件信息
custom_info = models.JSONField(verbose_name=_('Custom info'), default=dict)
@ -171,6 +172,13 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode
def __str__(self):
return '{0.name}({0.address})'.format(self)
def get_labels(self):
from labels.models import Label, LabeledResource
res_type = ContentType.objects.get_for_model(self.__class__)
label_ids = LabeledResource.objects.filter(res_type=res_type, res_id=self.id) \
.values_list('label_id', flat=True)
return Label.objects.filter(id__in=label_ids)
@staticmethod
def get_spec_values(instance, fields):
info = {}

3
apps/assets/models/domain.py
View File

@ -6,6 +6,7 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from common.utils import get_logger
from labels.mixins import LabeledMixin
from orgs.mixins.models import JMSOrgBaseModel
from .gateway import Gateway
@ -14,7 +15,7 @@ logger = get_logger(__file__)
__all__ = ['Domain']
class Domain(JMSOrgBaseModel):
class Domain(LabeledMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'))
class Meta:

3
apps/assets/models/platform.py
View File

@ -9,6 +9,7 @@ from common.db.models import JMSBaseModel
__all__ = ['Platform', 'PlatformProtocol', 'PlatformAutomation']
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
class PlatformProtocol(models.Model):
@ -74,7 +75,7 @@ class PlatformAutomation(models.Model):
platform = models.OneToOneField('Platform', on_delete=models.CASCADE, related_name='automation', null=True)
class Platform(JMSBaseModel):
class Platform(LabeledMixin, JMSBaseModel):
"""
对资产提供 约束和默认值
对资产进行抽象

9
apps/assets/serializers/__init__.py
View File

@ -2,11 +2,10 @@
#
from .asset import *
from .label import *
from .node import *
from .gateway import *
from .automations import *
from .cagegory import *
from .domain import *
from .favorite_asset import *
from .gateway import *
from .node import *
from .platform import *
from .cagegory import *
from .automations import *

11
apps/assets/serializers/asset/common.py
View File

@ -11,13 +11,14 @@ from accounts.serializers import AccountSerializer
from common.const import UUID_PATTERN
from common.serializers import (
WritableNestedModelSerializer, SecretReadableMixin,
CommonModelSerializer, MethodSerializer
CommonModelSerializer, MethodSerializer, ResourceLabelsMixin
)
from common.serializers.common import DictSerializer
from common.serializers.fields import LabeledChoiceField
from labels.models import Label
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from ...const import Category, AllTypes
from ...models import Asset, Node, Platform, Label, Protocol
from ...models import Asset, Node, Platform, Protocol
__all__ = [
'AssetSerializer', 'AssetSimpleSerializer', 'MiniAssetSerializer',
@ -117,10 +118,9 @@ class AccountSecretSerializer(SecretReadableMixin, CommonModelSerializer):
}
class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSerializer):
class AssetSerializer(BulkOrgResourceModelSerializer, ResourceLabelsMixin, WritableNestedModelSerializer):
category = LabeledChoiceField(choices=Category.choices, read_only=True, label=_('Category'))
type = LabeledChoiceField(choices=AllTypes.choices(), read_only=True, label=_('Type'))
labels = AssetLabelSerializer(many=True, required=False, label=_('Label'))
protocols = AssetProtocolsSerializer(many=True, required=False, label=_('Protocols'), default=())
accounts = AssetAccountSerializer(many=True, required=False, allow_null=True, write_only=True, label=_('Account'))
nodes_display = serializers.ListField(read_only=False, required=False, label=_("Node path"))
@ -201,8 +201,9 @@ class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSeriali
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.prefetch_related('domain', 'nodes', 'labels', 'protocols') \
queryset = queryset.prefetch_related('domain', 'nodes', 'protocols', ) \
.prefetch_related('platform', 'platform__automation') \
.prefetch_related('labels', 'labels__label') \
.annotate(category=F("platform__category")) \
.annotate(type=F("platform__type"))
return queryset

9
apps/assets/serializers/domain.py
View File

@ -3,6 +3,7 @@
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from .gateway import GatewayWithAccountSecretSerializer
@ -11,7 +12,7 @@ from ..models import Domain, Asset
__all__ = ['DomainSerializer', 'DomainWithGatewaySerializer']
class DomainSerializer(BulkOrgResourceModelSerializer):
class DomainSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
gateways = ObjectRelatedField(
many=True, required=False, label=_('Gateway'), read_only=True,
)
@ -41,6 +42,12 @@ class DomainSerializer(BulkOrgResourceModelSerializer):
instance = super().update(instance, validated_data)
return instance
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset \
.prefetch_related('labels', 'labels__label')
return queryset
class DomainWithGatewaySerializer(serializers.ModelSerializer):
gateways = GatewayWithAccountSecretSerializer(many=True, read_only=True)

47
apps/assets/serializers/label.py
View File

@ -1,47 +0,0 @@
# -*- coding: utf-8 -*-
#
from django.db.models import Count
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from ..models import Label
class LabelSerializer(BulkOrgResourceModelSerializer):
asset_count = serializers.ReadOnlyField(label=_("Assets amount"))
class Meta:
model = Label
fields_mini = ['id', 'name']
fields_small = fields_mini + [
'value', 'category', 'is_active',
'date_created', 'comment',
]
fields_m2m = ['asset_count', 'assets']
fields = fields_small + fields_m2m
read_only_fields = (
'category', 'date_created', 'asset_count',
)
extra_kwargs = {
'assets': {'required': False, 'label': _('Asset')}
}
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset.prefetch_related('assets') \
.annotate(asset_count=Count('assets'))
return queryset
class LabelDistinctSerializer(BulkOrgResourceModelSerializer):
value = serializers.SerializerMethodField()
class Meta:
model = Label
fields = ("name", "value")
@staticmethod
def get_value(obj):
labels = Label.objects.filter(name=obj["name"])
return ', '.join([label.value for label in labels])

10
apps/assets/serializers/platform.py
View File

@ -5,7 +5,7 @@ from rest_framework.validators import UniqueValidator
from common.serializers import (
WritableNestedModelSerializer, type_field_map, MethodSerializer,
DictSerializer, create_serializer_class
DictSerializer, create_serializer_class, ResourceLabelsMixin
)
from common.serializers.fields import LabeledChoiceField
from common.utils import lazyproperty
@ -123,7 +123,7 @@ class PlatformCustomField(serializers.Serializer):
choices = serializers.ListField(default=list, label=_("Choices"), required=False)
class PlatformSerializer(WritableNestedModelSerializer):
class PlatformSerializer(ResourceLabelsMixin, WritableNestedModelSerializer):
SU_METHOD_CHOICES = [
("sudo", "sudo su -"),
("su", "su - "),
@ -160,6 +160,7 @@ class PlatformSerializer(WritableNestedModelSerializer):
fields = fields_small + [
"protocols", "domain_enabled", "su_enabled",
"su_method", "automation", "comment", "custom_fields",
"labels"
] + read_only_fields
extra_kwargs = {
"su_enabled": {"label": _('Su enabled')},
@ -201,9 +202,8 @@ class PlatformSerializer(WritableNestedModelSerializer):
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset.prefetch_related(
'protocols', 'automation'
)
queryset = queryset.prefetch_related('protocols', 'automation') \
.prefetch_related('labels', 'labels__label')
return queryset
def validate_protocols(self, protocols):

1
apps/assets/urls/api_urls.py
View File

@ -17,7 +17,6 @@ router.register(r'clouds', api.CloudViewSet, 'cloud')
router.register(r'gpts', api.GPTViewSet, 'gpt')
router.register(r'customs', api.CustomViewSet, 'custom')
router.register(r'platforms', api.AssetPlatformViewSet, 'platform')
router.register(r'labels', api.LabelViewSet, 'label')
router.register(r'nodes', api.NodeViewSet, 'node')
router.register(r'domains', api.DomainViewSet, 'domain')
router.register(r'gateways', api.GatewayViewSet, 'gateway')

15
apps/audits/utils.py
View File

@ -2,6 +2,7 @@ import copy
from datetime import datetime
from itertools import chain
from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
from django.db import models
from common.db.fields import RelatedManager
@ -65,13 +66,19 @@ def _get_instance_field_value(
continue
data.setdefault(k, v)
continue
data.setdefault(str(f.verbose_name), value)
elif isinstance(f, GenericRelation):
value = [str(v) for v in value.all()]
elif isinstance(f, GenericForeignKey):
continue
try:
data.setdefault(str(f.verbose_name), value)
except Exception as e:
print(f.__dict__)
raise e
return data
def model_to_dict_for_operate_log(
instance, include_model_fields=True, include_related_fields=False
):
def model_to_dict_for_operate_log(instance, include_model_fields=True, include_related_fields=False):
model_need_continue_fields = ['date_updated']
m2m_need_continue_fields = ['history_passwords']

4
apps/common/api/mixin.py
View File

@ -11,7 +11,7 @@ from rest_framework.settings import api_settings
from common.drf.filters import (
IDSpmFilterBackend, CustomFilterBackend, IDInFilterBackend,
IDNotFilterBackend, NotOrRelFilterBackend
IDNotFilterBackend, NotOrRelFilterBackend, LabelFilterBackend
)
from common.utils import get_logger, lazyproperty
from .action import RenderToJsonMixin
@ -111,7 +111,7 @@ class ExtraFilterFieldsMixin:
"""
default_added_filters = (
CustomFilterBackend, IDSpmFilterBackend, IDInFilterBackend,
IDNotFilterBackend,
IDNotFilterBackend, LabelFilterBackend
)
filter_backends = api_settings.DEFAULT_FILTER_BACKENDS
extra_filter_fields = []

44
apps/common/drf/filters.py
View File

@ -21,7 +21,7 @@ __all__ = [
"DatetimeRangeFilterBackend", "IDSpmFilterBackend",
'IDInFilterBackend', "CustomFilterBackend",
"BaseFilterSet", 'IDNotFilterBackend',
'NotOrRelFilterBackend',
'NotOrRelFilterBackend', 'LabelFilterBackend',
]
@ -168,6 +168,48 @@ class IDNotFilterBackend(filters.BaseFilterBackend):
return queryset
class LabelFilterBackend(filters.BaseFilterBackend):
def get_schema_fields(self, view):
return [
coreapi.Field(
name='label', location='query', required=False,
type='string', example='/api/v1/users/users?label=abc',
description='Filter by label'
)
]
def filter_queryset(self, request, queryset, view):
label_id = request.query_params.get('label')
if not label_id:
return queryset
if not hasattr(queryset, 'model'):
return queryset
if not hasattr(queryset.model, 'labels'):
return queryset
kwargs = {}
if ':' in label_id:
k, v = label_id.split(':', 1)
kwargs['label__name'] = k
if v != '*':
kwargs['label__value'] = v
else:
kwargs['label_id'] = label_id
model = queryset.model
labeled_resource_cls = model.labels.field.related_model
app_label = model._meta.app_label
model_name = model._meta.model_name
res_ids = labeled_resource_cls.objects.filter(
res_type__app_label=app_label, res_type__model=model_name,
).filter(**kwargs).values_list('res_id', flat=True)
queryset = queryset.filter(id__in=set(res_ids))
return queryset
class CustomFilterBackend(filters.BaseFilterBackend):
def get_schema_fields(self, view):

30
apps/common/serializers/fields.py
View File

@ -20,7 +20,8 @@ __all__ = [
"TreeChoicesField",
"LabeledMultipleChoiceField",
"PhoneField",
"JSONManyToManyField"
"JSONManyToManyField",
"LabelRelatedField",
]
@ -99,6 +100,33 @@ class LabeledMultipleChoiceField(serializers.MultipleChoiceField):
return data
class LabelRelatedField(serializers.RelatedField):
def __init__(self, **kwargs):
queryset = kwargs.pop("queryset", None)
if queryset is None:
from labels.models import LabeledResource
queryset = LabeledResource.objects.all()
kwargs = {**kwargs}
read_only = kwargs.get("read_only", False)
if not read_only:
kwargs["queryset"] = queryset
super().__init__(**kwargs)
def to_representation(self, value):
if value is None:
return value
return str(value.label)
def to_internal_value(self, data):
from labels.models import LabeledResource, Label
if data is None:
return data
k, v = data.split(":", 1)
label, __ = Label.objects.get_or_create(name=k, value=v, defaults={'name': k, 'value': v})
return LabeledResource(label=label)
class ObjectRelatedField(serializers.RelatedField):
default_error_messages = {
"required": _("This field is required."),

28
apps/common/serializers/mixin.py
View File

@ -7,6 +7,7 @@ else:
from collections import Iterable
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import NOT_PROVIDED
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from rest_framework.fields import SkipField, empty
@ -14,14 +15,13 @@ from rest_framework.settings import api_settings
from rest_framework.utils import html
from common.db.fields import EncryptMixin
from common.serializers.fields import EncryptedField, LabeledChoiceField, ObjectRelatedField
from common.serializers.fields import EncryptedField, LabeledChoiceField, ObjectRelatedField, LabelRelatedField
__all__ = [
'BulkSerializerMixin', 'BulkListSerializerMixin',
'CommonSerializerMixin', 'CommonBulkSerializerMixin',
'SecretReadableMixin', 'CommonModelSerializer',
'CommonBulkModelSerializer',
'CommonBulkModelSerializer', 'ResourceLabelsMixin',
]
@ -391,3 +391,25 @@ class CommonBulkSerializerMixin(BulkSerializerMixin, CommonSerializerMixin):
class CommonBulkModelSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer):
pass
class ResourceLabelsMixin(serializers.Serializer):
labels = LabelRelatedField(many=True, label=_('Labels'), )
def update(self, instance, validated_data):
labels = validated_data.pop('labels', None)
res = super().update(instance, validated_data)
if labels is not None:
instance.labels.set(labels, bulk=False)
return res
def create(self, validated_data):
labels = validated_data.pop('labels', None)
instance = super().create(validated_data)
if labels is not None:
instance.labels.set(labels, bulk=False)
return instance
@classmethod
def setup_eager_loading(cls, queryset):
return queryset.prefetch_related('labels')

4
apps/common/signal_handlers.py
View File

@ -66,6 +66,10 @@ def digest_sql_query():
for table_name, queries in table_queries.items():
if table_name.startswith('rbac_') or table_name.startswith('auth_permission'):
continue
for query in queries:
sql = query['sql']
print(" # {}: {}".format(query['time'], sql))
if len(queries) < 3:
continue
print("- Table: {}".format(table_name))

5
apps/jumpserver/conf.py
View File

@ -586,8 +586,9 @@ class Config(dict):
# FTP 文件上传下载备份阈值，单位(M)，当值小于等于0时，不备份
'FTP_FILE_MAX_STORE': 100,
# API 请求次数限制
'MAX_LIMIT_PER_PAGE': 100,
# API 分页
'MAX_LIMIT_PER_PAGE': 10000,
'DEFAULT_PAGE_SIZE': None,
'LIMIT_SUPER_PRIV': False,

7
apps/jumpserver/rewriting/pagination.py
View File

@ -11,3 +11,10 @@ class MaxLimitOffsetPagination(LimitOffsetPagination):
return queryset.values_list('id').order_by().count()
except (AttributeError, TypeError, FieldError):
return len(queryset)
def paginate_queryset(self, queryset, request, view=None):
if view and hasattr(view, 'page_max_limit'):
self.max_limit = view.page_max_limit
if view and hasattr(view, 'page_default_limit'):
self.default_limit = view.page_default_limit
return super().paginate_queryset(queryset, request, view)

4
apps/jumpserver/settings/base.py
View File

@ -109,6 +109,7 @@ for host_port in ALLOWED_DOMAINS:
continue
CSRF_TRUSTED_ORIGINS.append('{}://*.{}'.format(schema, origin))
CORS_ALLOWED_ORIGINS = [o.replace('*.', '') for o in CSRF_TRUSTED_ORIGINS]
CSRF_FAILURE_VIEW = 'jumpserver.views.other.csrf_failure'
# print("CSRF_TRUSTED_ORIGINS: ")
# for origin in CSRF_TRUSTED_ORIGINS:
@ -134,6 +135,7 @@ INSTALLED_APPS = [
'acls.apps.AclsConfig',
'notifications.apps.NotificationsConfig',
'rbac.apps.RBACConfig',
'labels.apps.LabelsConfig',
'rest_framework',
'rest_framework_swagger',
'drf_yasg',
@ -142,6 +144,7 @@ INSTALLED_APPS = [
'django_filters',
'bootstrap3',
'captcha',
'corsheaders',
'private_storage',
'django_celery_beat',
'django.contrib.auth',
@ -160,6 +163,7 @@ MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.locale.LocaleMiddleware',
'corsheaders.middleware.CorsMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',

1
apps/jumpserver/settings/custom.py
View File

@ -207,6 +207,7 @@ SESSION_RSA_PUBLIC_KEY_NAME = 'jms_public_key'
OPERATE_LOG_ELASTICSEARCH_CONFIG = CONFIG.OPERATE_LOG_ELASTICSEARCH_CONFIG
MAX_LIMIT_PER_PAGE = CONFIG.MAX_LIMIT_PER_PAGE
DEFAULT_PAGE_SIZE = CONFIG.DEFAULT_PAGE_SIZE
# Magnus DB Port
MAGNUS_ORACLE_PORTS = CONFIG.MAGNUS_ORACLE_PORTS

1
apps/jumpserver/settings/libs.py
View File

@ -46,6 +46,7 @@ REST_FRAMEWORK = {
'DATETIME_FORMAT': '%Y/%m/%d %H:%M:%S %z',
'DATETIME_INPUT_FORMATS': ['%Y/%m/%d %H:%M:%S %z', 'iso-8601', '%Y-%m-%d %H:%M:%S %z'],
'DEFAULT_PAGINATION_CLASS': 'jumpserver.rewriting.pagination.MaxLimitOffsetPagination',
'PAGE_SIZE': CONFIG.DEFAULT_PAGE_SIZE,
'EXCEPTION_HANDLER': 'common.drf.exc_handlers.common_exception_handler',
}

1
apps/jumpserver/urls.py
View File

@ -28,6 +28,7 @@ api_v1 = [
path('acls/', include('acls.urls.api_urls', namespace='api-acls')),
path('notifications/', include('notifications.urls.api_urls', namespace='api-notifications')),
path('rbac/', include('rbac.urls.api_urls', namespace='api-rbac')),
path('labels/', include('labels.urls', namespace='api-label')),
path('prometheus/metrics/', api.PrometheusMetricsApi.as_view()),
]

0
apps/labels/__init__.py Normal file
View File

3
apps/labels/admin.py Normal file
View File

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

141
apps/labels/api.py Normal file
View File

@ -0,0 +1,141 @@
from django.shortcuts import get_object_or_404
from rest_framework.decorators import action
from rest_framework.response import Response
from common.api.generic import JMSModelViewSet
from common.utils import is_true
from orgs.mixins.api import OrgBulkModelViewSet
from orgs.mixins.models import OrgModelMixin
from orgs.utils import current_org
from rbac.models import ContentType
from rbac.serializers import ContentTypeSerializer
from . import serializers
from .models import Label, LabeledResource
__all__ = ['LabelViewSet']
class ContentTypeViewSet(JMSModelViewSet):
serializer_class = ContentTypeSerializer
http_method_names = ['get', 'head', 'options']
rbac_perms = {
'default': 'labels.view_contenttype',
'resources': 'labels.view_contenttype',
}
page_default_limit = None
can_labeled_content_type = []
model = ContentType
@classmethod
def get_can_labeled_content_type_ids(cls):
if cls.can_labeled_content_type:
return cls.can_labeled_content_type
content_types = ContentType.objects.all()
for ct in content_types:
model_cls = ct.model_class()
if not model_cls:
continue
if model_cls._meta.parents:
continue
if 'labels' in model_cls._meta._forward_fields_map.keys():
# if issubclass(model_cls, LabeledMixin):
cls.can_labeled_content_type.append(ct.id)
return cls.can_labeled_content_type
def get_queryset(self):
ids = self.get_can_labeled_content_type_ids()
queryset = ContentType.objects.filter(id__in=ids)
return queryset
@action(methods=['GET'], detail=True, serializer_class=serializers.ContentTypeResourceSerializer)
def resources(self, request, *args, **kwargs):
self.page_default_limit = 100
content_type = self.get_object()
model = content_type.model_class()
if issubclass(model, OrgModelMixin):
queryset = model.objects.filter(org_id=current_org.id)
else:
queryset = model.objects.all()
keyword = request.query_params.get('search')
if keyword:
queryset = content_type.filter_queryset(queryset, keyword)
return self.get_paginated_response_from_queryset(queryset)
class LabelContentTypeResourceViewSet(JMSModelViewSet):
serializer_class = serializers.ContentTypeResourceSerializer
rbac_perms = {
'default': 'labels.view_labeledresource',
'update': 'labels.change_labeledresource',
}
ordering_fields = ('res_type', 'date_created')
def get_queryset(self):
label_pk = self.kwargs.get('label')
res_type = self.kwargs.get('res_type')
label = get_object_or_404(Label, pk=label_pk)
content_type = get_object_or_404(ContentType, id=res_type)
bound = self.request.query_params.get('bound', '1')
res_ids = LabeledResource.objects.filter(res_type=content_type, label=label) \
.values_list('res_id', flat=True)
res_ids = set(res_ids)
model = content_type.model_class()
if is_true(bound):
queryset = model.objects.filter(id__in=list(res_ids))
else:
queryset = model.objects.exclude(id__in=list(res_ids))
keyword = self.request.query_params.get('search')
if keyword:
queryset = content_type.filter_queryset(queryset, keyword)
return queryset
def put(self, request, *args, **kwargs):
label_pk = self.kwargs.get('label')
res_type = self.kwargs.get('res_type')
content_type = get_object_or_404(ContentType, id=res_type)
label = get_object_or_404(Label, pk=label_pk)
res_ids = request.data.get('res_ids', [])
LabeledResource.objects \
.filter(res_type=content_type, label=label) \
.exclude(res_id__in=res_ids).delete()
resources = []
for res_id in res_ids:
resources.append(LabeledResource(res_type=content_type, res_id=res_id, label=label, org_id=current_org.id))
LabeledResource.objects.bulk_create(resources, ignore_conflicts=True)
return Response({"total": len(res_ids)})
class LabelViewSet(OrgBulkModelViewSet):
model = Label
filterset_fields = ("name", "value")
search_fields = filterset_fields
serializer_classes = {
'default': serializers.LabelSerializer,
'resource_types': ContentTypeSerializer,
}
rbac_perms = {
'resource_types': 'labels.view_label',
'keys': 'labels.view_label',
}
@action(methods=['GET'], detail=False)
def keys(self, request, *args, **kwargs):
queryset = Label.objects.all()
keyword = request.query_params.get('search')
if keyword:
queryset = queryset.filter(name__icontains=keyword)
keys = queryset.values_list('name', flat=True).distinct()
return Response(keys)
class LabeledResourceViewSet(OrgBulkModelViewSet):
model = LabeledResource
filterset_fields = ("label__name", "label__value", "res_type", "res_id", "label")
search_fields = filterset_fields
serializer_classes = {
'default': serializers.LabeledResourceSerializer,
}
ordering_fields = ('res_type', 'date_created')

6
apps/labels/apps.py Normal file
View File

@ -0,0 +1,6 @@
from django.apps import AppConfig
class LabelsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'labels'

0
apps/labels/const.py Normal file
View File

54
apps/labels/migrations/0001_initial.py Normal file
View File

@ -0,0 +1,54 @@
# Generated by Django 4.1.10 on 2023-11-06 10:38
from django.db import migrations, models
import django.db.models.deletion
import uuid
class Migration(migrations.Migration):
initial = True
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
]
operations = [
migrations.CreateModel(
name='Label',
fields=[
('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')),
('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')),
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('internal', models.BooleanField(default=False, verbose_name='Internal')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('name', models.CharField(db_index=True, max_length=64, verbose_name='Name')),
('value', models.CharField(max_length=64, verbose_name='Value')),
],
options={
'verbose_name': 'Label',
'unique_together': {('name', 'value', 'org_id')},
},
),
migrations.CreateModel(
name='LabeledResource',
fields=[
('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')),
('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')),
('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
('res_id', models.CharField(db_index=True, max_length=36, verbose_name='Resource ID')),
('label', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='labels.label')),
('res_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')),
],
options={
'abstract': False,
},
),
]

52
apps/labels/migrations/0002_auto_20231103_1659.py Normal file
View File

@ -0,0 +1,52 @@
# Generated by Django 4.1.10 on 2023-11-03 08:59
from django.db import migrations
def migrate_assets_labels(apps, schema_editor):
old_label_model = apps.get_model('assets', 'Label')
new_label_model = apps.get_model('labels', 'Label')
asset_model = apps.get_model('assets', 'Asset')
labeled_item_model = apps.get_model('labels', 'LabeledResource')
old_labels = old_label_model.objects.all()
new_labels = []
old_new_label_map = {}
for label in old_labels:
new_label = new_label_model(name=label.name, value=label.value, org_id=label.org_id)
old_new_label_map[label.id] = new_label
new_labels.append(new_label)
new_label_model.objects.bulk_create(new_labels, ignore_conflicts=True)
label_relations = asset_model.labels.through.objects.all()
bulk_size = 1000
count = 0
content_type = apps.get_model('contenttypes', 'contenttype').objects.get_for_model(asset_model)
while True:
relations = label_relations[count:count + bulk_size]
if not relations:
break
count += bulk_size
tagged_items = []
for relation in relations:
new_label = old_new_label_map[relation.label_id]
tagged_item = labeled_item_model(
label_id=new_label.id, res_type=content_type,
res_id=relation.asset_id, org_id=new_label.org_id
)
tagged_items.append(tagged_item)
labeled_item_model.objects.bulk_create(tagged_items, ignore_conflicts=True)
class Migration(migrations.Migration):
dependencies = [
('labels', '0001_initial'),
('assets', '0125_auto_20231011_1053')
]
operations = [
migrations.RunPython(migrate_assets_labels),
]

28
apps/labels/migrations/0003_alter_labeledresource_options_and_more.py Normal file
View File

@ -0,0 +1,28 @@
# Generated by Django 4.1.10 on 2023-11-15 10:58
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('contenttypes', '0002_remove_content_type_name'),
('labels', '0002_auto_20231103_1659'),
]
operations = [
migrations.AlterModelOptions(
name='labeledresource',
options={'verbose_name': 'Labeled resource'},
),
migrations.AlterField(
model_name='labeledresource',
name='label',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='labeled_resources', to='labels.label'),
),
migrations.AlterUniqueTogether(
name='labeledresource',
unique_together={('label', 'res_type', 'res_id', 'org_id')},
),
]

0
apps/labels/migrations/__init__.py Normal file
View File

13
apps/labels/mixins.py Normal file
View File

@ -0,0 +1,13 @@
from django.contrib.contenttypes.fields import GenericRelation
from django.db import models
from .models import LabeledResource
__all__ = ['LabeledMixin']
class LabeledMixin(models.Model):
labels = GenericRelation(LabeledResource, object_id_field='res_id', content_type_field='res_type')
class Meta:
abstract = True

38
apps/labels/models.py Normal file
View File

@ -0,0 +1,38 @@
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.translation import gettext_lazy as _
from common.utils import lazyproperty
from orgs.mixins.models import JMSOrgBaseModel
class Label(JMSOrgBaseModel):
name = models.CharField(max_length=64, verbose_name=_("Name"), db_index=True)
value = models.CharField(max_length=64, unique=False, verbose_name=_("Value"))
internal = models.BooleanField(default=False, verbose_name=_("Internal"))
class Meta:
unique_together = [('name', 'value', 'org_id')]
verbose_name = _('Label')
@lazyproperty
def res_count(self):
return self.labeled_resources.count()
def __str__(self):
return '{}:{}'.format(self.name, self.value)
class LabeledResource(JMSOrgBaseModel):
label = models.ForeignKey(Label, on_delete=models.CASCADE, related_name='labeled_resources')
res_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
res_id = models.CharField(max_length=36, verbose_name=_("Resource ID"), db_index=True)
resource = GenericForeignKey('res_type', 'res_id')
class Meta:
unique_together = [('label', 'res_type', 'res_id', 'org_id')]
verbose_name = _('Labeled resource')
def __str__(self):
return '{} => {}'.format(self.label, self.resource)

54
apps/labels/serializers.py Normal file
View File

@ -0,0 +1,54 @@
from django.contrib.contenttypes.models import ContentType
from django.db.models import Count
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers.fields import ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from .models import Label, LabeledResource
__all__ = ['LabelSerializer', 'LabeledResourceSerializer', 'ContentTypeResourceSerializer']
class LabelSerializer(BulkOrgResourceModelSerializer):
class Meta:
model = Label
fields = ['id', 'name', 'value', 'res_count', 'date_created', 'date_updated']
read_only_fields = ('date_created', 'date_updated', 'res_count')
extra_kwargs = {
'res_count': {'label': _('Resource count')},
}
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.annotate(res_count=Count('labeled_resources'))
return queryset
class LabeledResourceSerializer(serializers.ModelSerializer):
res_type = ObjectRelatedField(
queryset=ContentType.objects, attrs=('app_label', 'model', 'name'), label=_("Resource type")
)
label = ObjectRelatedField(queryset=Label.objects, attrs=('name', 'value'))
resource = serializers.CharField(label=_("Resource"))
class Meta:
model = LabeledResource
fields = ('id', 'label', 'res_type', 'res_id', 'date_created', 'resource', 'date_updated')
read_only_fields = ('date_created', 'date_updated', 'resource')
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.select_related('label', 'res_type')
return queryset
class ContentTypeResourceSerializer(serializers.Serializer):
id = serializers.CharField()
name = serializers.SerializerMethodField()
@staticmethod
def get_name(obj):
return str(obj)

3
apps/labels/tests.py Normal file
View File

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

21
apps/labels/urls.py Normal file
View File

@ -0,0 +1,21 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
from rest_framework_bulk.routes import BulkRouter
from . import api
app_name = 'labels'
router = BulkRouter()
router.register(r'labels', api.LabelViewSet, 'label')
router.register(r'labels/(?P<label>.*)/resource-types/(?P<res_type>.*)/resources',
api.LabelContentTypeResourceViewSet, 'label-content-type-resource')
router.register(r'labeled-resources', api.LabeledResourceViewSet, 'labeled-resource')
router.register(r'resource-types', api.ContentTypeViewSet, 'content-type')
urlpatterns = [
]
urlpatterns += router.urls

3
apps/labels/views.py Normal file
View File

@ -0,0 +1,3 @@
from django.shortcuts import render
# Create your views here.

4
apps/locale/ja/LC_MESSAGES/django.mo
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:185acf00dbd3e3ef37e9faf300c438e120f257fc876e72135cecb1234dd5e453
size 166353
oid sha256:790917753a2bc455aaa6a74322b18f7cbdb7ba860f4c08c46263e7762fb3fbe7
size 165198

803
apps/locale/ja/LC_MESSAGES/django.po
View File

File diff suppressed because it is too large Load Diff

4
apps/locale/zh/LC_MESSAGES/django.mo
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e4af35b685fe0eb73e4dd9790669164bb823065d994a24f5441c9cff7a6e99e7
size 135811
oid sha256:a46c7aff5f314cbab9849e4c11671f81b849f98fad73887ced5b2012635b8a97
size 135290

792
apps/locale/zh/LC_MESSAGES/django.po
View File

File diff suppressed because it is too large Load Diff

3
apps/ops/models/job.py
View File

@ -22,6 +22,7 @@ from acls.models import CommandFilterACL
from assets.models import Asset
from assets.automations.base.manager import SSHTunnelManager
from common.db.encoder import ModelJSONFieldEncoder
from labels.mixins import LabeledMixin
from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner, CommandInBlackListException
from ops.mixin import PeriodTaskModelMixin
from ops.variables import *
@ -132,7 +133,7 @@ class JMSPermedInventory(JMSInventory):
return mapper
class Job(JMSOrgBaseModel, PeriodTaskModelMixin):
class Job(LabeledMixin, JMSOrgBaseModel, PeriodTaskModelMixin):
name = models.CharField(max_length=128, null=True, verbose_name=_('Name'))
instant = models.BooleanField(default=False)

3
apps/ops/models/playbook.py
View File

@ -6,6 +6,7 @@ from django.db import models
from django.utils.translation import gettext_lazy as _
from private_storage.fields import PrivateFileField
from labels.mixins import LabeledMixin
from ops.const import CreateMethods
from ops.exception import PlaybookNoValidEntry
from orgs.mixins.models import JMSOrgBaseModel
@ -23,7 +24,7 @@ dangerous_keywords = (
)
class Playbook(JMSOrgBaseModel):
class Playbook(LabeledMixin, JMSOrgBaseModel):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
name = models.CharField(max_length=128, verbose_name=_('Name'), null=True)
path = PrivateFileField(upload_to='playbooks/')

15
apps/ops/serializers/job.py
View File

@ -3,23 +3,21 @@ import uuid
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from assets.models import Node, Asset
from perms.models import PermNode
from perms.utils.user_perm import UserPermAssetUtil
from assets.models import Asset
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import ReadableHiddenField
from ops.mixin import PeriodTaskSerializerMixin
from ops.models import Job, JobExecution
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
class JobSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
run_after_save = serializers.BooleanField(label=_("Run after save"), default=False, required=False)
nodes = serializers.ListField(required=False, child=serializers.CharField())
date_last_run = serializers.DateTimeField(label=_('Date last run'), read_only=True)
name = serializers.CharField(label=_('Name'), max_length=128, allow_blank=True, required=False)
assets = serializers.PrimaryKeyRelatedField(label=_('Assets'), queryset=Asset.objects, many=True,
required=False)
assets = serializers.PrimaryKeyRelatedField(label=_('Assets'), queryset=Asset.objects, many=True, required=False)
def to_internal_value(self, data):
instant = data.get('instant', False)
@ -36,11 +34,12 @@ class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
class Meta:
model = Job
read_only_fields = [
"id", "date_last_run", "date_created", "date_updated", "average_time_cost"
"id", "date_last_run", "date_created",
"date_updated", "average_time_cost"
]
fields = read_only_fields + [
"name", "instant", "type", "module",
"args", "playbook", "assets",
"args", "playbook", "assets", "labels",
"runas_policy", "runas", "creator",
"use_parameter_define", "parameters_define",
"timeout", "chdir", "comment", "summary",

7
apps/ops/serializers/playbook.py
View File

@ -2,10 +2,10 @@ import os
from rest_framework import serializers
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import ReadableHiddenField
from ops.models import Playbook
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from django.utils.translation import gettext_lazy as _
def parse_playbook_name(path):
@ -13,7 +13,7 @@ def parse_playbook_name(path):
return file_name.split(".")[-2]
class PlaybookSerializer(BulkOrgResourceModelSerializer):
class PlaybookSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
path = serializers.FileField(required=False)
@ -27,5 +27,6 @@ class PlaybookSerializer(BulkOrgResourceModelSerializer):
model = Playbook
read_only_fields = ["id", "date_created", "date_updated"]
fields = read_only_fields + [
"id", 'path', "name", "comment", "creator", 'create_method', 'vcs_url',
"id", 'path', "name", "comment", "creator",
'create_method', 'vcs_url', 'labels'
]

3
apps/perms/models/asset_permission.py
View File

@ -10,6 +10,7 @@ from accounts.models import Account
from assets.models import Asset
from common.utils import date_expired_default
from common.utils.timezone import local_now
from labels.mixins import LabeledMixin
from orgs.mixins.models import JMSOrgBaseModel
from orgs.mixins.models import OrgManager
from perms.const import ActionChoices
@ -56,7 +57,7 @@ def default_protocols():
return ['all']
class AssetPermission(JMSOrgBaseModel):
class AssetPermission(LabeledMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'))
users = models.ManyToManyField(
'users.User', related_name='%(class)ss', blank=True, verbose_name=_("User")

7
apps/perms/serializers/permission.py
View File

@ -7,6 +7,7 @@ from rest_framework import serializers
from accounts.models import AccountTemplate, Account
from accounts.tasks import push_accounts_to_assets_task
from assets.models import Asset, Node
from common.serializers import ResourceLabelsMixin
from common.serializers.fields import BitChoicesField, ObjectRelatedField
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from perms.models import ActionChoices, AssetPermission
@ -26,7 +27,7 @@ class ActionChoicesField(BitChoicesField):
return data
class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
class AssetPermissionSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
users = ObjectRelatedField(queryset=User.objects, many=True, required=False, label=_('User'))
user_groups = ObjectRelatedField(
queryset=UserGroup.objects, many=True, required=False, label=_('User group')
@ -50,7 +51,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
"is_valid", "comment", "from_ticket",
]
fields_small = fields_mini + fields_generic
fields_m2m = ["users", "user_groups", "assets", "nodes"]
fields_m2m = ["users", "user_groups", "assets", "nodes", "labels"]
fields = fields_mini + fields_m2m + fields_generic
read_only_fields = ["created_by", "date_created", "from_ticket"]
extra_kwargs = {
@ -130,7 +131,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
"""Perform necessary eager loading of data."""
queryset = queryset.prefetch_related(
"users", "user_groups", "assets", "nodes",
)
).prefetch_related('labels', 'labels__label')
return queryset
@staticmethod

2
apps/rbac/api/__init__.py
View File

@ -1,4 +1,4 @@
from .content_type import *
from .permission import *
from .role import *
from .rolebinding import *

11
apps/rbac/api/content_type.py Normal file
View File

@ -0,0 +1,11 @@
from rest_framework import viewsets
from .. import serializers
from ..models import ContentType
class ContentTypeViewSet(viewsets.ModelViewSet):
serializer_class = serializers.ContentTypeSerializer
filterset_fields = ("app_label", "model",)
search_fields = filterset_fields
queryset = ContentType.objects.all()

2
apps/rbac/const.py
View File

@ -73,7 +73,7 @@ exclude_permissions = (
('perms', 'rebuildusertreetask', '*', '*'),
('perms', 'permedasset', '*', 'permedasset'),
('perms', 'permedapplication', 'add,change,delete', 'permedapplication'),
('rbac', 'contenttype', '*', '*'),
('rbac', 'contenttype', 'add,change,delete', '*'),
('rbac', 'permission', 'add,delete,change', 'permission'),
('rbac', 'rolebinding', '*', '*'),
('rbac', 'systemrolebinding', 'change', 'systemrolebinding'),

51
apps/rbac/models/permission.py
View File

@ -1,8 +1,10 @@
from django.apps import apps
from django.contrib.auth.models import ContentType as DjangoContentType
from django.contrib.auth.models import Permission as DjangoPermission
from django.db.models import Q
from django.utils.translation import gettext_lazy as _
from common.utils import lazyproperty
from .. import const
Scope = const.Scope
@ -14,10 +16,59 @@ class ContentType(DjangoContentType):
class Meta:
proxy = True
_apps_map = {}
@property
def app_model(self):
return '%s.%s' % (self.app_label, self.model)
@classmethod
def apps_map(cls):
from ..tree import app_nodes_data
if cls._apps_map:
return cls._apps_map
mapper = {}
for d in app_nodes_data:
i = d['id']
name = d.get('name')
if not name:
config = apps.get_app_config(d['id'])
if config:
name = config.verbose_name
if name:
mapper[i] = name
cls._apps_map = mapper
return mapper
@property
def app_display(self):
return self.apps_map().get(self.app_label)
@lazyproperty
def fields(self):
model = self.model_class()
return model._meta.fields
@lazyproperty
def field_names(self):
return [f.name for f in self.fields]
@lazyproperty
def filter_field_names(self):
names = []
if 'name' in self.field_names:
names.append('name')
if 'address' in self.field_names:
names.append('address')
return names
def filter_queryset(self, queryset, keyword):
q = Q()
for name in self.filter_field_names:
q |= Q(**{name + '__icontains': keyword})
return queryset.filter(q)
class Permission(DjangoPermission):
""" 权限类 """

2
apps/rbac/serializers/__init__.py
View File

@ -1,4 +1,4 @@
from .content_type import *
from .permission import *
from .role import *
from .rolebinding import *

13
apps/rbac/serializers/content_type.py Normal file
View File

@ -0,0 +1,13 @@
from rest_framework import serializers
from ..models import ContentType
__all__ = ['ContentTypeSerializer']
class ContentTypeSerializer(serializers.ModelSerializer):
app_display = serializers.CharField()
class Meta:
model = ContentType
fields = ('id', 'app_label', 'app_display', 'model', 'name')

2
apps/rbac/tree.py
View File

@ -39,7 +39,9 @@ app_nodes_data = [
{'id': 'rbac', 'view': 'view_console'},
{'id': 'settings', 'view': 'view_setting'},
{'id': 'tickets', 'view': 'view_other'},
{'id': 'labels', 'view': 'view_label'},
{'id': 'authentication', 'view': 'view_other'},
{'id': 'ops', 'view': 'view_workbench'},
]
# 额外其他节点，可以在不同的层次，需要指定父节点，可以将一些 model 归类到这个节点下面

2
apps/rbac/urls/api_urls.py
View File

@ -6,7 +6,6 @@ from .. import api
app_name = 'rbac'
router = BulkRouter()
router.register(r'roles', api.RoleViewSet, 'role')
router.register(r'role-bindings', api.RoleBindingViewSet, 'role-binding')
@ -18,6 +17,7 @@ router.register(r'org-roles', api.OrgRoleViewSet, 'org-role')
router.register(r'org-role-bindings', api.OrgRoleBindingViewSet, 'org-role-binding')
router.register(r'permissions', api.PermissionViewSet, 'permission')
router.register(r'content-types', api.ContentTypeViewSet, 'content-type')
system_role_router = routers.NestedDefaultRouter(router, r'system-roles', lookup='system_role')
system_role_router.register(r'permissions', api.SystemRolePermissionsViewSet, 'system-role-permission')

4
apps/users/api/user.py
View File

@ -52,10 +52,6 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, SuggestionMixin, BulkModelV
'bulk_remove': 'users.remove_user',
}
def get_queryset(self):
queryset = super().get_queryset().prefetch_related('groups')
return queryset
def allow_bulk_destroy(self, qs, filtered):
is_valid = filtered.count() < qs.count()
if not is_valid:

4
apps/users/models/group.py
View File

@ -1,15 +1,15 @@
# -*- coding: utf-8 -*-
from django.db import models
from django.utils.translation import gettext_lazy as _
from common.utils import lazyproperty
from labels.mixins import LabeledMixin
from orgs.mixins.models import JMSOrgBaseModel
__all__ = ['UserGroup']
class UserGroup(JMSOrgBaseModel):
class UserGroup(LabeledMixin, JMSOrgBaseModel):
name = models.CharField(max_length=128, verbose_name=_('Name'))
def __str__(self):

3
apps/users/models/user.py
View File

@ -24,6 +24,7 @@ from common.utils import (
date_expired_default, get_logger, lazyproperty,
random_string, bulk_create_with_signal
)
from labels.mixins import LabeledMixin
from orgs.utils import current_org
from rbac.const import Scope
from ..signals import (
@ -734,7 +735,7 @@ class JSONFilterMixin:
return models.Q(id__in=user_id)
class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, JSONFilterMixin, AbstractUser):
class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, LabeledMixin, JSONFilterMixin, AbstractUser):
class Source(models.TextChoices):
local = 'local', _('Local')
ldap = 'ldap', 'LDAP/AD'

10
apps/users/serializers/group.py
View File

@ -3,7 +3,8 @@
from django.db.models import Count
from django.utils.translation import gettext_lazy as _
from common.serializers.mixin import ObjectRelatedField
from common.serializers.fields import ObjectRelatedField
from common.serializers.mixin import ResourceLabelsMixin
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from .. import utils
from ..models import User, UserGroup
@ -13,7 +14,7 @@ __all__ = [
]
class UserGroupSerializer(BulkOrgResourceModelSerializer):
class UserGroupSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer):
users = ObjectRelatedField(
required=False, many=True, queryset=User.objects, label=_('User'),
)
@ -24,7 +25,7 @@ class UserGroupSerializer(BulkOrgResourceModelSerializer):
fields_small = fields_mini + [
'comment', 'date_created', 'created_by'
]
fields = fields_mini + fields_small + ['users']
fields = fields_mini + fields_small + ['users', 'labels']
extra_kwargs = {
'created_by': {'label': _('Created by'), 'read_only': True},
'users_amount': {'label': _('Users amount')},
@ -43,5 +44,6 @@ class UserGroupSerializer(BulkOrgResourceModelSerializer):
@classmethod
def setup_eager_loading(cls, queryset):
""" Perform necessary eager loading of data. """
queryset = queryset.prefetch_related('users').annotate(users_amount=Count('users'))
queryset = queryset.prefetch_related('users', 'labels', 'labels__label') \
.annotate(users_amount=Count('users'))
return queryset

2
apps/users/serializers/profile.py
View File

@ -143,9 +143,9 @@ class UserProfileSerializer(UserSerializer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
system_roles_field = self.fields.get('system_roles')
org_roles_field = self.fields.get('org_roles')
if system_roles_field:
system_roles_field.read_only = True
org_roles_field = self.fields.get('org_roles')
if org_roles_field:
org_roles_field.read_only = True

11
apps/users/serializers/user.py
View File

@ -6,7 +6,7 @@ from functools import partial
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from common.serializers import CommonBulkSerializerMixin
from common.serializers import CommonBulkSerializerMixin, ResourceLabelsMixin
from common.serializers.fields import (
EncryptedField, ObjectRelatedField, LabeledChoiceField, PhoneField
)
@ -81,7 +81,7 @@ class RolesSerializerMixin(serializers.Serializer):
return fields
class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, serializers.ModelSerializer):
class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, ResourceLabelsMixin, serializers.ModelSerializer):
password_strategy = LabeledChoiceField(
choices=PasswordStrategy.choices,
default=PasswordStrategy.email,
@ -143,7 +143,7 @@ class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, serializer
# 外键的字段
fields_fk = []
# 多对多字段
fields_m2m = ["groups", "system_roles", "org_roles", ]
fields_m2m = ["groups", "system_roles", "org_roles", "labels"]
# 在serializer 上定义的字段
fields_custom = ["login_blocked", "password_strategy"]
fields = fields_verbose + fields_fk + fields_m2m + fields_custom
@ -259,6 +259,11 @@ class UserSerializer(RolesSerializerMixin, CommonBulkSerializerMixin, serializer
)
return instance
@classmethod
def setup_eager_loading(cls, queryset):
queryset = queryset.prefetch_related('groups', 'labels', 'labels__label')
return queryset
class UserRetrieveSerializer(UserSerializer):
login_confirm_settings = serializers.PrimaryKeyRelatedField(

19
poetry.lock generated
View File

@ -1919,6 +1919,25 @@ type = "legacy"
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
reference = "tsinghua"
[[package]]
name = "django-cors-headers"
version = "4.3.0"
description = "django-cors-headers is a Django application for handling the server headers required for Cross-Origin Resource Sharing (CORS)."
optional = false
python-versions = ">=3.8"
files = [
{file = "django_cors_headers-4.3.0-py3-none-any.whl", hash = "sha256:bd36c7aea0d070e462f3383f0dc9ef717e5fdc2b10a99c98c285f16da84ffba2"},
{file = "django_cors_headers-4.3.0.tar.gz", hash = "sha256:25aabc94d4837678c1edf442c7f68a5f5fd151f6767b0e0b01c61a2179d02711"},
]
[package.dependencies]
Django = ">=3.2"
[package.source]
type = "legacy"
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
reference = "tsinghua"
[[package]]
name = "django-debug-toolbar"
version = "4.1.0"

1
pyproject.toml
View File

@ -142,6 +142,7 @@ channels-redis = "4.1.0"
fido2 = "^1.1.2"
ua-parser = "^0.18.0"
user-agents = "^2.2.0"
django-cors-headers = "^4.3.0"
mistune = "0.8.4"
openai = "^1.3.7"