diff --git a/apps/assets/const/protocol.py b/apps/assets/const/protocol.py index d7367b85d..92ca5f3a1 100644 --- a/apps/assets/const/protocol.py +++ b/apps/assets/const/protocol.py @@ -20,6 +20,7 @@ class Protocol(ChoicesMixin, models.TextChoices): k8s = 'k8s', 'K8S' http = 'http', 'HTTP' + _settings = None @classmethod def device_protocols(cls): @@ -106,7 +107,7 @@ class Protocol(ChoicesMixin, models.TextChoices): @classmethod def settings(cls): return { - **cls.device_protocols(), - **cls.database_protocols(), - **cls.cloud_protocols() + **cls.device_protocols(), + **cls.database_protocols(), + **cls.cloud_protocols() } diff --git a/apps/assets/models/account.py b/apps/assets/models/account.py index 3159cc3b6..65e6c74f0 100644 --- a/apps/assets/models/account.py +++ b/apps/assets/models/account.py @@ -34,12 +34,8 @@ class Account(BaseAccount): ] @lazyproperty - def ip(self): - return self.asset.address - - @lazyproperty - def asset_name(self): - return self.asset.name + def platform(self): + return self.asset.platform def __str__(self): return '{}@{}'.format(self.username, self.asset.name) diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 429a4a0a5..7f96a07e9 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -73,11 +73,17 @@ class BaseAccount(OrgModelMixin): created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) @property - def public_key(self): - return '' + def has_secret(self): + return bool(self.secret) @property def private_key(self): + if self.secret_type == self.SecretType.ssh_key: + return self.secret + return None + + @property + def public_key(self): return '' @private_key.setter @@ -94,10 +100,6 @@ class BaseAccount(OrgModelMixin): self.secret = value self.secret_type = 'password' - def expire_assets_amount(self): - cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id) - cache.delete(cache_key) - @property def ssh_key_fingerprint(self): if self.public_key: @@ -152,53 +154,15 @@ class BaseAccount(OrgModelMixin): pass return None - def set_auth(self, **kwargs): - update_fields = [] - for k, v in kwargs.items(): - setattr(self, k, v) - update_fields.append(k) - if update_fields: - self.save(update_fields=update_fields) - - def _merge_auth(self, other): - if other.password: - self.password = other.password - if other.public_key or other.private_key: - self.private_key = other.private_key - self.public_key = other.public_key - - def clear_auth(self): - self.password = '' - self.private_key = '' - self.public_key = '' - self.token = '' - self.save() - @staticmethod def gen_password(length=36): return random_string(length, special_char=True) @staticmethod def gen_key(username): - private_key, public_key = ssh_key_gen( - username=username - ) + private_key, public_key = ssh_key_gen(username=username) return private_key, public_key - def auto_gen_auth(self, password=True, key=True): - _password = None - _private_key = None - _public_key = None - - if password: - _password = self.gen_password() - if key: - _private_key, _public_key = self.gen_key(self.username) - self.set_auth( - password=_password, private_key=_private_key, - public_key=_public_key - ) - def _to_secret_json(self): """Push system user use it""" return { diff --git a/apps/assets/models/platform.py b/apps/assets/models/platform.py index c55a0f9b6..30f377b00 100644 --- a/apps/assets/models/platform.py +++ b/apps/assets/models/platform.py @@ -4,6 +4,8 @@ from django.utils.translation import gettext_lazy as _ from assets.const import AllTypes from common.db.fields import JsonDictTextField +from assets.const import Protocol + __all__ = ['Platform', 'PlatformProtocol', 'PlatformAutomation'] @@ -20,6 +22,13 @@ class PlatformProtocol(models.Model): setting = models.JSONField(verbose_name=_('Setting'), default=dict) platform = models.ForeignKey('Platform', on_delete=models.CASCADE, related_name='protocols') + def __str__(self): + return '{}/{}'.format(self.name, self.port) + + @property + def secret_types(self): + return Protocol.settings().get(self.name, {}).get('secret_types') + class PlatformAutomation(models.Model): ping_enabled = models.BooleanField(default=False, verbose_name=_("Ping enabled")) diff --git a/apps/assets/serializers/account/account.py b/apps/assets/serializers/account/account.py index eb37e3bd9..ecf8e8490 100644 --- a/apps/assets/serializers/account/account.py +++ b/apps/assets/serializers/account/account.py @@ -1,12 +1,10 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from orgs.mixins.serializers import BulkOrgResourceModelSerializer from common.drf.serializers import SecretReadableMixin from common.drf.fields import ObjectRelatedField from assets.models import Account, AccountTemplate, Asset -from assets.serializers.base import AuthValidateMixin -from .common import AccountFieldsSerializerMixin +from .base import BaseAccountSerializer class AccountSerializerCreateMixin(serializers.ModelSerializer): @@ -28,7 +26,8 @@ class AccountSerializerCreateMixin(serializers.ModelSerializer): @staticmethod def replace_attrs(account_template: AccountTemplate, attrs: dict): exclude_fields = [ - '_state', 'org_id', 'id', 'date_created', 'date_updated' + '_state', 'org_id', 'id', 'date_created', + 'date_updated' ] template_attrs = { k: v for k, v in account_template.__dict__.items() @@ -52,34 +51,36 @@ class AccountSerializerCreateMixin(serializers.ModelSerializer): return instance -class AccountSerializer( - AuthValidateMixin, AccountSerializerCreateMixin, - AccountFieldsSerializerMixin, BulkOrgResourceModelSerializer -): +class AccountSerializer(AccountSerializerCreateMixin, BaseAccountSerializer): asset = ObjectRelatedField( required=False, queryset=Asset.objects, label=_('Asset'), attrs=('id', 'name', 'address') ) - platform = serializers.ReadOnlyField(label=_("Platform")) - class Meta(AccountFieldsSerializerMixin.Meta): + class Meta(BaseAccountSerializer.Meta): model = Account - fields = AccountFieldsSerializerMixin.Meta.fields \ + fields = BaseAccountSerializer.Meta.fields \ + + ['su_from', 'version', 'asset'] \ + ['template', 'push_now'] + extra_kwargs = { + **BaseAccountSerializer.Meta.extra_kwargs, + 'name': {'required': False, 'allow_null': True}, + } + + def __init__(self, *args, data=None, **kwargs): + super().__init__(*args, data=data, **kwargs) + if data and 'name' not in data: + data['name'] = data.get('username') @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('asset') + queryset = queryset.prefetch_related('asset', 'asset__platform') return queryset class AccountSecretSerializer(SecretReadableMixin, AccountSerializer): class Meta(AccountSerializer.Meta): - fields_backup = [ - 'name', 'address', 'platform', 'protocols', 'username', 'password', - 'private_key', 'public_key', 'date_created', 'date_updated', 'version' - ] extra_kwargs = { 'password': {'write_only': False}, 'private_key': {'write_only': False}, diff --git a/apps/assets/serializers/account/base.py b/apps/assets/serializers/account/base.py new file mode 100644 index 000000000..262272a0a --- /dev/null +++ b/apps/assets/serializers/account/base.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +from io import StringIO + +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from common.utils import validate_ssh_private_key, ssh_private_key_gen +from common.drf.fields import EncryptedField +from orgs.mixins.serializers import BulkOrgResourceModelSerializer +from assets.models import BaseAccount + +__all__ = ['BaseAccountSerializer'] + + +class BaseAccountSerializer(BulkOrgResourceModelSerializer): + secret = EncryptedField( + label=_('Secret'), required=False, allow_blank=True, + allow_null=True, max_length=40960 + ) + + class Meta: + model = BaseAccount + fields_mini = ['id', 'name', 'username'] + fields_small = fields_mini + ['privileged', 'secret_type', 'secret', 'has_secret'] + fields_other = ['created_by', 'date_created', 'date_updated', 'comment'] + fields = fields_small + fields_other + extra_kwargs = { + 'secret': {'write_only': True}, + 'passphrase': {'write_only': True}, + } + + def validate_private_key(self, private_key): + if not private_key: + return '' + passphrase = self.initial_data.get('passphrase') + passphrase = passphrase if passphrase else None + valid = validate_ssh_private_key(private_key, password=passphrase) + if not valid: + raise serializers.ValidationError(_("private key invalid or passphrase error")) + + private_key = ssh_private_key_gen(private_key, password=passphrase) + string_io = StringIO() + private_key.write_private_key(string_io) + private_key = string_io.getvalue() + return private_key + + def validate_secret(self, value): + secret_type = self.initial_data.get('secret_type') + if secret_type == 'ssh_key': + value = self.validate_private_key(value) + return value + diff --git a/apps/assets/serializers/account/common.py b/apps/assets/serializers/account/common.py deleted file mode 100644 index dff27bbc0..000000000 --- a/apps/assets/serializers/account/common.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# -from rest_framework import serializers - -__all__ = ['AccountFieldsSerializerMixin'] - - -class AccountFieldsSerializerMixin(serializers.ModelSerializer): - class Meta: - fields_mini = [ - 'id', 'name', 'username', 'privileged', - 'platform', 'version', 'secret_type', - ] - fields_write_only = ['secret', 'passphrase'] - fields_other = ['date_created', 'date_updated', 'comment'] - fields_small = fields_mini + fields_write_only + fields_other - fields_fk = ['asset'] - fields = fields_small + fields_fk - extra_kwargs = { - 'secret': {'write_only': True}, - 'passphrase': {'write_only': True}, - 'token': {'write_only': True}, - 'password': {'write_only': True}, - } - - def validate_name(self, value): - if not value: - return self.initial_data.get('username') - return '' diff --git a/apps/assets/serializers/account/history.py b/apps/assets/serializers/account/history.py index 0a6c2042f..a49636334 100644 --- a/apps/assets/serializers/account/history.py +++ b/apps/assets/serializers/account/history.py @@ -1,17 +1,14 @@ from assets.models import Account from common.drf.serializers import SecretReadableMixin -from .common import AccountFieldsSerializerMixin +from .base import BaseAccountSerializer from .account import AccountSerializer, AccountSecretSerializer class AccountHistorySerializer(AccountSerializer): class Meta: model = Account.history.model - fields = AccountFieldsSerializerMixin.Meta.fields_mini + \ - AccountFieldsSerializerMixin.Meta.fields_write_only + \ - AccountFieldsSerializerMixin.Meta.fields_fk + \ - ['history_id', 'date_created', 'date_updated'] + fields = BaseAccountSerializer.Meta.fields_mini read_only_fields = fields ref_name = 'AccountHistorySerializer' diff --git a/apps/assets/serializers/account/template.py b/apps/assets/serializers/account/template.py index 1df7ea65a..09c5b4541 100644 --- a/apps/assets/serializers/account/template.py +++ b/apps/assets/serializers/account/template.py @@ -2,31 +2,16 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from assets.models import AccountTemplate -from orgs.mixins.serializers import BulkOrgResourceModelSerializer -from assets.serializers.base import AuthValidateMixin -from .common import AccountFieldsSerializerMixin +from .base import BaseAccountSerializer -class AccountTemplateSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer): - class Meta: +class AccountTemplateSerializer(BaseAccountSerializer): + class Meta(BaseAccountSerializer.Meta): model = AccountTemplate - fields_mini = ['id', 'name', 'username', 'privileged'] - fields_write_only = AccountFieldsSerializerMixin.Meta.fields_write_only - fields_other = AccountFieldsSerializerMixin.Meta.fields_other - fields = fields_mini + fields_write_only + fields_other - extra_kwargs = { - 'username': {'required': True}, - 'name': {'required': True}, - 'private_key': {'write_only': True}, - 'public_key': {'write_only': True}, - } - - def validate(self, attrs): - attrs = self._validate_gen_key(attrs) - return attrs @classmethod def validate_required(cls, attrs): + # Todo: why ? required_field_dict = {} error = _('This field is required.') for k, v in cls().fields.items(): diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index 2ec7c95bb..b2746493c 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # +from io import StringIO + from rest_framework import serializers from django.utils.translation import ugettext_lazy as _ from django.db.transaction import atomic @@ -7,6 +9,7 @@ from django.db.models import F from common.drf.serializers import JMSWritableNestedModelSerializer from common.drf.fields import LabeledChoiceField, ObjectRelatedField +from common.utils import validate_ssh_private_key, ssh_private_key_gen from ..account import AccountSerializer from ...models import Asset, Node, Platform, Label, Domain, Account, Protocol from ...const import Category, AllTypes @@ -47,15 +50,17 @@ class AssetAccountSerializer(AccountSerializer): class Meta(AccountSerializer.Meta): fields_mini = [ - 'id', 'name', 'username', 'privileged', 'version', - 'secret_type', + 'id', 'name', 'username', 'privileged', + 'version', 'secret_type', ] fields_write_only = [ - 'secret', 'passphrase', 'push_now' + 'secret', 'push_now' ] fields = fields_mini + fields_write_only + + class AssetSerializer(JMSWritableNestedModelSerializer): category = LabeledChoiceField(choices=Category.choices, read_only=True, label=_('Category')) type = LabeledChoiceField(choices=AllTypes.choices, read_only=True, label=_('Type')) diff --git a/apps/assets/serializers/platform.py b/apps/assets/serializers/platform.py index 19611b5d6..c592f73e5 100644 --- a/apps/assets/serializers/platform.py +++ b/apps/assets/serializers/platform.py @@ -4,7 +4,7 @@ from django.utils.translation import gettext_lazy as _ from common.drf.fields import LabeledChoiceField from common.drf.serializers import JMSWritableNestedModelSerializer from ..models import Platform, PlatformProtocol, PlatformAutomation -from ..const import Category, AllTypes +from ..const import Category, AllTypes, Protocol __all__ = ['PlatformSerializer', 'PlatformOpsMethodSerializer'] @@ -64,7 +64,7 @@ class PlatformProtocolsSerializer(serializers.ModelSerializer): class Meta: model = PlatformProtocol - fields = ['id', 'name', 'port', 'setting'] + fields = ['id', 'name', 'port', 'secret_types', 'setting'] class PlatformSerializer(JMSWritableNestedModelSerializer): diff --git a/apps/common/drf/serializers.py b/apps/common/drf/serializers.py index aa0023b90..4f2080c95 100644 --- a/apps/common/drf/serializers.py +++ b/apps/common/drf/serializers.py @@ -23,7 +23,6 @@ __all__ = [ class MethodSerializer(serializers.Serializer): - def __init__(self, method_name=None, **kwargs): self.method_name = method_name super().__init__(**kwargs) diff --git a/apps/common/mixins/serializers.py b/apps/common/mixins/serializers.py index 72b7610d4..93c0a2050 100644 --- a/apps/common/mixins/serializers.py +++ b/apps/common/mixins/serializers.py @@ -2,13 +2,18 @@ # from collections import Iterable -from django.db.models import Prefetch, F, NOT_PROVIDED +from django.db.models import NOT_PROVIDED from django.core.exceptions import ObjectDoesNotExist from rest_framework.utils import html from rest_framework.settings import api_settings from rest_framework.exceptions import ValidationError from rest_framework.fields import SkipField, empty -__all__ = ['BulkSerializerMixin', 'BulkListSerializerMixin', 'CommonSerializerMixin', 'CommonBulkSerializerMixin'] + + +__all__ = [ + 'BulkSerializerMixin', 'BulkListSerializerMixin', + 'CommonSerializerMixin', 'CommonBulkSerializerMixin' +] class BulkSerializerMixin(object): @@ -281,20 +286,12 @@ class DynamicFieldsMixin: self.fields.pop(field, None) -class EagerLoadQuerySetFields: - def setup_eager_loading(self, queryset): - """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related( - Prefetch('nodes'), - Prefetch('labels'), - ).select_related('admin_user', 'domain', 'platform') \ - .annotate(platform_base=F('platform__base')) - return queryset - - class CommonSerializerMixin(DynamicFieldsMixin, DefaultValueFieldsMixin): instance: None initial_data: dict + common_fields = [ + 'comment', 'created_by', 'date_created', 'date_updated', + ] def get_initial_value(self, attr, default=None): value = self.initial_data.get(attr) @@ -305,6 +302,12 @@ class CommonSerializerMixin(DynamicFieldsMixin, DefaultValueFieldsMixin): return value return default + def get_field_names(self, declared_fields, info): + names = super().get_field_names(declared_fields, info) + common_names = [i for i in self.common_fields if i in names] + primary_names = [i for i in names if i not in self.common_fields] + return primary_names + common_names + class CommonBulkSerializerMixin(BulkSerializerMixin, CommonSerializerMixin): pass diff --git a/apps/orgs/caches.py b/apps/orgs/caches.py index e9e5bc700..c3a1cb86d 100644 --- a/apps/orgs/caches.py +++ b/apps/orgs/caches.py @@ -52,8 +52,6 @@ class OrgResourceStatisticsCache(OrgRelatedCache): assets_amount = IntegerField() nodes_amount = IntegerField(queryset=Node.objects) - admin_users_amount = IntegerField() - system_users_amount = IntegerField() domains_amount = IntegerField(queryset=Domain.objects) gateways_amount = IntegerField(queryset=Gateway.objects) asset_perms_amount = IntegerField(queryset=AssetPermission.objects) diff --git a/apps/orgs/serializers.py b/apps/orgs/serializers.py index fadf1ef8d..c080332dd 100644 --- a/apps/orgs/serializers.py +++ b/apps/orgs/serializers.py @@ -11,8 +11,6 @@ class ResourceStatisticsSerializer(serializers.Serializer): assets_amount = serializers.IntegerField(required=False) nodes_amount = serializers.IntegerField(required=False) - admin_users_amount = serializers.IntegerField(required=False) - system_users_amount = serializers.IntegerField(required=False) domains_amount = serializers.IntegerField(required=False) gateways_amount = serializers.IntegerField(required=False) diff --git a/apps/rbac/builtin.py b/apps/rbac/builtin.py index c56326601..3bca6138f 100644 --- a/apps/rbac/builtin.py +++ b/apps/rbac/builtin.py @@ -164,8 +164,9 @@ class BuiltinRole: @classmethod def sync_to_db(cls, show_msg=False): roles = cls.get_roles() + print("\n Update builtin roles") for pre_role in roles.values(): role, created = pre_role.update_or_create_role() if show_msg: - print("Update builtin Role: {} - {}".format(role.name, created)) + print(" - Update: {} - {}".format(role.name, created))