feat: 账号密钥用vault储存 (#10830)

* feat: 账号密钥用vault储存

* perf: 优化 Vault

* perf: 重构 Vault Backend 设计架构 (未完成)

* perf: 重构 Vault Backend 设计架构 (未完成2)

* perf: 重构 Vault Backend 设计架构 (未完成3)

* perf: 重构 Vault Backend 设计架构 (未完成4)

* perf: 重构 Vault Backend 设计架构 (未完成5)

* perf: 重构 Vault Backend 设计架构 (已完成)

* perf: 重构 Vault Backend 设计架构 (已完成)

* perf: 重构 Vault Backend 设计架构 (已完成)

* perf: 小优化

* perf: 优化

---------

Co-authored-by: feng <1304903146@qq.com>
Co-authored-by: Bai <baijiangjie@gmail.com>
Co-authored-by: feng626 <57284900+feng626@users.noreply.github.com>

apps/accounts/api/account/account.py

@@ -26,6 +26,7 @@ class AccountViewSet(OrgBulkModelViewSet):
     filterset_class = AccountFilterSet
     serializer_classes = {
         'default': serializers.AccountSerializer,
+        'retrieve': serializers.AccountDetailSerializer,
     }
     rbac_perms = {
         'partial_update': ['accounts.change_account'],
@@ -133,11 +134,13 @@ class AccountHistoriesSecretAPI(ExtraFilterFieldsMixin, RecordViewLogMixin, List
     def get_queryset(self):
         account = self.get_object()
         histories = account.history.all()
-        last_history = account.history.first()
+        latest_history = account.history.first()
-        if not last_history:
+        if not latest_history:
+            return histories
+        if account.secret != latest_history.secret:
+            return histories
+        if account.secret_type != latest_history.secret_type:
+            return histories
+        histories = histories.exclude(history_id=latest_history.history_id)
         return histories
 
-        if account.secret == last_history.secret \
-                and account.secret_type == last_history.secret_type:
-            histories = histories.exclude(history_id=last_history.history_id)
-        return histories

apps/accounts/backends/__init__.py

@@ -0,0 +1,41 @@
+import os
+from django.utils.functional import LazyObject
+from importlib import import_module
+
+from common.utils import get_logger
+from ..const import VaultTypeChoices
+
+__all__ = ['vault_client', 'get_vault_client']
+
+
+logger = get_logger(__file__)
+
+
+def get_vault_client(raise_exception=False, **kwargs):
+    try:
+        tp = kwargs.get('VAULT_TYPE')
+        module_path = f'apps.accounts.backends.{tp}.main'
+        client = import_module(module_path).Vault(**kwargs)
+    except Exception as e:
+        logger.error(f'Init vault client failed: {e}')
+        if raise_exception:
+            raise
+        tp = VaultTypeChoices.local
+        module_path = f'apps.accounts.backends.{tp}.main'
+        kwargs['VAULT_TYPE'] = tp
+        client = import_module(module_path).Vault(**kwargs)
+    return client
+
+
+class VaultClient(LazyObject):
+
+    def _setup(self):
+        from jumpserver import settings as js_settings
+        from django.conf import settings
+        vault_config_names = [k for k in js_settings.__dict__.keys() if k.startswith('VAULT_')]
+        vault_configs = {name: getattr(settings, name, None) for name in vault_config_names}
+        self._wrapped = get_vault_client(**vault_configs)
+
+
+""" 为了安全, 页面修改配置, 重启服务后才会重新初始化 vault_client """
+vault_client = VaultClient()

apps/accounts/backends/base.py

@@ -0,0 +1,77 @@
+from abc import ABC, abstractmethod
+
+from django.forms.models import model_to_dict
+
+__all__ = ['BaseVault']
+
+
+class BaseVault(ABC):
+
+    def __init__(self, *args, **kwargs):
+        self.type = kwargs.get('VAULT_TYPE')
+
+    def is_type(self, tp):
+        return self.type == tp
+
+    def get(self, instance):
+        """ 返回 secret 值 """
+        return self._get(instance)
+
+    def create(self, instance):
+        if not instance.secret_has_save_to_vault:
+            self._create(instance)
+            self._clean_db_secret(instance)
+            self.save_metadata(instance)
+
+        if instance.is_sync_metadata:
+            self.save_metadata(instance)
+
+    def update(self, instance):
+        if not instance.secret_has_save_to_vault:
+            self._update(instance)
+            self._clean_db_secret(instance)
+            self.save_metadata(instance)
+
+        if instance.is_sync_metadata:
+            self.save_metadata(instance)
+
+    def delete(self, instance):
+        self._delete(instance)
+
+    def save_metadata(self, instance):
+        metadata = model_to_dict(instance, fields=[
+            'name', 'username', 'secret_type',
+            'connectivity', 'su_from', 'privileged'
+        ])
+        metadata = {field: str(value) for field, value in metadata.items()}
+        return self._save_metadata(instance, metadata)
+
+    # -------- abstractmethod -------- #
+
+    @abstractmethod
+    def _get(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _create(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _update(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _delete(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _clean_db_secret(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _save_metadata(self, instance, metadata):
+        raise NotImplementedError
+
+    @abstractmethod
+    def is_active(self, *args, **kwargs) -> (bool, str):
+        raise NotImplementedError

apps/accounts/backends/hcp/__init__.py

@@ -0,0 +1 @@
+from .main import *

apps/accounts/backends/hcp/entries.py

@@ -0,0 +1,84 @@
+import sys
+from abc import ABC
+
+from common.db.utils import Encryptor
+from common.utils import lazyproperty
+
+current_module = sys.modules[__name__]
+
+__all__ = ['build_entry']
+
+
+class BaseEntry(ABC):
+
+    def __init__(self, instance):
+        self.instance = instance
+
+    @lazyproperty
+    def full_path(self):
+        path_base = self.path_base
+        path_spec = self.path_spec
+        path = f'{path_base}/{path_spec}'
+        return path
+
+    @property
+    def path_base(self):
+        path = f'orgs/{self.instance.org_id}'
+        return path
+
+    @property
+    def path_spec(self):
+        raise NotImplementedError
+
+    def to_internal_data(self):
+        secret = getattr(self.instance, '_secret', None)
+        if secret is not None:
+            secret = Encryptor(secret).encrypt()
+        data = {'secret': secret}
+        return data
+
+    @staticmethod
+    def to_external_data(data):
+        secret = data.pop('secret', None)
+        if secret is not None:
+            secret = Encryptor(secret).decrypt()
+        return secret
+
+
+class AccountEntry(BaseEntry):
+
+    @property
+    def path_spec(self):
+        path = f'assets/{self.instance.asset_id}/accounts/{self.instance.id}'
+        return path
+
+
+class AccountTemplateEntry(BaseEntry):
+
+    @property
+    def path_spec(self):
+        path = f'account-templates/{self.instance.id}'
+        return path
+
+
+class HistoricalAccountEntry(BaseEntry):
+
+    @property
+    def path_base(self):
+        account = self.instance.instance
+        path = f'accounts/{account.id}/'
+        return path
+
+    @property
+    def path_spec(self):
+        path = f'histories/{self.instance.history_id}'
+        return path
+
+
+def build_entry(instance) -> BaseEntry:
+    class_name = instance.__class__.__name__
+    entry_class_name = f'{class_name}Entry'
+    entry_class = getattr(current_module, entry_class_name, None)
+    if not entry_class:
+        raise Exception(f'Entry class {entry_class_name} is not found')
+    return entry_class(instance)

apps/accounts/backends/hcp/main.py

@@ -0,0 +1,47 @@
+from .entries import build_entry
+from .service import VaultKVClient
+from ..base import BaseVault
+
+__all__ = ['Vault']
+
+
+class Vault(BaseVault):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = VaultKVClient(
+            url=kwargs.get('VAULT_HCP_HOST'),
+            token=kwargs.get('VAULT_HCP_TOKEN'),
+            mount_point=kwargs.get('VAULT_HCP_MOUNT_POINT')
+        )
+
+    def is_active(self):
+        return self.client.is_active()
+
+    def _get(self, instance):
+        entry = build_entry(instance)
+        # TODO: get data 是不是层数太多了
+        data = self.client.get(path=entry.full_path).get('data', {})
+        data = entry.to_external_data(data)
+        return data
+
+    def _create(self, instance):
+        entry = build_entry(instance)
+        data = entry.to_internal_data()
+        self.client.create(path=entry.full_path, data=data)
+
+    def _update(self, instance):
+        entry = build_entry(instance)
+        data = entry.to_internal_data()
+        self.client.patch(path=entry.full_path, data=data)
+
+    def _delete(self, instance):
+        entry = build_entry(instance)
+        self.client.delete(path=entry.full_path)
+
+    def _clean_db_secret(self, instance):
+        instance.is_sync_metadata = False
+        instance.mark_secret_save_to_vault()
+
+    def _save_metadata(self, instance, metadata):
+        entry = build_entry(instance)
+        self.client.update_metadata(path=entry.full_path, metadata=metadata)

apps/accounts/backends/hcp/service.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+#
+import hvac
+from hvac import exceptions
+from requests.exceptions import ConnectionError
+
+from common.utils import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = ['VaultKVClient']
+
+
+class VaultKVClient(object):
+    max_versions = 20
+
+    def __init__(self, url, token, mount_point):
+        assert isinstance(self.max_versions, int) and self.max_versions >= 3, (
+            'max_versions must to be an integer that is greater than or equal to 3'
+        )
+        self.client = hvac.Client(url=url, token=token)
+        self.mount_point = mount_point
+        self.enable_secrets_engine_if_need()
+
+    def is_active(self):
+        try:
+            if not self.client.sys.is_initialized():
+                return False, 'Vault is not initialized'
+            if self.client.sys.is_sealed():
+                return False, 'Vault is sealed'
+            if not self.client.is_authenticated():
+                return False, 'Vault is not authenticated'
+        except ConnectionError as e:
+            logger.error(str(e))
+            return False, f'Vault is not reachable: {e}'
+        else:
+            return True, ''
+
+    def enable_secrets_engine_if_need(self):
+        secrets_engines = self.client.sys.list_mounted_secrets_engines()
+        mount_points = secrets_engines.keys()
+        if f'{self.mount_point}/' in mount_points:
+            return
+        self.client.sys.enable_secrets_engine(
+            backend_type='kv',
+            path=self.mount_point,
+            options={'version': 2}  # TODO: version 是否从配置中读取?
+        )
+        self.client.secrets.kv.v2.configure(
+            max_versions=self.max_versions,
+            mount_point=self.mount_point
+        )
+
+    def get(self, path, version=None):
+        try:
+            response = self.client.secrets.kv.v2.read_secret_version(
+                path=path,
+                version=version,
+                mount_point=self.mount_point
+            )
+        except exceptions.InvalidPath as e:
+            logger.error('Get secret error: {}'.format(e))
+            return {}
+        data = response.get('data', {})
+        return data
+
+    def create(self, path, data: dict):
+        self._update_or_create(path=path, data=data)
+
+    def update(self, path, data: dict):
+        """ 未更新的数据会被删除 """
+        self._update_or_create(path=path, data=data)
+
+    def patch(self, path, data: dict):
+        """ 未更新的数据不会被删除 """
+        self.client.secrets.kv.v2.patch(
+            path=path,
+            secret=data,
+            mount_point=self.mount_point
+        )
+
+    def delete(self, path):
+        self.client.secrets.kv.v2.delete_metadata_and_all_versions(
+            path=path,
+            mount_point=self.mount_point,
+        )
+
+    def _update_or_create(self, path, data: dict):
+        self.client.secrets.kv.v2.create_or_update_secret(
+            path=path,
+            secret=data,
+            mount_point=self.mount_point
+        )
+
+    def update_metadata(self, path, metadata: dict):
+        try:
+            self.client.secrets.kv.v2.update_metadata(
+                path=path,
+                mount_point=self.mount_point,
+                custom_metadata=metadata
+            )
+        except exceptions.InvalidPath as e:
+            logger.error('Update metadata error: {}'.format(e))

apps/accounts/backends/local/__init__.py

@@ -0,0 +1 @@
+from .main import *

apps/accounts/backends/local/main.py

@@ -0,0 +1,36 @@
+from common.utils import get_logger
+from ..base import BaseVault
+
+logger = get_logger(__name__)
+
+__all__ = ['Vault']
+
+
+class Vault(BaseVault):
+
+    def is_active(self):
+        return True, ''
+
+    def _get(self, instance):
+        secret = getattr(instance, '_secret', None)
+        return secret
+
+    def _create(self, instance):
+        """ Ignore """
+        pass
+
+    def _update(self, instance):
+        """ Ignore """
+        pass
+
+    def _delete(self, instance):
+        """ Ignore """
+        pass
+
+    def _save_metadata(self, instance, metadata):
+        """ Ignore """
+        pass
+
+    def _clean_db_secret(self, instance):
+        """ Ignore *重要* 不能删除本地 secret """
+        pass

apps/accounts/const/__init__.py

@@ -1,2 +1,3 @@
 from .account import *
 from .automation import *
+from .vault import *

apps/accounts/const/vault.py

@@ -0,0 +1,9 @@
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+__all__ = ['VaultTypeChoices']
+
+
+class VaultTypeChoices(models.TextChoices):
+    local = 'local', _('Local Vault')
+    hcp = 'hcp', _('HCP Vault')

apps/accounts/migrations/0012_auto_20230621_1456.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.19 on 2023-06-21 06:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0011_auto_20230506_1443'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='account',
+            old_name='secret',
+            new_name='_secret',
+        ),
+        migrations.RenameField(
+            model_name='accounttemplate',
+            old_name='secret',
+            new_name='_secret',
+        ),
+        migrations.RenameField(
+            model_name='historicalaccount',
+            old_name='secret',
+            new_name='_secret',
+        ),
+    ]

apps/accounts/models/account.py

@@ -7,9 +7,10 @@ from simple_history.models import HistoricalRecords
 from assets.models.base import AbsConnectivity
 from common.utils import lazyproperty
 from .base import BaseAccount
+from .mixins import VaultModelMixin
 from ..const import AliasAccount, Source
 
-__all__ = ['Account', 'AccountTemplate']
+__all__ = ['Account', 'AccountTemplate', 'AccountHistoricalRecords']
 
 
 class AccountHistoricalRecords(HistoricalRecords):
@@ -32,7 +33,7 @@ class AccountHistoricalRecords(HistoricalRecords):
         diff = attrs - history_attrs
         if not diff:
             return
-        super().post_save(instance, created, using=using, **kwargs)
+        return super().post_save(instance, created, using=using, **kwargs)
 
     def create_history_model(self, model, inherited):
         if self.included_fields and not self.excluded_fields:
@@ -53,7 +54,7 @@ class Account(AbsConnectivity, BaseAccount):
         on_delete=models.SET_NULL, verbose_name=_("Su from")
     )
     version = models.IntegerField(default=0, verbose_name=_('Version'))
-    history = AccountHistoricalRecords(included_fields=['id', 'secret', 'secret_type', 'version'])
+    history = AccountHistoricalRecords(included_fields=['id', '_secret', 'secret_type', 'version'])
     source = models.CharField(max_length=30, default=Source.LOCAL, verbose_name=_('Source'))
     source_id = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Source ID'))
 
@@ -198,3 +199,21 @@ class AccountTemplate(BaseAccount):
             return
         self.bulk_update_accounts(accounts, {'secret': self.secret})
         self.bulk_create_history_accounts(accounts, user_id)
+
+
+def replace_history_model_with_mixin():
+    """
+    替换历史模型中的父类为指定的Mixin类。
+
+    Parameters:
+        model (class): 历史模型类，例如 Account.history.model
+        mixin_class (class): 要替换为的Mixin类
+
+    Returns:
+        None
+    """
+    model = Account.history.model
+    model.__bases__ = (VaultModelMixin,) + model.__bases__
+
+
+replace_history_model_with_mixin()

apps/accounts/models/base.py

@@ -9,33 +9,32 @@ from django.db import models
 from django.utils.translation import gettext_lazy as _
 
 from accounts.const import SecretType
-from common.db import fields
 from common.utils import (
     ssh_key_string_to_obj, ssh_key_gen, get_logger,
     random_string, lazyproperty, parse_ssh_public_key_str, is_openssh_format_key
 )
+from accounts.models.mixins import VaultModelMixin, VaultManagerMixin, VaultQuerySetMixin
 from orgs.mixins.models import JMSOrgBaseModel, OrgManager
 
 logger = get_logger(__file__)
 
 
-class BaseAccountQuerySet(models.QuerySet):
+class BaseAccountQuerySet(VaultQuerySetMixin, models.QuerySet):
     def active(self):
         return self.filter(is_active=True)
 
 
-class BaseAccountManager(OrgManager):
+class BaseAccountManager(VaultManagerMixin, OrgManager):
     def active(self):
         return self.get_queryset().active()
 
 
-class BaseAccount(JMSOrgBaseModel):
+class BaseAccount(VaultModelMixin, JMSOrgBaseModel):
     name = models.CharField(max_length=128, verbose_name=_("Name"))
     username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True)
     secret_type = models.CharField(
         max_length=16, choices=SecretType.choices, default=SecretType.PASSWORD, verbose_name=_('Secret type')
     )
-    secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
     privileged = models.BooleanField(verbose_name=_("Privileged"), default=False)
     is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
 

apps/accounts/models/mixins/__init__.py

@@ -0,0 +1 @@
+from .vault import *

apps/accounts/models/mixins/vault.py

@@ -0,0 +1,88 @@
+from django.db import models
+from django.db.models.signals import post_save
+from django.utils.translation import ugettext_lazy as _
+
+from common.db import fields
+
+__all__ = ['VaultQuerySetMixin', 'VaultManagerMixin', 'VaultModelMixin']
+
+
+class VaultQuerySetMixin(models.QuerySet):
+
+    def update(self, **kwargs):
+        """
+           1. 替换 secret 为 _secret
+           2. 触发 post_save 信号
+        """
+        if 'secret' in kwargs:
+            kwargs.update({
+                '_secret': kwargs.pop('secret')
+            })
+        rows = super().update(**kwargs)
+
+        # 为了获取更新后的对象所以单独查询一次
+        ids = self.values_list('id', flat=True)
+        objs = self.model.objects.filter(id__in=ids)
+        for obj in objs:
+            post_save.send(obj.__class__, instance=obj, created=False)
+        return rows
+
+
+class VaultManagerMixin(models.Manager):
+    """ 触发 bulk_create 和 bulk_update 操作下的 post_save 信号 """
+
+    def bulk_create(self, objs, batch_size=None, ignore_conflicts=False):
+        objs = super().bulk_create(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts)
+        for obj in objs:
+            post_save.send(obj.__class__, instance=obj, created=True)
+        return objs
+
+    def bulk_update(self, objs, batch_size=None, ignore_conflicts=False):
+        objs = super().bulk_update(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts)
+        for obj in objs:
+            post_save.send(obj.__class__, instance=obj, created=False)
+        return objs
+
+
+class VaultModelMixin(models.Model):
+    _secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
+    is_sync_metadata = True
+
+    class Meta:
+        abstract = True
+
+    # 缓存 secret 值, lazy-property 不能用
+    __secret = False
+
+    @property
+    def secret(self):
+        if self.__secret is False:
+            from accounts.backends import vault_client
+            secret = vault_client.get(self)
+            if not secret and not self.secret_has_save_to_vault:
+                # vault_client 获取不到, 并且 secret 没有保存到 vault, 就从 self._secret 获取
+                secret = self._secret
+            self.__secret = secret
+        return self.__secret
+
+    @secret.setter
+    def secret(self, value):
+        """
+        保存的时候通过 post_save 信号监听进行处理,
+        先保存到 db, 再保存到 vault 同时删除本地 db _secret 值
+        """
+        self._secret = value
+
+    _secret_save_to_vault_mark = '# Secret-has-been-saved-to-vault #'
+
+    def mark_secret_save_to_vault(self):
+        self._secret = self._secret_save_to_vault_mark
+        self.save()
+
+    @property
+    def secret_has_save_to_vault(self):
+        return self._secret == self._secret_save_to_vault_mark
+
+    def save(self, *args, **kwargs):
+        """ 通过 post_save signal 处理 _secret 数据 """
+        return super().save(*args, **kwargs)

apps/accounts/serializers/account/account.py

@@ -198,7 +198,6 @@ class AccountAssetSerializer(serializers.ModelSerializer):
 
 class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerializer):
     asset = AccountAssetSerializer(label=_('Asset'))
-    has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
     source = LabeledChoiceField(
         choices=Source.choices, label=_("Source"), required=False,
         allow_null=True, default=Source.LOCAL
@@ -233,6 +232,15 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize
         return queryset
 
 
+class AccountDetailSerializer(AccountSerializer):
+    has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
+
+    class Meta(AccountSerializer.Meta):
+        model = Account
+        fields = AccountSerializer.Meta.fields + ['has_secret']
+        read_only_fields = AccountSerializer.Meta.read_only_fields + ['has_secret']
+
+
 class AssetAccountBulkSerializerResultSerializer(serializers.Serializer):
     asset = serializers.CharField(read_only=True, label=_('Asset'))
     state = serializers.CharField(read_only=True, label=_('State'))

apps/accounts/serializers/account/base.py

@@ -61,20 +61,18 @@ class AuthValidateMixin(serializers.Serializer):
 
 
 class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
-    has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
 
     class Meta:
         model = BaseAccount
         fields_mini = ['id', 'name', 'username']
         fields_small = fields_mini + [
-            'secret_type', 'secret', 'has_secret', 'passphrase',
+            'secret_type', 'secret', 'passphrase',
             'privileged', 'is_active', 'spec_info',
         ]
         fields_other = ['created_by', 'date_created', 'date_updated', 'comment']
         fields = fields_small + fields_other
         read_only_fields = [
-            'has_secret', 'spec_info',
+            'spec_info', 'date_verified', 'created_by', 'date_created',
-            'date_verified', 'created_by', 'date_created',
         ]
         extra_kwargs = {
             'spec_info': {'label': _('Spec info')},

apps/accounts/signal_handlers.py

@@ -1,8 +1,9 @@
-from django.db.models.signals import pre_save
+from django.db.models.signals import pre_save, post_save, post_delete
 from django.dispatch import receiver
 
+from accounts.backends import vault_client
 from common.utils import get_logger
-from .models import Account
+from .models import Account, AccountTemplate
 
 logger = get_logger(__name__)
 
@@ -13,3 +14,23 @@ def on_account_pre_save(sender, instance, **kwargs):
         instance.version = 1
     else:
         instance.version = instance.history.count()
+
+
+class VaultSignalHandler(object):
+    """ 处理 Vault 相关的信号 """
+
+    @staticmethod
+    def save_to_vault(sender, instance, created, **kwargs):
+        if created:
+            vault_client.create(instance)
+        else:
+            vault_client.update(instance)
+
+    @staticmethod
+    def delete_to_vault(sender, instance, **kwargs):
+        vault_client.delete(instance)
+
+
+for model in (Account, AccountTemplate, Account.history.model):
+    post_save.connect(VaultSignalHandler.save_to_vault, sender=model)
+    post_delete.connect(VaultSignalHandler.delete_to_vault, sender=model)

apps/accounts/tasks/vault.py

@@ -0,0 +1,56 @@
+import datetime
+
+from celery import shared_task
+from django.utils.translation import gettext_lazy as _
+
+from accounts.backends import vault_client
+from accounts.models import Account, AccountTemplate
+from common.utils import get_logger
+from orgs.utils import tmp_to_root_org
+from ..const import VaultTypeChoices
+
+logger = get_logger(__name__)
+
+
+@shared_task(verbose_name=_('Sync secret to vault'))
+def sync_secret_to_vault():
+    if vault_client.is_type(VaultTypeChoices.local):
+        # 这里不能判断 settings.VAULT_TYPE, 必须判断当前 vault_client 的类型
+        print('\033[35m>>> 当前 Vault 类型为本地数据库, 不需要同步')
+        return
+
+    print('\033[33m>>> 开始同步密钥数据到 Vault ({})'.format(datetime.datetime.now()))
+    with tmp_to_root_org():
+        to_sync_models = [Account, AccountTemplate, Account.history.model]
+        for model in to_sync_models:
+            print(f'\033[33m>>> 开始同步: {model.__module__}')
+            succeeded = []
+            failed = []
+            skipped = []
+            instances = model.objects.all()
+            for instance in instances:
+                instance_desc = f'[{instance}]'
+                if instance.secret_has_save_to_vault:
+                    print(f'\033[32m- 跳过同步: {instance_desc}, 原因: [已同步]')
+                    skipped.append(instance)
+                    continue
+                try:
+                    vault_client.create(instance)
+                except Exception as e:
+                    failed.append(instance)
+                    print(f'\033[31m- 同步失败: {instance_desc}, 原因: [{e}]')
+                else:
+                    succeeded.append(instance)
+                    print(f'\033[32m- 同步成功: {instance_desc}')
+
+            total = len(succeeded) + len(failed) + len(skipped)
+            print(
+                f'\033[33m>>> 同步完成: {model.__module__}, '
+                f'共计: {total}, '
+                f'成功: {len(succeeded)}, '
+                f'失败: {len(failed)}, '
+                f'跳过: {len(skipped)}'
+            )
+
+    print('\033[33m>>> 全部同步完成 ({})'.format(datetime.datetime.now()))
+    print('\033[0m')

apps/accounts/utils.py

@@ -1,9 +1,7 @@
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
-from accounts.const import (
+from accounts.const import SecretType, DEFAULT_PASSWORD_RULES
-    SecretType, DEFAULT_PASSWORD_RULES
-)
 from common.utils import ssh_key_gen, random_string
 from common.utils import validate_ssh_private_key, parse_ssh_private_key_str
 

apps/common/db/fields.py

@@ -11,12 +11,12 @@ from django.core.exceptions import ValidationError
 from django.core.validators import MinValueValidator, MaxValueValidator
 from django.db import models
 from django.db.models import Q, Manager, QuerySet
-from django.utils.encoding import force_str
 from django.utils.translation import gettext_lazy as _
 from rest_framework.utils.encoders import JSONEncoder
 
 from common.local import add_encrypted_field_set
-from common.utils import signer, crypto, contains_ip
+from common.utils import contains_ip
+from .utils import Encryptor
 from .validators import PortRangeValidator
 
 __all__ = [
@@ -139,20 +139,11 @@ class EncryptMixin:
     EncryptMixin要放在最前面
     """
 
-    def decrypt_from_signer(self, value):
-        return signer.unsign(value) or ""
-
     def from_db_value(self, value, expression, connection, context=None):
         if value is None:
             return value
 
-        value = force_str(value)
+        plain_value = Encryptor(value).decrypt()
-        plain_value = crypto.decrypt(value)
-
-        # 如果没有解开，使用原来的signer解密
-        if not plain_value:
-            plain_value = self.decrypt_from_signer(value)
-
         # 可能和Json mix，所以要先解密，再json
         sp = super()
         if hasattr(sp, "from_db_value"):
@@ -167,9 +158,9 @@ class EncryptMixin:
         sp = super()
         if hasattr(sp, "get_prep_value"):
             value = sp.get_prep_value(value)
-        value = force_str(value)
+
         # 替换新的加密方式
-        return crypto.encrypt(value)
+        return Encryptor(value).encrypt()
 
 
 class EncryptTextField(EncryptMixin, models.TextField):

apps/common/db/utils.py

@@ -1,8 +1,9 @@
 from contextlib import contextmanager
 
 from django.db import connections
+from django.utils.encoding import force_text
 
-from common.utils import get_logger
+from common.utils import get_logger, signer, crypto
 
 logger = get_logger(__file__)
 
@@ -55,3 +56,19 @@ def safe_db_connection():
     close_old_connections()
     yield
     close_old_connections()
+
+
+class Encryptor:
+    def __init__(self, value):
+        self.value = force_text(value)
+
+    def decrypt(self):
+        plain_value = crypto.decrypt(self.value)
+
+        # 如果没有解开，使用原来的signer解密
+        if not plain_value:
+            plain_value = signer.unsign(self.value) or ""
+        return plain_value
+
+    def encrypt(self):
+        return crypto.encrypt(self.value)

apps/common/decorators.py

@@ -67,12 +67,18 @@ class EventLoopThread(threading.Thread):
 _loop_thread = EventLoopThread()
 _loop_thread.setDaemon(True)
 _loop_thread.start()
-executor = ThreadPoolExecutor(max_workers=10,
+executor = ThreadPoolExecutor(
-                              thread_name_prefix='debouncer')
+    max_workers=10,
+    thread_name_prefix='debouncer'
+)
 _loop_debouncer_func_task_cache = {}
 _loop_debouncer_func_args_cache = {}
 
 
+def get_loop():
+    return _loop_thread.get_loop()
+
+
 def cancel_or_remove_debouncer_task(cache_key):
     task = _loop_debouncer_func_task_cache.get(cache_key, None)
     if not task:

apps/common/sdk/im/dingtalk/__init__.py

@@ -1,10 +1,10 @@
-import time
-import hmac
 import base64
+import hmac
+import time
 
-from common.utils import get_logger
-from common.sdk.im.utils import digest, as_request
 from common.sdk.im.mixin import BaseRequest
+from common.sdk.im.utils import digest, as_request
+from common.utils import get_logger
 from users.utils import construct_user_email
 
 logger = get_logger(__file__)

apps/jumpserver/conf.py

@@ -251,6 +251,12 @@ class Config(dict):
         # 临时密码
         'AUTH_TEMP_TOKEN': False,
 
+        # Vault
+        'VAULT_TYPE': 'local',
+        'VAULT_HCP_HOST': '',
+        'VAULT_HCP_TOKEN': '',
+        'VAULT_HCP_MOUNT_POINT': 'jumpserver',
+
         # Auth LDAP settings
         'AUTH_LDAP': False,
         'AUTH_LDAP_SERVER_URI': 'ldap://localhost:389',

apps/jumpserver/settings/auth.py

@@ -174,6 +174,12 @@ AUTH_OAUTH2_LOGOUT_URL_NAME = "authentication:oauth2:logout"
 # 临时 token
 AUTH_TEMP_TOKEN = CONFIG.AUTH_TEMP_TOKEN
 
+# Vault
+VAULT_TYPE = CONFIG.VAULT_TYPE
+VAULT_HCP_HOST = CONFIG.VAULT_HCP_HOST
+VAULT_HCP_TOKEN = CONFIG.VAULT_HCP_TOKEN
+VAULT_HCP_MOUNT_POINT = CONFIG.VAULT_HCP_MOUNT_POINT
+
 # Other setting
 # 这个是 User Login Private Token
 TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION

apps/orgs/mixins/models.py

@@ -119,3 +119,4 @@ class OrgModelMixin(models.Model):
 class JMSOrgBaseModel(JMSBaseModel, OrgModelMixin):
     class Meta:
         abstract = True
+

apps/orgs/utils.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 #
 import uuid
-from inspect import signature
-from functools import wraps
-from werkzeug.local import LocalProxy
 from contextlib import contextmanager
+from functools import wraps
+from inspect import signature
+
+from werkzeug.local import LocalProxy
 
 from common.local import thread_local
 from .models import Organization
@@ -133,6 +134,7 @@ def org_aware_func(org_arg_name):
     :param org_arg_name: 函数中包含org_id的对象是哪个参数
     :return:
     """
+
     def decorate(func):
         @wraps(func)
         def wrapper(*args, **kwargs):
@@ -149,7 +151,9 @@ def org_aware_func(org_arg_name):
             with tmp_to_org(org_aware_resource.org_id):
                 # print("Current org id: {}".format(org_aware_resource.org_id))
                 return func(*args, **kwargs)
+
         return wrapper
+
     return decorate
 
 
@@ -162,4 +166,5 @@ def ensure_in_real_or_default_org(func):
         if not current_org or current_org.is_root():
             raise ValueError('You must in a real or default org!')
         return func(*args, **kwargs)
+
     return wrapper

apps/settings/api/__init__.py

@@ -1,8 +1,9 @@
-from .settings import *
-from .ldap import *
-from .wecom import *
 from .dingtalk import *
-from .feishu import *
-from .public import *
 from .email import *
+from .feishu import *
+from .ldap import *
+from .public import *
+from .settings import *
 from .sms import *
+from .vault import *
+from .wecom import *

apps/settings/api/settings.py

@@ -50,6 +50,7 @@ class SettingsApi(generics.RetrieveUpdateAPIView):
         'huawei': serializers.HuaweiSMSSettingSerializer,
         'cmpp2': serializers.CMPP2SMSSettingSerializer,
         'custom': serializers.CustomSMSSettingSerializer,
+        'vault': serializers.VaultSettingSerializer,
     }
 
     rbac_category_permissions = {
@@ -75,6 +76,7 @@ class SettingsApi(generics.RetrieveUpdateAPIView):
         'sms': 'settings.change_sms',
         'alibaba': 'settings.change_sms',
         'tencent': 'settings.change_sms',
+        'vault': 'settings.change_vault',
     }
 
     def get_queryset(self):

apps/settings/api/vault.py

@@ -0,0 +1,60 @@
+from django.utils.translation import gettext_lazy as _
+from django.conf import settings
+from rest_framework import status
+from rest_framework.generics import GenericAPIView
+from rest_framework.views import Response, APIView
+
+from accounts.tasks.vault import sync_secret_to_vault
+from accounts.backends import get_vault_client
+from settings.models import Setting
+from .. import serializers
+
+
+class VaultTestingAPI(GenericAPIView):
+    serializer_class = serializers.VaultSettingSerializer
+    rbac_perms = {
+        'POST': 'settings.change_vault'
+    }
+
+    def get_config(self, request):
+        serializer = self.serializer_class(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        data = serializer.validated_data
+        for k, v in data.items():
+            if v:
+                continue
+            # 页面没有传递值, 从 settings 中获取
+            data[k] = getattr(settings, k, None)
+        return data
+
+    def post(self, request):
+        config = self.get_config(request)
+        try:
+            client = get_vault_client(raise_exception=True, **config)
+            ok, error = client.is_active()
+        except Exception as e:
+            ok, error = False, str(e)
+
+        if ok:
+            _status, msg = status.HTTP_200_OK, _('Test success')
+        else:
+            _status, msg = status.HTTP_400_BAD_REQUEST, error
+
+        return Response(status=_status, data={'msg': msg})
+
+
+class VaultSyncDataAPI(APIView):
+    perm_model = Setting
+    rbac_perms = {
+        'POST': 'settings.change_vault'
+    }
+
+    def post(self, request, *args, **kwargs):
+        task = self._run_task()
+        return Response({'task': task.id}, status=status.HTTP_201_CREATED)
+
+    @staticmethod
+    def _run_task():
+        task = sync_secret_to_vault.delay()
+        return task
+

apps/settings/migrations/0008_alter_setting_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.19 on 2023-06-30 10:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('settings', '0007_migrate_ldap_sync_org_ids'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='setting',
+            options={'permissions': [('change_email', 'Can change email setting'), ('change_auth', 'Can change auth setting'), ('change_vault', 'Can change vault setting'), ('change_systemmsgsubscription', 'Can change system msg sub setting'), ('change_sms', 'Can change sms setting'), ('change_security', 'Can change security setting'), ('change_clean', 'Can change clean setting'), ('change_interface', 'Can change interface setting'), ('change_license', 'Can change license setting'), ('change_terminal', 'Can change terminal setting'), ('change_other', 'Can change other setting')], 'verbose_name': 'System setting'},
+        ),
+    ]

apps/settings/models.py

@@ -159,6 +159,7 @@ class Setting(models.Model):
         permissions = [
             ('change_email', _('Can change email setting')),
             ('change_auth', _('Can change auth setting')),
+            ('change_vault', _('Can change vault setting')),
             ('change_systemmsgsubscription', _('Can change system msg sub setting')),
             ('change_sms', _('Can change sms setting')),
             ('change_security', _('Can change security setting')),

apps/settings/serializers/__init__.py

@@ -1,13 +1,14 @@
 # coding: utf-8
 #
 
-from .basic import *
 from .auth import *
-from .email import *
+from .basic import *
-from .public import *
-from .settings import *
-from .security import *
-from .terminal import *
 from .cleaning import *
+from .email import *
 from .other import *
+from .public import *
+from .security import *
+from .settings import *
+from .terminal import *
+from .vault import *
 

apps/settings/serializers/vault.py

@@ -0,0 +1,23 @@
+from django.utils.translation import ugettext_lazy as _
+from rest_framework import serializers
+
+from accounts.const import VaultTypeChoices
+from common.serializers.fields import EncryptedField
+
+__all__ = ['VaultSettingSerializer']
+
+
+class VaultSettingSerializer(serializers.Serializer):
+    VAULT_TYPE = serializers.ChoiceField(
+        default=VaultTypeChoices.local, choices=VaultTypeChoices.choices,
+        required=False, label=_('Type')
+    )
+    VAULT_HCP_HOST = serializers.CharField(
+        max_length=256, allow_blank=True, required=False, label=_('Host')
+    )
+    VAULT_HCP_TOKEN = EncryptedField(
+        max_length=256, allow_blank=True, required=False, label=_('Token')
+    )
+    VAULT_HCP_MOUNT_POINT = serializers.CharField(
+        max_length=256, allow_blank=True, required=False, label=_('Mount Point')
+    )

apps/settings/urls/api_urls.py

@@ -18,6 +18,8 @@ urlpatterns = [
     path('feishu/testing/', api.FeiShuTestingAPI.as_view(), name='feishu-testing'),
     path('sms/<str:backend>/testing/', api.SMSTestingAPI.as_view(), name='sms-testing'),
     path('sms/backend/', api.SMSBackendAPI.as_view(), name='sms-backend'),
+    path('vault/testing/', api.VaultTestingAPI.as_view(), name='vault-testing'),
+    path('vault/sync/', api.VaultSyncDataAPI.as_view(), name='vault-sync'),
 
     path('setting/', api.SettingsApi.as_view(), name='settings-setting'),
     path('logo/', api.SettingsLogoApi.as_view(), name='settings-logo'),

requirements/requirements.txt

@@ -133,3 +133,5 @@ ipython==8.14.0
 ForgeryPy3==0.3.1
 django-debug-toolbar==4.1.0
 Pympler==1.0.1
+hvac==1.1.1
+pyhcl==0.4.4