Merge pull request #11322 from jumpserver/dev

v3.6.0

Dockerfile

@@ -1,4 +1,4 @@
-FROM jumpserver/python:3.9-slim-buster as stage-build
+FROM python:3.11-slim-bullseye as stage-build
 ARG TARGETARCH
 
 ARG VERSION
@@ -8,9 +8,8 @@ WORKDIR /opt/jumpserver
 ADD . .
 RUN cd utils && bash -ixeu build.sh
 
-FROM jumpserver/python:3.9-slim-buster
+FROM python:3.11-slim-bullseye
 ARG TARGETARCH
-MAINTAINER JumpServer Team <ibuler@qq.com>
 
 ARG BUILD_DEPENDENCIES="              \
         g++                           \
@@ -22,6 +21,7 @@ ARG DEPENDENCIES="                    \
         libpq-dev                     \
         libffi-dev                    \
         libjpeg-dev                   \
+        libkrb5-dev                   \
         libldap2-dev                  \
         libsasl2-dev                  \
         libssl-dev                    \
@@ -37,13 +37,11 @@ ARG TOOLS="                           \
         default-libmysqlclient-dev    \
         default-mysql-client          \
         locales                       \
+        nmap                          \
         openssh-client                \
-        procps                        \
         sshpass                       \
         telnet                        \
-        unzip                         \
         vim                           \
-        git                           \
         wget"
 
 ARG APT_MIRROR=http://mirrors.ustc.edu.cn
@@ -65,46 +63,17 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \
     && sed -i "s@# alias @alias @g" ~/.bashrc \
     && rm -rf /var/lib/apt/lists/*
 
-ARG DOWNLOAD_URL=https://download.jumpserver.org
-
-RUN set -ex \
-    && \
-    if [ "${TARGETARCH}" == "amd64" ] || [ "${TARGETARCH}" == "arm64" ]; then \
-        mkdir -p /opt/oracle; \
-        cd /opt/oracle; \
-        wget ${DOWNLOAD_URL}/public/instantclient-basiclite-linux.${TARGETARCH}-19.10.0.0.0.zip; \
-        unzip instantclient-basiclite-linux.${TARGETARCH}-19.10.0.0.0.zip; \
-        echo "/opt/oracle/instantclient_19_10" > /etc/ld.so.conf.d/oracle-instantclient.conf; \
-        ldconfig; \
-        rm -f instantclient-basiclite-linux.${TARGETARCH}-19.10.0.0.0.zip; \
-    fi
-
-WORKDIR /tmp/build
-COPY ./requirements ./requirements
-
-ARG PIP_MIRROR=https://pypi.douban.com/simple
-ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    set -ex \
-    && pip config set global.index-url ${PIP_MIRROR} \
-    && pip install --upgrade pip \
-    && pip install --upgrade setuptools wheel \
-    && \
-    if [ "${TARGETARCH}" == "loong64" ]; then \
-        pip install https://download.jumpserver.org/pypi/simple/cryptography/cryptography-38.0.4-cp39-cp39-linux_loongarch64.whl; \
-        pip install https://download.jumpserver.org/pypi/simple/greenlet/greenlet-1.1.2-cp39-cp39-linux_loongarch64.whl; \
-        pip install https://download.jumpserver.org/pypi/simple/PyNaCl/PyNaCl-1.5.0-cp39-cp39-linux_loongarch64.whl; \
-        pip install https://download.jumpserver.org/pypi/simple/grpcio/grpcio-1.54.2-cp39-cp39-linux_loongarch64.whl; \
-    fi \
-    && pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
-    && pip install -r requirements/requirements.txt
-
 COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
-RUN echo > /opt/jumpserver/config.yml \
-    && rm -rf /tmp/build
-
 WORKDIR /opt/jumpserver
+
+ARG PIP_MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
+    && echo > /opt/jumpserver/config.yml \
+    && pip install poetry -i ${PIP_MIRROR} \
+    && poetry config virtualenvs.create false \
+    && poetry install --only=main
+
 VOLUME /opt/jumpserver/data
 VOLUME /opt/jumpserver/logs
 

Dockerfile-ee

@@ -1,10 +1,9 @@
 ARG VERSION
 FROM registry.fit2cloud.com/jumpserver/xpack:${VERSION} as build-xpack
 FROM jumpserver/core:${VERSION}
+
 COPY --from=build-xpack /opt/xpack /opt/jumpserver/apps/xpack
 
-WORKDIR /opt/jumpserver
-
-RUN --mount=type=cache,target=/root/.cache/pip \
+RUN --mount=type=cache,target=/root/.cache \
     set -ex \
-    && pip install -r requirements/requirements_xpack.txt
+    && poetry install --only=xpack

README.md

@@ -17,6 +17,7 @@
     9 年时间，倾情投入，用心做好一款开源堡垒机。
 </p>
 
+------------------------------
 JumpServer 是广受欢迎的开源堡垒机，是符合 4A 规范的专业运维安全审计系统。
 
 JumpServer 堡垒机帮助企业以更安全的方式管控和登录各种类型的资产，包括：
@@ -83,9 +84,7 @@ JumpServer 堡垒机帮助企业以更安全的方式管控和登录各种类型
 
 ### 参与贡献
 
-欢迎提交 PR 参与贡献。感谢以下贡献者，他们让 JumpServer 变的越来越好。
-
-<a href="https://github.com/jumpserver/jumpserver/graphs/contributors"><img src="https://opencollective.com/jumpserver/contributors.svg?width=890&button=false" /></a>
+欢迎提交 PR 参与贡献。 参考 [CONTRIBUTING.md](https://github.com/jumpserver/jumpserver/blob/dev/CONTRIBUTING.md)
 
 ## 组件项目
 

apps/accounts/api/account/__init__.py

@@ -1,3 +1,4 @@
 from .account import *
 from .task import *
 from .template import *
+from .virtual import *

apps/accounts/api/account/account.py

@@ -22,10 +22,11 @@ __all__ = [
 
 class AccountViewSet(OrgBulkModelViewSet):
     model = Account
-    search_fields = ('username', 'name', 'asset__name', 'asset__address')
+    search_fields = ('username', 'name', 'asset__name', 'asset__address', 'comment')
     filterset_class = AccountFilterSet
     serializer_classes = {
         'default': serializers.AccountSerializer,
+        'retrieve': serializers.AccountDetailSerializer,
     }
     rbac_perms = {
         'partial_update': ['accounts.change_account'],
@@ -52,20 +53,21 @@ class AccountViewSet(OrgBulkModelViewSet):
         return Response(data=serializer.data)
 
     @action(
-        methods=['get'], detail=False, url_path='username-suggestions',
+        methods=['post'], detail=False, url_path='username-suggestions',
         permission_classes=[IsValidUser]
     )
     def username_suggestions(self, request, *args, **kwargs):
-        asset_ids = request.query_params.get('assets')
-        node_keys = request.query_params.get('keys')
-        username = request.query_params.get('username')
+        asset_ids = request.data.get('assets')
+        node_ids = request.data.get('nodes')
+        username = request.data.get('username')
 
         assets = Asset.objects.all()
         if asset_ids:
-            assets = assets.filter(id__in=asset_ids.split(','))
-        if node_keys:
-            patten = Node.get_node_all_children_key_pattern(node_keys.split(','))
-            assets = assets.filter(nodes__key__regex=patten)
+            assets = assets.filter(id__in=asset_ids)
+        if node_ids:
+            nodes = Node.objects.filter(id__in=node_ids)
+            node_asset_ids = Node.get_nodes_all_assets(*nodes).values_list('id', flat=True)
+            assets = assets.filter(id__in=set(list(asset_ids) + list(node_asset_ids)))
 
         accounts = Account.objects.filter(asset__in=assets)
         if username:
@@ -132,11 +134,13 @@ class AccountHistoriesSecretAPI(ExtraFilterFieldsMixin, RecordViewLogMixin, List
     def get_queryset(self):
         account = self.get_object()
         histories = account.history.all()
-        last_history = account.history.first()
-        if not last_history:
+        latest_history = account.history.first()
+        if not latest_history:
             return histories
-
-        if account.secret == last_history.secret \
-                and account.secret_type == last_history.secret_type:
-            histories = histories.exclude(history_id=last_history.history_id)
+        if account.secret != latest_history.secret:
+            return histories
+        if account.secret_type != latest_history.secret_type:
+            return histories
+        histories = histories.exclude(history_id=latest_history.history_id)
         return histories
+

apps/accounts/api/account/virtual.py

@@ -0,0 +1,20 @@
+from django.shortcuts import get_object_or_404
+
+from accounts.models import VirtualAccount
+from accounts.serializers import VirtualAccountSerializer
+from common.utils import is_uuid
+from orgs.mixins.api import OrgBulkModelViewSet
+
+
+class VirtualAccountViewSet(OrgBulkModelViewSet):
+    serializer_class = VirtualAccountSerializer
+    search_fields = ('alias',)
+    filterset_fields = ('alias',)
+
+    def get_queryset(self):
+        return VirtualAccount.get_or_init_queryset()
+
+    def get_object(self, ):
+        pk = self.kwargs.get('pk')
+        kwargs = {'pk': pk} if is_uuid(pk) else {'alias': pk}
+        return get_object_or_404(VirtualAccount, **kwargs)

apps/accounts/api/automations/backup.py

@@ -26,8 +26,8 @@ class AccountBackupPlanViewSet(OrgBulkModelViewSet):
 
 class AccountBackupPlanExecutionViewSet(viewsets.ModelViewSet):
     serializer_class = serializers.AccountBackupPlanExecutionSerializer
-    search_fields = ('trigger',)
-    filterset_fields = ('trigger', 'plan_id')
+    search_fields = ('trigger', 'plan__name')
+    filterset_fields = ('trigger', 'plan_id', 'plan__name')
     http_method_names = ['get', 'post', 'options']
 
     def get_queryset(self):

apps/accounts/api/automations/base.py

@@ -1,5 +1,5 @@
 from django.shortcuts import get_object_or_404
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import status, mixins, viewsets
 from rest_framework.response import Response
 
@@ -95,8 +95,8 @@ class AutomationExecutionViewSet(
     mixins.CreateModelMixin, mixins.ListModelMixin,
     mixins.RetrieveModelMixin, viewsets.GenericViewSet
 ):
-    search_fields = ('trigger',)
-    filterset_fields = ('trigger', 'automation_id')
+    search_fields = ('trigger', 'automation__name')
+    filterset_fields = ('trigger', 'automation_id', 'automation__name')
     serializer_class = serializers.AutomationExecutionSerializer
 
     tp: str

apps/accounts/apps.py

@@ -6,6 +6,5 @@ class AccountsConfig(AppConfig):
     name = 'accounts'
 
     def ready(self):
-        from . import signal_handlers
-        from . import tasks
-        __all__ = signal_handlers
+        from . import signal_handlers  # noqa
+        from . import tasks  # noqa

apps/accounts/automations/backup_account/handlers.py

@@ -1,22 +1,17 @@
 import os
 import time
-from openpyxl import Workbook
 from collections import defaultdict, OrderedDict
 
 from django.conf import settings
-from django.db.models import F
+from openpyxl import Workbook
 from rest_framework import serializers
 
-from accounts.models import Account
-from assets.const import AllTypes
-from accounts.serializers import AccountSecretSerializer
 from accounts.notifications import AccountBackupExecutionTaskMsg
-from users.models import User
-from common.utils import get_logger
-from common.utils.timezone import local_now_display
+from accounts.serializers import AccountSecretSerializer
+from assets.const import AllTypes
 from common.utils.file import encrypt_and_compress_zip_file
-
-logger = get_logger(__file__)
+from common.utils.timezone import local_now_display
+from users.models import User
 
 PATH = os.path.join(os.path.dirname(settings.BASE_DIR), 'tmp')
 
@@ -76,8 +71,22 @@ class AssetAccountHandler(BaseAccountHandler):
         )
         return filename
 
+    @staticmethod
+    def handler_secret(data, section):
+        for account_data in data:
+            secret = account_data.get('secret')
+            if not secret:
+                continue
+            length = len(secret)
+            index = length // 2
+            if section == "front":
+                secret = secret[:index] + '*' * (length - index)
+            elif section == "back":
+                secret = '*' * (length - index) + secret[index:]
+            account_data['secret'] = secret
+
     @classmethod
-    def create_data_map(cls, accounts):
+    def create_data_map(cls, accounts, section):
         data_map = defaultdict(list)
 
         if not accounts.exists():
@@ -97,9 +106,10 @@ class AssetAccountHandler(BaseAccountHandler):
         for tp, _accounts in account_type_map.items():
             sheet_name = type_dict.get(tp, tp)
             data = AccountSecretSerializer(_accounts, many=True).data
+            cls.handler_secret(data, section)
             data_map.update(cls.add_rows(data, header_fields, sheet_name))
 
-        logger.info('\n\033[33m- 共备份 {} 条账号\033[0m'.format(accounts.count()))
+        print('\n\033[33m- 共备份 {} 条账号\033[0m'.format(accounts.count()))
         return data_map
 
 
@@ -109,8 +119,8 @@ class AccountBackupHandler:
         self.plan_name = self.execution.plan.name
         self.is_frozen = False  # 任务状态冻结标志
 
-    def create_excel(self):
-        logger.info(
+    def create_excel(self, section='complete'):
+        print(
             '\n'
             '\033[32m>>> 正在生成资产或应用相关备份信息文件\033[0m'
             ''
@@ -119,7 +129,7 @@ class AccountBackupHandler:
         time_start = time.time()
         files = []
         accounts = self.execution.backup_accounts
-        data_map = AssetAccountHandler.create_data_map(accounts)
+        data_map = AssetAccountHandler.create_data_map(accounts, section)
         if not data_map:
             return files
 
@@ -133,14 +143,14 @@ class AccountBackupHandler:
         wb.save(filename)
         files.append(filename)
         timedelta = round((time.time() - time_start), 2)
-        logger.info('步骤完成: 用时 {}s'.format(timedelta))
+        print('步骤完成: 用时 {}s'.format(timedelta))
         return files
 
     def send_backup_mail(self, files, recipients):
         if not files:
             return
         recipients = User.objects.filter(id__in=list(recipients))
-        logger.info(
+        print(
             '\n'
             '\033[32m>>> 发送备份邮件\033[0m'
             ''
@@ -155,7 +165,7 @@ class AccountBackupHandler:
                 encrypt_and_compress_zip_file(attachment, password, files)
                 attachment_list = [attachment, ]
             AccountBackupExecutionTaskMsg(plan_name, user).publish(attachment_list)
-            logger.info('邮件已发送至{}({})'.format(user, user.email))
+            print('邮件已发送至{}({})'.format(user, user.email))
         for file in files:
             os.remove(file)
 
@@ -163,33 +173,42 @@ class AccountBackupHandler:
         self.execution.reason = reason[:1024]
         self.execution.is_success = is_success
         self.execution.save()
-        logger.info('已完成对任务状态的更新')
+        print('已完成对任务状态的更新')
 
-    def step_finished(self, is_success):
+    @staticmethod
+    def step_finished(is_success):
         if is_success:
-            logger.info('任务执行成功')
+            print('任务执行成功')
         else:
-            logger.error('任务执行失败')
+            print('任务执行失败')
 
     def _run(self):
         is_success = False
         error = '-'
         try:
-            recipients = self.execution.plan_snapshot.get('recipients')
-            if not recipients:
-                logger.info(
+            recipients_part_one = self.execution.snapshot.get('recipients_part_one', [])
+            recipients_part_two = self.execution.snapshot.get('recipients_part_two', [])
+            if not recipients_part_one and not recipients_part_two:
+                print(
                     '\n'
                     '\033[32m>>> 该备份任务未分配收件人\033[0m'
                     ''
                 )
+            if recipients_part_one and recipients_part_two:
+                files = self.create_excel(section='front')
+                self.send_backup_mail(files, recipients_part_one)
+
+                files = self.create_excel(section='back')
+                self.send_backup_mail(files, recipients_part_two)
             else:
+                recipients = recipients_part_one or recipients_part_two
                 files = self.create_excel()
                 self.send_backup_mail(files, recipients)
         except Exception as e:
             self.is_frozen = True
-            logger.error('任务执行被异常中断')
-            logger.info('下面打印发生异常的 Traceback 信息 : ')
-            logger.error(e, exc_info=True)
+            print('任务执行被异常中断')
+            print('下面打印发生异常的 Traceback 信息 : ')
+            print(e)
             error = str(e)
         else:
             is_success = True
@@ -199,15 +218,15 @@ class AccountBackupHandler:
             self.step_finished(is_success)
 
     def run(self):
-        logger.info('任务开始: {}'.format(local_now_display()))
+        print('任务开始: {}'.format(local_now_display()))
         time_start = time.time()
         try:
             self._run()
         except Exception as e:
-            logger.error('任务运行出现异常')
-            logger.error('下面显示异常 Traceback 信息: ')
-            logger.error(e, exc_info=True)
+            print('任务运行出现异常')
+            print('下面显示异常 Traceback 信息: ')
+            print(e)
         finally:
-            logger.info('\n任务结束: {}'.format(local_now_display()))
+            print('\n任务结束: {}'.format(local_now_display()))
             timedelta = round((time.time() - time_start), 2)
-            logger.info('用时: {}'.format(timedelta))
+            print('用时: {}'.format(timedelta))

apps/accounts/automations/backup_account/manager.py

@@ -4,13 +4,9 @@ import time
 
 from django.utils import timezone
 
-from common.utils import get_logger
 from common.utils.timezone import local_now_display
-
 from .handlers import AccountBackupHandler
 
-logger = get_logger(__name__)
-
 
 class AccountBackupManager:
     def __init__(self, execution):
@@ -23,7 +19,7 @@ class AccountBackupManager:
 
     def do_run(self):
         execution = self.execution
-        logger.info('\n\033[33m# 账号备份计划正在执行\033[0m')
+        print('\n\033[33m# 账号备份计划正在执行\033[0m')
         handler = AccountBackupHandler(execution)
         handler.run()
 
@@ -35,10 +31,10 @@ class AccountBackupManager:
         self.time_end = time.time()
         self.date_end = timezone.now()
 
-        logger.info('\n\n' + '-' * 80)
-        logger.info('计划执行结束 {}\n'.format(local_now_display()))
+        print('\n\n' + '-' * 80)
+        print('计划执行结束 {}\n'.format(local_now_display()))
         self.timedelta = self.time_end - self.time_start
-        logger.info('用时: {}s'.format(self.timedelta))
+        print('用时: {}s'.format(self.timedelta))
         self.execution.timedelta = self.timedelta
         self.execution.save()
 

apps/accounts/automations/change_secret/custom/ssh/main.yml

@@ -2,9 +2,10 @@
   gather_facts: no
   vars:
     ansible_connection: local
+    ansible_become: false
 
   tasks:
-    - name: Test privileged account
+    - name: Test privileged account (paramiko)
       ssh_ping:
         login_host: "{{ jms_asset.address }}"
         login_port: "{{ jms_asset.port }}"
@@ -12,9 +13,14 @@
         login_password: "{{ jms_account.secret }}"
         login_secret_type: "{{ jms_account.secret_type }}"
         login_private_key_path: "{{ jms_account.private_key_path }}"
+        become: "{{ custom_become | default(False) }}"
+        become_method: "{{ custom_become_method | default('su') }}"
+        become_user: "{{ custom_become_user | default('') }}"
+        become_password: "{{ custom_become_password | default('') }}"
+        become_private_key_path: "{{ custom_become_private_key_path | default(None) }}"
       register: ping_info
 
-    - name: Change asset password
+    - name: Change asset password (paramiko)
       custom_command:
         login_user: "{{ jms_account.username }}"
         login_password: "{{ jms_account.secret }}"
@@ -22,6 +28,11 @@
         login_port: "{{ jms_asset.port }}"
         login_secret_type: "{{ jms_account.secret_type }}"
         login_private_key_path: "{{ jms_account.private_key_path }}"
+        become: "{{ custom_become | default(False) }}"
+        become_method: "{{ custom_become_method | default('su') }}"
+        become_user: "{{ custom_become_user | default('') }}"
+        become_password: "{{ custom_become_password | default('') }}"
+        become_private_key_path: "{{ custom_become_private_key_path | default(None) }}"
         name: "{{ account.username }}"
         password: "{{ account.secret }}"
         commands: "{{ params.commands }}"
@@ -30,9 +41,10 @@
       when: ping_info is succeeded
       register: change_info
 
-    - name: Verify password
+    - name: Verify password (paramiko)
       ssh_ping:
         login_user: "{{ account.username }}"
         login_password: "{{ account.secret }}"
         login_host: "{{ jms_asset.address }}"
         login_port: "{{ jms_asset.port }}"
+        become: false

apps/accounts/automations/change_secret/host/aix/main.yml

@@ -1,10 +1,41 @@
 - hosts: demo
   gather_facts: no
   tasks:
-    - name: Test privileged account
+    - name: "Test privileged {{ jms_account.username }} account"
       ansible.builtin.ping:
 
-    - name: Change password
+    - name: "Check if {{ account.username }} user exists"
+      getent:
+        database: passwd
+        key: "{{ account.username }}"
+      register: user_info
+      ignore_errors: yes  # 忽略错误，如果用户不存在时不会导致playbook失败
+
+    - name: "Add {{ account.username }} user"
+      ansible.builtin.user:
+        name: "{{ account.username }}"
+        shell: "{{ params.shell }}"
+        home: "{{ params.home | default('/home/' + account.username, true) }}"
+        groups: "{{ params.groups }}"
+        expires: -1
+        state: present
+      when: user_info.failed
+
+    - name: "Add {{ account.username }} group"
+      ansible.builtin.group:
+        name: "{{ account.username }}"
+        state: present
+      when: user_info.failed
+
+    - name: "Add {{ account.username }} user to group"
+      ansible.builtin.user:
+        name: "{{ account.username }}"
+        groups: "{{ params.groups }}"
+      when:
+        - user_info.failed
+        - params.groups
+
+    - name: "Change {{ account.username }} password"
       ansible.builtin.user:
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('des') }}"
@@ -12,44 +43,54 @@
       ignore_errors: true
       when: account.secret_type == "password"
 
-    - name: create user If it already exists, no operation will be performed
-      ansible.builtin.user:
-        name: "{{ account.username }}"
-      when: account.secret_type == "ssh_key"
-
     - name: remove jumpserver ssh key
       ansible.builtin.lineinfile:
         dest: "{{ ssh_params.dest }}"
         regexp: "{{ ssh_params.regexp }}"
         state: absent
       when:
-      - account.secret_type == "ssh_key"
-      - ssh_params.strategy == "set_jms"
+        - account.secret_type == "ssh_key"
+        - ssh_params.strategy == "set_jms"
 
-    - name: Change SSH key
+    - name: "Change {{ account.username }} SSH key"
       ansible.builtin.authorized_key:
         user: "{{ account.username }}"
         key: "{{ account.secret }}"
         exclusive: "{{ ssh_params.exclusive }}"
       when: account.secret_type == "ssh_key"
 
+    - name: "Set {{ account.username }} sudo setting"
+      ansible.builtin.lineinfile:
+        dest: /etc/sudoers
+        state: present
+        regexp: "^{{ account.username }} ALL="
+        line: "{{ account.username + ' ALL=(ALL) NOPASSWD: ' + params.sudo }}"
+        validate: visudo -cf %s
+      when:
+        - user_info.failed
+        - params.sudo
+
     - name: Refresh connection
       ansible.builtin.meta: reset_connection
 
-    - name: Verify password
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_password: "{{ account.secret }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} password (paramiko)"
+      ssh_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "password"
+      delegate_to: localhost
 
-    - name: Verify SSH key
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_ssh_private_key_file: "{{ account.private_key_path }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} SSH KEY (paramiko)"
+      ssh_ping:
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_user: "{{ account.username }}"
+        login_private_key_path: "{{ account.private_key_path  }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "ssh_key"
+      delegate_to: localhost

apps/accounts/automations/change_secret/host/posix/main.yml

@@ -1,10 +1,17 @@
 - hosts: demo
   gather_facts: no
   tasks:
-    - name: Test privileged account
+    - name: "Test privileged {{ jms_account.username }} account"
       ansible.builtin.ping:
 
-    - name: Check user
+    - name: "Check if {{ account.username }} user exists"
+      getent:
+        database: passwd
+        key: "{{ account.username }}"
+      register: user_info
+      ignore_errors: yes  # 忽略错误，如果用户不存在时不会导致playbook失败
+
+    - name: "Add {{ account.username }} user"
       ansible.builtin.user:
         name: "{{ account.username }}"
         shell: "{{ params.shell }}"
@@ -12,19 +19,23 @@
         groups: "{{ params.groups }}"
         expires: -1
         state: present
+      when: user_info.failed
 
     - name: "Add {{ account.username }} group"
       ansible.builtin.group:
         name: "{{ account.username }}"
         state: present
+      when: user_info.failed
 
-    - name: Add user groups
+    - name: "Add {{ account.username }} user to group"
       ansible.builtin.user:
         name: "{{ account.username }}"
         groups: "{{ params.groups }}"
-      when: params.groups
+      when:
+        - user_info.failed
+        - params.groups
 
-    - name: Change password
+    - name: "Change {{ account.username }} password"
       ansible.builtin.user:
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('sha512') }}"
@@ -32,11 +43,6 @@
       ignore_errors: true
       when: account.secret_type == "password"
 
-    - name: create user If it already exists, no operation will be performed
-      ansible.builtin.user:
-        name: "{{ account.username }}"
-      when: account.secret_type == "ssh_key"
-
     - name: remove jumpserver ssh key
       ansible.builtin.lineinfile:
         dest: "{{ ssh_params.dest }}"
@@ -46,14 +52,14 @@
         - account.secret_type == "ssh_key"
         - ssh_params.strategy == "set_jms"
 
-    - name: Change SSH key
+    - name: "Change {{ account.username }} SSH key"
       ansible.builtin.authorized_key:
         user: "{{ account.username }}"
         key: "{{ account.secret }}"
         exclusive: "{{ ssh_params.exclusive }}"
       when: account.secret_type == "ssh_key"
 
-    - name: Set sudo setting
+    - name: "Set {{ account.username }} sudo setting"
       ansible.builtin.lineinfile:
         dest: /etc/sudoers
         state: present
@@ -61,25 +67,30 @@
         line: "{{ account.username + ' ALL=(ALL) NOPASSWD: ' + params.sudo }}"
         validate: visudo -cf %s
       when:
+        - user_info.failed
         - params.sudo
 
     - name: Refresh connection
       ansible.builtin.meta: reset_connection
 
-    - name: Verify password
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_password: "{{ account.secret }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} password (paramiko)"
+      ssh_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "password"
+      delegate_to: localhost
 
-    - name: Verify SSH key
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_ssh_private_key_file: "{{ account.private_key_path }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} SSH KEY (paramiko)"
+      ssh_ping:
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_user: "{{ account.username }}"
+        login_private_key_path: "{{ account.private_key_path  }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "ssh_key"
+      delegate_to: localhost

apps/accounts/automations/push_account/host/aix/main.yml

@@ -1,10 +1,17 @@
 - hosts: demo
   gather_facts: no
   tasks:
-    - name: Test privileged account
+    - name: "Test privileged {{ jms_account.username }} account"
       ansible.builtin.ping:
 
-    - name: Push user
+    - name: "Check if {{ account.username }} user exists"
+      getent:
+        database: passwd
+        key: "{{ account.username }}"
+      register: user_info
+      ignore_errors: yes  # 忽略错误，如果用户不存在时不会导致playbook失败
+
+    - name: "Add {{ account.username }} user"
       ansible.builtin.user:
         name: "{{ account.username }}"
         shell: "{{ params.shell }}"
@@ -12,22 +19,26 @@
         groups: "{{ params.groups }}"
         expires: -1
         state: present
+      when: user_info.failed
 
     - name: "Add {{ account.username }} group"
       ansible.builtin.group:
         name: "{{ account.username }}"
         state: present
+      when: user_info.failed
 
-    - name: Add user groups
+    - name: "Add {{ account.username }} user to group"
       ansible.builtin.user:
         name: "{{ account.username }}"
         groups: "{{ params.groups }}"
-      when: params.groups
+      when:
+        - user_info.failed
+        - params.groups
 
-    - name: Push user password
+    - name: "Change {{ account.username }} password"
       ansible.builtin.user:
         name: "{{ account.username }}"
-        password: "{{ account.secret | password_hash('sha512') }}"
+        password: "{{ account.secret | password_hash('des') }}"
         update_password: always
       ignore_errors: true
       when: account.secret_type == "password"
@@ -41,14 +52,14 @@
         - account.secret_type == "ssh_key"
         - ssh_params.strategy == "set_jms"
 
-    - name: Push SSH key
+    - name: "Change {{ account.username }} SSH key"
       ansible.builtin.authorized_key:
         user: "{{ account.username }}"
         key: "{{ account.secret }}"
         exclusive: "{{ ssh_params.exclusive }}"
       when: account.secret_type == "ssh_key"
 
-    - name: Set sudo setting
+    - name: "Set {{ account.username }} sudo setting"
       ansible.builtin.lineinfile:
         dest: /etc/sudoers
         state: present
@@ -56,25 +67,31 @@
         line: "{{ account.username + ' ALL=(ALL) NOPASSWD: ' + params.sudo }}"
         validate: visudo -cf %s
       when:
+        - user_info.failed
         - params.sudo
 
     - name: Refresh connection
       ansible.builtin.meta: reset_connection
 
-    - name: Verify password
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_password: "{{ account.secret }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} password (paramiko)"
+      ssh_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "password"
+      delegate_to: localhost
 
-    - name: Verify SSH key
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_ssh_private_key_file: "{{ account.private_key_path }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} SSH KEY (paramiko)"
+      ssh_ping:
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_user: "{{ account.username }}"
+        login_private_key_path: "{{ account.private_key_path  }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "ssh_key"
+      delegate_to: localhost
+

apps/accounts/automations/push_account/host/posix/main.yml

@@ -1,10 +1,17 @@
 - hosts: demo
   gather_facts: no
   tasks:
-    - name: Test privileged account
+    - name: "Test privileged {{ jms_account.username }} account"
       ansible.builtin.ping:
 
-    - name: Push user
+    - name: "Check if {{ account.username }} user exists"
+      getent:
+        database: passwd
+        key: "{{ account.username }}"
+      register: user_info
+      ignore_errors: yes  # 忽略错误，如果用户不存在时不会导致playbook失败
+
+    - name: "Add {{ account.username }} user"
       ansible.builtin.user:
         name: "{{ account.username }}"
         shell: "{{ params.shell }}"
@@ -12,19 +19,23 @@
         groups: "{{ params.groups }}"
         expires: -1
         state: present
+      when: user_info.failed
 
     - name: "Add {{ account.username }} group"
       ansible.builtin.group:
         name: "{{ account.username }}"
         state: present
+      when: user_info.failed
 
-    - name: Add user groups
+    - name: "Add {{ account.username }} user to group"
       ansible.builtin.user:
         name: "{{ account.username }}"
         groups: "{{ params.groups }}"
-      when: params.groups
+      when:
+        - user_info.failed
+        - params.groups
 
-    - name: Push user password
+    - name: "Change {{ account.username }} password"
       ansible.builtin.user:
         name: "{{ account.username }}"
         password: "{{ account.secret | password_hash('sha512') }}"
@@ -41,14 +52,14 @@
         - account.secret_type == "ssh_key"
         - ssh_params.strategy == "set_jms"
 
-    - name: Push SSH key
+    - name: "Change {{ account.username }} SSH key"
       ansible.builtin.authorized_key:
         user: "{{ account.username }}"
         key: "{{ account.secret }}"
         exclusive: "{{ ssh_params.exclusive }}"
       when: account.secret_type == "ssh_key"
 
-    - name: Set sudo setting
+    - name: "Set {{ account.username }} sudo setting"
       ansible.builtin.lineinfile:
         dest: /etc/sudoers
         state: present
@@ -56,25 +67,31 @@
         line: "{{ account.username + ' ALL=(ALL) NOPASSWD: ' + params.sudo }}"
         validate: visudo -cf %s
       when:
+        - user_info.failed
         - params.sudo
 
     - name: Refresh connection
       ansible.builtin.meta: reset_connection
 
-    - name: Verify password
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_password: "{{ account.secret }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} password (paramiko)"
+      ssh_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "password"
+      delegate_to: localhost
 
-    - name: Verify SSH key
-      ansible.builtin.ping:
-      become: no
-      vars:
-        ansible_user: "{{ account.username }}"
-        ansible_ssh_private_key_file: "{{ account.private_key_path }}"
-        ansible_become: no
+    - name: "Verify {{ account.username }} SSH KEY (paramiko)"
+      ssh_ping:
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_user: "{{ account.username }}"
+        login_private_key_path: "{{ account.private_key_path  }}"
+        gateway_args: "{{ jms_asset.ansible_ssh_common_args | default('') }}"
+        become: false
       when: account.secret_type == "ssh_key"
+      delegate_to: localhost
+

apps/accounts/automations/verify_account/custom/ssh/main.yml

@@ -2,6 +2,7 @@
   gather_facts: no
   vars:
     ansible_connection: local
+    ansible_become: false
 
   tasks:
     - name: Verify account (paramiko)
@@ -12,3 +13,8 @@
         login_password: "{{ account.secret }}"
         login_secret_type: "{{ account.secret_type }}"
         login_private_key_path: "{{ account.private_key_path }}"
+        become: "{{ custom_become | default(False) }}"
+        become_method: "{{ custom_become_method | default('su') }}"
+        become_user: "{{ custom_become_user | default('') }}"
+        become_password: "{{ custom_become_password | default('') }}"
+        become_private_key_path: "{{ custom_become_private_key_path | default(None) }}"

apps/accounts/backends/__init__.py

@@ -0,0 +1,41 @@
+from importlib import import_module
+
+from django.utils.functional import LazyObject
+
+from common.utils import get_logger
+from ..const import VaultTypeChoices
+
+__all__ = ['vault_client', 'get_vault_client']
+
+
+logger = get_logger(__file__)
+
+
+def get_vault_client(raise_exception=False, **kwargs):
+    enabled = kwargs.get('VAULT_ENABLED')
+    tp = 'hcp' if enabled else 'local'
+    try:
+        module_path = f'apps.accounts.backends.{tp}.main'
+        client = import_module(module_path).Vault(**kwargs)
+    except Exception as e:
+        logger.error(f'Init vault client failed: {e}')
+        if raise_exception:
+            raise
+        tp = VaultTypeChoices.local
+        module_path = f'apps.accounts.backends.{tp}.main'
+        client = import_module(module_path).Vault(**kwargs)
+    return client
+
+
+class VaultClient(LazyObject):
+
+    def _setup(self):
+        from jumpserver import settings as js_settings
+        from django.conf import settings
+        vault_config_names = [k for k in js_settings.__dict__.keys() if k.startswith('VAULT_')]
+        vault_configs = {name: getattr(settings, name, None) for name in vault_config_names}
+        self._wrapped = get_vault_client(**vault_configs)
+
+
+""" 为了安全, 页面修改配置, 重启服务后才会重新初始化 vault_client """
+vault_client = VaultClient()

apps/accounts/backends/base.py

@@ -0,0 +1,74 @@
+from abc import ABC, abstractmethod
+
+from django.forms.models import model_to_dict
+
+__all__ = ['BaseVault']
+
+
+class BaseVault(ABC):
+
+    def __init__(self, *args, **kwargs):
+        self.enabled = kwargs.get('VAULT_ENABLED')
+
+    def get(self, instance):
+        """ 返回 secret 值 """
+        return self._get(instance)
+
+    def create(self, instance):
+        if not instance.secret_has_save_to_vault:
+            self._create(instance)
+            self._clean_db_secret(instance)
+            self.save_metadata(instance)
+
+        if instance.is_sync_metadata:
+            self.save_metadata(instance)
+
+    def update(self, instance):
+        if not instance.secret_has_save_to_vault:
+            self._update(instance)
+            self._clean_db_secret(instance)
+            self.save_metadata(instance)
+
+        if instance.is_sync_metadata:
+            self.save_metadata(instance)
+
+    def delete(self, instance):
+        self._delete(instance)
+
+    def save_metadata(self, instance):
+        metadata = model_to_dict(instance, fields=[
+            'name', 'username', 'secret_type',
+            'connectivity', 'su_from', 'privileged'
+        ])
+        metadata = {k: str(v)[:500] for k, v in metadata.items() if v}
+        return self._save_metadata(instance, metadata)
+
+    # -------- abstractmethod -------- #
+
+    @abstractmethod
+    def _get(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _create(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _update(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _delete(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _clean_db_secret(self, instance):
+        raise NotImplementedError
+
+    @abstractmethod
+    def _save_metadata(self, instance, metadata):
+        raise NotImplementedError
+
+    @abstractmethod
+    def is_active(self, *args, **kwargs) -> (bool, str):
+        raise NotImplementedError

apps/accounts/backends/hcp/__init__.py

@@ -0,0 +1 @@
+from .main import *

apps/accounts/backends/hcp/entries.py

@@ -0,0 +1,84 @@
+import sys
+from abc import ABC
+
+from common.db.utils import Encryptor
+from common.utils import lazyproperty
+
+current_module = sys.modules[__name__]
+
+__all__ = ['build_entry']
+
+
+class BaseEntry(ABC):
+
+    def __init__(self, instance):
+        self.instance = instance
+
+    @lazyproperty
+    def full_path(self):
+        path_base = self.path_base
+        path_spec = self.path_spec
+        path = f'{path_base}/{path_spec}'
+        return path
+
+    @property
+    def path_base(self):
+        path = f'orgs/{self.instance.org_id}'
+        return path
+
+    @property
+    def path_spec(self):
+        raise NotImplementedError
+
+    def to_internal_data(self):
+        secret = getattr(self.instance, '_secret', None)
+        if secret is not None:
+            secret = Encryptor(secret).encrypt()
+        data = {'secret': secret}
+        return data
+
+    @staticmethod
+    def to_external_data(data):
+        secret = data.pop('secret', None)
+        if secret is not None:
+            secret = Encryptor(secret).decrypt()
+        return secret
+
+
+class AccountEntry(BaseEntry):
+
+    @property
+    def path_spec(self):
+        path = f'assets/{self.instance.asset_id}/accounts/{self.instance.id}'
+        return path
+
+
+class AccountTemplateEntry(BaseEntry):
+
+    @property
+    def path_spec(self):
+        path = f'account-templates/{self.instance.id}'
+        return path
+
+
+class HistoricalAccountEntry(BaseEntry):
+
+    @property
+    def path_base(self):
+        account = self.instance.instance
+        path = f'accounts/{account.id}/'
+        return path
+
+    @property
+    def path_spec(self):
+        path = f'histories/{self.instance.history_id}'
+        return path
+
+
+def build_entry(instance) -> BaseEntry:
+    class_name = instance.__class__.__name__
+    entry_class_name = f'{class_name}Entry'
+    entry_class = getattr(current_module, entry_class_name, None)
+    if not entry_class:
+        raise Exception(f'Entry class {entry_class_name} is not found')
+    return entry_class(instance)

apps/accounts/backends/hcp/main.py

@@ -0,0 +1,53 @@
+from common.db.utils import get_logger
+from .entries import build_entry
+from .service import VaultKVClient
+from ..base import BaseVault
+
+__all__ = ['Vault']
+
+logger = get_logger(__name__)
+
+
+class Vault(BaseVault):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = VaultKVClient(
+            url=kwargs.get('VAULT_HCP_HOST'),
+            token=kwargs.get('VAULT_HCP_TOKEN'),
+            mount_point=kwargs.get('VAULT_HCP_MOUNT_POINT')
+        )
+
+    def is_active(self):
+        return self.client.is_active()
+
+    def _get(self, instance):
+        entry = build_entry(instance)
+        # TODO: get data 是不是层数太多了
+        data = self.client.get(path=entry.full_path).get('data', {})
+        data = entry.to_external_data(data)
+        return data
+
+    def _create(self, instance):
+        entry = build_entry(instance)
+        data = entry.to_internal_data()
+        self.client.create(path=entry.full_path, data=data)
+
+    def _update(self, instance):
+        entry = build_entry(instance)
+        data = entry.to_internal_data()
+        self.client.patch(path=entry.full_path, data=data)
+
+    def _delete(self, instance):
+        entry = build_entry(instance)
+        self.client.delete(path=entry.full_path)
+
+    def _clean_db_secret(self, instance):
+        instance.is_sync_metadata = False
+        instance.mark_secret_save_to_vault()
+
+    def _save_metadata(self, instance, metadata):
+        try:
+            entry = build_entry(instance)
+            self.client.update_metadata(path=entry.full_path, metadata=metadata)
+        except Exception as e:
+            logger.error(f'save metadata error: {e}')

apps/accounts/backends/hcp/service.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+#
+import hvac
+from hvac import exceptions
+from requests.exceptions import ConnectionError
+
+from common.utils import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = ['VaultKVClient']
+
+
+class VaultKVClient(object):
+    max_versions = 20
+
+    def __init__(self, url, token, mount_point):
+        assert isinstance(self.max_versions, int) and self.max_versions >= 3, (
+            'max_versions must to be an integer that is greater than or equal to 3'
+        )
+        self.client = hvac.Client(url=url, token=token)
+        self.mount_point = mount_point
+        self.enable_secrets_engine_if_need()
+
+    def is_active(self):
+        try:
+            if not self.client.sys.is_initialized():
+                return False, 'Vault is not initialized'
+            if self.client.sys.is_sealed():
+                return False, 'Vault is sealed'
+            if not self.client.is_authenticated():
+                return False, 'Vault is not authenticated'
+        except ConnectionError as e:
+            logger.error(str(e))
+            return False, f'Vault is not reachable: {e}'
+        else:
+            return True, ''
+
+    def enable_secrets_engine_if_need(self):
+        secrets_engines = self.client.sys.list_mounted_secrets_engines()
+        mount_points = secrets_engines.keys()
+        if f'{self.mount_point}/' in mount_points:
+            return
+        self.client.sys.enable_secrets_engine(
+            backend_type='kv',
+            path=self.mount_point,
+            options={'version': 2}  # TODO: version 是否从配置中读取?
+        )
+        self.client.secrets.kv.v2.configure(
+            max_versions=self.max_versions,
+            mount_point=self.mount_point
+        )
+
+    def get(self, path, version=None):
+        try:
+            response = self.client.secrets.kv.v2.read_secret_version(
+                path=path,
+                version=version,
+                mount_point=self.mount_point
+            )
+        except exceptions.InvalidPath as e:
+            return {}
+        data = response.get('data', {})
+        return data
+
+    def create(self, path, data: dict):
+        self._update_or_create(path=path, data=data)
+
+    def update(self, path, data: dict):
+        """ 未更新的数据会被删除 """
+        self._update_or_create(path=path, data=data)
+
+    def patch(self, path, data: dict):
+        """ 未更新的数据不会被删除 """
+        self.client.secrets.kv.v2.patch(
+            path=path,
+            secret=data,
+            mount_point=self.mount_point
+        )
+
+    def delete(self, path):
+        self.client.secrets.kv.v2.delete_metadata_and_all_versions(
+            path=path,
+            mount_point=self.mount_point,
+        )
+
+    def _update_or_create(self, path, data: dict):
+        self.client.secrets.kv.v2.create_or_update_secret(
+            path=path,
+            secret=data,
+            mount_point=self.mount_point
+        )
+
+    def update_metadata(self, path, metadata: dict):
+        try:
+            self.client.secrets.kv.v2.update_metadata(
+                path=path,
+                mount_point=self.mount_point,
+                custom_metadata=metadata
+            )
+        except exceptions.InvalidPath as e:
+            logger.error('Update metadata error: {}'.format(e))

apps/accounts/backends/local/__init__.py

@@ -0,0 +1 @@
+from .main import *

apps/accounts/backends/local/main.py

@@ -0,0 +1,36 @@
+from common.utils import get_logger
+from ..base import BaseVault
+
+logger = get_logger(__name__)
+
+__all__ = ['Vault']
+
+
+class Vault(BaseVault):
+
+    def is_active(self):
+        return True, ''
+
+    def _get(self, instance):
+        secret = getattr(instance, '_secret', None)
+        return secret
+
+    def _create(self, instance):
+        """ Ignore """
+        pass
+
+    def _update(self, instance):
+        """ Ignore """
+        pass
+
+    def _delete(self, instance):
+        """ Ignore """
+        pass
+
+    def _save_metadata(self, instance, metadata):
+        """ Ignore """
+        pass
+
+    def _clean_db_secret(self, instance):
+        """ Ignore *重要* 不能删除本地 secret """
+        pass

apps/accounts/const/__init__.py

@@ -1,2 +1,3 @@
 from .account import *
 from .automation import *
+from .vault import *

apps/accounts/const/account.py

@@ -1,5 +1,5 @@
 from django.db.models import TextChoices
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 
 class SecretType(TextChoices):
@@ -16,6 +16,10 @@ class AliasAccount(TextChoices):
     USER = '@USER', _('Dynamic user')
     ANON = '@ANON', _('Anonymous account')
 
+    @classmethod
+    def virtual_choices(cls):
+        return [(k, v) for k, v in cls.choices if k not in (cls.ALL,)]
+
 
 class Source(TextChoices):
     LOCAL = 'local', _('Local')

apps/accounts/const/automation.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.const import Connectivity
 from common.db.fields import TreeChoices

apps/accounts/const/vault.py

@@ -0,0 +1,9 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+__all__ = ['VaultTypeChoices']
+
+
+class VaultTypeChoices(models.TextChoices):
+    local = 'local', _('Database')
+    hcp = 'hcp', _('HCP Vault')

apps/accounts/filters.py

@@ -13,7 +13,8 @@ class AccountFilterSet(BaseFilterSet):
     hostname = drf_filters.CharFilter(field_name='name', lookup_expr='exact')
     username = drf_filters.CharFilter(field_name="username", lookup_expr='exact')
     address = drf_filters.CharFilter(field_name="asset__address", lookup_expr='exact')
-    asset = drf_filters.CharFilter(field_name="asset_id", lookup_expr='exact')
+    asset_id = drf_filters.CharFilter(field_name="asset", lookup_expr='exact')
+    asset = drf_filters.CharFilter(field_name='asset', lookup_expr='exact')
     assets = drf_filters.CharFilter(field_name='asset_id', lookup_expr='exact')
     nodes = drf_filters.CharFilter(method='filter_nodes')
     node_id = drf_filters.CharFilter(method='filter_nodes')
@@ -45,7 +46,7 @@ class AccountFilterSet(BaseFilterSet):
 
     class Meta:
         model = Account
-        fields = ['id', 'asset_id', 'source_id', 'secret_type']
+        fields = ['id', 'asset', 'source_id', 'secret_type']
 
 
 class GatheredAccountFilterSet(BaseFilterSet):

apps/accounts/migrations/0012_auto_20230621_1456.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.19 on 2023-06-21 06:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0011_auto_20230506_1443'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='account',
+            old_name='secret',
+            new_name='_secret',
+        ),
+        migrations.RenameField(
+            model_name='accounttemplate',
+            old_name='secret',
+            new_name='_secret',
+        ),
+        migrations.RenameField(
+            model_name='historicalaccount',
+            old_name='secret',
+            new_name='_secret',
+        ),
+    ]

apps/accounts/migrations/0013_account_backup_recipients.py

@@ -0,0 +1,77 @@
+# Generated by Django 4.1.10 on 2023-08-03 08:28
+from django.conf import settings
+from django.db import migrations, models
+
+import common.db.encoder
+
+
+def migrate_recipients(apps, schema_editor):
+    account_backup_model = apps.get_model('accounts', 'AccountBackupAutomation')
+    execution_model = apps.get_model('accounts', 'AccountBackupExecution')
+    for account_backup in account_backup_model.objects.all():
+        recipients = list(account_backup.recipients.all())
+        if not recipients:
+            continue
+        account_backup.recipients_part_one.set(recipients)
+
+    objs = []
+    for execution in execution_model.objects.all():
+        snapshot = execution.snapshot
+        recipients = snapshot.pop('recipients', {})
+        snapshot.update({'recipients_part_one': recipients, 'recipients_part_two': {}})
+        objs.append(execution)
+    execution_model.objects.bulk_update(objs, ['snapshot'])
+
+
+def migrate_snapshot(apps, schema_editor):
+    model = apps.get_model('accounts', 'AccountBackupExecution')
+    objs = []
+    for execution in model.objects.all():
+        execution.snapshot = execution.plan_snapshot
+        objs.append(execution)
+    model.objects.bulk_update(objs, ['snapshot'])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('accounts', '0012_auto_20230621_1456'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='accountbackupautomation',
+            name='recipients_part_one',
+            field=models.ManyToManyField(
+                blank=True, related_name='recipient_part_one_plans',
+                to=settings.AUTH_USER_MODEL, verbose_name='Recipient part one'
+            ),
+        ),
+        migrations.AddField(
+            model_name='accountbackupautomation',
+            name='recipients_part_two',
+            field=models.ManyToManyField(
+                blank=True, related_name='recipient_part_two_plans',
+                to=settings.AUTH_USER_MODEL, verbose_name='Recipient part two'
+            ),
+        ),
+        migrations.AddField(
+            model_name='accountbackupexecution',
+            name='snapshot',
+            field=models.JSONField(
+                default=dict, encoder=common.db.encoder.ModelJSONFieldEncoder,
+                null=True, blank=True, verbose_name='Account backup snapshot'
+            ),
+        ),
+        migrations.RunPython(migrate_snapshot),
+        migrations.RunPython(migrate_recipients),
+        migrations.RemoveField(
+            model_name='accountbackupexecution',
+            name='plan_snapshot',
+        ),
+        migrations.RemoveField(
+            model_name='accountbackupautomation',
+            name='recipients',
+        ),
+
+    ]

apps/accounts/migrations/0014_virtualaccount.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.1.10 on 2023-08-01 09:12
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('accounts', '0013_account_backup_recipients'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='VirtualAccount',
+            fields=[
+                ('created_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Created by')),
+                ('updated_by', models.CharField(blank=True, max_length=128, null=True, verbose_name='Updated by')),
+                ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('alias', models.CharField(choices=[('@INPUT', 'Manual input'), ('@USER', 'Dynamic user'), ('@ANON', 'Anonymous account')], max_length=128, verbose_name='Alias')),
+                ('secret_from_login', models.BooleanField(default=None, null=True, verbose_name='Secret from login')),
+            ],
+            options={
+                'unique_together': {('alias', 'org_id')},
+            },
+        ),
+    ]

apps/accounts/models/__init__.py

@@ -1,3 +1,5 @@
 from .account import *
 from .automations import *
 from .base import *
+from .template import *
+from .virtual import *

apps/accounts/models/account.py

@@ -1,15 +1,14 @@
 from django.db import models
-from django.db.models import Count, Q
-from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from simple_history.models import HistoricalRecords
 
 from assets.models.base import AbsConnectivity
 from common.utils import lazyproperty
 from .base import BaseAccount
-from ..const import AliasAccount, Source
+from .mixins import VaultModelMixin
+from ..const import Source
 
-__all__ = ['Account', 'AccountTemplate']
+__all__ = ['Account', 'AccountHistoricalRecords']
 
 
 class AccountHistoricalRecords(HistoricalRecords):
@@ -32,7 +31,7 @@ class AccountHistoricalRecords(HistoricalRecords):
         diff = attrs - history_attrs
         if not diff:
             return
-        super().post_save(instance, created, using=using, **kwargs)
+        return super().post_save(instance, created, using=using, **kwargs)
 
     def create_history_model(self, model, inherited):
         if self.included_fields and not self.excluded_fields:
@@ -53,7 +52,7 @@ class Account(AbsConnectivity, BaseAccount):
         on_delete=models.SET_NULL, verbose_name=_("Su from")
     )
     version = models.IntegerField(default=0, verbose_name=_('Version'))
-    history = AccountHistoricalRecords(included_fields=['id', 'secret', 'secret_type', 'version'])
+    history = AccountHistoricalRecords(included_fields=['id', '_secret', 'secret_type', 'version'])
     source = models.CharField(max_length=30, default=Source.LOCAL, verbose_name=_('Source'))
     source_id = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Source ID'))
 
@@ -88,29 +87,6 @@ class Account(AbsConnectivity, BaseAccount):
     def has_secret(self):
         return bool(self.secret)
 
-    @classmethod
-    def get_special_account(cls, name):
-        if name == AliasAccount.INPUT.value:
-            return cls.get_manual_account()
-        elif name == AliasAccount.ANON.value:
-            return cls.get_anonymous_account()
-        else:
-            return cls(name=name, username=name, secret=None)
-
-    @classmethod
-    def get_manual_account(cls):
-        """ @INPUT 手动登录的账号(any) """
-        return cls(name=AliasAccount.INPUT.label, username=AliasAccount.INPUT.value, secret=None)
-
-    @classmethod
-    def get_anonymous_account(cls):
-        return cls(name=AliasAccount.ANON.label, username=AliasAccount.ANON.value, secret=None)
-
-    @classmethod
-    def get_user_account(cls):
-        """ @USER 动态用户的账号(self) """
-        return cls(name=AliasAccount.USER.label, username=AliasAccount.USER.value, secret=None)
-
     @lazyproperty
     def versions(self):
         return self.history.count()
@@ -120,81 +96,19 @@ class Account(AbsConnectivity, BaseAccount):
         return self.asset.accounts.exclude(id=self.id).exclude(su_from=self)
 
 
-class AccountTemplate(BaseAccount):
-    su_from = models.ForeignKey(
-        'self', related_name='su_to', null=True,
-        on_delete=models.SET_NULL, verbose_name=_("Su from")
-    )
+def replace_history_model_with_mixin():
+    """
+    替换历史模型中的父类为指定的Mixin类。
 
-    class Meta:
-        verbose_name = _('Account template')
-        unique_together = (
-            ('name', 'org_id'),
-        )
-        permissions = [
-            ('view_accounttemplatesecret', _('Can view asset account template secret')),
-            ('change_accounttemplatesecret', _('Can change asset account template secret')),
-        ]
+    Parameters:
+        model (class): 历史模型类，例如 Account.history.model
+        mixin_class (class): 要替换为的Mixin类
 
-    @classmethod
-    def get_su_from_account_templates(cls, pk=None):
-        if pk is None:
-            return cls.objects.all()
-        return cls.objects.exclude(Q(id=pk) | Q(su_from_id=pk))
+    Returns:
+        None
+    """
+    model = Account.history.model
+    model.__bases__ = (VaultModelMixin,) + model.__bases__
 
-    def __str__(self):
-        return f'{self.name}({self.username})'
 
-    def get_su_from_account(self, asset):
-        su_from = self.su_from
-        if su_from and asset.platform.su_enabled:
-            account = asset.accounts.filter(
-                username=su_from.username,
-                secret_type=su_from.secret_type
-            ).first()
-            return account
-
-    def __str__(self):
-        return self.username
-
-    @staticmethod
-    def bulk_update_accounts(accounts, data):
-        history_model = Account.history.model
-        account_ids = accounts.values_list('id', flat=True)
-        history_accounts = history_model.objects.filter(id__in=account_ids)
-        account_id_count_map = {
-            str(i['id']): i['count']
-            for i in history_accounts.values('id').order_by('id')
-            .annotate(count=Count(1)).values('id', 'count')
-        }
-
-        for account in accounts:
-            account_id = str(account.id)
-            account.version = account_id_count_map.get(account_id) + 1
-            for k, v in data.items():
-                setattr(account, k, v)
-        Account.objects.bulk_update(accounts, ['version', 'secret'])
-
-    @staticmethod
-    def bulk_create_history_accounts(accounts, user_id):
-        history_model = Account.history.model
-        history_account_objs = []
-        for account in accounts:
-            history_account_objs.append(
-                history_model(
-                    id=account.id,
-                    version=account.version,
-                    secret=account.secret,
-                    secret_type=account.secret_type,
-                    history_user_id=user_id,
-                    history_date=timezone.now()
-                )
-            )
-        history_model.objects.bulk_create(history_account_objs)
-
-    def bulk_sync_account_secret(self, accounts, user_id):
-        """ 批量同步账号密码 """
-        if not accounts:
-            return
-        self.bulk_update_accounts(accounts, {'secret': self.secret})
-        self.bulk_create_history_accounts(accounts, user_id)
+replace_history_model_with_mixin()

apps/accounts/models/automations/backup_account.py

@@ -6,7 +6,7 @@ import uuid
 from celery import current_task
 from django.db import models
 from django.db.models import F
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.const.choices import Trigger
 from common.db.encoder import ModelJSONFieldEncoder
@@ -22,9 +22,13 @@ logger = get_logger(__file__)
 
 class AccountBackupAutomation(PeriodTaskModelMixin, JMSOrgBaseModel):
     types = models.JSONField(default=list)
-    recipients = models.ManyToManyField(
-        'users.User', related_name='recipient_escape_route_plans', blank=True,
-        verbose_name=_("Recipient")
+    recipients_part_one = models.ManyToManyField(
+        'users.User', related_name='recipient_part_one_plans', blank=True,
+        verbose_name=_("Recipient part one")
+    )
+    recipients_part_two = models.ManyToManyField(
+        'users.User', related_name='recipient_part_two_plans', blank=True,
+        verbose_name=_("Recipient part two")
     )
 
     def __str__(self):
@@ -52,9 +56,13 @@ class AccountBackupAutomation(PeriodTaskModelMixin, JMSOrgBaseModel):
             'org_id': self.org_id,
             'created_by': self.created_by,
             'types': self.types,
-            'recipients': {
-                str(recipient.id): (str(recipient), bool(recipient.secret_key))
-                for recipient in self.recipients.all()
+            'recipients_part_one': {
+                str(user.id): (str(user), bool(user.secret_key))
+                for user in self.recipients_part_one.all()
+            },
+            'recipients_part_two': {
+                str(user.id): (str(user), bool(user.secret_key))
+                for user in self.recipients_part_two.all()
             }
         }
 
@@ -68,7 +76,7 @@ class AccountBackupAutomation(PeriodTaskModelMixin, JMSOrgBaseModel):
         except AttributeError:
             hid = str(uuid.uuid4())
         execution = AccountBackupExecution.objects.create(
-            id=hid, plan=self, plan_snapshot=self.to_attr_json(), trigger=trigger
+            id=hid, plan=self, snapshot=self.to_attr_json(), trigger=trigger
         )
         return execution.start()
 
@@ -85,7 +93,7 @@ class AccountBackupExecution(OrgModelMixin):
     timedelta = models.FloatField(
         default=0.0, verbose_name=_('Time'), null=True
     )
-    plan_snapshot = models.JSONField(
+    snapshot = models.JSONField(
         encoder=ModelJSONFieldEncoder, default=dict,
         blank=True, null=True, verbose_name=_('Account backup snapshot')
     )
@@ -108,16 +116,9 @@ class AccountBackupExecution(OrgModelMixin):
 
     @property
     def types(self):
-        types = self.plan_snapshot.get('types')
+        types = self.snapshot.get('types')
         return types
 
-    @property
-    def recipients(self):
-        recipients = self.plan_snapshot.get('recipients')
-        if not recipients:
-            return []
-        return recipients.values()
-
     @lazyproperty
     def backup_accounts(self):
         from accounts.models import Account

apps/accounts/models/automations/change_secret.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from accounts.const import (
     AutomationTypes, SecretType, SecretStrategy, SSHKeyStrategy
@@ -86,7 +86,7 @@ class ChangeSecretRecord(JMSBaseModel):
     asset = models.ForeignKey('assets.Asset', on_delete=models.CASCADE, null=True)
     account = models.ForeignKey('accounts.Account', on_delete=models.CASCADE, null=True)
     old_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Old secret'))
-    new_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
+    new_secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('New secret'))
     date_started = models.DateTimeField(blank=True, null=True, verbose_name=_('Date started'))
     date_finished = models.DateTimeField(blank=True, null=True, verbose_name=_('Date finished'))
     status = models.CharField(max_length=16, default='pending')

apps/accounts/models/automations/gather_account.py

@@ -1,6 +1,6 @@
 from django.db import models
 from django.db.models import Q
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from accounts.const import AutomationTypes, Source
 from accounts.models import Account

apps/accounts/models/automations/push_account.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from accounts.const import AutomationTypes
 from accounts.models import Account

apps/accounts/models/automations/verify_account.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from accounts.const import AutomationTypes
 from .base import AccountBaseAutomation

apps/accounts/models/base.py

@@ -6,36 +6,35 @@ from hashlib import md5
 import sshpubkeys
 from django.conf import settings
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from accounts.const import SecretType
-from common.db import fields
 from common.utils import (
     ssh_key_string_to_obj, ssh_key_gen, get_logger,
     random_string, lazyproperty, parse_ssh_public_key_str, is_openssh_format_key
 )
+from accounts.models.mixins import VaultModelMixin, VaultManagerMixin, VaultQuerySetMixin
 from orgs.mixins.models import JMSOrgBaseModel, OrgManager
 
 logger = get_logger(__file__)
 
 
-class BaseAccountQuerySet(models.QuerySet):
+class BaseAccountQuerySet(VaultQuerySetMixin, models.QuerySet):
     def active(self):
         return self.filter(is_active=True)
 
 
-class BaseAccountManager(OrgManager):
+class BaseAccountManager(VaultManagerMixin, OrgManager):
     def active(self):
         return self.get_queryset().active()
 
 
-class BaseAccount(JMSOrgBaseModel):
+class BaseAccount(VaultModelMixin, JMSOrgBaseModel):
     name = models.CharField(max_length=128, verbose_name=_("Name"))
     username = models.CharField(max_length=128, blank=True, verbose_name=_('Username'), db_index=True)
     secret_type = models.CharField(
         max_length=16, choices=SecretType.choices, default=SecretType.PASSWORD, verbose_name=_('Secret type')
     )
-    secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
     privileged = models.BooleanField(verbose_name=_("Privileged"), default=False)
     is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
 

apps/accounts/models/mixins/__init__.py

@@ -0,0 +1 @@
+from .vault import *

apps/accounts/models/mixins/vault.py

@@ -0,0 +1,94 @@
+from django.db import models
+from django.db.models.signals import post_save
+from django.utils.translation import gettext_lazy as _
+
+from common.db import fields
+
+__all__ = ['VaultQuerySetMixin', 'VaultManagerMixin', 'VaultModelMixin']
+
+
+class VaultQuerySetMixin(models.QuerySet):
+
+    def update(self, **kwargs):
+        """
+           1. 替换 secret 为 _secret
+           2. 触发 post_save 信号
+        """
+        if 'secret' in kwargs:
+            kwargs.update({
+                '_secret': kwargs.pop('secret')
+            })
+        rows = super().update(**kwargs)
+
+        # 为了获取更新后的对象所以单独查询一次
+        ids = self.values_list('id', flat=True)
+        objs = self.model.objects.filter(id__in=ids)
+        for obj in objs:
+            post_save.send(obj.__class__, instance=obj, created=False)
+        return rows
+
+
+class VaultManagerMixin(models.Manager):
+    """ 触发 bulk_create 和 bulk_update 操作下的 post_save 信号 """
+
+    def bulk_create(self, objs, batch_size=None, ignore_conflicts=False):
+        objs = super().bulk_create(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts)
+        for obj in objs:
+            post_save.send(obj.__class__, instance=obj, created=True)
+        return objs
+
+    def bulk_update(self, objs, batch_size=None, ignore_conflicts=False):
+        objs = super().bulk_update(objs, batch_size=batch_size, ignore_conflicts=ignore_conflicts)
+        for obj in objs:
+            post_save.send(obj.__class__, instance=obj, created=False)
+        return objs
+
+
+class VaultModelMixin(models.Model):
+    _secret = fields.EncryptTextField(blank=True, null=True, verbose_name=_('Secret'))
+    is_sync_metadata = True
+
+    class Meta:
+        abstract = True
+
+    # 缓存 secret 值, lazy-property 不能用
+    __secret = None
+
+    @property
+    def secret(self):
+        if self.__secret:
+            return self.__secret
+        from accounts.backends import vault_client
+        secret = vault_client.get(self)
+        if not secret and not self.secret_has_save_to_vault:
+            # vault_client 获取不到, 并且 secret 没有保存到 vault, 就从 self._secret 获取
+            secret = self._secret
+        self.__secret = secret
+        return self.__secret
+
+    @secret.setter
+    def secret(self, value):
+        """
+        保存的时候通过 post_save 信号监听进行处理,
+        先保存到 db, 再保存到 vault 同时删除本地 db _secret 值
+        """
+        self._secret = value
+        self.__secret = value
+
+    _secret_save_to_vault_mark = '# Secret-has-been-saved-to-vault #'
+
+    def mark_secret_save_to_vault(self):
+        self._secret = self._secret_save_to_vault_mark
+        self.save()
+
+    @property
+    def secret_has_save_to_vault(self):
+        return self._secret == self._secret_save_to_vault_mark
+
+    def save(self, *args, **kwargs):
+        """ 通过 post_save signal 处理 _secret 数据 """
+        update_fields = kwargs.get('update_fields')
+        if update_fields and 'secret' in update_fields:
+            update_fields.remove('secret')
+            update_fields.append('_secret')
+        return super().save(*args, **kwargs)

apps/accounts/models/template.py

@@ -0,0 +1,86 @@
+from django.db import models
+from django.db.models import Count, Q
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+from .account import Account
+from .base import BaseAccount
+
+__all__ = ['AccountTemplate', ]
+
+
+class AccountTemplate(BaseAccount):
+    su_from = models.ForeignKey(
+        'self', related_name='su_to', null=True,
+        on_delete=models.SET_NULL, verbose_name=_("Su from")
+    )
+
+    class Meta:
+        verbose_name = _('Account template')
+        unique_together = (
+            ('name', 'org_id'),
+        )
+        permissions = [
+            ('view_accounttemplatesecret', _('Can view asset account template secret')),
+            ('change_accounttemplatesecret', _('Can change asset account template secret')),
+        ]
+
+    @classmethod
+    def get_su_from_account_templates(cls, pk=None):
+        if pk is None:
+            return cls.objects.all()
+        return cls.objects.exclude(Q(id=pk) | Q(su_from_id=pk))
+
+    def __str__(self):
+        return f'{self.name}({self.username})'
+
+    def get_su_from_account(self, asset):
+        su_from = self.su_from
+        if su_from and asset.platform.su_enabled:
+            account = asset.accounts.filter(
+                username=su_from.username,
+                secret_type=su_from.secret_type
+            ).first()
+            return account
+
+    @staticmethod
+    def bulk_update_accounts(accounts, data):
+        history_model = Account.history.model
+        account_ids = accounts.values_list('id', flat=True)
+        history_accounts = history_model.objects.filter(id__in=account_ids)
+        account_id_count_map = {
+            str(i['id']): i['count']
+            for i in history_accounts.values('id').order_by('id')
+            .annotate(count=Count(1)).values('id', 'count')
+        }
+
+        for account in accounts:
+            account_id = str(account.id)
+            account.version = account_id_count_map.get(account_id) + 1
+            for k, v in data.items():
+                setattr(account, k, v)
+        Account.objects.bulk_update(accounts, ['version', 'secret'])
+
+    @staticmethod
+    def bulk_create_history_accounts(accounts, user_id):
+        history_model = Account.history.model
+        history_account_objs = []
+        for account in accounts:
+            history_account_objs.append(
+                history_model(
+                    id=account.id,
+                    version=account.version,
+                    secret=account.secret,
+                    secret_type=account.secret_type,
+                    history_user_id=user_id,
+                    history_date=timezone.now()
+                )
+            )
+        history_model.objects.bulk_create(history_account_objs)
+
+    def bulk_sync_account_secret(self, accounts, user_id):
+        """ 批量同步账号密码 """
+        if not accounts:
+            return
+        self.bulk_update_accounts(accounts, {'secret': self.secret})
+        self.bulk_create_history_accounts(accounts, user_id)

apps/accounts/models/virtual.py

@@ -0,0 +1,103 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from accounts.const import AliasAccount
+from orgs.mixins.models import JMSOrgBaseModel
+
+__all__ = ['VirtualAccount']
+
+from orgs.utils import tmp_to_org
+
+
+class VirtualAccount(JMSOrgBaseModel):
+    alias = models.CharField(max_length=128, choices=AliasAccount.virtual_choices(), verbose_name=_('Alias'), )
+    secret_from_login = models.BooleanField(default=None, null=True, verbose_name=_("Secret from login"), )
+
+    class Meta:
+        unique_together = [('alias', 'org_id')]
+
+    @property
+    def name(self):
+        return self.get_alias_display()
+
+    @property
+    def username(self):
+        usernames_map = {
+            AliasAccount.INPUT: _("Manual input"),
+            AliasAccount.USER: _("Same with user"),
+            AliasAccount.ANON: ''
+        }
+        usernames_map = {str(k): v for k, v in usernames_map.items()}
+        return usernames_map.get(self.alias, '')
+
+    @property
+    def comment(self):
+        comments_map = {
+            AliasAccount.INPUT: _('Non-asset account, Input username/password on connect'),
+            AliasAccount.USER: _('The account username name same with user on connect'),
+            AliasAccount.ANON: _('Connect asset without using a username and password, '
+                                 'and it only supports web-based and custom-type assets'),
+        }
+        comments_map = {str(k): v for k, v in comments_map.items()}
+        return comments_map.get(self.alias, '')
+
+    @classmethod
+    def get_or_init_queryset(cls):
+        aliases = [i[0] for i in AliasAccount.virtual_choices()]
+        alias_created = cls.objects.all().values_list('alias', flat=True)
+        need_created = set(aliases) - set(alias_created)
+
+        if need_created:
+            accounts = [cls(alias=alias) for alias in need_created]
+            cls.objects.bulk_create(accounts, ignore_conflicts=True)
+        return cls.objects.all()
+
+    @classmethod
+    def get_special_account(cls, alias, user, asset, input_username='', input_secret='', from_permed=True):
+        if alias == AliasAccount.INPUT.value:
+            account = cls.get_manual_account(input_username, input_secret, from_permed)
+        elif alias == AliasAccount.ANON.value:
+            account = cls.get_anonymous_account()
+        elif alias == AliasAccount.USER.value:
+            account = cls.get_same_account(user, asset, input_secret=input_secret, from_permed=from_permed)
+        else:
+            account = cls(name=alias, username=alias, secret=None)
+        account.alias = alias
+        if asset:
+            account.asset = asset
+            account.org_id = asset.org_id
+        return account
+
+    @classmethod
+    def get_manual_account(cls, input_username='', input_secret='', from_permed=True):
+        """ @INPUT 手动登录的账号(any) """
+        from .account import Account
+        if from_permed:
+            username = AliasAccount.INPUT.value
+            secret = ''
+        else:
+            username = input_username
+            secret = input_secret
+        return Account(name=AliasAccount.INPUT.label, username=username, secret=secret)
+
+    @classmethod
+    def get_anonymous_account(cls):
+        from .account import Account
+        return Account(name=AliasAccount.ANON.label, username=AliasAccount.ANON.value, secret=None)
+
+    @classmethod
+    def get_same_account(cls, user, asset, input_secret='', from_permed=True):
+        """ @USER 动态用户的账号(self) """
+        from .account import Account
+        username = user.username
+
+        with tmp_to_org(asset.org):
+            same_account = cls.objects.filter(alias='@USER').first()
+
+        secret = ''
+        if same_account and same_account.secret_from_login:
+            secret = user.get_cached_password_if_has()
+
+        if not secret and not from_permed:
+            secret = input_secret
+        return Account(name=AliasAccount.USER.label, username=username, secret=secret)

apps/accounts/notifications.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.tasks import send_mail_attachment_async
 from users.models import User

apps/accounts/serializers/account/__init__.py

@@ -1,5 +1,6 @@
 from .account import *
 from .backup import *
 from .base import *
-from .template import *
 from .gathered_account import *
+from .template import *
+from .virtual import *

apps/accounts/serializers/account/account.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 
 from django.db import IntegrityError
 from django.db.models import Q
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 from rest_framework.generics import get_object_or_404
 from rest_framework.validators import UniqueTogetherValidator
@@ -95,6 +95,8 @@ class AccountCreateUpdateSerializerMixin(serializers.Serializer):
             field.name for field in template._meta.fields
             if field.name not in ignore_fields
         ]
+        field_names = [name if name != '_secret' else 'secret' for name in field_names]
+
         attrs = {}
         for name in field_names:
             value = getattr(template, name, None)
@@ -198,7 +200,6 @@ class AccountAssetSerializer(serializers.ModelSerializer):
 
 class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerializer):
     asset = AccountAssetSerializer(label=_('Asset'))
-    has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
     source = LabeledChoiceField(
         choices=Source.choices, label=_("Source"), required=False,
         allow_null=True, default=Source.LOCAL
@@ -233,6 +234,15 @@ class AccountSerializer(AccountCreateUpdateSerializerMixin, BaseAccountSerialize
         return queryset
 
 
+class AccountDetailSerializer(AccountSerializer):
+    has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
+
+    class Meta(AccountSerializer.Meta):
+        model = Account
+        fields = AccountSerializer.Meta.fields + ['has_secret']
+        read_only_fields = AccountSerializer.Meta.read_only_fields + ['has_secret']
+
+
 class AssetAccountBulkSerializerResultSerializer(serializers.Serializer):
     asset = serializers.CharField(read_only=True, label=_('Asset'))
     state = serializers.CharField(read_only=True, label=_('State'))

apps/accounts/serializers/account/backup.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from accounts.models import AccountBackupAutomation, AccountBackupExecution
@@ -24,7 +24,7 @@ class AccountBackupSerializer(PeriodTaskSerializerMixin, BulkOrgResourceModelSer
         ]
         fields = read_only_fields + [
             'id', 'name', 'is_periodic', 'interval', 'crontab',
-            'comment', 'recipients', 'types'
+            'comment', 'types', 'recipients_part_one', 'recipients_part_two'
         ]
         extra_kwargs = {
             'name': {'required': True},
@@ -44,7 +44,7 @@ class AccountBackupPlanExecutionSerializer(serializers.ModelSerializer):
     class Meta:
         model = AccountBackupExecution
         read_only_fields = [
-            'id', 'date_start', 'timedelta', 'plan_snapshot',
-            'trigger', 'reason', 'is_success', 'org_id', 'recipients'
+            'id', 'date_start', 'timedelta', 'snapshot',
+            'trigger', 'reason', 'is_success', 'org_id'
         ]
         fields = read_only_fields + ['plan']

apps/accounts/serializers/account/base.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from accounts.const import SecretType
@@ -61,20 +61,18 @@ class AuthValidateMixin(serializers.Serializer):
 
 
 class BaseAccountSerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
-    has_secret = serializers.BooleanField(label=_("Has secret"), read_only=True)
 
     class Meta:
         model = BaseAccount
         fields_mini = ['id', 'name', 'username']
         fields_small = fields_mini + [
-            'secret_type', 'secret', 'has_secret', 'passphrase',
+            'secret_type', 'secret', 'passphrase',
             'privileged', 'is_active', 'spec_info',
         ]
         fields_other = ['created_by', 'date_created', 'date_updated', 'comment']
         fields = fields_small + fields_other
         read_only_fields = [
-            'has_secret', 'spec_info',
-            'date_verified', 'created_by', 'date_created',
+            'spec_info', 'date_verified', 'created_by', 'date_created',
         ]
         extra_kwargs = {
             'spec_info': {'label': _('Spec info')},

apps/accounts/serializers/account/gathered_account.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from accounts.models import GatheredAccount
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer

apps/accounts/serializers/account/template.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from accounts.models import AccountTemplate, Account

apps/accounts/serializers/account/virtual.py

@@ -0,0 +1,26 @@
+from django.utils.translation import gettext_lazy as _
+from rest_framework import serializers
+
+from accounts.models import VirtualAccount
+
+__all__ = ['VirtualAccountSerializer']
+
+
+class VirtualAccountSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VirtualAccount
+        field_mini = ['id', 'alias', 'username', 'name']
+        common_fields = ['date_created', 'date_updated', 'comment']
+        fields = field_mini + [
+            'secret_from_login',
+        ] + common_fields
+        read_only_fields = common_fields + common_fields
+        extra_kwargs = {
+            'comment': {'label': _('Comment')},
+            'name': {'label': _('Name')},
+            'username': {'label': _('Username')},
+            'secret_from_login': {'help_text': _('Current only support login from AD/LDAP. Secret priority: '
+                                                 'Same account in asset secret > Login secret > Manual input')
+                                  },
+            'alias': {'required': False},
+        }

apps/accounts/serializers/automations/base.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from accounts.models import AutomationExecution

apps/accounts/serializers/automations/change_secret.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from accounts.const import (

apps/accounts/signal_handlers.py

@@ -1,8 +1,9 @@
-from django.db.models.signals import pre_save
+from django.db.models.signals import pre_save, post_save, post_delete
 from django.dispatch import receiver
 
+from accounts.backends import vault_client
 from common.utils import get_logger
-from .models import Account
+from .models import Account, AccountTemplate
 
 logger = get_logger(__name__)
 
@@ -13,3 +14,23 @@ def on_account_pre_save(sender, instance, **kwargs):
         instance.version = 1
     else:
         instance.version = instance.history.count()
+
+
+class VaultSignalHandler(object):
+    """ 处理 Vault 相关的信号 """
+
+    @staticmethod
+    def save_to_vault(sender, instance, created, **kwargs):
+        if created:
+            vault_client.create(instance)
+        else:
+            vault_client.update(instance)
+
+    @staticmethod
+    def delete_to_vault(sender, instance, **kwargs):
+        vault_client.delete(instance)
+
+
+for model in (Account, AccountTemplate, Account.history.model):
+    post_save.connect(VaultSignalHandler.save_to_vault, sender=model)
+    post_delete.connect(VaultSignalHandler.delete_to_vault, sender=model)

apps/accounts/tasks/backup_account.py

@@ -23,7 +23,7 @@ def task_activity_callback(self, pid, trigger, *args, **kwargs):
 
 
 @shared_task(verbose_name=_('Execute account backup plan'), activity_callback=task_activity_callback)
-def execute_account_backup_task(pid, trigger):
+def execute_account_backup_task(pid, trigger, **kwargs):
     from accounts.models import AccountBackupAutomation
     with tmp_to_root_org():
         plan = get_object_or_none(AccountBackupAutomation, pk=pid)

apps/accounts/tasks/push_account.py

@@ -1,5 +1,5 @@
 from celery import shared_task
-from django.utils.translation import gettext_noop, ugettext_lazy as _
+from django.utils.translation import gettext_noop, gettext_lazy as _
 
 from accounts.const import AutomationTypes
 from accounts.tasks.common import quickstart_automation_by_snapshot

apps/accounts/tasks/vault.py

@@ -0,0 +1,68 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+
+from celery import shared_task
+from django.utils.translation import gettext_lazy as _
+
+from accounts.backends import vault_client
+from accounts.models import Account, AccountTemplate
+from common.utils import get_logger
+from orgs.utils import tmp_to_root_org
+
+logger = get_logger(__name__)
+
+
+def sync_instance(instance):
+    instance_desc = f'[{instance._meta.verbose_name}-{instance.id}-{instance}]'
+    if instance.secret_has_save_to_vault:
+        msg = f'\033[32m- 跳过同步: {instance_desc}, 原因: [已同步]'
+        return "skipped", msg
+
+    try:
+        vault_client.create(instance)
+    except Exception as e:
+        msg = f'\033[31m- 同步失败: {instance_desc}, 原因: [{e}]'
+        return "failed", msg
+    else:
+        msg = f'\033[32m- 同步成功: {instance_desc}'
+        return "succeeded", msg
+
+
+@shared_task(verbose_name=_('Sync secret to vault'))
+def sync_secret_to_vault():
+    if not vault_client.enabled:
+        # 这里不能判断 settings.VAULT_ENABLED, 必须判断当前 vault_client 的类型
+        print('\033[35m>>> 当前 Vault 功能未开启, 不需要同步')
+        return
+
+    failed, skipped, succeeded = 0, 0, 0
+    to_sync_models = [Account, AccountTemplate, Account.history.model]
+    print(f'\033[33m>>> 开始同步密钥数据到 Vault ({datetime.now().strftime("%Y-%m-%d %H:%M:%S")})')
+    with tmp_to_root_org():
+        instances = []
+        for model in to_sync_models:
+            instances += list(model.objects.all())
+
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            tasks = [executor.submit(sync_instance, instance) for instance in instances]
+
+            for future in as_completed(tasks):
+                status, msg = future.result()
+                print(msg)
+                if status == "succeeded":
+                    succeeded += 1
+                elif status == "failed":
+                    failed += 1
+                elif status == "skipped":
+                    skipped += 1
+
+    total = succeeded + failed + skipped
+    print(
+        f'\033[33m>>> 同步完成: {model.__module__}, '
+        f'共计: {total}, '
+        f'成功: {succeeded}, '
+        f'失败: {failed}, '
+        f'跳过: {skipped}'
+    )
+    print(f'\033[33m>>> 全部同步完成 ({datetime.now().strftime("%Y-%m-%d %H:%M:%S")})')
+    print('\033[0m')

apps/accounts/urls.py

@@ -9,6 +9,7 @@ app_name = 'accounts'
 router = BulkRouter()
 
 router.register(r'accounts', api.AccountViewSet, 'account')
+router.register(r'virtual-accounts', api.VirtualAccountViewSet, 'virtual-account')
 router.register(r'gathered-accounts', api.GatheredAccountViewSet, 'gathered-account')
 router.register(r'account-secrets', api.AccountSecretsViewSet, 'account-secret')
 router.register(r'account-templates', api.AccountTemplateViewSet, 'account-template')

apps/accounts/utils.py

@@ -1,9 +1,7 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
-from accounts.const import (
-    SecretType, DEFAULT_PASSWORD_RULES
-)
+from accounts.const import SecretType, DEFAULT_PASSWORD_RULES
 from common.utils import ssh_key_gen, random_string
 from common.utils import validate_ssh_private_key, parse_ssh_private_key_str
 

apps/acls/apps.py

@@ -1,5 +1,5 @@
 from django.apps import AppConfig
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 
 class AclsConfig(AppConfig):

apps/acls/models/command_acl.py

@@ -3,7 +3,7 @@
 import re
 
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.utils import lazyproperty, get_logger
 from orgs.mixins.models import JMSOrgBaseModel

apps/acls/models/login_acl.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.utils import get_request_ip, get_ip_city
 from common.utils.timezone import local_now_display

apps/acls/models/login_asset_acl.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from .base import UserAssetAccountBaseACL
 

apps/acls/serializers/base.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from acls.models.base import BaseACL

apps/acls/serializers/command_acl.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from acls.models import CommandGroup, CommandFilterACL

apps/acls/serializers/login_acl.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 
 from common.serializers import MethodSerializer
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer

apps/acls/serializers/rules/rules.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 #
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
-from django.utils.translation import ugettext_lazy as _
 
 from common.utils import get_logger
 from common.utils.ip import is_ip_address, is_ip_network, is_ip_segment

apps/applications/apps.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
-from django.utils.translation import ugettext_lazy as _
 from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
 
 
 class ApplicationsConfig(AppConfig):
@@ -9,5 +9,4 @@ class ApplicationsConfig(AppConfig):
     verbose_name = _('Applications')
 
     def ready(self):
-        from . import signal_handlers
         super().ready()

apps/applications/migrations/0006_application.py

@@ -2,7 +2,6 @@
 
 from django.db import migrations, models
 import django.db.models.deletion
-import django_mysql.models
 import uuid
 
 
@@ -127,7 +126,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(max_length=128, verbose_name='Name')),
                 ('category', models.CharField(choices=[('db', 'Database'), ('remote_app', 'Remote app'), ('cloud', 'Cloud')], max_length=16, verbose_name='Category')),
                 ('type', models.CharField(choices=[('mysql', 'MySQL'), ('oracle', 'Oracle'), ('postgresql', 'PostgreSQL'), ('mariadb', 'MariaDB'), ('chrome', 'Chrome'), ('mysql_workbench', 'MySQL Workbench'), ('vmware_client', 'vSphere Client'), ('custom', 'Custom'), ('k8s', 'Kubernetes')], max_length=16, verbose_name='Type')),
-                ('attrs', django_mysql.models.JSONField(default=dict)),
+                ('attrs', models.JSONField(default=dict)),
                 ('comment', models.TextField(blank=True, default='', max_length=128, verbose_name='Comment')),
                 ('domain', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='applications', to='assets.Domain', verbose_name='Domain')),
             ],

apps/applications/models.py

@@ -1,5 +1,5 @@
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.db.models import JMSBaseModel
 from orgs.mixins.models import OrgModelMixin

apps/assets/api/domain.py

@@ -1,5 +1,5 @@
 # ~*~ coding: utf-8 ~*~
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.views.generic.detail import SingleObjectMixin
 from rest_framework.serializers import ValidationError
 from rest_framework.views import APIView, Response
@@ -29,6 +29,7 @@ class DomainViewSet(OrgBulkModelViewSet):
     def get_queryset(self):
         return super().get_queryset().prefetch_related('assets')
 
+
 class GatewayViewSet(HostViewSet):
     perm_model = Gateway
     filterset_fields = ("domain__name", "name", "domain")

apps/assets/api/mixin.py

@@ -2,7 +2,7 @@ from typing import List
 
 from rest_framework.request import Request
 
-from assets.models import Node, PlatformProtocol, Protocol
+from assets.models import Node, Protocol
 from assets.utils import get_node_from_request, is_query_node_all_assets
 from common.utils import lazyproperty, timeit
 
@@ -42,7 +42,7 @@ class SerializeToTreeNodeMixin:
                 'name': _name(node),
                 'title': _name(node),
                 'pId': node.parent_key,
-                'isParent': True,
+                'isParent': node.assets_amount > 0,
                 'open': _open(node),
                 'meta': {
                     'data': {
@@ -70,25 +70,18 @@ class SerializeToTreeNodeMixin:
 
     @timeit
     def serialize_assets(self, assets, node_key=None, pid=None):
-        sftp_enabled_platform = PlatformProtocol.objects \
-            .filter(name='ssh', setting__sftp_enabled=True) \
-            .values_list('platform', flat=True) \
-            .distinct()
         if node_key is None:
             get_pid = lambda asset: getattr(asset, 'parent_key', '')
         else:
             get_pid = lambda asset: node_key
-        ssh_asset_ids = [
-            str(i) for i in
-            Protocol.objects.filter(name='ssh').values_list('asset_id', flat=True)
-        ]
+        sftp_asset_ids = Protocol.objects.filter(name='sftp') \
+            .values_list('asset_id', flat=True)
+        sftp_asset_ids = list(sftp_asset_ids)
         data = [
             {
                 'id': str(asset.id),
                 'name': asset.name,
-                'title':
-                    f'{asset.address}\n{asset.comment}'
-                    if asset.comment else asset.address,
+                'title': f'{asset.address}\n{asset.comment}',
                 'pId': pid or get_pid(asset),
                 'isParent': False,
                 'open': False,
@@ -99,8 +92,7 @@ class SerializeToTreeNodeMixin:
                     'data': {
                         'platform_type': asset.platform.type,
                         'org_name': asset.org_name,
-                        'sftp': (asset.platform_id in sftp_enabled_platform) \
-                                and (str(asset.id) in ssh_asset_ids),
+                        'sftp': asset.id in sftp_asset_ids,
                         'name': asset.name,
                         'address': asset.address
                     },

apps/assets/api/node.py

@@ -3,7 +3,7 @@ from collections import namedtuple, defaultdict
 from functools import partial
 
 from django.db.models.signals import m2m_changed
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.generics import get_object_or_404

apps/assets/apps.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
-from django.utils.translation import ugettext_lazy as _
 
 from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
 
 
 class AssetsConfig(AppConfig):
@@ -12,7 +12,6 @@ class AssetsConfig(AppConfig):
         super().__init__(*args, **kwargs)
 
     def ready(self):
+        from . import signal_handlers  # noqa
+        from . import tasks  # noqa
         super().ready()
-        from . import signal_handlers
-        from . import tasks
-

apps/assets/automations/base/manager.py

@@ -9,7 +9,7 @@ import yaml
 from django.conf import settings
 from django.utils import timezone
 from django.utils.translation import gettext as _
-from sshtunnel import SSHTunnelForwarder, BaseSSHTunnelForwarderError
+from sshtunnel import SSHTunnelForwarder
 
 from assets.automations.methods import platform_automation_methods
 from common.utils import get_logger, lazyproperty, is_openssh_format_key, ssh_pubkey_gen
@@ -262,22 +262,21 @@ class BasePlaybookManager:
         info = self.file_to_json(runner.inventory)
         servers, not_valid = [], []
         for k, host in info['all']['hosts'].items():
-            jms_asset, jms_gateway = host['jms_asset'], host.get('gateway')
+            jms_asset, jms_gateway = host.get('jms_asset'), host.get('gateway')
             if not jms_gateway:
                 continue
-
-            server = SSHTunnelForwarder(
-                (jms_gateway['address'], jms_gateway['port']),
-                ssh_username=jms_gateway['username'],
-                ssh_password=jms_gateway['secret'],
-                ssh_pkey=jms_gateway['private_key_path'],
-                remote_bind_address=(jms_asset['address'], jms_asset['port'])
-            )
             try:
+                server = SSHTunnelForwarder(
+                    (jms_gateway['address'], jms_gateway['port']),
+                    ssh_username=jms_gateway['username'],
+                    ssh_password=jms_gateway['secret'],
+                    ssh_pkey=jms_gateway['private_key_path'],
+                    remote_bind_address=(jms_asset['address'], jms_asset['port'])
+                )
                 server.start()
-            except BaseSSHTunnelForwarderError:
+            except Exception as e:
                 err_msg = 'Gateway is not active: %s' % jms_asset.get('name', '')
-                print('\033[31m %s \033[0m\n' % err_msg)
+                print(f'\033[31m {err_msg} 原因: {e} \033[0m\n')
                 not_valid.append(k)
             else:
                 host['ansible_host'] = jms_asset['address'] = '127.0.0.1'

apps/assets/automations/ping/custom/ssh/main.yml

@@ -2,6 +2,7 @@
   gather_facts: no
   vars:
     ansible_connection: local
+    ansible_become: false
 
   tasks:
     - name: Test asset connection (paramiko)
@@ -12,3 +13,8 @@
         login_port: "{{ jms_asset.port }}"
         login_secret_type: "{{ jms_account.secret_type }}"
         login_private_key_path: "{{ jms_account.private_key_path }}"
+        become: "{{ custom_become | default(False) }}"
+        become_method: "{{ custom_become_method | default('su') }}"
+        become_user: "{{ custom_become_user | default('') }}"
+        become_password: "{{ custom_become_password | default('') }}"
+        become_private_key_path: "{{ custom_become_private_key_path | default(None) }}"

apps/assets/automations/ping_gateway/manager.py

@@ -2,7 +2,7 @@ import socket
 
 import paramiko
 from django.utils import timezone
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.const import AutomationTypes, Connectivity
 from assets.models import Gateway

apps/assets/const/automation.py

@@ -1,5 +1,5 @@
 from django.db.models import TextChoices
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 
 class Connectivity(TextChoices):

apps/assets/const/host.py

@@ -33,10 +33,10 @@ class HostTypes(BaseType):
     def _get_protocol_constrains(cls) -> dict:
         return {
             '*': {
-                'choices': ['ssh', 'telnet', 'vnc', 'rdp']
+                'choices': ['ssh', 'sftp', 'telnet', 'vnc', 'rdp']
             },
             cls.WINDOWS: {
-                'choices': ['rdp', 'ssh', 'vnc', 'winrm']
+                'choices': ['rdp', 'ssh', 'sftp', 'vnc', 'winrm']
             }
         }
 

apps/assets/const/protocol.py

@@ -11,6 +11,7 @@ __all__ = ['Protocol']
 
 class Protocol(ChoicesMixin, models.TextChoices):
     ssh = 'ssh', 'SSH'
+    sftp = 'sftp', 'SFTP'
     rdp = 'rdp', 'RDP'
     telnet = 'telnet', 'Telnet'
     vnc = 'vnc', 'VNC'
@@ -36,17 +37,16 @@ class Protocol(ChoicesMixin, models.TextChoices):
             cls.ssh: {
                 'port': 22,
                 'secret_types': ['password', 'ssh_key'],
+            },
+            cls.sftp: {
+                'port': 22,
+                'secret_types': ['password', 'ssh_key'],
                 'setting': {
-                    'sftp_enabled': {
-                        'type': 'bool',
-                        'default': True,
-                        'label': _('SFTP enabled')
-                    },
                     'sftp_home': {
                         'type': 'str',
                         'default': '/tmp',
                         'label': _('SFTP home')
-                    },
+                    }
                 }
             },
             cls.rdp: {
@@ -81,6 +81,26 @@ class Protocol(ChoicesMixin, models.TextChoices):
             cls.telnet: {
                 'port': 23,
                 'secret_types': ['password'],
+                'setting': {
+                    'username_prompt': {
+                        'type': 'str',
+                        'default': 'username:|login:',
+                        'label': _('Username prompt'),
+                        'help_text': _('We will send username when we see this prompt')
+                    },
+                    'password_prompt': {
+                        'type': 'str',
+                        'default': 'password:',
+                        'label': _('Password prompt'),
+                        'help_text': _('We will send password when we see this prompt')
+                    },
+                    'success_prompt': {
+                        'type': 'str',
+                        'default': 'success|成功|#|>|\$',
+                        'label': _('Success prompt'),
+                        'help_text': _('We will consider login success when we see this prompt')
+                    }
+                }
             },
             cls.winrm: {
                 'port': 5985,
@@ -119,7 +139,15 @@ class Protocol(ChoicesMixin, models.TextChoices):
                 'port': 1521,
                 'required': True,
                 'secret_types': ['password'],
-                'xpack': True
+                'xpack': True,
+                'setting': {
+                    'sysdba': {
+                        'type': 'bool',
+                        'default': False,
+                        'label': _('SYSDBA'),
+                        'help_text': _('Connect as SYSDBA')
+                    },
+                }
             },
             cls.sqlserver: {
                 'port': 1433,
@@ -166,6 +194,15 @@ class Protocol(ChoicesMixin, models.TextChoices):
                 'port_from_addr': True,
                 'secret_types': ['password'],
                 'setting': {
+                    'safe_mode': {
+                        'type': 'bool',
+                        'default': False,
+                        'label': _('Safe mode'),
+                        'help_text': _(
+                            'When safe mode is enabled, some operations will be disabled, such as: '
+                            'New tab, right click, visit other website, etc.'
+                        )
+                    },
                     'autofill': {
                         'label': _('Autofill'),
                         'type': 'choice',

apps/assets/const/types.py

@@ -224,7 +224,7 @@ class AllTypes(ChoicesMixin):
         return dict(id='ROOT', name=_('All types'), title=_('All types'), open=True, isParent=True)
 
     @classmethod
-    def get_tree_nodes(cls, resource_platforms, include_asset=False):
+    def get_tree_nodes(cls, resource_platforms, include_asset=False, get_root=True):
         from ..models import Platform
         platform_count = defaultdict(int)
         for platform_id in resource_platforms:
@@ -239,10 +239,10 @@ class AllTypes(ChoicesMixin):
             category_type_mapper[p.category] += platform_count[p.id]
             tp_platforms[p.category + '_' + p.type].append(p)
 
-        nodes = [cls.get_root_nodes()]
+        nodes = [cls.get_root_nodes()] if get_root else []
         for category, type_cls in cls.category_types():
             # Category 格式化
-            meta = {'type': 'category', 'category': category.value}
+            meta = {'type': 'category', 'category': category.value, '_type': category.value}
             category_node = cls.choice_to_node(category, 'ROOT', meta=meta)
             category_count = category_type_mapper.get(category, 0)
             category_node['name'] += f'({category_count})'

apps/assets/exceptions.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 from rest_framework import status
 
 from common.exceptions import JMSException

apps/assets/filters.py

@@ -1,14 +1,11 @@
 # -*- coding: utf-8 -*-
 #
 from django.db.models import Q
-from django_filters import rest_framework as drf_filters
 from rest_framework import filters
 from rest_framework.compat import coreapi, coreschema
 
 from assets.utils import get_node_from_request, is_query_node_all_assets
-from common.drf.filters import BaseFilterSet
-
-from .models import Label, Node
+from .models import Label
 
 
 class AssetByNodeFilterBackend(filters.BaseFilterBackend):

apps/assets/migrations/0121_auto_20230725_1458.py

@@ -0,0 +1,100 @@
+# Generated by Django 4.1.10 on 2023-07-25 06:58
+
+from django.db import migrations
+import json
+
+
+def migrate_platforms_sftp_protocol(apps, schema_editor):
+    platform_protocol_cls = apps.get_model('assets', 'PlatformProtocol')
+    platform_cls = apps.get_model('assets', 'Platform')
+    ssh_protocols = platform_protocol_cls.objects \
+        .filter(name='ssh', setting__sftp_enabled=True) \
+        .exclude(name__in=('Gateway', 'RemoteAppHost')) \
+        .filter(platform__type='linux')
+    platforms_has_sftp = platform_cls.objects.filter(protocols__name='sftp')
+
+    new_protocols = []
+    print("\nPlatform add sftp protocol: ")
+    for protocol in ssh_protocols:
+        protocol_setting = protocol.setting or {}
+        if protocol.platform in platforms_has_sftp:
+            continue
+
+        kwargs = {
+            'name': 'sftp',
+            'port': protocol.port,
+            'primary': False,
+            'required': False,
+            'default': True,
+            'public': True,
+            'setting': {
+                'sftp_home': protocol_setting.get('sftp_home', '/tmp'),
+            },
+            'platform': protocol.platform,
+        }
+        new_protocol = platform_protocol_cls(**kwargs)
+        new_protocols.append(new_protocol)
+        print(" - {}".format(protocol.platform.name))
+
+    new_protocols_dict = {(protocol.name, protocol.platform): protocol for protocol in new_protocols}
+    new_protocols = list(new_protocols_dict.values())
+    platform_protocol_cls.objects.bulk_create(new_protocols, ignore_conflicts=True)
+
+
+def migrate_assets_sftp_protocol(apps, schema_editor):
+    asset_cls = apps.get_model('assets', 'Asset')
+    platform_cls = apps.get_model('assets', 'Platform')
+    protocol_cls = apps.get_model('assets', 'Protocol')
+    sftp_platforms = list(platform_cls.objects.filter(protocols__name='sftp').values_list('id'))
+
+    count = 0
+    print("\nAsset add sftp protocol: ")
+    asset_ids = asset_cls.objects\
+        .filter(platform__in=sftp_platforms)\
+        .exclude(protocols__name='sftp')\
+        .distinct()\
+        .values_list('id', flat=True)
+    while True:
+        _asset_ids = asset_ids[count:count + 1000]
+        if not _asset_ids:
+            break
+        count += 1000
+
+        new_protocols = []
+        ssh_protocols = protocol_cls.objects.filter(name='ssh', asset_id__in=_asset_ids).distinct()
+        ssh_protocols_map = {protocol.asset_id: protocol for protocol in ssh_protocols}
+        for asset_id, protocol in ssh_protocols_map.items():
+            new_protocols.append(protocol_cls(name='sftp', port=protocol.port, asset_id=asset_id))
+        protocol_cls.objects.bulk_create(new_protocols, ignore_conflicts=True)
+        print(" - Add {}".format(len(new_protocols)))
+
+
+def migrate_telnet_regex(apps, schema_editor):
+    setting_cls = apps.get_model('settings', 'Setting')
+    setting = setting_cls.objects.filter(name='TERMINAL_TELNET_REGEX').first()
+    if not setting:
+        print("Not found telnet regex setting, skip")
+        return
+    try:
+        value = json.loads(setting.value)
+    except Exception:
+        print("Invalid telnet regex setting, skip")
+        return
+    platform_protocol_cls = apps.get_model('assets', 'PlatformProtocol')
+    telnets = platform_protocol_cls.objects.filter(name='telnet')
+    if telnets.count() > 0:
+        telnets.update(setting={'success_prompt': value})
+        print("Migrate telnet regex setting success: ", telnets.count())
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0120_auto_20230630_1613'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_platforms_sftp_protocol),
+        migrations.RunPython(migrate_assets_sftp_protocol),
+        migrations.RunPython(migrate_telnet_regex),
+    ]

apps/assets/migrations/0122_auto_20230803_1553.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.10 on 2023-08-03 07:53
+
+from django.db import migrations
+
+
+def migrate_web_setting_safe_mode(apps, schema_editor):
+    platform_protocol_cls = apps.get_model('assets', 'PlatformProtocol')
+    protocols = platform_protocol_cls.objects.filter(name='http')
+    for protocol in protocols:
+        setting = protocol.setting or {}
+        setting['safe_mode'] = False
+        protocol.setting = setting
+        protocol.save(update_fields=['setting'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0121_auto_20230725_1458'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_web_setting_safe_mode),
+    ]

apps/assets/models/asset/common.py

@@ -8,7 +8,7 @@ from collections import defaultdict
 from django.db import models
 from django.db.models import Q
 from django.forms import model_to_dict
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets import const
 from common.db.fields import EncryptMixin
@@ -221,8 +221,11 @@ class Asset(NodesRelationMixin, AbsConnectivity, JSONFilterMixin, JMSOrgBaseMode
         return self.address
 
     def get_target_ssh_port(self):
-        protocol = self.protocols.all().filter(name='ssh').first()
-        return protocol.port if protocol else 22
+        return self.get_protocol_port('ssh')
+
+    def get_protocol_port(self, protocol):
+        protocol = self.protocols.all().filter(name=protocol).first()
+        return protocol.port if protocol else 0
 
     @property
     def is_valid(self):

apps/assets/models/asset/host.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from .common import Asset
 

apps/assets/models/automations/base.py

@@ -2,7 +2,7 @@ import uuid
 
 from celery import current_task
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.models.asset import Asset
 from assets.models.node import Node

apps/assets/models/automations/gather_facts.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.const import AutomationTypes
 from .base import AssetBaseAutomation

apps/assets/models/automations/ping.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.const import AutomationTypes
 from .base import AssetBaseAutomation

apps/assets/models/base.py

@@ -3,7 +3,7 @@
 
 from django.db import models
 from django.utils import timezone
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.const import Connectivity
 from common.utils import (

apps/assets/models/cmd_filter.py

@@ -4,7 +4,7 @@ import uuid
 
 from django.core.validators import MinValueValidator, MaxValueValidator
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.utils import get_logger
 from orgs.mixins.models import OrgModelMixin

apps/assets/models/domain.py

@@ -3,7 +3,7 @@
 import random
 
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.utils import get_logger
 from orgs.mixins.models import JMSOrgBaseModel

apps/assets/models/favorite_asset.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.db.models import JMSBaseModel
 

apps/assets/models/gateway.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from assets.const import GATEWAY_NAME
 from assets.models.platform import Platform

apps/assets/models/group.py

@@ -7,7 +7,7 @@ from __future__ import unicode_literals
 import uuid
 
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 __all__ = ['AssetGroup']
 

apps/assets/models/label.py

@@ -2,7 +2,7 @@
 #
 
 from django.db import models
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from common.utils import lazyproperty
 from orgs.mixins.models import JMSOrgBaseModel

apps/assets/models/node.py

@@ -10,8 +10,7 @@ from django.core.cache import cache
 from django.db import models, transaction
 from django.db.models import Q, Manager
 from django.db.transaction import atomic
-from django.utils.translation import ugettext
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _, gettext
 
 from common.db.models import output_as_string
 from common.utils import get_logger
@@ -163,7 +162,7 @@ class FamilyMixin:
         return key
 
     def get_next_child_preset_name(self):
-        name = ugettext("New node")
+        name = gettext("New node")
         values = [
             child.value[child.value.rfind(' '):]
             for child in self.get_children()