Merge pull request #10213 from jumpserver/dev

v3.2.0 rc2

apps/accounts/api/account/task.py

@@ -24,10 +24,11 @@ class AccountsTaskCreateAPI(CreateAPIView):
     def perform_create(self, serializer):
         data = serializer.validated_data
         accounts = data.get('accounts', [])
+        params = data.get('params')
         account_ids = [str(a.id) for a in accounts]
 
         if data['action'] == 'push':
-            task = push_accounts_to_assets_task.delay(account_ids)
+            task = push_accounts_to_assets_task.delay(account_ids, params)
         else:
             account = accounts[0]
             asset = account.asset

apps/accounts/automations/change_secret/custom/ssh/main.yml

@@ -0,0 +1,40 @@
+- hosts: custom
+  gather_facts: no
+  vars:
+    ansible_connection: local
+
+  tasks:
+    - name: Test privileged account
+      ssh_ping:
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_secret_type: "{{ jms_account.secret_type }}"
+        login_private_key_path: "{{ jms_account.private_key_path }}"
+      register: ping_info
+
+    - name: Change asset password
+      custom_command:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_secret_type: "{{ jms_account.secret_type }}"
+        login_private_key_path: "{{ jms_account.private_key_path }}"
+        name: "{{ account.username }}"
+        password: "{{ account.secret }}"
+        commands: "{{ params.commands }}"
+        first_conn_delay_time: "{{ first_conn_delay_time | default(0.5) }}"
+      when: ping_info is succeeded
+      register: change_info
+
+    - name: Verify password
+      ssh_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+      when:
+        - ping_info is succeeded
+        - change_info is succeeded

apps/accounts/automations/change_secret/custom/ssh/manifest.yml

@@ -0,0 +1,14 @@
+id: change_secret_by_ssh
+name: Change secret by SSH
+category:
+  - device
+  - host
+type:
+  - all
+method: change_secret
+params:
+  - name: commands
+    type: list
+    label: '自定义命令'
+    default: ['']
+    help_text: '自定义命令中如需包含账号的 username 和 password 字段，请使用 &#123;username&#125;、&#123;password&#125;格式，执行任务时会进行替换 。<br />比如针对 Linux 主机进行改密，一般需要配置三条命令：<br />1.passwd &#123;username&#125; <br />2.&#123;password&#125; <br />3.&#123;password&#125;'

apps/accounts/automations/push_account/manager.py

@@ -1,6 +1,6 @@
 from copy import deepcopy
 
-from accounts.const import AutomationTypes, SecretType
+from accounts.const import AutomationTypes, SecretType, Connectivity
 from assets.const import HostTypes
 from common.utils import get_logger
 from ..base.manager import AccountBasePlaybookManager
@@ -74,6 +74,7 @@ class PushAccountManager(ChangeSecretManager, AccountBasePlaybookManager):
             return
         account.secret = new_secret
         account.save(update_fields=['secret'])
+        account.set_connectivity(Connectivity.OK)
 
     def on_host_error(self, host, error, result):
         pass

apps/accounts/automations/verify_account/custom/main.yml

@@ -0,0 +1,14 @@
+- hosts: custom
+  gather_facts: no
+  vars:
+    ansible_connection: local
+
+  tasks:
+    - name: Verify account
+      ssh_ping:
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_secret_type: "{{ jms_account.secret_type }}"
+        login_private_key_path: "{{ jms_account.private_key_path }}"

apps/accounts/automations/verify_account/custom/manifest.yml

@@ -0,0 +1,8 @@
+id: verify_account_by_ssh
+name: Verify account by SSH
+category:
+  - device
+  - host
+type:
+  - all
+method: verify_account

apps/accounts/serializers/account/account.py

@@ -97,7 +97,7 @@ class AccountCreateUpdateSerializerMixin(serializers.Serializer):
 
     @staticmethod
     def push_account_if_need(instance, push_now, params, stat):
-        if not push_now or stat != 'created':
+        if not push_now or stat not in ['created', 'updated']:
             return
         push_accounts_to_assets_task.delay([str(instance.id)], params)
 
@@ -407,3 +407,7 @@ class AccountTaskSerializer(serializers.Serializer):
         queryset=Account.objects, required=False, allow_empty=True, many=True
     )
     task = serializers.CharField(read_only=True)
+    params = serializers.JSONField(
+        decoder=None, encoder=None, required=False,
+        style={'base_template': 'textarea.html'}
+    )

apps/accounts/serializers/automations/change_secret.py

@@ -58,6 +58,7 @@ class ChangeSecretAutomationSerializer(AuthValidateMixin, BaseAutomationSerializ
                 "Currently only mail sending is supported"
             )},
         }}
+
     @property
     def model_type(self):
         return AutomationTypes.change_secret

apps/assets/automations/base/manager.py

@@ -54,7 +54,9 @@ class BasePlaybookManager:
         if serializer is None:
             return {}
 
-        data = self.params.get(method_id, {})
+        data = self.params.get(method_id)
+        if not data:
+            data = automation_params.get(method_id, {})
         params = serializer(data).data
         return {
             field_name: automation_params.get(field_name, '')

apps/assets/automations/ping/custom/main.yml

@@ -0,0 +1,14 @@
+- hosts: custom
+  gather_facts: no
+  vars:
+    ansible_connection: local
+
+  tasks:
+    - name: Test asset connection
+      ssh_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_secret_type: "{{ jms_account.secret_type }}"
+        login_private_key_path: "{{ jms_account.private_key_path }}"

apps/assets/automations/ping/custom/manifest.yml

@@ -0,0 +1,8 @@
+id: ping_by_ssh
+name: Ping by SSH
+category:
+  - device
+  - host
+type:
+  - all
+method: ping

apps/assets/automations/ping/manager.py

@@ -14,8 +14,10 @@ class PingManager(BasePlaybookManager):
     def method_type(cls):
         return AutomationTypes.ping
 
-    def host_callback(self, host, asset=None, account=None, **kwargs):
-        super().host_callback(host, asset=asset, account=account, **kwargs)
+    def host_callback(self, host, asset=None, account=None, automation=None, **kwargs):
+        super().host_callback(
+            host, asset=asset, account=account, automation=automation, **kwargs
+        )
         self.host_asset_and_account_mapper[host['name']] = (asset, account)
         return host
 

apps/assets/const/base.py

@@ -6,12 +6,27 @@ from .protocol import Protocol
 
 class Type:
     def __init__(self, label, value):
+        self.name = value
         self.label = label
         self.value = value
 
     def __str__(self):
         return self.value
 
+    def __add__(self, other):
+        if isinstance(other, str):
+            return str(str(self) + other)
+        raise TypeError("unsupported operand type(s) for +: '{}' and '{}'".format(
+            type(self), type(other))
+        )
+
+    def __radd__(self, other):
+        if isinstance(other, str):
+            return str(other + str(self))
+        raise TypeError("unsupported operand type(s) for +(r): '{}' and '{}'".format(
+            type(self), type(other))
+        )
+
 
 class BaseType(TextChoices):
     """
@@ -77,10 +92,7 @@ class BaseType(TextChoices):
 
     @classmethod
     def get_types(cls):
-        tps = cls._get_choices_to_types()
-        if not has_valid_xpack_license():
-            tps = cls.get_community_types()
-        return tps
+        return cls._get_choices_to_types()
 
     @classmethod
     def get_community_types(cls):
@@ -88,4 +100,9 @@ class BaseType(TextChoices):
 
     @classmethod
     def get_choices(cls):
+        if not has_valid_xpack_license():
+            return [
+                (tp.value, tp.label)
+                for tp in cls.get_community_types()
+            ]
         return cls.choices

apps/assets/const/device.py

@@ -32,15 +32,16 @@ class DeviceTypes(BaseType):
     def _get_automation_constrains(cls) -> dict:
         return {
             '*': {
-                'ansible_enabled': False,
+                'ansible_enabled': True,
                 'ansible_config': {
                     'ansible_connection': 'local',
+                    'first_conn_delay_time': 0.5,
                 },
-                'ping_enabled': False,
+                'ping_enabled': True,
                 'gather_facts_enabled': False,
                 'gather_accounts_enabled': False,
-                'verify_account_enabled': False,
-                'change_secret_enabled': False,
+                'verify_account_enabled': True,
+                'change_secret_enabled': True,
                 'push_account_enabled': False
             }
         }

apps/assets/models/asset/common.py

@@ -90,6 +90,7 @@ class Protocol(models.Model):
     name = models.CharField(max_length=32, verbose_name=_("Name"))
     port = models.IntegerField(verbose_name=_("Port"))
     asset = models.ForeignKey('Asset', on_delete=models.CASCADE, related_name='protocols', verbose_name=_("Asset"))
+    _setting = None
 
     def __str__(self):
         return '{}/{}'.format(self.name, self.port)
@@ -102,8 +103,14 @@ class Protocol(models.Model):
 
     @property
     def setting(self):
+        if self._setting is not None:
+            return self._setting
         return self.asset_platform_protocol.get('setting', {})
 
+    @setting.setter
+    def setting(self, value):
+        self._setting = value
+
     @property
     def public(self):
         return self.asset_platform_protocol.get('public', True)

apps/assets/serializers/platform.py

@@ -168,7 +168,7 @@ class PlatformSerializer(WritableNestedModelSerializer):
         su_enabled = attrs.get('su_enabled', False) and self.constraints.get('su_enabled', False)
         automation = attrs.get('automation', {})
         automation['ansible_enabled'] = automation.get('ansible_enabled', False) \
-                                        and self.constraints.get('ansible_enabled', False)
+                                        and self.constraints['automation'].get('ansible_enabled', False)
         attrs.update({
             'domain_enabled': domain_enabled,
             'su_enabled': su_enabled,

apps/assets/utils/k8s.py

@@ -4,6 +4,7 @@ from urllib.parse import urlencode
 from kubernetes import client
 from kubernetes.client import api_client
 from kubernetes.client.api import core_v1_api
+from kubernetes.client.exceptions import ApiException
 
 from common.utils import get_logger
 from ..const import CloudTypes, Category
@@ -65,9 +66,13 @@ class KubernetesClient:
         proxy_url = cls.get_proxy_url(asset)
         k8s = cls(k8s_url, secret, proxy=proxy_url)
         func_name = f'get_{tp}s'
+        data = []
         if hasattr(k8s, func_name):
-            return getattr(k8s, func_name)(*args)
-        return []
+            try:
+                data = getattr(k8s, func_name)(*args)
+            except ApiException as e:
+                logger.error(e.reason)
+        return data
 
 
 class KubernetesTree:

apps/ops/ansible/inventory.py

@@ -150,7 +150,8 @@ class JMSInventory:
             },
             'jms_account': {
                 'id': str(account.id), 'username': account.username,
-                'secret': account.secret, 'secret_type': account.secret_type
+                'secret': account.secret, 'secret_type': account.secret_type,
+                'private_key_path': account.private_key_path
             } if account else None
         }
 

apps/ops/ansible/modules/custom_command.py

@@ -0,0 +1,115 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+---
+module: custom_command
+short_description: Adds or removes a user with custom commands by ssh
+description:
+    - You can add or edit users using ssh with custom commands.
+
+options:
+  protocol:
+    default: ssh
+    choices: [ssh]
+    description:
+      - C(ssh) The remote asset is connected using ssh.
+    type: str
+  name:
+    description:
+      - The name of the user to add or remove.
+    required: true
+    aliases: [user]
+    type: str
+  password:
+    description:
+      - The password to use for the user.
+    type: str
+    aliases: [pass]
+  commands:
+    description:
+      - Custom change password commands.
+    type: list
+    required: true
+  first_conn_delay_time:
+    description:
+      - Delay for executing the command after SSH connection(unit: s)
+    type: float
+    required: false
+'''
+
+EXAMPLES = '''
+- name: Create user with name 'jms' and password '123456'.
+  custom_command:
+    login_host: "localhost"
+    login_port: 22
+    login_user: "admin"
+    login_password: "123456"
+    name: "jms"
+    password: "123456"
+    commands: ['passwd {username}', '{password}', '{password}']
+'''
+
+RETURN = '''
+name:
+    description: The name of the user to add.
+    returned: success
+    type: str
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+from ops.ansible.modules_utils.custom_common import (
+    SSHClient, ssh_common_argument_spec
+)
+
+
+def get_commands(module):
+    username = module.params['name']
+    password = module.params['password']
+    commands = module.params['commands'] or []
+    for index, command in enumerate(commands):
+        commands[index] = command.format(
+            username=username, password=password
+        )
+    return commands
+
+# =========================================
+# Module execution.
+#
+
+
+def main():
+    argument_spec = ssh_common_argument_spec()
+    argument_spec.update(
+        name=dict(required=True, aliases=['user']),
+        password=dict(aliases=['pass'], no_log=True),
+        commands=dict(type='list', required=False),
+        first_conn_delay_time=dict(
+            type='float', required=False, default=0.5
+        ),
+    )
+    module = AnsibleModule(argument_spec=argument_spec)
+
+    ssh_client = SSHClient(module)
+    commands = get_commands(module)
+    if not commands:
+        module.fail_json(
+            msg='No command found, please go to the platform details to add'
+        )
+    err = ssh_client.execute(commands)
+    if err:
+        module.fail_json(
+            msg='There was a problem executing the command: %s' % err
+        )
+
+    user = module.params['name']
+    module.exit_json(changed=True, user=user)
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules/oracle_user.py

@@ -69,7 +69,7 @@ EXAMPLES = '''
   oracle_user:
     hostname: "remote server"
     login_database: "helowin"
-    login_username: "system"
+    login_user: "system"
     login_password: "123456"
     name: "jms"
     password: "123456"
@@ -78,7 +78,7 @@ EXAMPLES = '''
   oracle_user:
     hostname: "remote server"
     login_database: "helowin"
-    login_username: "system"
+    login_user: "system"
     login_password: "123456"
     name: "jms"
     state: "absent"

apps/ops/ansible/modules/ssh_ping.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+---
+module: custom_ssh_ping
+short_description: Use ssh to probe whether an asset is connectable
+description:
+    - Use ssh to probe whether an asset is connectable
+'''
+
+EXAMPLES = '''
+- name: >
+    Ping asset server.
+  custom_ssh_ping:
+    login_host: 127.0.0.1
+    login_port: 22
+    login_user: jms
+    login_password: password
+'''
+
+RETURN = '''
+is_available:
+  description: MongoDB server availability.
+  returned: always
+  type: bool
+  sample: true
+conn_err_msg:
+  description: Connection error message.
+  returned: always
+  type: str
+  sample: ''
+'''
+
+
+from ansible.module_utils.basic import AnsibleModule
+
+from ops.ansible.modules_utils.custom_common import (
+    SSHClient, ssh_common_argument_spec
+)
+
+
+# =========================================
+# Module execution.
+#
+
+
+def main():
+    options = ssh_common_argument_spec()
+    module = AnsibleModule(argument_spec=options, supports_check_mode=True,)
+
+    result = {
+        'changed': False, 'is_available': True
+    }
+    client = SSHClient(module)
+    err = client.connect()
+    if err:
+        module.fail_json(msg='Unable to connect to asset: %s' % err)
+        result['is_available'] = False
+
+    return module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules_utils/custom_common.py

@@ -0,0 +1,68 @@
+import time
+
+import paramiko
+
+from paramiko.ssh_exception import SSHException, NoValidConnectionsError
+
+
+def ssh_common_argument_spec():
+    options = dict(
+        login_host=dict(type='str', required=False, default='localhost'),
+        login_port=dict(type='int', required=False, default=22),
+        login_user=dict(type='str', required=False, default='root'),
+        login_password=dict(type='str', required=False, no_log=True),
+        login_secret_type=dict(type='str', required=False, default='password'),
+        login_private_key_path=dict(type='str', required=False, no_log=True),
+    )
+    return options
+
+
+class SSHClient:
+    def __init__(self, module):
+        self.module = module
+        self.is_connect = False
+        self.client = paramiko.SSHClient()
+        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    def get_connect_params(self):
+        params = {
+            'allow_agent': False, 'look_for_keys': False,
+            'hostname': self.module.params['login_host'],
+            'port': self.module.params['login_port'],
+            'username': self.module.params['login_user'],
+        }
+        secret_type = self.module.params['login_secret_type']
+        if secret_type == 'ssh_key':
+            params['key_filename'] = self.module.params['login_private_key_path']
+        else:
+            params['password'] = self.module.params['login_password']
+        return params
+
+    def connect(self):
+        try:
+            self.client.connect(**self.get_connect_params())
+        except (SSHException, NoValidConnectionsError) as err:
+            err_msg = str(err)
+        else:
+            self.is_connect = True
+            err_msg = ''
+        return err_msg
+
+    def execute(self, commands):
+        if not self.is_connect:
+            self.connect()
+
+        channel = self.client.invoke_shell()
+        # 读取首次登陆终端返回的消息
+        channel.recv(2048)
+        # 网络设备一般登录有延迟，等终端有返回后再执行命令
+        delay_time = self.module.params['first_conn_delay_time']
+        time.sleep(delay_time)
+        err_msg = ''
+        try:
+            for command in commands:
+                channel.send(command + '\n')
+                time.sleep(0.3)
+        except SSHException as e:
+            err_msg = str(e)
+        return err_msg

apps/terminal/api/applet/applet.py

@@ -64,7 +64,7 @@ class DownloadUploadMixin:
         if instance and not update:
             return Response({'error': 'Applet already exists: {}'.format(name)}, status=400)
 
-        applet, serializer = Applet.install_from_dir(tmp_dir)
+        applet, serializer = Applet.install_from_dir(tmp_dir, builtin=False)
         return Response(serializer.data, status=201)
 
     @action(detail=True, methods=['get'])