Merge branch 'v3' of github.com:jumpserver/jumpserver into v3

Dockerfile

@@ -99,6 +99,7 @@ VOLUME /opt/jumpserver/data
 VOLUME /opt/jumpserver/logs
 
 ENV LANG=zh_CN.UTF-8
+ENV ANSIBLE_LIBRARY=/opt/jumpserver/apps/ops/ansible/modules
 
 EXPOSE 8080
 

apps/acls/api/connect_acl.py

@@ -0,0 +1,52 @@
+from rest_framework.views import APIView
+from rest_framework import status
+from django.http.response import JsonResponse
+from django.utils.translation import ugettext_lazy as _
+
+from common.drf.api import JMSBulkModelViewSet
+from common.const.choices import ConnectMethodChoices
+from ..models import ConnectACL
+from .. import serializers
+
+__all__ = ['ConnectACLViewSet', 'ConnectMethodsAPI', 'ConnectMethodPermissionsAPI']
+
+
+class ConnectACLViewSet(JMSBulkModelViewSet):
+    queryset = ConnectACL.objects.all()
+    filterset_fields = ('name', )
+    search_fields = ('name',)
+    serializer_class = serializers.ConnectACLSerializer
+
+
+class ConnectMethodsAPI(APIView):
+    rbac_perms = {
+        'GET': 'acls.view_connnectacl',
+    }
+
+    @staticmethod
+    def get(request, *args, **kwargs):
+        data = []
+        for m in ConnectMethodChoices.choices:
+            data.append({'label': m[1], 'value': m[0]})
+        return JsonResponse(data, safe=False)
+
+
+class ConnectMethodPermissionsAPI(APIView):
+    rbac_perms = {
+        'GET': 'acls.view_connnectacl',
+    }
+
+    @staticmethod
+    def get(request, *args, **kwargs):
+        login_type = request.query_params.get('login_type')
+        if not login_type:
+            rules = ConnectACL().all_rules(request.user)
+            return JsonResponse({'rules': rules})
+
+        acl = ConnectACL.match(request.user, login_type)
+        if acl:
+            err = _('The current user is not allowed to login in this way')
+            return JsonResponse({'error': err})
+        else:
+            return JsonResponse({'msg': 'ok'})
+

apps/acls/migrations/0004_connectacl.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.16 on 2022-11-30 02:46
+
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0040_alter_user_source'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('acls', '0003_auto_20211130_1037'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ConnectACL',
+            fields=[
+                ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
+                ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('name', models.CharField(max_length=128, verbose_name='Name')),
+                ('priority', models.IntegerField(default=50, help_text='1-100, the lower the value will be match first', validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(100)], verbose_name='Priority')),
+                ('is_active', models.BooleanField(default=True, verbose_name='Active')),
+                ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
+                ('rules', models.JSONField(default=list, verbose_name='Rule')),
+                ('action', models.CharField(choices=[('reject', 'Reject'), ('allow', 'Allow')], default='reject', max_length=64, verbose_name='Action')),
+                ('user_groups', models.ManyToManyField(blank=True, related_name='connect_acls', to='users.UserGroup', verbose_name='User group')),
+                ('users', models.ManyToManyField(blank=True, related_name='connect_acls', to=settings.AUTH_USER_MODEL, verbose_name='User')),
+            ],
+            options={
+                'verbose_name': 'Connect acl',
+                'ordering': ('priority', '-date_updated', 'name'),
+            },
+        ),
+    ]

apps/acls/models/connect_acl.py

@@ -0,0 +1,120 @@
+from django.db import models
+from django.core.cache import cache
+from django.utils.translation import ugettext_lazy as _
+
+from common.utils.connection import get_redis_client
+from common.const.choices import ConnectMethodChoices
+from orgs.mixins.models import OrgManager, OrgModelMixin
+from .base import BaseACL, BaseACLQuerySet
+
+
+class ACLManager(OrgManager):
+
+    def valid(self):
+        return self.get_queryset().valid()
+
+
+class ConnectACL(BaseACL, OrgModelMixin):
+    ConnectACLUserCacheKey = 'CONNECT_ACL_USER_{}'
+    ConnectACLUserCacheTTL = 600
+
+    class ActionChoices(models.TextChoices):
+        reject = 'reject', _('Reject')
+
+    # 用户
+    users = models.ManyToManyField(
+        'users.User', related_name='connect_acls', blank=True,
+        verbose_name=_("User")
+    )
+    user_groups = models.ManyToManyField(
+        'users.UserGroup', related_name='connect_acls', blank=True,
+        verbose_name=_("User group"),
+    )
+    rules = models.JSONField(default=list, verbose_name=_('Rule'))
+    # 动作
+    action = models.CharField(
+        max_length=64, verbose_name=_('Action'),
+        choices=ActionChoices.choices, default=ActionChoices.reject
+    )
+
+    objects = ACLManager.from_queryset(BaseACLQuerySet)()
+
+    class Meta:
+        ordering = ('priority', '-date_updated', 'name')
+        verbose_name = _('Connect acl')
+
+    def __str__(self):
+        return self.name
+
+    @property
+    def rules_display(self):
+        return ', '.join(
+            [ConnectMethodChoices.get_label(i) for i in self.rules]
+        )
+
+    def is_action(self, action):
+        return self.action == action
+
+    @staticmethod
+    def match(user, connect_type):
+        if not user:
+            return
+
+        user_acls = user.connect_acls.all().valid().distinct()
+        for acl in user_acls:
+            if connect_type in acl.rules:
+                return acl
+
+        for user_group in user.groups.all():
+            acls = user_group.connect_acls.all().valid().distinct()
+            for acl in acls:
+                if connect_type in acl.rules:
+                    return acl
+
+    def _get_all_rules_from_cache(self, user):
+        find = False
+        cache_key = self.ConnectACLUserCacheKey.format(user.id)
+        rules = cache.get(cache_key)
+        if rules is not None:
+            find = True
+        return rules, find
+
+    @staticmethod
+    def _get_all_rules_from_db(user):
+        connect_rules = set()
+        user_acls = user.connect_acls.all().valid()
+        user_acl_rules = user_acls.values_list('id', 'rules')
+        for r_id, rule in user_acl_rules:
+            connect_rules.update(rule)
+
+        for ug in user.groups.all():
+            user_group_acls = ug.connect_acls.all().valid()
+            user_group_rules = user_group_acls.values_list('id', 'rules')
+            for r_id, rule in user_group_rules:
+                connect_rules.update(rule)
+        return list(connect_rules)
+
+    def set_all_rules_to_cache(self, key, rules):
+        cache.set(key, rules, self.ConnectACLUserCacheTTL)
+
+    def all_rules(self, user):
+        rules, find = self._get_all_rules_from_cache(user)
+        if not find:
+            rules = self._get_all_rules_from_db(user)
+            self.set_all_rules_to_cache(
+                self.ConnectACLUserCacheKey.format(user.id), rules
+            )
+        return rules
+
+    def clear_rules_cache(self):
+        cache.delete_pattern(
+            self.ConnectACLUserCacheKey.format('*')
+        )
+
+    def save(self, *args, **kwargs):
+        self.clear_rules_cache()
+        return super().save(*args, **kwargs)
+
+    def delete(self, using=None, keep_parents=False):
+        self.clear_rules_cache()
+        return super().delete(using=using, keep_parents=keep_parents)

apps/acls/serializers/connnect_acl.py

@@ -0,0 +1,36 @@
+from django.utils.translation import ugettext as _
+from rest_framework import serializers
+
+from common.drf.serializers import BulkModelSerializer
+from common.const.choices import ConnectMethodChoices
+from ..models import ConnectACL
+
+
+__all__ = ['ConnectACLSerializer', ]
+
+
+class ConnectACLSerializer(BulkModelSerializer):
+    action_display = serializers.ReadOnlyField(source='get_action_display', label=_('Action'))
+
+    class Meta:
+        model = ConnectACL
+        fields_mini = ['id', 'name']
+        fields_small = fields_mini + [
+            'priority', 'rules', 'rules_display', 'action', 'action_display', 'is_active',
+            'date_created', 'date_updated', 'comment', 'created_by'
+        ]
+        fields_m2m = ['users', 'user_groups']
+        fields = fields_small + fields_m2m
+        extra_kwargs = {
+            'priority': {'default': 50},
+            'is_active': {'default': True}
+        }
+
+    @staticmethod
+    def validate_rules(rules):
+        for r in rules:
+            label = ConnectMethodChoices.get_label(r)
+            if not label:
+                error = _('Invalid connection method: {}').format(r)
+                raise serializers.ValidationError(error)
+        return rules

apps/assets/automations/change_secret/database/mongodb/main.yml

@@ -0,0 +1,43 @@
+- hosts: mongodb
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Test MongoDB connection
+      mongodb_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+      register: db_info
+
+    - name: Display MongoDB version
+      debug:
+        var: db_info.server_version
+      when: db_info is succeeded
+
+    - name: Change MongoDB password
+      mongodb_user:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        db: "{{ jms_asset.specific.db_name }}"
+        name: "{{ account.username }}"
+        password: "{{ account.secret }}"
+      when: db_info is succeeded
+      register: change_info
+
+    - name: Verify password
+      mongodb_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+      when:
+      - db_info is succeeded
+      - change_info is succeeded

apps/assets/automations/change_secret/database/mongodb/manifest.yml

@@ -0,0 +1,6 @@
+id: change_secret_mongodb
+name: Change password for MongoDB
+category: database
+type:
+  - mongodb
+method: change_secret

apps/assets/automations/change_secret/database/oracle/main.yml

@@ -0,0 +1,45 @@
+- hosts: oracle
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Test Oracle connection
+      oracle_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"
+      register: db_info
+
+    - name: Display Oracle version
+      debug:
+        var: db_info.server_version
+      when: db_info is succeeded
+
+    - name: Change Oracle password
+      oracle_user:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"
+        name: "{{ account.username }}"
+        password: "{{ account.secret }}"
+      when: db_info is succeeded
+      register: change_info
+
+    - name: Verify password
+      oracle_ping:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ account.mode }}"
+      when:
+      - db_info is succeeded
+      - change_info is succeeded

apps/assets/automations/change_secret/database/oracle/manifest.yml

@@ -0,0 +1,6 @@
+id: change_secret_oracle
+name: Change password for Oracle
+category: database
+type:
+  - oracle
+method: change_secret

apps/assets/automations/change_secret/database/sqlserver/main.yml

@@ -0,0 +1,47 @@
+- hosts: sqlserver
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Test SQLServer connection
+      community.general.mssql_script:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        name: '{{ jms_asset.specific.db_name }}'
+        script: |
+          SELECT @@version
+      register: db_info
+
+  - name: SQLServer version
+    set_fact:
+      info:
+        version: "{{ db_info.query_results[0][0][0][0].splitlines()[0] }}"
+  - debug:
+      var: info
+
+    - name: Change SQLServer password
+      community.general.mssql_script:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        name: '{{ jms_asset.specific.db_name }}'
+        script: "ALTER LOGIN {{ account.username }} WITH PASSWORD = '{{ account.secret }}'; select @@version"
+      when: db_info is succeeded
+      register: change_info
+
+    - name: Verify password
+      community.general.mssql_script:
+        login_user: "{{ account.username }}"
+        login_password: "{{ account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        name: '{{ jms_asset.specific.db_name }}'
+        script: |
+          SELECT @@version
+      when:
+      - db_info is succeeded
+      - change_info is succeeded

apps/assets/automations/change_secret/database/sqlserver/manifest.yml

@@ -0,0 +1,6 @@
+id: change_secret_sqlserver
+name: Change password for SQLServer
+category: database
+type:
+  - sqlserver
+method: change_secret

apps/assets/automations/change_secret/manager.py

@@ -155,6 +155,8 @@ class ChangeSecretManager(BasePlaybookManager):
                 'secret': new_secret,
                 'private_key_path': private_key_path
             }
+            if asset.platform.type == 'oracle':
+                h['account']['mode'] = 'sysdba' if account.privileged else None
             inventory_hosts.append(h)
             method_hosts.append(h['name'])
         self.method_hosts_mapper[method_attr] = method_hosts

apps/assets/automations/gather_accounts/database/mongodb/main.yml

@@ -0,0 +1,22 @@
+- hosts: mongodb
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Get info
+      community.mongodb.mongodb_info:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        filter: users
+      register: db_info
+
+    - name: Define info by set_fact
+      set_fact:
+        info: "{{ db_info.users }}"
+
+    - debug:
+        var: info

apps/assets/automations/gather_accounts/database/mongodb/manifest.yml

@@ -0,0 +1,6 @@
+id: gather_accounts_mongodb
+name: Gather account from MongoDB
+category: database
+type:
+  - mongodb
+method: gather_accounts

apps/assets/automations/gather_accounts/database/mysql/main.yml

@@ -9,7 +9,7 @@
         login_user: "{{ jms_account.username }}"
         login_password: "{{ jms_account.secret }}"
         login_host: "{{ jms_asset.address }}"
-        login_port: 1234
+        login_port: "{{ jms_asset.port }}"
         filter: users
       register: db_info
 

apps/assets/automations/gather_accounts/database/oracle/main.yml

@@ -0,0 +1,23 @@
+- hosts: oralce
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Get info
+      oracle_info:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"
+        filter: users
+      register: db_info
+
+    - name: Define info by set_fact
+      set_fact:
+        info: "{{ db_info.users }}"
+
+    - debug:
+        var: info

apps/assets/automations/gather_accounts/database/oracle/manifest.yml

@@ -0,0 +1,6 @@
+id: gather_accounts_oracle
+name: Gather account from Oracle
+category: database
+type:
+  - oracle
+method: gather_accounts

apps/assets/automations/gather_facts/database/mongodb/main.yml

@@ -0,0 +1,22 @@
+- hosts: mongodb
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Get info
+      mongodb_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+      register: db_info
+
+    - name: Define info by set_fact
+      set_fact:
+        info:
+          version: "{{ db_info.server_version }}"
+
+    - debug:
+        var: info

apps/assets/automations/gather_facts/database/mongodb/manifest.yml

@@ -0,0 +1,6 @@
+id: gather_facts_mongodb
+name: Gather facts from MongoDB
+category: database
+type:
+  - mongodb
+method: gather_facts

apps/assets/automations/gather_facts/database/oracle/main.yml

@@ -0,0 +1,23 @@
+- hosts: oracle
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Get info
+      oracle_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"
+      register: db_info
+
+    - name: Define info by set_fact
+      set_fact:
+        info:
+          version: "{{ db_info.server_version }}"
+
+    - debug:
+        var: info

apps/assets/automations/gather_facts/database/oracle/manifest.yml

@@ -0,0 +1,6 @@
+id: gather_facts_oracle
+name: Gather facts from Oracle
+category: database
+type:
+  - oracle
+method: gather_facts

apps/assets/automations/ping/database/mongodb/main.yml

@@ -0,0 +1,13 @@
+- hosts: mongodb
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Test MongoDB connection
+      mongodb_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"

apps/assets/automations/ping/database/mongodb/manifest.yml

@@ -0,0 +1,6 @@
+id: mongodb_ping
+name: Ping MongoDB
+category: database
+type:
+  - mongodb
+method: ping

apps/assets/automations/ping/database/oracle/main.yml

@@ -0,0 +1,14 @@
+- hosts: oracle
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Test Oracle connection
+      oracle_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"

apps/assets/automations/ping/database/oracle/manifest.yml

@@ -0,0 +1,6 @@
+id: oracle_ping
+name: Ping Oracle
+category: database
+type:
+  - oracle
+method: ping

apps/assets/automations/ping/database/sqlserver/main.yml

@@ -0,0 +1,15 @@
+- hosts: sqlserver
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Test SQLServer connection
+      community.general.mssql_script:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        name: '{{ jms_asset.specific.db_name }}'
+        script: |
+          SELECT @@version

apps/assets/automations/ping/database/sqlserver/manifest.yml

@@ -0,0 +1,6 @@
+id: sqlserver_ping
+name: Ping SQLServer
+category: database
+type:
+  - sqlserver
+method: ping

apps/assets/automations/push_account/database/mongodb/main.yml

@@ -0,0 +1,16 @@
+- hosts: mongodb
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Add user account.username
+      mongodb_user:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        db: "{{ jms_asset.specific.db_name }}"
+        name: "{{ account.username }}"
+        password: "{{ account.secret }}"

apps/assets/automations/push_account/database/mongodb/manifest.yml

@@ -0,0 +1,6 @@
+id: push_account_mongodb
+name: Push account from MongoDB
+category: database
+type:
+  - mongodb
+method: push_account

apps/assets/automations/push_account/database/oracle/main.yml

@@ -0,0 +1,16 @@
+- hosts: oracle
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Add user account.username
+      oracle_user:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"
+        name: "{{ account.username }}"
+        password: "{{ account.secret }}"

apps/assets/automations/push_account/database/oracle/manifest.yml

@@ -0,0 +1,6 @@
+id: push_account_oracle
+name: Push account from Oracle
+category: database
+type:
+  - oracle
+method: push_account

apps/assets/automations/verify_account/database/mongodb/main.yml

@@ -0,0 +1,13 @@
+- hosts: mongdb
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Verify account
+      mongodb_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"

apps/assets/automations/verify_account/database/mongodb/manifest.yml

@@ -0,0 +1,6 @@
+id: verify_account_mongodb
+name: Verify account from MongoDB
+category: database
+type:
+  - mongodb
+method: verify_account

apps/assets/automations/verify_account/database/oracle/main.yml

@@ -0,0 +1,14 @@
+- hosts: oracle
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Verify account
+      oracle_ping:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        login_database: "{{ jms_asset.specific.db_name }}"
+        mode: "{{ jms_account.mode }}"

apps/assets/automations/verify_account/database/oracle/manifest.yml

@@ -0,0 +1,6 @@
+id: verify_account_oracle
+name: Verify account from Oracle
+category: database
+type:
+  - oracle
+method: verify_account

apps/assets/automations/verify_account/database/sqlserver/main.yml

@@ -0,0 +1,15 @@
+- hosts: sqlserver
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/local/bin/python
+
+  tasks:
+    - name: Verify account
+      community.general.mssql_script:
+        login_user: "{{ jms_account.username }}"
+        login_password: "{{ jms_account.secret }}"
+        login_host: "{{ jms_asset.address }}"
+        login_port: "{{ jms_asset.port }}"
+        name: '{{ jms_asset.specific.db_name }}'
+        script: |
+          SELECT @@version

apps/assets/automations/verify_account/database/sqlserver/manifest.yml

@@ -0,0 +1,6 @@
+id: verify_account_sqlserver
+name: Verify account from SQLServer
+category: database
+type:
+  - sqlserver
+method: verify_account

apps/assets/models/base.py

@@ -86,8 +86,8 @@ class BaseAccount(JMSOrgBaseModel):
 
     @lazyproperty
     def public_key(self):
-        if self.secret_type == SecretType.SSH_KEY:
-            return parse_ssh_public_key_str(self.private_key)
+        if self.secret_type == SecretType.SSH_KEY and self.private_key:
+            return parse_ssh_public_key_str(private_key=self.private_key)
         return None
 
     @property

apps/assets/serializers/account/account.py

@@ -72,7 +72,9 @@ class AccountSerializer(AccountSerializerCreateMixin, BaseAccountSerializer):
     def __init__(self, *args, data=None, **kwargs):
         super().__init__(*args, data=data, **kwargs)
         if data and 'name' not in data:
-            data['name'] = data.get('username')
+            username = data.get('username')
+            if username is not None:
+                data['name'] = username
         if hasattr(self, 'initial_data') and \
                 not getattr(self, 'initial_data', None):
             delattr(self, 'initial_data')

apps/assets/serializers/base.py

@@ -28,7 +28,7 @@ class AuthValidateMixin(serializers.Serializer):
 
     def validate_secret(self, secret):
         if not secret:
-            return
+            return ''
         secret_type = self.initial_secret_type
         if secret_type == SecretType.PASSWORD:
             validate_password_for_ansible(secret)
@@ -44,7 +44,7 @@ class AuthValidateMixin(serializers.Serializer):
     def clean_auth_fields(validated_data):
         for field in ('secret',):
             value = validated_data.get(field)
-            if not value:
+            if value is None:
                 validated_data.pop(field, None)
         validated_data.pop('passphrase', None)
 

apps/audits/migrations/0016_alter_userloginlog_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-11-30 07:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0015_auto_20221011_1745'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userloginlog',
+            name='type',
+            field=models.CharField(choices=[('web_cli', 'Web Client'), ('web_gui', 'Web GUI'), ('db_cli', 'DB Client'), ('db_gui', 'DB GUI'), ('rdp_cli', 'RDP Client'), ('rdp_file', 'RDP File'), ('ssh_cli', 'SSH Client'), ('web_sftp', 'Web SFTP')], max_length=128, verbose_name='Login type'),
+        ),
+    ]

apps/common/management/commands/services/command.py

@@ -1,3 +1,4 @@
+import multiprocessing
 from django.core.management.base import BaseCommand, CommandError
 from django.db.models import TextChoices
 from .utils import ServicesUtil
@@ -91,11 +92,15 @@ class BaseActionCommand(BaseCommand):
         super().__init__(*args, **kwargs)
 
     def add_arguments(self, parser):
+        cores = 10
+        if (multiprocessing.cpu_count() * 2 + 1) < cores:
+            cores = multiprocessing.cpu_count() * 2 + 1
+
         parser.add_argument(
             'services',  nargs='+', choices=Services.export_services_values(), help='Service',
         )
         parser.add_argument('-d', '--daemon', nargs="?", const=True)
-        parser.add_argument('-w', '--worker', type=int, nargs="?", default=4)
+        parser.add_argument('-w', '--worker', type=int, nargs="?", default=cores)
         parser.add_argument('-f', '--force', nargs="?", const=True)
 
     def initial_util(self, *args, **options):

apps/common/management/commands/services/services/gunicorn.py

@@ -16,11 +16,11 @@ class GunicornService(BaseService):
 
         log_format = '%(h)s %(t)s %(L)ss "%(r)s" %(s)s %(b)s '
         bind = f'{HTTP_HOST}:{HTTP_PORT}'
+
         cmd = [
             'gunicorn', 'jumpserver.asgi:application',
             '-b', bind,
             '-k', 'uvicorn.workers.UvicornWorker',
-            '--threads', '10',
             '-w', str(self.worker),
             '--max-requests', '4096',
             '--access-logformat', log_format,

apps/ops/ansible/inventory.py

@@ -112,6 +112,9 @@ class JMSInventory:
                 'secret': account.secret, 'secret_type': account.secret_type
             } if account else None
         }
+        if host['jms_account'] and asset.platform.type == 'oracle':
+            host['jms_account']['mode'] = 'sysdba' if account.privileged else None
+
         ansible_config = dict(automation.ansible_config)
         ansible_connection = ansible_config.get('ansible_connection', 'ssh')
         host.update(ansible_config)

apps/ops/ansible/modules/mongodb_ping.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: mongodb_ping
+short_description: Check remote MongoDB server availability
+description:
+- Simple module to check remote MongoDB server availability.
+
+requirements:
+  - "pymongo"
+'''
+
+EXAMPLES = '''
+- name: >
+    Ping MongoDB server using non-default credentials and SSL
+    registering the return values into the result variable for future use
+  mongodb_ping:
+    login_db: test_db
+    login_host: jumpserver
+    login_user: jms
+    login_password: secret_pass
+    ssl: True
+    ssl_ca_certs: "/tmp/ca.crt"
+    ssl_certfile: "/tmp/tls.key" #cert and key in one file
+    connection_options:
+     - "tlsAllowInvalidHostnames=true"
+'''
+
+RETURN = '''
+is_available:
+  description: MongoDB server availability.
+  returned: always
+  type: bool
+  sample: true
+server_version:
+  description: MongoDB server version.
+  returned: always
+  type: str
+  sample: '4.0.0'
+conn_err_msg:
+  description: Connection error message.
+  returned: always
+  type: str
+  sample: ''
+'''
+
+
+from pymongo.errors import PyMongoError
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils._text import to_native
+from ansible_collections.community.mongodb.plugins.module_utils.mongodb_common import (
+    mongodb_common_argument_spec,
+    mongo_auth,
+    get_mongodb_client,
+)
+
+
+class MongoDBPing(object):
+    def __init__(self, module, client):
+        self.module = module
+        self.client = client
+        self.is_available = False
+        self.conn_err_msg = ''
+        self.version = ''
+
+    def do(self):
+        self.get_mongodb_version()
+        return self.is_available, self.version
+
+    def get_err(self):
+        return self.conn_err_msg
+
+    def get_mongodb_version(self):
+        try:
+            server_info = self.client.server_info()
+            self.is_available = True
+            self.version = server_info.get('version', '')
+        except PyMongoError as err:
+            self.is_available = False
+            self.version = ''
+            self.conn_err_msg = err
+
+
+# =========================================
+# Module execution.
+#
+
+
+def main():
+    argument_spec = mongodb_common_argument_spec()
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    client = None
+    result = {
+        'changed': False, 'is_available': False, 'server_version': ''
+    }
+    try:
+        client = get_mongodb_client(module, directConnection=True)
+        client = mongo_auth(module, client, directConnection=True)
+    except Exception as e:
+        module.fail_json(msg='Unable to connect to database: %s' % to_native(e))
+
+    mongodb_ping = MongoDBPing(module, client)
+    result["is_available"], result["server_version"] = mongodb_ping.do()
+    conn_err_msg = mongodb_ping.get_err()
+    if conn_err_msg:
+        module.fail_json(msg='Unable to connect to database: %s' % conn_err_msg)
+
+    try:
+        client.close()
+    except Exception:
+        pass
+
+    return module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules/mongodb_user.py

@@ -0,0 +1,426 @@
+#!/usr/bin/python
+
+# Modified from ansible_collections.community.mongodb.plugins.modules.mongodb_user
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+---
+module: mongodb_user
+short_description: Adds or removes a user from a MongoDB database
+description:
+    - Adds or removes a user from a MongoDB database.
+version_added: "1.0.0"
+
+extends_documentation_fragment:
+  - community.mongodb.login_options
+  - community.mongodb.ssl_options
+
+options:
+  replica_set:
+    description:
+      - Replica set to connect to (automatically connects to primary for writes).
+    type: str
+  database:
+    description:
+      - The name of the database to add/remove the user from.
+    required: true
+    type: str
+    aliases: [db]
+  name:
+    description:
+      - The name of the user to add or remove.
+    required: true
+    aliases: [user]
+    type: str
+  password:
+    description:
+      - The password to use for the user.
+    type: str
+    aliases: [pass]
+  roles:
+    type: list
+    elements: raw
+    description:
+      - >
+          The database user roles valid values could either be one or more of the following strings:
+          'read', 'readWrite', 'dbAdmin', 'userAdmin', 'clusterAdmin', 'readAnyDatabase', 'readWriteAnyDatabase', 'userAdminAnyDatabase',
+          'dbAdminAnyDatabase'
+      - "Or the following dictionary '{ db: DATABASE_NAME, role: ROLE_NAME }'."
+      - "This param requires pymongo 2.5+. If it is a string, mongodb 2.4+ is also required. If it is a dictionary, mongo 2.6+ is required."
+  state:
+    description:
+      - The database user state.
+    default: present
+    choices: [absent, present]
+    type: str
+  update_password:
+    default: always
+    choices: [always, on_create]
+    description:
+      - C(always) will always update passwords and cause the module to return changed.
+      - C(on_create) will only set the password for newly created users.
+      - This must be C(always) to use the localhost exception when adding the first admin user.
+      - This option is effectively ignored when using x.509 certs. It is defaulted to 'on_create' to maintain a \
+          a specific module behaviour when the login_database is '$external'.
+    type: str
+  create_for_localhost_exception:
+    type: path
+    description:
+      - This is parmeter is only useful for handling special treatment around the localhost exception.
+      - If C(login_user) is defined, then the localhost exception is not active and this parameter has no effect.
+      - If this file is NOT present (and C(login_user) is not defined), then touch this file after successfully adding the user.
+      - If this file is present (and C(login_user) is not defined), then skip this task.
+
+notes:
+    - Requires the pymongo Python package on the remote host, version 2.4.2+. This
+      can be installed using pip or the OS package manager. Newer mongo server versions require newer
+      pymongo versions. @see http://api.mongodb.org/python/current/installation.html
+requirements:
+  - "pymongo"
+author:
+    - "Elliott Foster (@elliotttf)"
+    - "Julien Thebault (@Lujeni)"
+'''
+
+EXAMPLES = '''
+- name: Create 'burgers' database user with name 'bob' and password '12345'.
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: bob
+    password: 12345
+    state: present
+
+- name: Create a database user via SSL (MongoDB must be compiled with the SSL option and configured properly)
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: bob
+    password: 12345
+    state: present
+    ssl: True
+
+- name: Delete 'burgers' database user with name 'bob'.
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: bob
+    state: absent
+
+- name: Define more users with various specific roles (if not defined, no roles is assigned, and the user will be added via pre mongo 2.2 style)
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: ben
+    password: 12345
+    roles: read
+    state: present
+
+- name: Define roles
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: jim
+    password: 12345
+    roles: readWrite,dbAdmin,userAdmin
+    state: present
+
+- name: Define roles
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: joe
+    password: 12345
+    roles: readWriteAnyDatabase
+    state: present
+
+- name: Add a user to database in a replica set, the primary server is automatically discovered and written to
+  community.mongodb.mongodb_user:
+    database: burgers
+    name: bob
+    replica_set: belcher
+    password: 12345
+    roles: readWriteAnyDatabase
+    state: present
+
+# add a user 'oplog_reader' with read only access to the 'local' database on the replica_set 'belcher'. This is useful for oplog access (MONGO_OPLOG_URL).
+# please notice the credentials must be added to the 'admin' database because the 'local' database is not synchronized and can't receive user credentials
+# To login with such user, the connection string should be MONGO_OPLOG_URL="mongodb://oplog_reader:oplog_reader_password@server1,server2/local?authSource=admin"
+# This syntax requires mongodb 2.6+ and pymongo 2.5+
+- name: Roles as a dictionary
+  community.mongodb.mongodb_user:
+    login_user: root
+    login_password: root_password
+    database: admin
+    user: oplog_reader
+    password: oplog_reader_password
+    state: present
+    replica_set: belcher
+    roles:
+      - db: local
+        role: read
+
+- name: Adding a user with X.509 Member Authentication
+  community.mongodb.mongodb_user:
+    login_host: "mongodb-host.test"
+    login_port: 27001
+    login_database: "$external"
+    database: "admin"
+    name: "admin"
+    password: "test"
+    roles:
+    - dbAdminAnyDatabase
+    ssl: true
+    ssl_ca_certs: "/tmp/ca.crt"
+    ssl_certfile: "/tmp/tls.key" #cert and key in one file
+    state: present
+    auth_mechanism: "MONGODB-X509"
+    connection_options:
+     - "tlsAllowInvalidHostnames=true"
+'''
+
+RETURN = '''
+user:
+    description: The name of the user to add or remove.
+    returned: success
+    type: str
+'''
+
+import os
+import traceback
+from operator import itemgetter
+
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.six import binary_type, text_type
+from ansible.module_utils._text import to_native, to_bytes
+from ansible_collections.community.mongodb.plugins.module_utils.mongodb_common import (
+    missing_required_lib,
+    mongodb_common_argument_spec,
+    mongo_auth,
+    PYMONGO_IMP_ERR,
+    pymongo_found,
+    get_mongodb_client,
+)
+
+
+def user_find(client, user, db_name):
+    """Check if the user exists.
+
+    Args:
+        client (cursor): Mongodb cursor on admin database.
+        user (str): User to check.
+        db_name (str): User's database.
+
+    Returns:
+        dict: when user exists, False otherwise.
+    """
+    try:
+        for mongo_user in client[db_name].command('usersInfo')['users']:
+            if mongo_user['user'] == user:
+                # NOTE: there is no 'db' field in mongo 2.4.
+                if 'db' not in mongo_user:
+                    return mongo_user
+                # Workaround to make the condition works with AWS DocumentDB,
+                # since all users are in the admin database.
+                if mongo_user["db"] in [db_name, "admin"]:
+                    return mongo_user
+    except Exception as excep:
+        if hasattr(excep, 'code') and excep.code == 11:  # 11=UserNotFound
+            pass  # Allow return False
+        else:
+            raise
+    return False
+
+
+def user_add(module, client, db_name, user, password, roles):
+    # pymongo's user_add is a _create_or_update_user so we won't know if it was changed or updated
+    # without reproducing a lot of the logic in database.py of pymongo
+    db = client[db_name]
+
+    try:
+        exists = user_find(client, user, db_name)
+    except Exception as excep:
+        # We get this exception: "not authorized on admin to execute command"
+        # when auth is enabled on a new instance. The loalhost exception should
+        # allow us to create the first user. If the localhost exception does not apply,
+        # then user creation will also fail with unauthorized. So, ignore Unauthorized here.
+        if hasattr(excep, 'code') and excep.code == 13:  # 13=Unauthorized
+            exists = False
+        else:
+            raise
+
+    if exists:
+        user_add_db_command = 'updateUser'
+        if not roles:
+            roles = None
+    else:
+        user_add_db_command = 'createUser'
+
+    user_dict = {}
+
+    if password is not None:
+        user_dict["pwd"] = password
+    if roles is not None:
+        user_dict["roles"] = roles
+
+    db.command(user_add_db_command, user, **user_dict)
+
+
+def user_remove(module, client, db_name, user):
+    exists = user_find(client, user, db_name)
+    if exists:
+        if module.check_mode:
+            module.exit_json(changed=True, user=user)
+        db = client[db_name]
+        db.command("dropUser", user)
+    else:
+        module.exit_json(changed=False, user=user)
+
+
+def check_if_roles_changed(uinfo, roles, db_name):
+    # We must be aware of users which can read the oplog on a replicaset
+    # Such users must have access to the local DB, but since this DB does not store users credentials
+    # and is not synchronized among replica sets, the user must be stored on the admin db
+    # Therefore their structure is the following :
+    # {
+    #     "_id" : "admin.oplog_reader",
+    #     "user" : "oplog_reader",
+    #     "db" : "admin",                    # <-- admin DB
+    #     "roles" : [
+    #         {
+    #             "role" : "read",
+    #             "db" : "local"             # <-- local DB
+    #         }
+    #     ]
+    # }
+
+    def make_sure_roles_are_a_list_of_dict(roles, db_name):
+        output = list()
+        for role in roles:
+            if isinstance(role, (binary_type, text_type)):
+                new_role = {"role": role, "db": db_name}
+                output.append(new_role)
+            else:
+                output.append(role)
+        return output
+
+    roles_as_list_of_dict = make_sure_roles_are_a_list_of_dict(roles, db_name)
+    uinfo_roles = uinfo.get('roles', [])
+
+    if sorted(roles_as_list_of_dict, key=itemgetter('db')) == sorted(uinfo_roles, key=itemgetter('db')):
+        return False
+    return True
+
+
+# =========================================
+# Module execution.
+#
+
+def main():
+    argument_spec = mongodb_common_argument_spec()
+    argument_spec.update(
+        database=dict(required=True, aliases=['db']),
+        name=dict(required=True, aliases=['user']),
+        password=dict(aliases=['pass'], no_log=True),
+        replica_set=dict(default=None),
+        roles=dict(default=None, type='list', elements='raw'),
+        state=dict(default='present', choices=['absent', 'present']),
+        update_password=dict(default="always", choices=["always", "on_create"], no_log=False),
+        create_for_localhost_exception=dict(default=None, type='path'),
+    )
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+    login_user = module.params['login_user']
+
+    # Certs don't have a password but we want this module behaviour
+    if module.params['login_database'] == '$external':
+        module.params['update_password'] = 'on_create'
+
+    if not pymongo_found:
+        module.fail_json(msg=missing_required_lib('pymongo'),
+                         exception=PYMONGO_IMP_ERR)
+
+    create_for_localhost_exception = module.params['create_for_localhost_exception']
+    b_create_for_localhost_exception = (
+        to_bytes(create_for_localhost_exception, errors='surrogate_or_strict')
+        if create_for_localhost_exception is not None else None
+    )
+
+    db_name = module.params['database']
+    user = module.params['name']
+    password = module.params['password']
+    roles = module.params['roles'] or []
+    state = module.params['state']
+    update_password = module.params['update_password']
+
+    try:
+        directConnection = False
+        if module.params['replica_set'] is None:
+            directConnection = True
+        client = get_mongodb_client(module, directConnection=directConnection)
+        client = mongo_auth(module, client, directConnection=directConnection)
+    except Exception as e:
+        module.fail_json(msg='Unable to connect to database: %s' % to_native(e))
+
+    if state == 'present':
+        if password is None and update_password == 'always':
+            module.fail_json(msg='password parameter required when adding a user unless update_password is set to on_create')
+
+        if login_user is None and create_for_localhost_exception is not None:
+            if os.path.exists(b_create_for_localhost_exception):
+                try:
+                    client.close()
+                except Exception:
+                    pass
+                module.exit_json(changed=False, user=user, skipped=True, msg="The path in create_for_localhost_exception exists.")
+
+        try:
+            if update_password != 'always':
+                uinfo = user_find(client, user, db_name)
+                if uinfo:
+                    password = None
+                    if not check_if_roles_changed(uinfo, roles, db_name):
+                        module.exit_json(changed=False, user=user)
+
+            if module.check_mode:
+                module.exit_json(changed=True, user=user)
+            user_add(module, client, db_name, user, password, roles)
+        except Exception as e:
+            module.fail_json(msg='Unable to add or update user: %s' % to_native(e), exception=traceback.format_exc())
+        finally:
+            try:
+                client.close()
+            except Exception:
+                pass
+            # Here we can  check password change if mongo provide a query for that : https://jira.mongodb.org/browse/SERVER-22848
+            # newuinfo = user_find(client, user, db_name)
+            # if uinfo['role'] == newuinfo['role'] and CheckPasswordHere:
+            #    module.exit_json(changed=False, user=user)
+
+        if login_user is None and create_for_localhost_exception is not None:
+            # localhost exception applied.
+            try:
+                # touch the file
+                open(b_create_for_localhost_exception, 'wb').close()
+            except Exception as e:
+                module.fail_json(
+                    changed=True,
+                    msg='Added user but unable to touch create_for_localhost_exception file %s: %s' % (create_for_localhost_exception, to_native(e)),
+                    exception=traceback.format_exc()
+                )
+
+    elif state == 'absent':
+        try:
+            user_remove(module, client, db_name, user)
+        except Exception as e:
+            module.fail_json(msg='Unable to remove user: %s' % to_native(e), exception=traceback.format_exc())
+        finally:
+            try:
+                client.close()
+            except Exception:
+                pass
+    module.exit_json(changed=True, user=user)
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules/oracle_info.py

@@ -0,0 +1,261 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: oracle_info
+short_description: Gather information about Oracle servers
+description:
+- Gathers information about Oracle servers.
+
+options:
+  filter:
+    description:
+    - Limit the collected information by comma separated string or YAML list.
+    - Allowable values are C(version), C(databases), C(settings), C(users).
+    - By default, collects all subsets.
+    - You can use '!' before value (for example, C(!users)) to exclude it from the information.
+    - If you pass including and excluding values to the filter, for example, I(filter=!settings,version),
+      the excluding values, C(!settings) in this case, will be ignored.
+    type: list
+    elements: str
+  login_db:
+    description:
+    - Database name to connect to.
+    - It makes sense if I(login_user) is allowed to connect to a specific database only.
+    type: str
+  exclude_fields:
+    description:
+    - List of fields which are not needed to collect.
+    - "Supports elements: C(db_size). Unsupported elements will be ignored."
+    type: list
+    elements: str
+'''
+
+EXAMPLES = r'''
+- name: Get Oracle version with non-default credentials
+  oracle_info:
+    login_user: mysuperuser
+    login_password: mysuperpass
+    login_database: service_name
+    filter: version
+
+- name: Collect all info except settings and users by sys
+  oracle_info:
+    login_user: sys
+    login_password: sys_pass
+    login_database: service_name
+    filter: "!settings,!users"
+    exclude_fields: db_size
+'''
+
+RETURN = r'''
+version:
+  description: Database server version.
+  returned: if not excluded by filter
+  type: dict
+  sample: { "version": {"full": "11.2.0.1.0"} }
+  contains:
+    full:
+      description: Full server version.
+      returned: if not excluded by filter
+      type: str
+      sample: "11.2.0.1.0"
+databases:
+  description: Information about databases.
+  returned: if not excluded by filter
+  type: dict
+  sample:
+  - { "USERS": { "size": 5242880 }, "EXAMPLE": { "size": 104857600 } }
+  contains:
+    size:
+      description: Database size in bytes.
+      returned: if not excluded by filter
+      type: dict
+      sample: { 'size': 656594 }
+settings:
+  description: Global settings (variables) information.
+  returned: if not excluded by filter
+  type: dict
+  sample:
+  - { "result_cache_mode": "MANUAL", "instance_type": "RDBMS" }
+users:
+  description: Users information.
+  returned: if not excluded by filter
+  type: dict
+  sample:
+  - { "USERS": { "TEST": { "USERNAME": "TEST", "ACCOUNT_STATUS": "OPEN" } } }
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+from ops.ansible.modules_utils.oracle_common import (
+    OracleClient, oracle_common_argument_spec
+)
+
+
+class OracleInfo(object):
+    def __init__(self, module, oracle_client):
+        self.module = module
+        self.oracle_client = oracle_client
+        self.info = {
+            'version': {}, 'databases': {},
+            'settings': {}, 'users': {},
+        }
+
+    def get_info(self, filter_, exclude_fields):
+        include_list = []
+        exclude_list = []
+
+        if filter_:
+            partial_info = {}
+
+            for fi in filter_:
+                if fi.lstrip('!') not in self.info:
+                    self.module.warn('filter element: %s is not allowable, ignored' % fi)
+                    continue
+
+                if fi[0] == '!':
+                    exclude_list.append(fi.lstrip('!'))
+                else:
+                    include_list.append(fi)
+
+            if include_list:
+                self.__collect(exclude_fields, set(include_list))
+
+                for i in self.info:
+                    if i in include_list:
+                        partial_info[i] = self.info[i]
+            else:
+                not_in_exclude_list = list(set(self.info) - set(exclude_list))
+                self.__collect(exclude_fields, set(not_in_exclude_list))
+
+                for i in self.info:
+                    if i not in exclude_list:
+                        partial_info[i] = self.info[i]
+            return partial_info
+        else:
+            self.__collect(exclude_fields, set(self.info))
+            return self.info
+
+    def __collect(self, exclude_fields, wanted):
+        """Collect all possible subsets."""
+        if 'version' in wanted:
+            self.__get_version()
+
+        if 'settings' in wanted:
+            self.__get_settings()
+
+        if 'databases' in wanted:
+            self.__get_databases(exclude_fields)
+        #
+        if 'users' in wanted:
+            self.__get_users()
+
+    def __get_version(self):
+        version_sql = 'SELECT VERSION FROM PRODUCT_COMPONENT_VERSION where ROWNUM=1'
+        rtn, err = self.oracle_client.execute(version_sql, exception_to_fail=True)
+        self.info['version'] = {'full': rtn.get('version')}
+
+    def __get_settings(self):
+        """Get global variables (instance settings)."""
+        def _set_settings_value(item_dict):
+            try:
+                self.info['settings'][item_dict['name']] = item_dict['value']
+            except KeyError:
+                pass
+
+        settings_sql = "SELECT name, value FROM V$PARAMETER"
+        rtn, err = self.oracle_client.execute(settings_sql, exception_to_fail=True)
+
+        if isinstance(rtn, dict):
+            _set_settings_value(rtn)
+        elif isinstance(rtn, list):
+            for i in rtn:
+                _set_settings_value(i)
+
+    def __get_users(self):
+        """Get user info."""
+        def _set_users_value(item_dict):
+            try:
+                tablespace = item_dict.pop('default_tablespace')
+                username = item_dict.pop('username')
+                partial_users = self.info['users'].get(tablespace, {})
+                partial_users[username] = item_dict
+                self.info['users'][tablespace] = partial_users
+            except KeyError:
+                pass
+
+        users_sql = "SELECT * FROM dba_users"
+        rtn, err = self.oracle_client.execute(users_sql, exception_to_fail=True)
+        if isinstance(rtn, dict):
+            _set_users_value(rtn)
+        elif isinstance(rtn, list):
+            for i in rtn:
+                _set_users_value(i)
+
+    def __get_databases(self, exclude_fields):
+        """Get info about databases."""
+        def _set_databases_value(item_dict):
+            try:
+                tablespace_name = item_dict.pop('tablespace_name')
+                size = item_dict.get('size')
+                partial_params = {}
+                if size:
+                    partial_params['size'] = size
+                self.info['databases'][tablespace_name] = partial_params
+            except KeyError:
+                pass
+
+        database_sql = 'SELECT ' \
+                       '      tablespace_name, sum(bytes) as "size"' \
+                       'FROM dba_data_files GROUP BY tablespace_name'
+        if exclude_fields and 'db_size' in exclude_fields:
+            database_sql = "SELECT " \
+                           "      tablespace_name " \
+                           "FROM dba_data_files GROUP BY tablespace_name"
+
+        rtn, err = self.oracle_client.execute(database_sql, exception_to_fail=True)
+        if isinstance(rtn, dict):
+            _set_databases_value(rtn)
+        elif isinstance(rtn, list):
+            for i in rtn:
+                _set_databases_value(i)
+
+
+# ===========================================
+# Module execution.
+#
+
+
+def main():
+    argument_spec = oracle_common_argument_spec()
+    argument_spec.update(
+        filter=dict(type='list'),
+        exclude_fields=dict(type='list'),
+    )
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    filter_ = module.params['filter']
+    exclude_fields = module.params['exclude_fields']
+
+    if filter_:
+        filter_ = [f.strip() for f in filter_]
+
+    if exclude_fields:
+        exclude_fields = set([f.strip() for f in exclude_fields])
+
+    oracle_client = OracleClient(module)
+    oracle = OracleInfo(module, oracle_client)
+
+    module.exit_json(changed=False, **oracle.get_info(filter_, exclude_fields))
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules/oracle_ping.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = '''
+---
+module: oracle_ping
+short_description: Check remote Oracle server availability
+description:
+- Simple module to check remote Oracle server availability.
+
+requirements:
+  - "oracledb"
+'''
+
+EXAMPLES = '''
+- name: >
+    Ping Oracle server using non-default credentials and SSL
+    registering the return values into the result variable for future use
+  oracle_ping:
+    login_host: jumpserver
+    login_port: 1521
+    login_user: jms
+    login_password: secret_pass
+    login_database: test_db
+'''
+
+RETURN = '''
+is_available:
+  description: Oracle server availability.
+  returned: always
+  type: bool
+  sample: true
+server_version:
+  description: Oracle server version.
+  returned: always
+  type: str
+  sample: '4.0.0'
+conn_err_msg:
+  description: Connection error message.
+  returned: always
+  type: str
+  sample: ''
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+from ops.ansible.modules_utils.oracle_common import (
+    OracleClient, oracle_common_argument_spec
+)
+
+
+class OracleDBPing(object):
+    def __init__(self, module, oracle_client):
+        self.module = module
+        self.oracle_client = oracle_client
+        self.is_available = False
+        self.conn_err_msg = ''
+        self.version = ''
+
+    def do(self):
+        self.get_oracle_version()
+        return self.is_available, self.version
+
+    def get_err(self):
+        return self.conn_err_msg
+
+    def get_oracle_version(self):
+        version_sql = 'SELECT VERSION FROM PRODUCT_COMPONENT_VERSION where ROWNUM=1'
+        rtn, err = self.oracle_client.execute(version_sql)
+        if err:
+            self.conn_err_msg = err
+        else:
+            self.version = rtn.get('version')
+            self.is_available = True
+
+
+# =========================================
+# Module execution.
+#
+
+
+def main():
+    argument_spec = oracle_common_argument_spec()
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    result = {
+        'changed': False, 'is_available': False, 'server_version': ''
+    }
+    oracle_client = OracleClient(module)
+
+    oracle_ping = OracleDBPing(module, oracle_client)
+    result["is_available"], result["server_version"] = oracle_ping.do()
+    conn_err_msg = oracle_ping.get_err()
+    oracle_client.close()
+    if conn_err_msg:
+        module.fail_json(msg='Unable to connect to database: %s' % conn_err_msg)
+
+    return module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules/oracle_user.py

@@ -0,0 +1,215 @@
+#!/usr/bin/python
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+
+DOCUMENTATION = '''
+---
+module: oracle_user
+short_description: Adds or removes a user from a Oracle database
+description:
+    - Adds or removes a user from a Oracle database.
+
+options:
+  authentication_type:
+    description:
+        - Authentication type of the user(default password)
+    required: false
+    type: str
+    choices: ['external', 'global', 'no_authentication', 'password']
+  default_tablespace:
+    description:
+        - The default tablespace for the user
+        - If not provided, the default is used
+    required: false
+    type: str
+  oracle_home:
+    description:
+        - Define the directory into which all Oracle software is installed.
+        - Define ORACLE_HOME environment variable if set.
+    type: str
+  state:
+    description:
+      - The database user state.
+    default: present
+    choices: [absent, present]
+    type: str
+  update_password:
+    default: always
+    choices: [always, on_create]
+    description:
+      - C(always) will always update passwords and cause the module to return changed.
+      - C(on_create) will only set the password for newly created users.
+    type: str
+  temporary_tablespace:
+    description:
+        - The default temporary tablespace for the user
+        - If not provided, the default is used
+    required: false
+    type: str
+  name:
+    description:
+      - The name of the user to add or remove.
+    required: true
+    aliases: [user]
+    type: str
+  password:
+    description:
+      - The password to use for the user.
+    type: str
+    aliases: [pass]
+    
+requirements:
+  - "oracledb"
+'''
+
+EXAMPLES = '''
+- name: Create default tablespace user with name 'jms' and password '123456'.
+  oracle_user:
+    hostname: "remote server"
+    login_database: "helowin"
+    login_username: "system"
+    login_password: "123456"
+    name: "jms"
+    password: "123456"
+
+- name: Delete user with name 'jms'.
+  oracle_user:
+    hostname: "remote server"
+    login_database: "helowin"
+    login_username: "system"
+    login_password: "123456"
+    name: "jms"
+    state: "absent"
+'''
+
+RETURN = '''
+name:
+    description: The name of the user to add or remove.
+    returned: success
+    type: str
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+from ops.ansible.modules_utils.oracle_common import (
+    OracleClient, oracle_common_argument_spec
+)
+
+
+def user_find(oracle_client, username):
+    user = None
+    username = username.upper()
+    user_find_sql = "select username, " \
+                    "       authentication_type, " \
+                    "       default_tablespace, " \
+                    "       temporary_tablespace " \
+                    "from dba_users where username='%s'" % username
+    rtn, err = oracle_client.execute(user_find_sql)
+    if isinstance(rtn, dict):
+        user = rtn
+    return user
+
+
+def user_add(
+        module, oracle_client, username, password, auth_type,
+        default_tablespace, temporary_tablespace
+):
+    username = username.upper()
+    extend_sql = None
+    user = user_find(oracle_client, username)
+    auth_type = auth_type.lower()
+    identified_suffix_map = {
+        'external': 'identified externally ',
+        'global': 'identified globally ',
+        'password': 'identified by "%s" ',
+    }
+    if user:
+        user_sql = "alter user %s " % username
+        user_sql += identified_suffix_map.get(auth_type, 'no authentication ') % password
+
+        if default_tablespace and default_tablespace.lower() != user['default_tablespace'].lower():
+            user_sql += 'default tablespace %s quota unlimited on %s ' % (default_tablespace, default_tablespace)
+        if temporary_tablespace and temporary_tablespace.lower() != user['temporary_tablespace'].lower():
+            user_sql += 'temporary tablespace %s ' % temporary_tablespace
+    else:
+        user_sql = "create user %s " % username
+        user_sql += identified_suffix_map.get(auth_type, 'no authentication ') % password
+        if default_tablespace:
+            user_sql += 'default tablespace %s quota unlimited on %s ' % (default_tablespace, default_tablespace)
+        if temporary_tablespace:
+            user_sql += 'temporary tablespace %s ' % temporary_tablespace
+        extend_sql = 'grant connect to %s' % username
+
+    rtn, err = oracle_client.execute(user_sql)
+    if err:
+        module.fail_json(msg='Cannot add/edit user %s: %s' % (username, err), changed=False)
+    else:
+        if extend_sql:
+            oracle_client.execute(extend_sql)
+        module.exit_json(msg='User %s has been created.' % username, changed=True, name=username)
+
+
+def user_remove(module, oracle_client, username):
+    user = user_find(oracle_client, username)
+
+    if user:
+        rtn, err = oracle_client.execute('drop user %s cascade' % username)
+        if err:
+            module.fail_json(msg='Cannot drop user %s: %s' % (username, err), changed=False)
+        else:
+            module.exit_json(msg='User %s dropped.' % username, changed=True, name=username)
+    else:
+        module.exit_json(msg="User %s doesn't exist." % username, changed=False, name=username)
+
+
+# =========================================
+# Module execution.
+#
+
+def main():
+    argument_spec = oracle_common_argument_spec()
+    argument_spec.update(
+        authentication_type=dict(
+            type='str', required=False,
+            choices=['external', 'global', 'no_authentication', 'password']
+        ),
+        default_tablespace=dict(required=False, aliases=['db']),
+        name=dict(required=True, aliases=['user']),
+        password=dict(aliases=['pass'], no_log=True),
+        state=dict(type='str', default='present', choices=['absent', 'present']),
+        update_password=dict(default="always", choices=["always", "on_create"], no_log=False),
+        temporary_tablespace=dict(type='str', default=None),
+    )
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        supports_check_mode=True,
+    )
+
+    authentication_type = module.params['authentication_type'] or 'password'
+    default_tablespace = module.params['default_tablespace']
+    user = module.params['name']
+    password = module.params['password']
+    state = module.params['state']
+    update_password = module.params['update_password']
+    temporary_tablespace = module.params['temporary_tablespace']
+
+    oracle_client = OracleClient(module)
+    if state == 'present':
+        if password is None and update_password == 'always':
+            module.fail_json(
+                msg='password parameter required when adding a user unless update_password is set to on_create'
+            )
+        user_add(
+            module, oracle_client, username=user, password=password,
+            auth_type=authentication_type, default_tablespace=default_tablespace,
+            temporary_tablespace=temporary_tablespace
+        )
+    elif state == 'absent':
+        user_remove(oracle_client)
+    module.exit_json(changed=True, user=user)
+
+
+if __name__ == '__main__':
+    main()

apps/ops/ansible/modules_utils/oracle_common.py

@@ -0,0 +1,94 @@
+import os
+
+import oracledb
+
+from oracledb.exceptions import DatabaseError
+from ansible.module_utils._text import to_native
+
+
+def oracle_common_argument_spec():
+    """
+    Returns a dict containing common options shared across the Oracle modules.
+    """
+    options = dict(
+        login_user=dict(type='str', required=False),
+        login_password=dict(type='str', required=False, no_log=True),
+        login_database=dict(type='str', required=False, default='test'),
+        login_host=dict(type='str', required=False, default='localhost'),
+        login_port=dict(type='int', required=False, default=1521),
+        oracle_home=dict(type='str', required=False),
+        mode=dict(type='str', required=False),
+    )
+    return options
+
+
+class OracleClient(object):
+    def __init__(self, module):
+        self.module = module
+        self._conn = None
+        self._cursor = None
+        self.connect_params = {}
+
+        self.init_params()
+
+    def init_params(self):
+        params = self.module.params
+        hostname = params['login_host']
+        port = params['login_port']
+        service_name = params['login_database']
+        username = params['login_user']
+        password = params['login_password']
+        oracle_home = params['oracle_home']
+        mode = params['mode']
+
+        if oracle_home:
+            os.environ.setdefault('ORACLE_HOME', oracle_home)
+        if mode == 'sysdba':
+            self.connect_params['mode'] = oracledb.SYSDBA
+
+        self.connect_params['host'] = hostname
+        self.connect_params['port'] = port
+        self.connect_params['user'] = username
+        self.connect_params['password'] = password
+        self.connect_params['service_name'] = service_name
+
+    @property
+    def cursor(self):
+        if self._cursor is None:
+            try:
+                oracledb.init_oracle_client(lib_dir='/Users/jiangweidong/Downloads/instantclient_19_8')
+                self._conn = oracledb.connect(**self.connect_params)
+                self._cursor = self._conn.cursor()
+            except DatabaseError as err:
+                self.module.fail_json(
+                    msg="Unable to connect to database: %s, %s" % (to_native(err), self.connect_params)
+                )
+        return self._cursor
+
+    def execute(self, sql, exception_to_fail=False):
+        sql = sql[:-1] if sql.endswith(';') else sql
+        result, error = None, None
+        try:
+            self.cursor.execute(sql)
+            sql_header = self.cursor.description or []
+            column_names = [description[0].lower() for description in sql_header]
+            if column_names:
+                result = [dict(zip(column_names, row)) for row in self.cursor]
+                result = result[0] if len(result) == 1 else result
+            else:
+                result = None
+        except DatabaseError as err:
+            error = err
+        if exception_to_fail and error:
+            self.module.fail_json(msg='Cannot execute sql: %s' % to_native(error))
+        return result, error
+
+    def close(self):
+        try:
+            if self._cursor:
+                self._cursor.close()
+            if self._conn:
+                self._conn.close()
+        except:
+            pass
+

apps/ops/ws.py

@@ -1,18 +1,18 @@
-import time
+import asyncio
 import os
-import threading
-import json
-from channels.generic.websocket import JsonWebsocketConsumer
 
-from common.utils import get_logger
+import aiofiles
+from channels.generic.websocket import AsyncJsonWebsocketConsumer
+
 from common.db.utils import close_old_connections
-from .celery.utils import get_celery_task_log_path
+from common.utils import get_logger
 from .ansible.utils import get_ansible_task_log_path
+from .celery.utils import get_celery_task_log_path
 
 logger = get_logger(__name__)
 
 
-class TaskLogWebsocket(JsonWebsocketConsumer):
+class TaskLogWebsocket(AsyncJsonWebsocketConsumer):
     disconnected = False
 
     log_types = {
@@ -20,70 +20,59 @@ class TaskLogWebsocket(JsonWebsocketConsumer):
         'ansible': get_ansible_task_log_path
     }
 
-    def connect(self):
+    async def connect(self):
         user = self.scope["user"]
         if user.is_authenticated:
-            self.accept()
+            await self.accept()
         else:
-            self.close()
+            await self.close()
 
-    def get_log_path(self, task_id):
-        func = self.log_types.get(self.log_type)
+    def get_log_path(self, task_id, log_type):
+        func = self.log_types.get(log_type)
         if func:
             return func(task_id)
 
-    def receive(self, text_data=None, bytes_data=None, **kwargs):
-        data = json.loads(text_data)
-        task_id = data.get('task')
-        self.log_type = data.get('type', 'celery')
-        if task_id:
-            self.handle_task(task_id)
+    async def receive_json(self, content, **kwargs):
+        task_id = content.get('task')
+        task_typ = content.get('type', 'celery')
+        log_path = self.get_log_path(task_id, task_typ)
+        await self.async_handle_task(task_id, log_path)
 
-    def wait_util_log_path_exist(self, task_id):
-        log_path = self.get_log_path(task_id)
+    async def async_handle_task(self, task_id, log_path):
+        logger.info("Task id: {}".format(task_id))
         while not self.disconnected:
             if not os.path.exists(log_path):
-                self.send_json({'message': '.', 'task': task_id})
-                time.sleep(0.5)
-                continue
-            self.send_json({'message': '\r\n'})
-            try:
-                logger.debug('Task log path: {}'.format(log_path))
-                task_log_f = open(log_path, 'rb')
-                return task_log_f
-            except OSError:
-                return None
-
-    def read_log_file(self, task_id):
-        task_log_f = self.wait_util_log_path_exist(task_id)
-        if not task_log_f:
-            logger.debug('Task log file is None: {}'.format(task_id))
-            return
-
-        task_end_mark = []
-        while not self.disconnected:
-            data = task_log_f.read(4096)
-            if data:
-                data = data.replace(b'\n', b'\r\n')
-                self.send_json(
-                    {'message': data.decode(errors='ignore'), 'task': task_id}
-                )
-                if data.find(b'succeeded in') != -1:
-                    task_end_mark.append(1)
-                if data.find(bytes(task_id, 'utf8')) != -1:
-                    task_end_mark.append(1)
-            elif len(task_end_mark) == 2:
-                logger.debug('Task log end: {}'.format(task_id))
+                await self.send_json({'message': '.', 'task': task_id})
+                await asyncio.sleep(0.5)
+            else:
+                await self.send_task_log(task_id, log_path)
                 break
-            time.sleep(0.2)
-        task_log_f.close()
 
-    def handle_task(self, task_id):
-        logger.info("Task id: {}".format(task_id))
-        thread = threading.Thread(target=self.read_log_file, args=(task_id,))
-        thread.start()
+    async def send_task_log(self, task_id, log_path):
+        await self.send_json({'message': '\r\n'})
+        try:
+            logger.debug('Task log path: {}'.format(log_path))
+            task_end_mark = []
+            async with aiofiles.open(log_path, 'rb') as task_log_f:
+                while not self.disconnected:
+                    data = await task_log_f.read(4096)
+                    if data:
+                        data = data.replace(b'\n', b'\r\n')
+                        await self.send_json(
+                            {'message': data.decode(errors='ignore'), 'task': task_id}
+                        )
+                        if data.find(b'succeeded in') != -1:
+                            task_end_mark.append(1)
+                        if data.find(bytes(task_id, 'utf8')) != -1:
+                            task_end_mark.append(1)
+                    elif len(task_end_mark) == 2:
+                        logger.debug('Task log end: {}'.format(task_id))
+                        break
+                    await asyncio.sleep(0.2)
+        except OSError as e:
+            logger.warn('Task log path open failed: {}'.format(e))
 
-    def disconnect(self, close_code):
+    async def disconnect(self, close_code):
         self.disconnected = True
-        self.close()
+        await self.close()
         close_old_connections()

jms

@@ -177,7 +177,7 @@ if __name__ == '__main__':
         help="The service to start",
     )
     parser.add_argument('-d', '--daemon', nargs="?", const=True)
-    parser.add_argument('-w', '--worker', type=int, nargs="?", default=4)
+    parser.add_argument('-w', '--worker', type=int, nargs="?")
     parser.add_argument('-f', '--force', nargs="?", const=True)
 
     args = parser.parse_args()

requirements/requirements.txt

@@ -1,3 +1,4 @@
+aiofiles==22.1.0
 amqp==5.0.9
 ansible==6.4.0
 ansible-runner==2.2.1