Merge branch 'v3' of http://github.com/jumpserver/jumpserver into pr@v3@feat_support_clear_private_key

apps/assets/models/asset/host.py

@@ -3,7 +3,6 @@ from .common import Asset
 
 
 class Host(Asset):
-    pass
 
     @classmethod
     def get_gateway_queryset(cls):

apps/assets/models/domain.py

@@ -13,8 +13,9 @@ from django.utils.translation import ugettext_lazy as _
 from common.db import fields
 from common.utils import get_logger, lazyproperty
 from orgs.mixins.models import OrgModelMixin
+from assets.models import Host
 from .base import BaseAccount
-from ..const import SecretType, GATEWAY_NAME
+from ..const import SecretType
 
 logger = get_logger(__file__)
 
@@ -37,7 +38,7 @@ class Domain(OrgModelMixin):
 
     @lazyproperty
     def gateways(self):
-        return self.assets.filter(platform__name=GATEWAY_NAME, is_active=True)
+        return Host.get_gateway_queryset().filter(domain=self, is_active=True)
 
     def select_gateway(self):
         return self.random_gateway()

apps/assets/serializers/domain.py

@@ -1,28 +1,33 @@
 # -*- coding: utf-8 -*-
 #
 from rest_framework import serializers
+from rest_framework.generics import get_object_or_404
 from django.utils.translation import ugettext_lazy as _
 
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from common.drf.serializers import SecretReadableMixin
-from ..models import Domain, Asset
+from common.drf.fields import ObjectRelatedField, EncryptedField
+from assets.const import SecretType
+from ..models import Domain, Asset, Account
+from ..serializers import HostSerializer
+from .utils import validate_password_for_ansible, validate_ssh_key
 
 
 class DomainSerializer(BulkOrgResourceModelSerializer):
     asset_count = serializers.SerializerMethodField(label=_('Assets amount'))
     gateway_count = serializers.SerializerMethodField(label=_('Gateways count'))
+    assets = ObjectRelatedField(
+        many=True, required=False, queryset=Asset.objects, label=_('Asset')
+    )
 
     class Meta:
         model = Domain
         fields_mini = ['id', 'name']
-        fields_small = fields_mini + [
+        fields_small = fields_mini + ['comment']
-            'comment', 'date_created'
+        fields_m2m = ['assets']
-        ]
+        read_only_fields = ['asset_count', 'gateway_count', 'date_created']
-        fields_m2m = [
+        fields = fields_small + fields_m2m + read_only_fields
-            'asset_count', 'assets', 'gateway_count',
+
-        ]
-        fields = fields_small + fields_m2m
-        read_only_fields = ('asset_count', 'gateway_count', 'date_created')
         extra_kwargs = {
             'assets': {'required': False, 'label': _('Assets')},
         }
@@ -36,20 +41,86 @@ class DomainSerializer(BulkOrgResourceModelSerializer):
         return obj.gateways.count()
 
 
-class GatewaySerializer(BulkOrgResourceModelSerializer):
+class GatewaySerializer(HostSerializer):
-    is_connective = serializers.BooleanField(required=False, label=_('Connectivity'))
+    password = EncryptedField(
+        label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024,
+        validators=[validate_password_for_ansible], write_only=True
+    )
+    private_key = EncryptedField(
+        label=_('SSH private key'), required=False, allow_blank=True, allow_null=True,
+        max_length=16384, write_only=True
+    )
+    passphrase = serializers.CharField(
+        label=_('Key password'), allow_blank=True, allow_null=True, required=False, write_only=True,
+        max_length=512,
+    )
+    username = serializers.CharField(
+        label=_('Username'), allow_blank=True, max_length=128, required=True,
+    )
 
-    class Meta:
+    class Meta(HostSerializer.Meta):
-        model = Asset
+        fields = HostSerializer.Meta.fields + [
-        fields_mini = ['id']
+            'username', 'password', 'private_key', 'passphrase'
-        fields_small = fields_mini + [
-            'address', 'port', 'protocol',
-            'is_active', 'is_connective',
-            'date_created', 'date_updated',
-            'created_by', 'comment',
         ]
-        fields_fk = ['domain']
+
-        fields = fields_small + fields_fk
+    def validate_private_key(self, secret):
+        if not secret:
+            return
+        passphrase = self.initial_data.get('passphrase')
+        passphrase = passphrase if passphrase else None
+        validate_ssh_key(secret, passphrase)
+        return secret
+
+    @staticmethod
+    def clean_auth_fields(validated_data):
+        username = validated_data.pop('username', None)
+        password = validated_data.pop('password', None)
+        private_key = validated_data.pop('private_key', None)
+        validated_data.pop('passphrase', None)
+        return username, password, private_key
+
+    @staticmethod
+    def create_accounts(instance, username, password, private_key):
+        account_name = f'{instance.name}-{_("Gateway")}'
+        account_data = {
+            'privileged': True,
+            'name': account_name,
+            'username': username,
+            'asset_id': instance.id,
+            'created_by': instance.created_by
+        }
+        if password:
+            Account.objects.create(
+                **account_data, secret=password, secret_type=SecretType.PASSWORD
+            )
+        if private_key:
+            Account.objects.create(
+                **account_data, secret=private_key, secret_type=SecretType.SSH_KEY
+            )
+
+    @staticmethod
+    def update_accounts(instance, username, password, private_key):
+        accounts = instance.accounts.filter(username=username)
+        if password:
+            account = get_object_or_404(accounts, SecretType.PASSWORD)
+            account.secret = password
+            account.save()
+        if private_key:
+            account = get_object_or_404(accounts, SecretType.SSH_KEY)
+            account.secret = private_key
+            account.save()
+
+    def create(self, validated_data):
+        auth_fields = self.clean_auth_fields(validated_data)
+        instance = super().create(validated_data)
+        self.create_accounts(instance, *auth_fields)
+        return instance
+
+    def update(self, instance, validated_data):
+        auth_fields = self.clean_auth_fields(validated_data)
+        instance = super().update(instance, validated_data)
+        self.update_accounts(instance, *auth_fields)
+        return instance
 
 
 class GatewayWithAuthSerializer(SecretReadableMixin, GatewaySerializer):

apps/authentication/migrations/0015_alter_connectiontoken_login.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.14 on 2022-11-23 02:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('authentication', '0014_auto_20221122_2152'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='connectiontoken',
+            name='login',
+            field=models.CharField(max_length=128, verbose_name='Login account'),
+        ),
+    ]

apps/ops/api/job.py

@@ -1,4 +1,3 @@
-from django.shortcuts import get_object_or_404
 from rest_framework import viewsets
 
 from ops.models import Job, JobExecution
@@ -7,14 +6,17 @@ from ops.serializers.job import JobSerializer, JobExecutionSerializer
 __all__ = ['JobViewSet', 'JobExecutionViewSet']
 
 from ops.tasks import run_ops_job, run_ops_job_executions
+from orgs.mixins.api import OrgBulkModelViewSet
 
 
-class JobViewSet(viewsets.ModelViewSet):
+class JobViewSet(OrgBulkModelViewSet):
     serializer_class = JobSerializer
-    queryset = Job.objects.all()
+    model = Job
+    permission_classes = ()
 
     def get_queryset(self):
-        return self.queryset.filter(instant=False)
+        query_set = super().get_queryset()
+        return query_set.filter(instant=False)
 
     def perform_create(self, serializer):
         instance = serializer.save()
@@ -22,20 +24,20 @@ class JobViewSet(viewsets.ModelViewSet):
             run_ops_job.delay(instance.id)
 
 
-class JobExecutionViewSet(viewsets.ModelViewSet):
+class JobExecutionViewSet(OrgBulkModelViewSet):
     serializer_class = JobExecutionSerializer
-    queryset = JobExecution.objects.all()
     http_method_names = ('get', 'post', 'head', 'options',)
+    # filter_fields = ('type',)
+    permission_classes = ()
+    model = JobExecution
 
     def perform_create(self, serializer):
         instance = serializer.save()
         run_ops_job_executions.delay(instance.id)
 
     def get_queryset(self):
+        query_set = super().get_queryset()
         job_id = self.request.query_params.get('job_id')
-        job_type = self.request.query_params.get('type')
         if job_id:
-            self.queryset = self.queryset.filter(job_id=job_id)
+            self.queryset = query_set.filter(job_id=job_id)
-        if job_type:
+        return query_set
-            self.queryset = self.queryset.filter(job__type=job_type)
-        return self.queryset

apps/ops/migrations/0034_job_org_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.14 on 2022-11-23 09:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0033_auto_20221118_1431'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='job',
+            name='org_id',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
+        ),
+    ]

apps/ops/migrations/0035_jobexecution_org_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.14 on 2022-11-23 10:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0034_job_org_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jobexecution',
+            name='org_id',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
+        ),
+    ]

apps/ops/models/job.py

@@ -9,16 +9,14 @@ from django.utils.translation import gettext_lazy as _
 from django.utils import timezone
 from celery import current_task
 
-from common.const.choices import Trigger
-from common.db.models import BaseCreateUpdateModel
-
 __all__ = ["Job", "JobExecution"]
 
 from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner
 from ops.mixin import PeriodTaskModelMixin
+from orgs.mixins.models import JMSOrgBaseModel
 
 
-class Job(BaseCreateUpdateModel, PeriodTaskModelMixin):
+class Job(JMSOrgBaseModel, PeriodTaskModelMixin):
     class Types(models.TextChoices):
         adhoc = 'adhoc', _('Adhoc')
         playbook = 'playbook', _('Playbook')
@@ -94,7 +92,7 @@ class Job(BaseCreateUpdateModel, PeriodTaskModelMixin):
         return self.executions.create()
 
 
-class JobExecution(BaseCreateUpdateModel):
+class JobExecution(JMSOrgBaseModel):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     task_id = models.UUIDField(null=True)
     status = models.CharField(max_length=16, verbose_name=_('Status'), default='running')

apps/ops/serializers/job.py

@@ -1,14 +1,13 @@
-from django.db import transaction
 from rest_framework import serializers
-
 from common.drf.fields import ReadableHiddenField
 from ops.mixin import PeriodTaskSerializerMixin
 from ops.models import Job, JobExecution
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 _all_ = []
 
 
-class JobSerializer(serializers.ModelSerializer, PeriodTaskSerializerMixin):
+class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
     owner = ReadableHiddenField(default=serializers.CurrentUserDefault())
 
     class Meta:

apps/ops/tasks.py

@@ -10,6 +10,7 @@ from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
 from common.utils import get_logger, get_object_or_none, get_log_keep_day
+from orgs.utils import tmp_to_org
 from .celery.decorator import (
     register_as_period_task, after_app_shutdown_clean_periodic,
     after_app_ready_start
@@ -27,28 +28,30 @@ logger = get_logger(__file__)
 @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task"))
 def run_ops_job(job_id):
     job = get_object_or_none(Job, id=job_id)
-    execution = job.create_execution()
+    with tmp_to_org(job.org):
-    try:
+        execution = job.create_execution()
-        execution.start()
+        try:
-    except SoftTimeLimitExceeded:
+            execution.start()
-        execution.set_error('Run timeout')
+        except SoftTimeLimitExceeded:
-        logger.error("Run adhoc timeout")
+            execution.set_error('Run timeout')
-    except Exception as e:
+            logger.error("Run adhoc timeout")
-        execution.set_error(e)
+        except Exception as e:
-        logger.error("Start adhoc execution error: {}".format(e))
+            execution.set_error(e)
+            logger.error("Start adhoc execution error: {}".format(e))
 
 
 @shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task execution"))
 def run_ops_job_executions(execution_id, **kwargs):
     execution = get_object_or_none(JobExecution, id=execution_id)
-    try:
+    with tmp_to_org(execution.org):
-        execution.start()
+        try:
-    except SoftTimeLimitExceeded:
+            execution.start()
-        execution.set_error('Run timeout')
+        except SoftTimeLimitExceeded:
-        logger.error("Run adhoc timeout")
+            execution.set_error('Run timeout')
-    except Exception as e:
+            logger.error("Run adhoc timeout")
-        execution.set_error(e)
+        except Exception as e:
-        logger.error("Start adhoc execution error: {}".format(e))
+            execution.set_error(e)
+            logger.error("Start adhoc execution error: {}".format(e))
 
 
 @shared_task(verbose_name=_('Periodic clear celery tasks'))