Merge branch 'v3' of github.com:jumpserver/jumpserver into v3

apps/assets/utils/k8s.py

@@ -10,7 +10,6 @@ from kubernetes.client.exceptions import ApiException
 from rest_framework.generics import get_object_or_404
 
 from common.utils import get_logger
-from common.tree import TreeNode
 from assets.models import Account, Asset
 
 from ..const import CloudTypes, Category
@@ -19,13 +18,15 @@ logger = get_logger(__file__)
 
 
 class KubernetesClient:
-    def __init__(self, url, token):
+    def __init__(self, url, token, proxy=None):
         self.url = url
         self.token = token
+        self.proxy = proxy
 
     def get_api(self):
         configuration = client.Configuration()
         configuration.host = self.url
+        configuration.proxy = self.proxy
         configuration.verify_ssl = False
         configuration.api_key = {"authorization": "Bearer " + self.token}
         c = api_client.ApiClient(configuration=configuration)
@@ -82,11 +83,23 @@ class KubernetesClient:
                 data[namespace] = [pod_info, ]
         return data
 
-    @staticmethod
+    @classmethod
-    def get_kubernetes_data(app_id, username):
+    def get_proxy_url(cls, asset):
+        if not asset.domain:
+            return None
+
+        gateway = asset.domain.select_gateway()
+        if not gateway:
+            return None
+        return f'{gateway.address}:{gateway.port}'
+
+    @classmethod
+    def get_kubernetes_data(cls, app_id, username):
         asset = get_object_or_404(Asset, id=app_id)
         account = get_object_or_404(Account, asset=asset, username=username)
-        k8s = KubernetesClient(asset.address, account.secret)
+        k8s_url = f'{asset.address}:{asset.port}'
+        proxy_url = cls.get_proxy_url(asset)
+        k8s = cls(k8s_url, account.secret, proxy=proxy_url)
         return k8s.get_pods()
 
 
@@ -112,7 +125,7 @@ class KubernetesTree:
     def as_asset_tree_node(self, asset):
         i = urlencode({'asset_id': self.tree_id})
         node = self.create_tree_node(
-            i, str(asset.id), str(asset), 'asset',
+            i, str(asset.id), str(asset), 'asset', is_open=True,
         )
         return node
 
@@ -136,14 +149,14 @@ class KubernetesTree:
         return node
 
     @staticmethod
-    def create_tree_node(id_, pid, name, identity, icon='', is_container=False):
+    def create_tree_node(id_, pid, name, identity, icon='', is_container=False, is_open=False):
         node = {
             'id': id_,
             'name': name,
             'title': name,
             'pId': pid,
             'isParent': not is_container,
-            'open': False,
+            'open': is_open,
             'iconSkin': icon,
             'meta': {
                 'type': 'k8s',

apps/audits/api.py

@@ -13,19 +13,25 @@ from common.drf.api import JMSReadOnlyModelViewSet
 from common.plugins.es import QuerySet as ESQuerySet
 from common.drf.filters import DatetimeRangeFilter
 from common.api import CommonGenericViewSet
-from orgs.mixins.api import OrgGenericViewSet, OrgBulkModelViewSet, OrgRelationMixin
+from ops.models.job import JobAuditLog
+from orgs.mixins.api import OrgGenericViewSet, OrgBulkModelViewSet
 from orgs.utils import current_org
-# from ops.models import CommandExecution
-from . import filters
 from .backends import TYPE_ENGINE_MAPPING
 from .models import FTPLog, UserLoginLog, OperateLog, PasswordChangeLog
-from .serializers import FTPLogSerializer, UserLoginLogSerializer
+from .serializers import FTPLogSerializer, UserLoginLogSerializer, JobAuditLogSerializer
 from .serializers import (
     OperateLogSerializer, OperateLogActionDetailSerializer,
     PasswordChangeLogSerializer
 )
 
 
+class JobAuditViewSet(OrgBulkModelViewSet):
+    serializer_class = JobAuditLogSerializer
+    http_method_names = ('get', 'head', 'options',)
+    permission_classes = ()
+    model = JobAuditLog
+
+
 class FTPLogViewSet(CreateModelMixin, ListModelMixin, OrgGenericViewSet):
     model = FTPLog
     serializer_class = FTPLogSerializer

apps/audits/serializers.py

@@ -4,6 +4,8 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
 from common.drf.fields import LabeledChoiceField
+from ops.models.job import JobAuditLog
+from ops.serializers.job import JobExecutionSerializer
 from terminal.models import Session
 from . import models
 from .const import (
@@ -15,6 +17,17 @@ from .const import (
 )
 
 
+class JobAuditLogSerializer(JobExecutionSerializer):
+    class Meta:
+        model = JobAuditLog
+        read_only_fields = ["timedelta", "time_cost", 'is_finished', 'date_start',
+                            'date_finished',
+                            'date_created',
+                            'is_success',
+                            'creator_name']
+        fields = read_only_fields + []
+
+
 class FTPLogSerializer(serializers.ModelSerializer):
     operate = LabeledChoiceField(choices=OperateChoices.choices, label=_("Operate"))
 

apps/audits/urls/api_urls.py

@@ -7,7 +7,6 @@ from rest_framework.routers import DefaultRouter
 from common import api as capi
 from .. import api
 
-
 app_name = "audits"
 
 router = DefaultRouter()
@@ -15,9 +14,7 @@ router.register(r'ftp-logs', api.FTPLogViewSet, 'ftp-log')
 router.register(r'login-logs', api.UserLoginLogViewSet, 'login-log')
 router.register(r'operate-logs', api.OperateLogViewSet, 'operate-log')
 router.register(r'password-change-logs', api.PasswordChangeLogViewSet, 'password-change-log')
-# router.register(r'command-execution-logs', api.CommandExecutionViewSet, 'command-execution-log')
+router.register(r'job-logs', api.JobAuditViewSet, 'job-log')
-# router.register(r'command-executions-hosts-relations', api.CommandExecutionHostRelationViewSet, 'command-executions-hosts-relation')
-
 
 urlpatterns = [
     path('my-login-logs/', api.MyLoginLogAPIView.as_view(), name='my-login-log'),

apps/common/management/commands/services/services/base.py

@@ -44,9 +44,12 @@ class BaseService(object):
         if self.is_running:
             msg = f'{self.name} is running: {self.pid}.'
         else:
-            msg = '\033[31m{} is stopped.\033[0m\nYou can manual start it to find the error: \n' \
+            msg = f'{self.name} is stopped.'
-                  '  $ cd {}\n' \
+            if DEBUG:
-                  '  $ {}'.format(self.name, self.cwd, ' '.join(self.cmd))
+                msg = '\033[31m{} is stopped.\033[0m\nYou can manual start it to find the error: \n' \
+                      '  $ cd {}\n' \
+                      '  $ {}'.format(self.name, self.cwd, ' '.join(self.cmd))
+
         print(msg)
 
     # -- log --
@@ -147,7 +150,6 @@ class BaseService(object):
                 self.remove_pid()
                 break
             else:
-                time.sleep(1)
                 continue
 
     def watch(self):
@@ -203,4 +205,3 @@ class BaseService(object):
             logging.info(f'Remove old log: {to_delete_dir}')
             shutil.rmtree(to_delete_dir, ignore_errors=True)
     # -- end action --
-

apps/common/management/commands/services/utils.py

@@ -40,7 +40,8 @@ class ServicesUtil(object):
             service: BaseService
             service.start()
             self.files_preserve_map[service.name] = service.log_file
-            time.sleep(1)
+
+        time.sleep(1)
 
     def stop(self):
         for service in self._services:

apps/ops/ansible/inventory.py

@@ -10,7 +10,7 @@ __all__ = ['JMSInventory']
 
 class JMSInventory:
     def __init__(self, assets, account_policy='privileged_first',
-                 account_prefer='root,Administrator', host_callback=None, unique_host_name=False):
+                 account_prefer='root,Administrator', host_callback=None):
         """
         :param assets:
         :param account_prefer: account username name if not set use account_policy
@@ -21,7 +21,6 @@ class JMSInventory:
         self.account_policy = account_policy
         self.host_callback = host_callback
         self.exclude_hosts = {}
-        self.unique_host_name = unique_host_name
 
     @staticmethod
     def clean_assets(assets):
@@ -114,8 +113,6 @@ class JMSInventory:
                 'secret': account.secret, 'secret_type': account.secret_type
             } if account else None
         }
-        if self.unique_host_name:
-            host['name'] += '({})'.format(asset.id)
 
         if host['jms_account'] and asset.platform.type == 'oracle':
             host['jms_account']['mode'] = 'sysdba' if account.privileged else None

apps/ops/api/adhoc.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from .base import SelfBulkModelViewSet
+from orgs.mixins.api import OrgBulkModelViewSet
 from ..models import AdHoc
 from ..serializers import (
     AdHocSerializer
@@ -10,7 +10,11 @@ __all__ = [
 ]
 
 
-class AdHocViewSet(SelfBulkModelViewSet):
+class AdHocViewSet(OrgBulkModelViewSet):
     serializer_class = AdHocSerializer
     permission_classes = ()
     model = AdHoc
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        return queryset.filter(creator=self.request.user)

apps/ops/api/base.py

@@ -1,17 +0,0 @@
-from rest_framework_bulk import BulkModelViewSet
-
-from common.mixins import CommonApiMixin
-
-__all__ = ['SelfBulkModelViewSet']
-
-
-class SelfBulkModelViewSet(CommonApiMixin, BulkModelViewSet):
-
-    def get_queryset(self):
-        if hasattr(self, 'model'):
-            return self.model.objects.filter(creator=self.request.user)
-        else:
-            assert self.queryset is None, (
-                    "'%s' should not include a `queryset` attribute"
-                    % self.__class__.__name__
-            )

apps/ops/api/job.py

@@ -2,14 +2,15 @@ from rest_framework.views import APIView
 from django.shortcuts import get_object_or_404
 from rest_framework.response import Response
 
-from ops.api.base import SelfBulkModelViewSet
 from ops.models import Job, JobExecution
+from ops.models.job import JobAuditLog
 from ops.serializers.job import JobSerializer, JobExecutionSerializer
 
-__all__ = ['JobViewSet', 'JobExecutionViewSet', 'JobRunVariableHelpAPIView', 'JobAssetDetail']
+__all__ = ['JobViewSet', 'JobExecutionViewSet', 'JobRunVariableHelpAPIView', 'JobAssetDetail', ]
 
 from ops.tasks import run_ops_job_execution
 from ops.variables import JMS_JOB_VARIABLE_HELP
+from orgs.mixins.api import OrgBulkModelViewSet
 
 
 def set_task_to_serializer_data(serializer, task):
@@ -18,16 +19,17 @@ def set_task_to_serializer_data(serializer, task):
     setattr(serializer, "_data", data)
 
 
-class JobViewSet(SelfBulkModelViewSet):
+class JobViewSet(OrgBulkModelViewSet):
     serializer_class = JobSerializer
     permission_classes = ()
     model = Job
 
     def get_queryset(self):
-        query_set = super().get_queryset()
+        queryset = super().get_queryset()
+        queryset = queryset.filter(creator=self.request.user)
         if self.action != 'retrieve':
-            return query_set.filter(instant=False)
+            return queryset.filter(instant=False)
-        return query_set
+        return queryset
 
     def perform_create(self, serializer):
         instance = serializer.save()
@@ -48,7 +50,7 @@ class JobViewSet(SelfBulkModelViewSet):
         set_task_to_serializer_data(serializer, task)
 
 
-class JobExecutionViewSet(SelfBulkModelViewSet):
+class JobExecutionViewSet(OrgBulkModelViewSet):
     serializer_class = JobExecutionSerializer
     http_method_names = ('get', 'post', 'head', 'options',)
     permission_classes = ()
@@ -60,11 +62,12 @@ class JobExecutionViewSet(SelfBulkModelViewSet):
         set_task_to_serializer_data(serializer, task)
 
     def get_queryset(self):
-        query_set = super().get_queryset()
+        queryset = super().get_queryset()
+        queryset = queryset.filter(creator=self.request.user)
         job_id = self.request.query_params.get('job_id')
         if job_id:
-            query_set = query_set.filter(job_id=job_id)
+            queryset = queryset.filter(job_id=job_id)
-        return query_set
+        return queryset
 
 
 class JobRunVariableHelpAPIView(APIView):

apps/ops/api/playbook.py

@@ -2,11 +2,7 @@ import os
 import zipfile
 
 from django.conf import settings
-from rest_framework_bulk import BulkModelViewSet
-
-from common.mixins import CommonApiMixin
 from orgs.mixins.api import OrgBulkModelViewSet
-from .base import SelfBulkModelViewSet
 from ..exception import PlaybookNoValidEntry
 from ..models import Playbook
 from ..serializers.playbook import PlaybookSerializer
@@ -20,11 +16,16 @@ def unzip_playbook(src, dist):
         fz.extract(file, dist)
 
 
-class PlaybookViewSet(SelfBulkModelViewSet):
+class PlaybookViewSet(OrgBulkModelViewSet):
     serializer_class = PlaybookSerializer
     permission_classes = ()
     model = Playbook
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.filter(creator=self.request.user)
+        return queryset
+
     def perform_create(self, serializer):
         instance = serializer.save()
         src_path = os.path.join(settings.MEDIA_ROOT, instance.path.name)

apps/ops/migrations/0029_auto_20221215_1712.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.14 on 2022-12-15 09:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0028_auto_20221205_1627'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='adhoc',
+            name='org_id',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='org_id',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
+        ),
+        migrations.AddField(
+            model_name='jobexecution',
+            name='org_id',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
+        ),
+        migrations.AddField(
+            model_name='playbook',
+            name='org_id',
+            field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
+        ),
+    ]

apps/ops/models/adhoc.py

@@ -9,10 +9,12 @@ from common.utils import get_logger
 
 __all__ = ["AdHoc"]
 
+from orgs.mixins.models import JMSOrgBaseModel
+
 logger = get_logger(__file__)
 
 
-class AdHoc(JMSBaseModel):
+class AdHoc(JMSOrgBaseModel):
     class Modules(models.TextChoices):
         shell = 'shell', _('Shell')
         winshell = 'win_shell', _('Powershell')

apps/ops/models/job.py

@@ -9,15 +9,15 @@ from django.utils.translation import gettext_lazy as _
 from django.utils import timezone
 from celery import current_task
 
-__all__ = ["Job", "JobExecution"]
+__all__ = ["Job", "JobExecution", "JobAuditLog"]
 
-from common.db.models import JMSBaseModel
 from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner
 from ops.mixin import PeriodTaskModelMixin
 from ops.variables import *
+from orgs.mixins.models import JMSOrgBaseModel
 
 
-class Job(JMSBaseModel, PeriodTaskModelMixin):
+class Job(JMSOrgBaseModel, PeriodTaskModelMixin):
     class Types(models.TextChoices):
         adhoc = 'adhoc', _('Adhoc')
         playbook = 'playbook', _('Playbook')
@@ -88,7 +88,7 @@ class Job(JMSBaseModel, PeriodTaskModelMixin):
 
     @property
     def inventory(self):
-        return JMSInventory(self.assets.all(), self.runas_policy, self.runas, unique_host_name=True)
+        return JMSInventory(self.assets.all(), self.runas_policy, self.runas)
 
     def create_execution(self):
         return self.executions.create()
@@ -97,7 +97,7 @@ class Job(JMSBaseModel, PeriodTaskModelMixin):
         ordering = ['date_created']
 
 
-class JobExecution(JMSBaseModel):
+class JobExecution(JMSOrgBaseModel):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     task_id = models.UUIDField(null=True)
     status = models.CharField(max_length=16, verbose_name=_('Status'), default='running')
@@ -133,26 +133,25 @@ class JobExecution(JMSBaseModel):
                     "status": "ok",
                     "tasks": [],
                 }
-                host_name = "{}({})".format(asset.name, asset.id)
+                if self.summary["excludes"].get(asset.name, None):
-                if self.summary["excludes"].get(host_name, None):
                     asset_detail.update({"status": "excludes"})
                     result["detail"].append(asset_detail)
                     break
-                if self.result["dark"].get(host_name, None):
+                if self.result["dark"].get(asset.name, None):
                     asset_detail.update({"status": "failed"})
-                    for key, task in self.result["dark"][host_name].items():
+                    for key, task in self.result["dark"][asset.name].items():
                         task_detail = {"name": key,
                                        "output": "{}{}".format(task.get("stdout", ""), task.get("stderr", ""))}
                         asset_detail["tasks"].append(task_detail)
-                if self.result["failures"].get(host_name, None):
+                if self.result["failures"].get(asset.name, None):
                     asset_detail.update({"status": "failed"})
-                    for key, task in self.result["failures"][host_name].items():
+                    for key, task in self.result["failures"][asset.name].items():
                         task_detail = {"name": key,
                                        "output": "{}{}".format(task.get("stdout", ""), task.get("stderr", ""))}
                         asset_detail["tasks"].append(task_detail)
 
-                if self.result["ok"].get(host_name, None):
+                if self.result["ok"].get(asset.name, None):
-                    for key, task in self.result["ok"][host_name].items():
+                    for key, task in self.result["ok"][asset.name].items():
                         task_detail = {"name": key,
                                        "output": "{}{}".format(task.get("stdout", ""), task.get("stderr", ""))}
                         asset_detail["tasks"].append(task_detail)
@@ -202,10 +201,11 @@ class JobExecution(JMSBaseModel):
 
     def gather_static_variables(self):
         default = {
-            JMS_USERNAME: self.creator.username,
+            JMS_JOB_ID: str(self.job.id),
-            JMS_JOB_ID: self.job.id,
             JMS_JOB_NAME: self.job.name,
         }
+        if self.creator:
+            default.update({JMS_USERNAME: self.creator.username})
         return default
 
     @property
@@ -255,7 +255,10 @@ class JobExecution(JMSBaseModel):
         this = self.__class__.objects.get(id=self.id)
         this.status = status_mapper.get(cb.status, cb.status)
         this.summary.update(cb.summary)
-        this.result.update(cb.result)
+        if this.result:
+            this.result.update(cb.result)
+        else:
+            this.result = cb.result
         this.finish_task()
 
     def finish_task(self):
@@ -283,3 +286,12 @@ class JobExecution(JMSBaseModel):
 
     class Meta:
         ordering = ['-date_created']
+
+
+class JobAuditLog(JobExecution):
+    @property
+    def creator_name(self):
+        return self.creator.name
+
+    class Meta:
+        proxy = True

apps/ops/models/playbook.py

@@ -5,11 +5,11 @@ from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from common.db.models import JMSBaseModel
 from ops.exception import PlaybookNoValidEntry
+from orgs.mixins.models import JMSOrgBaseModel
 
 
-class Playbook(JMSBaseModel):
+class Playbook(JMSOrgBaseModel):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, verbose_name=_('Name'), null=True)
     path = models.FileField(upload_to='playbooks/')

apps/ops/serializers/adhoc.py

@@ -4,10 +4,11 @@ from __future__ import unicode_literals
 from rest_framework import serializers
 
 from common.drf.fields import ReadableHiddenField
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from ..models import AdHoc
 
 
-class AdHocSerializer(serializers.ModelSerializer):
+class AdHocSerializer(BulkOrgResourceModelSerializer):
     creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
     row_count = serializers.IntegerField(read_only=True)
     size = serializers.IntegerField(read_only=True)

apps/ops/serializers/job.py

@@ -3,9 +3,11 @@ from rest_framework import serializers
 from common.drf.fields import ReadableHiddenField
 from ops.mixin import PeriodTaskSerializerMixin
 from ops.models import Job, JobExecution
+from ops.models.job import JobAuditLog
+from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 
-class JobSerializer(serializers.ModelSerializer, PeriodTaskSerializerMixin):
+class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
     creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
     run_after_save = serializers.BooleanField(label=_("Run after save"), read_only=True, default=False, required=False)
 
@@ -25,7 +27,7 @@ class JobSerializer(serializers.ModelSerializer, PeriodTaskSerializerMixin):
         ]
 
 
-class JobExecutionSerializer(serializers.ModelSerializer):
+class JobExecutionSerializer(BulkOrgResourceModelSerializer):
     creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
     job_type = serializers.ReadOnlyField(label=_("Job type"))
     count = serializers.ReadOnlyField(label=_("Count"))
@@ -39,3 +41,6 @@ class JobExecutionSerializer(serializers.ModelSerializer):
         fields = read_only_fields + [
             "job", "parameters"
         ]
+
+
+

apps/ops/serializers/playbook.py

@@ -12,7 +12,7 @@ def parse_playbook_name(path):
     return file_name.split(".")[-2]
 
 
-class PlaybookSerializer(serializers.ModelSerializer):
+class PlaybookSerializer(BulkOrgResourceModelSerializer):
     creator = ReadableHiddenField(default=serializers.CurrentUserDefault())
     path = serializers.FileField(required=False)
 

apps/ops/tasks.py

@@ -36,7 +36,8 @@ def run_ops_job(job_id):
 def run_ops_job_execution(execution_id, **kwargs):
     execution = get_object_or_none(JobExecution, id=execution_id)
     try:
-        execution.start()
+        with tmp_to_org(execution.org):
+            execution.start()
     except SoftTimeLimitExceeded:
         execution.set_error('Run timeout')
         logger.error("Run adhoc timeout")

apps/perms/api/user_permission/tree/node_with_asset.py

@@ -158,7 +158,7 @@ class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView):
         asset_id = parent_info.get('asset_id')
         asset_id = tree_id if not asset_id else asset_id
 
-        if tree_id and not account_username:
+        if tree_id and not key and not account_username:
             asset = self.asset(asset_id)
             accounts = self.get_accounts(asset)
             asset_node = KubernetesTree(tree_id).as_asset_tree_node(asset)
@@ -168,6 +168,6 @@ class UserGrantedK8sAsTreeApi(SelfOrPKUserMixin, ListAPIView):
                     account, parent_info,
                 )
                 tree.append(account_node)
-        else:
+        elif key and account_username:
             tree = KubernetesTree(key).async_tree_node(parent_info)
         return Response(data=tree)