Merge branch 'v3' of github.com:jumpserver/jumpserver into v3

apps/authentication/api/connection_token.py

@@ -210,7 +210,7 @@ class ConnectionTokenMixin:
 
 class ConnectionTokenViewSet(ConnectionTokenMixin, RootOrgViewMixin, JMSModelViewSet):
     filterset_fields = (
-        'type', 'user_display', 'asset_display'
+        'user_display', 'asset_display'
     )
     search_fields = filterset_fields
     serializer_classes = {

apps/authentication/migrations/0012_auto_20220816_1629.py

@@ -49,5 +49,9 @@ class Migration(migrations.Migration):
         migrations.RemoveField(
             model_name='connectiontoken',
             name='system_user',
-        )
+        ),
+        migrations.RemoveField(
+            model_name='connectiontoken',
+            name='type',
+        ),
     ]

apps/authentication/migrations/0013_remove_connectiontoken_type.py

@@ -1,17 +0,0 @@
-# Generated by Django 3.2.14 on 2022-10-26 08:07
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('authentication', '0012_auto_20220816_1629'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='connectiontoken',
-            name='type',
-        ),
-    ]

apps/authentication/serializers/connection_token.py

@@ -23,7 +23,7 @@ class ConnectionTokenSerializer(OrgResourceModelSerializerMixin):
 
     class Meta:
         model = ConnectionToken
-        fields_mini = ['id', 'type']
+        fields_mini = ['id']
         fields_small = fields_mini + [
             'secret', 'date_expired', 'date_created', 'date_updated',
             'created_by', 'updated_by', 'org_id', 'org_name',

apps/common/apps.py

@@ -10,6 +10,8 @@ class CommonConfig(AppConfig):
     def ready(self):
         from . import signal_handlers
         from .signals import django_ready
-        if 'migrate' in sys.argv or 'compilemessages' in sys.argv:
+        excludes = ['migrate', 'compilemessages', 'makemigrations']
-            return
+        for i in excludes:
+            if i in sys.argv:
+                return
         django_ready.send(CommonConfig)

apps/common/drf/serializers/common.py

@@ -12,7 +12,7 @@ from .mixin import BulkListSerializerMixin, BulkSerializerMixin
 
 __all__ = [
     'MethodSerializer', 'EmptySerializer', 'BulkModelSerializer',
-    'AdaptedBulkListSerializer', 'CeleryTaskSerializer',
+    'AdaptedBulkListSerializer', 'CeleryTaskExecutionSerializer',
     'WritableNestedModelSerializer',
     'GroupedChoiceSerializer',
 ]
@@ -73,7 +73,7 @@ class AdaptedBulkListSerializer(BulkListSerializerMixin, BulkListSerializer):
     pass
 
 
-class CeleryTaskSerializer(serializers.Serializer):
+class CeleryTaskExecutionSerializer(serializers.Serializer):
     task = serializers.CharField(read_only=True)
 
 

apps/locale/zh/LC_MESSAGES/django.mo

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab1c609cc4c83a223835be0eab2a5a5b9050c853b66ccd2b2fa480073c8fc763
+oid sha256:5d945fc6151d6f6354052a0a09706a52e7a2fa2f9e02254965cf26b134d78c3a
-size 103346
+size 103377

apps/ops/api/adhoc.py

@@ -5,7 +5,7 @@ from django.shortcuts import get_object_or_404
 from rest_framework import viewsets, generics
 from rest_framework.views import Response
 
-from common.drf.serializers import CeleryTaskSerializer
+from common.drf.serializers import CeleryTaskExecutionSerializer
 from ..models import AdHoc, AdHocExecution
 from ..serializers import (
     AdHocSerializer,

apps/ops/api/celery.py

@@ -4,6 +4,7 @@
 import os
 import re
 
+from django.shortcuts import get_object_or_404
 from django.utils.translation import ugettext as _
 from rest_framework import viewsets
 from celery.result import AsyncResult
@@ -12,20 +13,21 @@ from django_celery_beat.models import PeriodicTask
 
 from common.permissions import IsValidUser
 from common.api import LogTailApi
-from ..models import CeleryTask
+from ..models import CeleryTaskExecution, CeleryTask
 from ..serializers import CeleryResultSerializer, CeleryPeriodTaskSerializer
 from ..celery.utils import get_celery_task_log_path
 from ..ansible.utils import get_ansible_task_log_path
 from common.mixins.api import CommonApiMixin
 
-
 __all__ = [
-    'CeleryTaskLogApi', 'CeleryResultApi', 'CeleryPeriodTaskViewSet',
+    'CeleryTaskExecutionLogApi', 'CeleryResultApi', 'CeleryPeriodTaskViewSet',
-    'AnsibleTaskLogApi',
+    'AnsibleTaskLogApi', 'CeleryTaskViewSet', 'CeleryTaskExecutionViewSet'
 ]
 
+from ..serializers.celery import CeleryTaskSerializer, CeleryTaskExecutionSerializer
 
-class CeleryTaskLogApi(LogTailApi):
+
+class CeleryTaskExecutionLogApi(LogTailApi):
     permission_classes = (IsValidUser,)
     task = None
     task_id = ''
@@ -46,8 +48,8 @@ class CeleryTaskLogApi(LogTailApi):
         if new_path and os.path.isfile(new_path):
             return new_path
         try:
-            task = CeleryTask.objects.get(id=self.task_id)
+            task = CeleryTaskExecution.objects.get(id=self.task_id)
-        except CeleryTask.DoesNotExist:
+        except CeleryTaskExecution.DoesNotExist:
             return None
         return task.full_log_path
 
@@ -94,3 +96,22 @@ class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet):
         queryset = super().get_queryset()
         queryset = queryset.exclude(description='')
         return queryset
+
+
+class CeleryTaskViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
+    queryset = CeleryTask.objects.all()
+    serializer_class = CeleryTaskSerializer
+    http_method_names = ('get', 'head', 'options',)
+
+
+class CeleryTaskExecutionViewSet(CommonApiMixin, viewsets.ReadOnlyModelViewSet):
+    serializer_class = CeleryTaskExecutionSerializer
+    http_method_names = ('get', 'head', 'options',)
+
+    def get_queryset(self):
+        task_id = self.kwargs.get("task_pk")
+        if task_id:
+            task = CeleryTask.objects.get(pk=task_id)
+            return CeleryTaskExecution.objects.filter(name=task.name)
+        else:
+            return CeleryTaskExecution.objects.none()

apps/ops/migrations/0027_auto_20221024_1709.py

@@ -0,0 +1,36 @@
+# Generated by Django 3.2.14 on 2022-10-24 09:09
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ops', '0026_auto_20221009_2050'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(name='CeleryTask'),
+        migrations.CreateModel(
+            name='CeleryTask',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=1024)),
+            ]
+        ),
+        migrations.CreateModel(
+            name='CeleryTaskExecution',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=1024)),
+                ('args', models.JSONField(verbose_name='Args')),
+                ('kwargs', models.JSONField(verbose_name='Kwargs')),
+                ('state', models.CharField(max_length=16, verbose_name='State')),
+                ('is_finished', models.BooleanField(default=False, verbose_name='Finished')),
+                ('date_published', models.DateTimeField(auto_now_add=True)),
+                ('date_start', models.DateTimeField(null=True)),
+                ('date_finished', models.DateTimeField(null=True)),
+            ],
+        ),
+    ]

apps/ops/models/celery.py

@@ -7,8 +7,27 @@ from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.db import models
 
+from ops.celery import app
+
 
 class CeleryTask(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4)
+    name = models.CharField(max_length=1024)
+
+    @property
+    def verbose_name(self):
+        task = app.tasks.get(self.name, None)
+        if task:
+            return getattr(task, 'verbose_name', None)
+
+    @property
+    def description(self):
+        task = app.tasks.get(self.name, None)
+        if task:
+            return getattr(task, 'description', None)
+
+
+class CeleryTaskExecution(models.Model):
     LOG_DIR = os.path.join(settings.PROJECT_DIR, 'data', 'celery')
     id = models.UUIDField(primary_key=True, default=uuid.uuid4)
     name = models.CharField(max_length=1024)

apps/ops/serializers/celery.py

@@ -5,10 +5,12 @@ from rest_framework import serializers
 from django_celery_beat.models import PeriodicTask
 
 __all__ = [
-    'CeleryResultSerializer', 'CeleryTaskSerializer',
+    'CeleryResultSerializer', 'CeleryTaskExecutionSerializer',
-    'CeleryPeriodTaskSerializer'
+    'CeleryPeriodTaskSerializer', 'CeleryTaskSerializer'
 ]
 
+from ops.models import CeleryTask, CeleryTaskExecution
+
 
 class CeleryResultSerializer(serializers.Serializer):
     id = serializers.UUIDField()
@@ -16,10 +18,6 @@ class CeleryResultSerializer(serializers.Serializer):
     state = serializers.CharField(max_length=16)
 
 
-class CeleryTaskSerializer(serializers.Serializer):
-    pass
-
-
 class CeleryPeriodTaskSerializer(serializers.ModelSerializer):
     class Meta:
         model = PeriodicTask
@@ -27,3 +25,19 @@ class CeleryPeriodTaskSerializer(serializers.ModelSerializer):
             'name', 'task', 'enabled', 'description',
             'last_run_at', 'total_run_count'
         ]
+
+
+class CeleryTaskSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = CeleryTask
+        fields = [
+            'id', 'name', 'verbose_name', 'description',
+        ]
+
+
+class CeleryTaskExecutionSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = CeleryTaskExecution
+        fields = [
+            "id", "name", "args", "kwargs", "state", "is_finished", "date_published", "date_start", "date_finished"
+        ]

apps/ops/signal_handlers.py

@@ -1,12 +1,15 @@
 import ast
 
+from django.db import transaction
+from django.dispatch import receiver
 from django.utils import translation, timezone
 from django.core.cache import cache
-from celery import signals
+from celery import signals, current_app
 
 from common.db.utils import close_old_connections, get_logger
-from .models import CeleryTask
+from common.signals import django_ready
-
+from .celery import app
+from .models import CeleryTaskExecution, CeleryTask
 
 logger = get_logger(__name__)
 
@@ -14,6 +17,22 @@ TASK_LANG_CACHE_KEY = 'TASK_LANG_{}'
 TASK_LANG_CACHE_TTL = 1800
 
 
+@receiver(django_ready)
+def sync_registered_tasks(*args, **kwargs):
+    with transaction.atomic():
+        try:
+            db_tasks = CeleryTask.objects.all()
+        except Exception as e:
+            return
+        celery_task_names = [key for key in app.tasks]
+        db_task_names = db_tasks.values_list('name', flat=True)
+
+        db_tasks.exclude(name__in=celery_task_names).delete()
+        not_in_db_tasks = set(celery_task_names) - set(db_task_names)
+        tasks_to_create = [CeleryTask(name=name) for name in not_in_db_tasks]
+        CeleryTask.objects.bulk_create(tasks_to_create)
+
+
 @signals.before_task_publish.connect
 def before_task_publish(headers=None, **kwargs):
     task_id = headers.get('id')
@@ -25,7 +44,7 @@ def before_task_publish(headers=None, **kwargs):
 @signals.task_prerun.connect
 def on_celery_task_pre_run(task_id='', **kwargs):
     # 更新状态
-    CeleryTask.objects.filter(id=task_id).update(state='RUNNING', date_start=timezone.now())
+    CeleryTaskExecution.objects.filter(id=task_id).update(state='RUNNING', date_start=timezone.now())
     # 关闭之前的数据库连接
     close_old_connections()
 
@@ -41,7 +60,7 @@ def on_celery_task_post_run(task_id='', state='', **kwargs):
     close_old_connections()
     print("Task post run: ", task_id, state)
 
-    CeleryTask.objects.filter(id=task_id).update(
+    CeleryTaskExecution.objects.filter(id=task_id).update(
         state=state, date_finished=timezone.now(), is_finished=True
     )
 
@@ -72,4 +91,4 @@ def task_sent_handler(headers=None, body=None, **kwargs):
         'args': args,
         'kwargs': kwargs
     }
-    CeleryTask.objects.create(**data)
+    CeleryTaskExecution.objects.create(**data)

apps/ops/tasks.py

@@ -20,7 +20,7 @@ from .celery.utils import (
     create_or_update_celery_periodic_tasks, get_celery_periodic_task,
     disable_celery_periodic_task, delete_celery_periodic_task
 )
-from .models import CeleryTask, AdHoc, Playbook
+from .models import CeleryTaskExecution, AdHoc, Playbook
 from .notifications import ServerPerformanceCheckUtil
 
 logger = get_logger(__file__)
@@ -94,9 +94,9 @@ def clean_celery_tasks_period():
     logger.debug("Start clean celery task history")
     expire_days = get_log_keep_day('TASK_LOG_KEEP_DAYS')
     days_ago = timezone.now() - timezone.timedelta(days=expire_days)
-    tasks = CeleryTask.objects.filter(date_start__lt=days_ago)
+    tasks = CeleryTaskExecution.objects.filter(date_start__lt=days_ago)
     tasks.delete()
-    tasks = CeleryTask.objects.filter(date_start__isnull=True)
+    tasks = CeleryTaskExecution.objects.filter(date_start__isnull=True)
     tasks.delete()
     command = "find %s -mtime +%s -name '*.log' -type f -exec rm -f {} \\;" % (
         settings.CELERY_LOG_DIR, expire_days

apps/ops/urls/api_urls.py

@@ -4,8 +4,9 @@ from __future__ import unicode_literals
 from django.urls import path
 from rest_framework.routers import DefaultRouter
 from rest_framework_bulk.routes import BulkRouter
-from .. import api
+from rest_framework_nested import routers
 
+from .. import api
 
 app_name = "ops"
 
@@ -16,12 +17,19 @@ router.register(r'adhoc', api.AdHocViewSet, 'adhoc')
 router.register(r'adhoc-executions', api.AdHocExecutionViewSet, 'execution')
 router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task')
 
-urlpatterns = [
+router.register(r'tasks', api.CeleryTaskViewSet, 'task')
-    path('celery/task/<uuid:pk>/log/', api.CeleryTaskLogApi.as_view(), name='celery-task-log'),
-    path('celery/task/<uuid:pk>/result/', api.CeleryResultApi.as_view(), name='celery-result'),
 
-    path('ansible/task/<uuid:pk>/log/', api.AnsibleTaskLogApi.as_view(), name='ansible-task-log'),
+task_router = routers.NestedDefaultRouter(router, r'tasks', lookup='task')
+task_router.register(r'executions', api.CeleryTaskExecutionViewSet, 'task-execution')
+
+urlpatterns = [
+
+    path('celery/task/<uuid:name>/task-execution/<uuid:pk>/log/', api.CeleryTaskExecutionLogApi.as_view(),
+         name='celery-task-execution-log'),
+    path('celery/task/<uuid:name>/task-execution/<uuid:pk>/result/', api.CeleryResultApi.as_view(),
+         name='celery-task-execution-result'),
+
+    path('ansible/task-execution/<uuid:pk>/log/', api.AnsibleTaskLogApi.as_view(), name='ansible-task-log'),
 ]
 
-urlpatterns += router.urls
+urlpatterns += (router.urls + bulk_router.urls + task_router.urls)
-urlpatterns += bulk_router.urls

apps/orgs/migrations/0014_organization_builtin.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.2.14 on 2022-10-26 09:07
+
+from django.db import migrations, models
+
+
+def update_builtin_org(apps, schema_editor):
+    org_model = apps.get_model('orgs', 'Organization')
+    org_model.objects.create(
+        id='00000000-0000-0000-0000-000000000004',
+        name='SYSTEM', builtin=True
+    )
+
+    # 更新 Default
+    org_model.objects.filter(name='DEFAULT').update(builtin=True)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orgs', '0013_alter_organization_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organization',
+            name='builtin',
+            field=models.BooleanField(default=False, verbose_name='Builtin'),
+        ),
+        migrations.RunPython(update_builtin_org),
+    ]

apps/orgs/models.py

@@ -69,6 +69,7 @@ class Organization(OrgRoleMixin, models.Model):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     name = models.CharField(max_length=128, unique=True, verbose_name=_("Name"))
     created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by'))
+    builtin = models.BooleanField(default=False, verbose_name=_('Builtin'))
     date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created'))
     comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
     members = models.ManyToManyField(
@@ -139,13 +140,13 @@ class Organization(OrgRoleMixin, models.Model):
     @classmethod
     def default(cls):
         defaults = dict(id=cls.DEFAULT_ID, name=cls.DEFAULT_NAME)
-        obj, created = cls.objects.get_or_create(defaults=defaults, id=cls.DEFAULT_ID)
+        obj, created = cls.objects.get_or_create(defaults=defaults, id=cls.DEFAULT_ID, builtin=True)
         return obj
 
     @classmethod
     def root(cls):
         name = settings.GLOBAL_ORG_DISPLAY_NAME or cls.ROOT_NAME
-        return cls(id=cls.ROOT_ID, name=name)
+        return cls(id=cls.ROOT_ID, name=name, builtin=True)
 
     def is_root(self):
         return self.id == self.ROOT_ID

apps/orgs/signal_handlers/common.py

@@ -4,6 +4,7 @@ import threading
 from collections import defaultdict
 from functools import partial
 
+import django.db.utils
 from django.dispatch import receiver
 from django.conf import settings
 from django.utils.functional import LazyObject
@@ -45,7 +46,10 @@ def expire_orgs_mapping_for_memory(org_id):
 def subscribe_orgs_mapping_expire(sender, **kwargs):
     logger.debug("Start subscribe for expire orgs mapping from memory")
     if settings.DEBUG:
-        set_to_default_org()
+        try:
+            set_to_default_org()
+        except django.db.utils.OperationalError:
+            pass
 
     def keep_subscribe_org_mapping():
         orgs_mapping_for_memory_pub_sub.subscribe(

apps/terminal/automations/deploy_applet_host/playbook.yml

@@ -0,0 +1,56 @@
+---
+- hosts: windows
+  vars:
+    - DownloadHost: https://demo.jumpserver.org/download
+    - RDS_Licensing: enabled
+    - RDS_LicenseServer: 127.0.0.1
+    - RDS_LicensingMode: 4
+    - RDS_fSingleSessionPerUser: 0
+    - RDS_MaxDisconnectionTime: 60000
+    - RDS_RemoteAppLogoffTimeLimit: 0
+  tasks:
+    - name: Install RDS-Licensing (RDS)
+      ansible.windows.win_feature:
+        name: RDS-Licensing
+        state: present
+        include_management_tools: yes
+      when: RDS_Licensing == "enabled"
+    - name: Install RDS-RD-Server (RDS)
+      ansible.windows.win_feature:
+        name: RDS-RD-Server
+        state: present
+        include_management_tools: yes
+      register: win_feature
+    - name: Reboot if installing RDS feature requires it
+      ansible.windows.win_reboot:
+      when: win_feature.reboot_required
+    - name: Set RDS LicenseServer (regedit)
+      ansible.windows.win_regedit:
+        path: HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services
+        name: LicenseServers
+        data: "{{ RDS_LicenseServer }}"
+        type: string
+    - name: Set RDS LicensingMode (regedit)
+      ansible.windows.win_regedit:
+        path: HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services
+        name: LicensingMode
+        data: "{{ RDS_LicensingMode }}"
+        type: dword
+    - name: Set RDS fSingleSessionPerUser (regedit)
+      ansible.windows.win_regedit:
+        path: HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services
+        name: fSingleSessionPerUser
+        data: "{{ RDS_fSingleSessionPerUser }}"
+        type: dword
+    - name: Set RDS MaxDisconnectionTime (regedit)
+      ansible.windows.win_regedit:
+        path: HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services
+        name: MaxDisconnectionTime
+        data: "{{ RDS_MaxDisconnectionTime }}"
+        type: dword
+      when: RDS_MaxDisconnectionTime >= 60000
+    - name: Set RDS RemoteAppLogoffTimeLimit (regedit)
+      ansible.windows.win_regedit:
+        path: HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\Terminal Services
+        name: RemoteAppLogoffTimeLimit
+        data: "{{ RDS_RemoteAppLogoffTime }}"

apps/terminal/migrations/0055_auto_20221026_1631.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.2.14 on 2022-10-26 08:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('terminal', '0054_auto_20221024_1452'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='applet',
+            name='vcs_type',
+        ),
+        migrations.RemoveField(
+            model_name='applet',
+            name='vcs_url',
+        ),
+        migrations.AddField(
+            model_name='applet',
+            name='is_active',
+            field=models.BooleanField(default=True, verbose_name='Is active'),
+        ),
+    ]

apps/terminal/models/applet/applet.py

@@ -27,8 +27,7 @@ class Applet(JMSBaseModel):
     version = models.CharField(max_length=16, verbose_name=_('Version'))
     author = models.CharField(max_length=128, verbose_name=_('Author'))
     type = models.CharField(max_length=16, verbose_name=_('Type'), default='general', choices=Type.choices)
-    vcs_type = models.CharField(max_length=16, verbose_name=_('VCS type'), null=True)
+    is_active = models.BooleanField(default=True, verbose_name=_('Is active'))
-    vcs_url = models.CharField(max_length=256, verbose_name=_('URL'), null=True)
     protocols = models.JSONField(default=list, verbose_name=_('Protocol'))
     tags = models.JSONField(default=list, verbose_name=_('Tags'))
     comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))