pref: 修改 applet

apps/assets/serializers/asset/common.py

@@ -107,6 +107,8 @@ class AssetSerializer(OrgResourceSerializerMixin, WritableNestedModelSerializer)
             return
         nodes_to_set = []
         for full_value in nodes_display:
+            if not full_value.startswith('/'):
+                full_value = '/' + instance.org.name + '/' + full_value
             node = Node.objects.filter(full_value=full_value).first()
             if node:
                 nodes_to_set.append(node)

apps/common/utils/lock.py

@@ -76,7 +76,6 @@ class DistributedLock(RedisLock):
             # 要创建一个新的锁对象
             with self.__class__(**self.kwargs_copy):
                 return func(*args, **kwds)
-
         return inner
 
     @classmethod
@@ -105,22 +104,21 @@ class DistributedLock(RedisLock):
         if self._reentrant:
             if self.locked_by_current_thread():
                 self._acquired_reentrant_lock = True
-                logger.debug(f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
                 return True
 
-            logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name}')
             acquired = super().acquire(blocking=blocking, timeout=timeout)
             if acquired:
-                logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name}')
                 setattr(thread_local, self.name, self.id)
             else:
-                logger.debug(
+                logger.debug(f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name}')
-                    f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id}')
             return acquired
         else:
-            logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name}')
             acquired = super().acquire(blocking=blocking, timeout=timeout)
-            logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name}')
             return acquired
 
     @property
@@ -139,17 +137,17 @@ class DistributedLock(RedisLock):
     def _release_on_reentrant_locked_by_brother(self):
         if self._acquired_reentrant_lock:
             self._acquired_reentrant_lock = False
-            logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
             return
         else:
-            self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
 
     def _release_on_reentrant_locked_by_me(self):
-        logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+        logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name}')
 
         id = getattr(thread_local, self.name, None)
         if id != self.id:
-            raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
         try:
             # 这里要保证先删除 thread_local 的标记，
             delattr(thread_local, self.name)
@@ -171,9 +169,9 @@ class DistributedLock(RedisLock):
     def _release(self):
         try:
             self._release_redis_lock()
-            logger.debug(f'Released lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Released lock: lock_id={self.id} lock={self.name}')
         except NotAcquired as e:
-            logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id} error: {e}')
+            logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} error: {e}')
             self._raise_exc(e)
 
     def release(self):
@@ -188,12 +186,12 @@ class DistributedLock(RedisLock):
                     _release = self._release_on_reentrant_locked_by_brother
             else:
                 self._raise_exc_with_log(
-                    f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                    f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name}')
 
         # 处理是否在事务提交时才释放锁
         if self._release_on_transaction_commit:
             logger.debug(
-                f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name}')
             transaction.on_commit(_release)
         else:
             _release()

apps/orgs/utils.py

@@ -88,6 +88,21 @@ def tmp_to_org(org):
         set_current_org(ori_org)
 
 
+@contextmanager
+def tmp_to_builtin_org(system=0, default=0):
+    if system:
+        org_id = Organization.SYSTEM_ID
+    elif default:
+        org_id = Organization.DEFAULT_ID
+    else:
+        raise ValueError("Must set system or default")
+    ori_org = get_current_org()
+    set_current_org(org_id)
+    yield
+    if ori_org is not None:
+        set_current_org(ori_org)
+
+
 def get_org_filters():
     kwargs = {}
 

apps/terminal/api/applet/applet.py

@@ -1,12 +1,13 @@
 import os.path
 import shutil
 import zipfile
-
 import yaml
+
 from django.core.files.storage import default_storage
 from rest_framework import viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
+from rest_framework.serializers import ValidationError
 
 from terminal import serializers, models
 from terminal.serializers import AppletUploadSerializer
@@ -19,9 +20,16 @@ class AppletViewSet(viewsets.ModelViewSet):
         'upload': 'terminal.add_applet',
     }
 
-    @action(detail=False, methods=['post'], serializer_class=AppletUploadSerializer)
+    def perform_destroy(self, instance):
-    def upload(self, request, *args, **kwargs):
+        if not instance.name:
-        serializer = self.get_serializer(data=request.data)
+            raise ValidationError('Applet is not null')
+        path = default_storage.path('applets/{}'.format(instance.name))
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        instance.delete()
+
+    def extract_and_check_file(self, request):
+        serializer = self.get_serializer(data=self.request.data)
         serializer.is_valid(raise_exception=True)
 
         file = serializer.validated_data['file']
@@ -29,13 +37,11 @@ class AppletViewSet(viewsets.ModelViewSet):
         if default_storage.exists(save_to):
             default_storage.delete(save_to)
         rel_path = default_storage.save(save_to, file)
-
         path = default_storage.path(rel_path)
         extract_to = default_storage.path('applets/{}.tmp'.format(file.name))
         if os.path.exists(extract_to):
             shutil.rmtree(extract_to)
 
-        update = request.query_params.get('update')
         with zipfile.ZipFile(path) as zp:
             if zp.testzip() is not None:
                 return Response({'msg': 'Invalid Zip file'}, status=400)
@@ -46,12 +52,21 @@ class AppletViewSet(viewsets.ModelViewSet):
         for name in files:
             path = os.path.join(tmp_dir, name)
             if not os.path.exists(path):
-                return Response({'error': 'Missing file: {}'.format(path)}, status=400)
+                raise ValidationError({'error': 'Missing file {}'.format(name)})
 
         with open(os.path.join(tmp_dir, 'manifest.yml')) as f:
             manifest = yaml.safe_load(f)
 
-        name = manifest.get('name', '')
+        if not manifest.get('name', ''):
+            raise ValidationError({'error': 'Missing name in manifest.yml'})
+        return manifest, tmp_dir
+
+    @action(detail=False, methods=['post'], serializer_class=AppletUploadSerializer)
+    def upload(self, request, *args, **kwargs):
+        manifest, tmp_dir = self.extract_and_check_file(request)
+        name = manifest['name']
+        update = request.query_params.get('update')
+
         instance = models.Applet.objects.filter(name=name).first()
         if instance and not update:
             return Response({'error': 'Applet already exists: {}'.format(name)}, status=400)

apps/terminal/api/applet/host.py

@@ -1,5 +1,9 @@
 from rest_framework import viewsets
+from rest_framework.decorators import action
 
+from orgs.utils import tmp_to_root_org
+from orgs.models import Organization
+from assets.models import Host
 from terminal import serializers, models
 
 __all__ = ['AppletHostViewSet', 'AppletHostDeploymentViewSet']
@@ -9,6 +13,15 @@ class AppletHostViewSet(viewsets.ModelViewSet):
     queryset = models.AppletHost.objects.all()
     serializer_class = serializers.AppletHostSerializer
 
+    @action(methods=['get'], detail=False)
+    def hosts(self, request):
+        with tmp_to_root_org():
+            kwargs = {
+                'platform__name': 'RemoteAppHost',
+                'org_id': Organization.SYSTEM_ID
+            }
+            return Host.objects.filter(**kwargs)
+
 
 class AppletHostDeploymentViewSet(viewsets.ModelViewSet):
     queryset = models.AppletHostDeployment.objects.all()

apps/terminal/migrations/0054_auto_20221024_1452.py

@@ -22,10 +22,10 @@ class Migration(migrations.Migration):
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
                 ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
                 ('name', models.CharField(max_length=128, unique=True, verbose_name='Name')),
+                ('display_name', models.CharField(max_length=128, unique=True, verbose_name='Display name')),
                 ('version', models.CharField(max_length=16, verbose_name='Version')),
                 ('author', models.CharField(max_length=128, verbose_name='Author')),
                 ('type', models.CharField(choices=[('general', 'General'), ('web', 'Web')], default='general', max_length=16, verbose_name='Type')),
-                ('path', models.FilePathField(verbose_name='Path')),
                 ('vcs_type', models.CharField(max_length=16, null=True, verbose_name='VCS type')),
                 ('vcs_url', models.CharField(max_length=256, null=True, verbose_name='URL')),
                 ('protocols', models.JSONField(default=list, verbose_name='Protocol')),

apps/terminal/models/applet/applet.py

@@ -1,7 +1,8 @@
 import yaml
 import os.path
-from rest_framework.exceptions import ValidationError
 
+from django.conf import settings
+from django.core.files.storage import default_storage
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
@@ -22,9 +23,9 @@ class Applet(JMSBaseModel):
         archive = 'archive', _('Remote gzip')
 
     name = models.CharField(max_length=128, verbose_name=_('Name'), unique=True)
+    display_name = models.CharField(max_length=128, verbose_name=_('Display name'))
     version = models.CharField(max_length=16, verbose_name=_('Version'))
     author = models.CharField(max_length=128, verbose_name=_('Author'))
-    path = models.FilePathField(verbose_name=_('Path'))
     type = models.CharField(max_length=16, verbose_name=_('Type'), default='general', choices=Type.choices)
     vcs_type = models.CharField(max_length=16, verbose_name=_('VCS type'), null=True)
     vcs_url = models.CharField(max_length=256, verbose_name=_('URL'), null=True)
@@ -35,6 +36,10 @@ class Applet(JMSBaseModel):
     def __str__(self):
         return self.name
 
+    @property
+    def path(self):
+        return default_storage.path('applets/{}'.format(self.name))
+
     @property
     def manifest(self):
         path = os.path.join(self.path, 'manifest.yml')
@@ -48,27 +53,7 @@ class Applet(JMSBaseModel):
         path = os.path.join(self.path, 'icon.png')
         if not os.path.exists(path):
             return None
-        with open(path, 'rb') as f:
+        return os.path.join(settings.MEDIA_URL, 'applets', self.name, 'icon.png')
-            return f.read()
-
-    @classmethod
-    def validate_manifest(cls, manifest):
-        fields = ['name', 'display_name', 'version', 'author', 'type', 'tags', 'protocols']
-        for field in fields:
-            if field not in manifest:
-                raise ValidationError(f'Missing field {field}')
-        if manifest['type'] not in [i[0] for i in cls.Type.choices]:
-            raise ValidationError('Invalid type')
-        if not isinstance(manifest['protocols'], list):
-            raise ValidationError('Invalid protocols')
-
-    @classmethod
-    def create_by_manifest(cls, manifest):
-        obj = cls()
-        for k, v in manifest.items():
-            setattr(obj, k, v)
-        obj.save()
-        return obj
 
 
 class AppletPublication(JMSBaseModel):

apps/terminal/serializers/applet.py

@@ -1,7 +1,10 @@
 from rest_framework import serializers
+from django.utils.translation import gettext_lazy as _
 
-from common.drf.fields import ObjectRelatedField
+from common.drf.fields import ObjectRelatedField, LabeledChoiceField
-from assets.models import Host
+from assets.models import Host, Platform
+from assets.serializers import HostSerializer
+from orgs.utils import tmp_to_builtin_org
 from ..models import Applet, AppletPublication, AppletHost, AppletHostDeployment
 
 
@@ -13,14 +16,18 @@ __all__ = [
 
 
 class AppletSerializer(serializers.ModelSerializer):
+    icon = serializers.ReadOnlyField(label=_("Icon"))
+    type = LabeledChoiceField(choices=Applet.Type.choices, label=_("Type"))
+
     class Meta:
         model = Applet
-        fields_mini = ['id', 'name']
+        fields_mini = ['id', 'name', 'display_name']
         read_only_fields = [
-            'date_created', 'date_updated'
+            'icon', 'date_created', 'date_updated'
         ]
         fields = fields_mini + [
-            'version', 'author', 'type', 'protocols', 'comment'
+            'version', 'author', 'type', 'protocols',
+            'tags', 'comment'
         ] + read_only_fields
 
 
@@ -42,15 +49,40 @@ class AppletPublicationSerializer(serializers.ModelSerializer):
 
 
 class AppletHostSerializer(serializers.ModelSerializer):
-    host = ObjectRelatedField(queryset=Host.objects.all())
+    host = HostSerializer(allow_null=True, required=False)
 
     class Meta:
         model = AppletHost
         fields_mini = ['id', 'host']
-        read_only_fields = ['date_created', 'date_updated']
+        read_only_fields = ['date_synced', 'status', 'date_created', 'date_updated']
-        fields = fields_mini + [
+        fields = fields_mini + ['comment', 'account_automation'] + read_only_fields
-            'comment', 'account_automation', 'date_synced', 'status',
+
-        ] + read_only_fields
+    def __init__(self, *args, **kwargs):
+        self.host_data = kwargs.get('data', {}).pop('host', {})
+        super().__init__(*args, **kwargs)
+
+    def _create_host(self):
+        platform = Platform.objects.get(name='RemoteAppHost')
+        data = {
+            **self.host_data,
+            'platform': platform.id,
+            'nodes_display': [
+                'RemoteAppHosts'
+            ]
+        }
+        serializer = HostSerializer(data=data)
+        try:
+            serializer.is_valid(raise_exception=True)
+        except serializers.ValidationError:
+            raise serializers.ValidationError({'host': serializer.errors})
+        host = serializer.save()
+        return host
+
+    def create(self, validated_data):
+        with tmp_to_builtin_org(system=1):
+            host = self._create_host()
+        instance = super().create({**validated_data, 'host': host})
+        return instance
 
 
 class AppletHostDeploymentSerializer(serializers.ModelSerializer):