Merge remote-tracking branch 'origin/v3' into v3

apps/assets/automations/gather_facts/database/mysql/main.yml

@@ -2,27 +2,21 @@
   gather_facts: no
   vars:
     ansible_python_interpreter: /usr/local/bin/python
-    jms_account:
-      username: root
-      secret: redhat
-    jms_asset:
-      address: 127.0.0.1
-      port: 3306
 
   tasks:
-    - name: Gather facts info
+    - name: Get info
       community.mysql.mysql_info:
         login_user: "{{ jms_account.username }}"
         login_password: "{{ jms_account.secret }}"
         login_host: "{{ jms_asset.address }}"
         login_port: "{{ jms_asset.port }}"
+        filter: version
       register: db_info
 
-    - name: Get info
+    - name: Define Mysql info by set_fact
       set_fact:
         info:
           version: "{{ db_info.version.full }}"
 
     - debug:
-        var: db_info
+        var: info
-

apps/assets/automations/gather_facts/database/postgresql/main.yml

@@ -2,19 +2,9 @@
   gather_facts: no
   vars:
     ansible_python_interpreter: /usr/local/bin/python
-    jms_account:
-      username: postgre
-      secret: postgre
-    jms_asset:
-      address: 127.0.0.1
-      port: 5432
-      database: testdb
-    account:
-      username: test
-      secret: jumpserver
 
   tasks:
-    - name: Test PostgreSQL connection
+    - name: Get info
       community.postgresql.postgresql_info:
         login_user: "{{ jms_account.username }}"
         login_password: "{{ jms_account.secret }}"
@@ -23,6 +13,10 @@
         login_db: "{{ jms_asset.database }}"
       register: db_info
 
-    - name: Debug it
+    - name: Define Postgresql info by set_fact
-      debug:
+      set_fact:
-        var: db_info
+        info:
+          version: "{{ db_info.server_version.raw }}"
+
+    - debug:
+        var: info

apps/assets/automations/gather_facts/database/sqlserver/main.yml

@@ -1,10 +0,0 @@
-{% for account in accounts %}
-- hosts: {{ account.asset.name }}
-  vars:
-    account:
-      username: {{ account.username }}
-      password: {{ account.password }}
-      public_key: {{ account.public_key }}
-  roles:
-    - change_secret
-{% endfor %}

apps/assets/automations/gather_facts/database/sqlserver/manifest.yml

@@ -1,8 +0,0 @@
-id: gather_facts_sqlserver
-name: Change password for SQLServer
-version: 1
-category: database
-type:
-  - sqlserver
-method: gather_facts
-

apps/assets/automations/gather_facts/database/sqlserver/roles/change_password/tasks/main.yml

@@ -1,27 +0,0 @@
-- name: ping
-  ansible.builtin.ping:
-
-#- name: print variables
-#  debug:
-#    msg: "Username: {{ account.username }}, Password: {{ account.password }}"
-
-- name: Change password
-  user:
-    name: "{{ account.username }}"
-    password: "{{ account.password | password_hash('des') }}"
-    update_password: always
-  when: account.password
-
-- name: Change public key
-  authorized_key:
-    user: "{{ account.username }}"
-    key: "{{ account.public_key }}"
-    state: present
-  when: account.public_key
-
-- name: Verify password
-  ansible.builtin.ping:
-  vars:
-    ansible_user: "{{ account.username }}"
-    ansible_pass: "{{ account.password }}"
-    ansible_ssh_connection: paramiko

apps/assets/automations/gather_facts/manager.py

@@ -1,4 +1,5 @@
 from common.utils import get_logger
+from assets.const import AutomationTypes
 from ..base.manager import BasePlaybookManager
 
 logger = get_logger(__name__)
@@ -11,7 +12,7 @@ class GatherFactsManager(BasePlaybookManager):
 
     @classmethod
     def method_type(cls):
-        return 'gather_facts'
+        return AutomationTypes.gather_facts
 
     def host_callback(self, host, asset=None, **kwargs):
         super().host_callback(host, asset=asset, **kwargs)
@@ -19,14 +20,10 @@ class GatherFactsManager(BasePlaybookManager):
         return host
 
     def on_host_success(self, host, result):
-        info = result.get('Get info', {}).get('res', {}).get('ansible_facts', {}).get('info', {})
+        info = result.get('debug', {}).get('res', {}).get('info', {})
         asset = self.host_asset_mapper.get(host)
         if asset and info:
             asset.info = info
             asset.save()
         else:
             logger.error("Not found info, task name must be 'Get info': {}".format(host))
-
-
-
-

apps/assets/const/host.py

@@ -67,6 +67,7 @@ class HostTypes(BaseType):
         return {
             cls.LINUX: [
                 {'name': 'Linux'},
+                {'name': 'Gateway'}
             ],
             cls.UNIX: [
                 {'name': 'Unix'},
@@ -75,16 +76,31 @@ class HostTypes(BaseType):
                 {'name': 'AIX', 'automation': {
                     'push_account_method': 'push_account_aix',
                     'change_secret_method': 'push_secret_aix'
-                }},
+                }}
             ],
             cls.WINDOWS: [
                 {'name': 'Windows'},
-                {'name': 'Windows-TLS', 'protocols_setting': {
+                {
-                    'rdp': {'security': 'tls'},
+                    'name': 'Windows-TLS',
-                }},
+                    'protocols_setting': {
-                {'name': 'Windows-RDP', 'protocols_setting': {
+                        'rdp': {'security': 'tls'},
-                    'rdp': {'security': 'rdp'},
+                    }
-                }}
+                },
+                {
+                    'name': 'Windows-RDP',
+                    'protocols_setting': {
+                        'rdp': {'security': 'rdp'},
+                    }
+                },
+                {
+                    'name': 'RemoteAppHost',
+                    '_protocols': ['rdp', 'ssh'],
+                    'protocols_setting':  {
+                        'ssh': {
+                            'required': True
+                        }
+                    }
+                }
             ],
             cls.OTHER_HOST: []
         }

apps/assets/const/types.py

@@ -224,7 +224,10 @@ class AllTypes(ChoicesMixin):
                     if _protocols:
                         protocols_data = [p for p in protocols_data if p['name'] in _protocols]
                     for p in protocols_data:
-                        p['setting'] = {**_protocols_setting.get(p['name'], {}), **p.get('setting', {})}
+                        setting = _protocols_setting.get(p['name'], {})
+                        p['required'] = setting.pop('required', False)
+                        p['default'] = setting.pop('default', False)
+                        p['setting'] = {**setting, **p.get('setting', {})}
 
                     platform_data = {
                         **default_platform_data, **d,

apps/assets/migrations/0108_auto_20221019_1706.py

@@ -55,7 +55,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='changesecretautomation',
             name='secret_strategy',
-            field=models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='random_one', max_length=16, verbose_name='Secret strategy'),
+            field=models.CharField(choices=[('specific', 'Specific'), ('random_one', 'All assets use the same random password'), ('random_all', 'All assets use different random password')], default='specific', max_length=16, verbose_name='Secret strategy'),
         ),
         migrations.AddField(
             model_name='changesecretautomation',

apps/assets/models/automations/base.py

@@ -9,6 +9,7 @@ from common.db.fields import EncryptJsonDictTextField
 from orgs.mixins.models import OrgModelMixin
 from ops.mixin import PeriodTaskModelMixin
 from assets.models import Node, Asset
+from assets.tasks import execute_automation
 
 
 class BaseAutomation(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin):
@@ -38,7 +39,11 @@ class BaseAutomation(CommonModelMixin, PeriodTaskModelMixin, OrgModelMixin):
         return assets.group_by_platform()
 
     def get_register_task(self):
-        raise NotImplementedError
+        name = f"automation_{self.type}_strategy_period_{str(self.id)[:8]}"
+        task = execute_automation.name
+        args = (str(self.id), Trigger.timing, self._meta.model)
+        kwargs = {}
+        return name, task, args, kwargs
 
     def get_many_to_many_ids(self, field: str):
         return [str(i) for i in getattr(self, field).all().values_list('id', flat=True)]

apps/assets/models/automations/change_secret.py

@@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
 from common.db import fields
 from common.const.choices import Trigger
 from common.db.models import JMSBaseModel
-from assets.tasks import execute_change_secret_automation
 from assets.const import AutomationTypes, SecretType, SecretStrategy, SSHKeyStrategy
 from .base import BaseAutomation
 
@@ -35,13 +34,6 @@ class ChangeSecretAutomation(BaseAutomation):
     class Meta:
         verbose_name = _("Change secret automation")
 
-    def get_register_task(self):
-        name = "automation_change_secret_strategy_period_{}".format(str(self.id)[:8])
-        task = execute_change_secret_automation.name
-        args = (str(self.id), Trigger.timing)
-        kwargs = {}
-        return name, task, args, kwargs
-
     def to_attr_json(self):
         attr_json = super().to_attr_json()
         attr_json.update({

apps/assets/models/automations/gather_facts.py

@@ -1,16 +1,15 @@
 from django.utils.translation import ugettext_lazy as _
 
+from assets.const import AutomationTypes
 from .base import BaseAutomation
 
-
 __all__ = ['GatherFactsAutomation']
 
 
 class GatherFactsAutomation(BaseAutomation):
-    class Meta:
-        verbose_name = _("Gather asset facts")
-
     def save(self, *args, **kwargs):
-        self.type = 'gather_facts'
+        self.type = AutomationTypes.gather_facts
         super().save(*args, **kwargs)
 
+    class Meta:
+        verbose_name = _("Gather asset facts")

apps/assets/serializers/asset/common.py

@@ -107,6 +107,8 @@ class AssetSerializer(OrgResourceSerializerMixin, WritableNestedModelSerializer)
             return
         nodes_to_set = []
         for full_value in nodes_display:
+            if not full_value.startswith('/'):
+                full_value = '/' + instance.org.name + '/' + full_value
             node = Node.objects.filter(full_value=full_value).first()
             if node:
                 nodes_to_set.append(node)

apps/assets/tasks/automation.py

@@ -7,12 +7,11 @@ logger = get_logger(__file__)
 
 
 @shared_task
-def execute_change_secret_automation(pid, trigger):
+def execute_automation(pid, trigger, mode):
-    from assets.models import ChangeSecretAutomation
     with tmp_to_root_org():
-        instance = get_object_or_none(ChangeSecretAutomation, pk=pid)
+        instance = get_object_or_none(mode, pk=pid)
     if not instance:
-        logger.error("No automation plan found: {}".format(pid))
+        logger.error("No automation task found: {}".format(pid))
         return
     with tmp_to_org(instance.org):
         instance.execute(trigger)

apps/authentication/migrations/0013_auto_20221025_1908.py

@@ -0,0 +1,29 @@
+# Generated by Django 3.2.14 on 2022-10-25 11:08
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('assets', '0111_alter_changesecretautomation_secret_strategy'),
+        ('authentication', '0012_auto_20220816_1629'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='connectiontoken',
+            name='type',
+        ),
+        migrations.AddField(
+            model_name='connectiontoken',
+            name='account_display',
+            field=models.CharField(default='', max_length=128, verbose_name='Account display'),
+        ),
+        migrations.AlterField(
+            model_name='connectiontoken',
+            name='account',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='connection_tokens', to='assets.account', verbose_name='Account'),
+        ),
+    ]

apps/authentication/models.py

@@ -63,13 +63,6 @@ def date_expired_default():
 
 
 class ConnectionToken(OrgModelMixin, JMSBaseModel):
-    class Type(models.TextChoices):
-        asset = 'asset', _('Asset')
-        application = 'application', _('Application')
-
-    type = models.CharField(
-        max_length=16, default=Type.asset, choices=Type.choices, verbose_name=_("Type")
-    )
     secret = models.CharField(max_length=64, default='', verbose_name=_("Secret"))
     date_expired = models.DateTimeField(
         default=date_expired_default, verbose_name=_("Date expired")
@@ -85,7 +78,11 @@ class ConnectionToken(OrgModelMixin, JMSBaseModel):
         related_name='connection_tokens', null=True, blank=True
     )
     asset_display = models.CharField(max_length=128, default='', verbose_name=_("Asset display"))
-    account = models.CharField(max_length=128, default='', verbose_name=_("Account"))
+    account = models.ForeignKey(
+        'assets.Account', on_delete=models.SET_NULL, verbose_name=_('Account'),
+        related_name='connection_tokens', null=True, blank=True
+    )
+    account_display = models.CharField(max_length=128, default='', verbose_name=_("Account display"))
 
     class Meta:
         ordering = ('-date_expired',)

apps/common/utils/lock.py

@@ -76,7 +76,6 @@ class DistributedLock(RedisLock):
             # 要创建一个新的锁对象
             with self.__class__(**self.kwargs_copy):
                 return func(*args, **kwds)
-
         return inner
 
     @classmethod
@@ -105,22 +104,21 @@ class DistributedLock(RedisLock):
         if self._reentrant:
             if self.locked_by_current_thread():
                 self._acquired_reentrant_lock = True
-                logger.debug(f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
                 return True
 
-            logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name}')
             acquired = super().acquire(blocking=blocking, timeout=timeout)
             if acquired:
-                logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name}')
                 setattr(thread_local, self.name, self.id)
             else:
-                logger.debug(
+                logger.debug(f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name}')
-                    f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id}')
             return acquired
         else:
-            logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name}')
             acquired = super().acquire(blocking=blocking, timeout=timeout)
-            logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name}')
             return acquired
 
     @property
@@ -139,17 +137,17 @@ class DistributedLock(RedisLock):
     def _release_on_reentrant_locked_by_brother(self):
         if self._acquired_reentrant_lock:
             self._acquired_reentrant_lock = False
-            logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
             return
         else:
-            self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
 
     def _release_on_reentrant_locked_by_me(self):
-        logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+        logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name}')
 
         id = getattr(thread_local, self.name, None)
         if id != self.id:
-            raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}')
+            raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name}')
         try:
             # 这里要保证先删除 thread_local 的标记，
             delattr(thread_local, self.name)
@@ -171,9 +169,9 @@ class DistributedLock(RedisLock):
     def _release(self):
         try:
             self._release_redis_lock()
-            logger.debug(f'Released lock: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+            logger.debug(f'Released lock: lock_id={self.id} lock={self.name}')
         except NotAcquired as e:
-            logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id} error: {e}')
+            logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} error: {e}')
             self._raise_exc(e)
 
     def release(self):
@@ -188,12 +186,12 @@ class DistributedLock(RedisLock):
                     _release = self._release_on_reentrant_locked_by_brother
             else:
                 self._raise_exc_with_log(
-                    f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                    f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name}')
 
         # 处理是否在事务提交时才释放锁
         if self._release_on_transaction_commit:
             logger.debug(
-                f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name} thread={self._thread_id}')
+                f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name}')
             transaction.on_commit(_release)
         else:
             _release()

apps/ops/mixin.py

@@ -1,10 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 import abc
-import uuid
 from django.utils.translation import ugettext_lazy as _
 from django.db import models
-from django import forms
 from rest_framework import serializers
 
 from .celery.utils import (

apps/orgs/utils.py

@@ -88,6 +88,21 @@ def tmp_to_org(org):
         set_current_org(ori_org)
 
 
+@contextmanager
+def tmp_to_builtin_org(system=0, default=0):
+    if system:
+        org_id = Organization.SYSTEM_ID
+    elif default:
+        org_id = Organization.DEFAULT_ID
+    else:
+        raise ValueError("Must set system or default")
+    ori_org = get_current_org()
+    set_current_org(org_id)
+    yield
+    if ori_org is not None:
+        set_current_org(ori_org)
+
+
 def get_org_filters():
     kwargs = {}
 

apps/terminal/api/__init__.py

@@ -1,10 +1,5 @@
 # -*- coding: utf-8 -*-
 #
-from .terminal import *
 from .session import *
-from .command import *
+from .component import *
-from .task import *
+from .applet import *
-from .storage import *
-from .status import *
-from .sharing import *
-from .endpoint import *

apps/terminal/api/applet/__init__.py

@@ -0,0 +1,2 @@
+from .applet import *
+from .host import *

apps/terminal/api/applet/applet.py

@@ -0,0 +1,86 @@
+import os.path
+import shutil
+import zipfile
+import yaml
+
+from django.core.files.storage import default_storage
+from rest_framework import viewsets
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.serializers import ValidationError
+
+from terminal import serializers, models
+from terminal.serializers import AppletUploadSerializer
+
+
+class AppletViewSet(viewsets.ModelViewSet):
+    queryset = models.Applet.objects.all()
+    serializer_class = serializers.AppletSerializer
+    rbac_perms = {
+        'upload': 'terminal.add_applet',
+    }
+
+    def perform_destroy(self, instance):
+        if not instance.name:
+            raise ValidationError('Applet is not null')
+        path = default_storage.path('applets/{}'.format(instance.name))
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        instance.delete()
+
+    def extract_and_check_file(self, request):
+        serializer = self.get_serializer(data=self.request.data)
+        serializer.is_valid(raise_exception=True)
+
+        file = serializer.validated_data['file']
+        save_to = 'applets/{}'.format(file.name + '.tmp.zip')
+        if default_storage.exists(save_to):
+            default_storage.delete(save_to)
+        rel_path = default_storage.save(save_to, file)
+        path = default_storage.path(rel_path)
+        extract_to = default_storage.path('applets/{}.tmp'.format(file.name))
+        if os.path.exists(extract_to):
+            shutil.rmtree(extract_to)
+
+        with zipfile.ZipFile(path) as zp:
+            if zp.testzip() is not None:
+                return Response({'msg': 'Invalid Zip file'}, status=400)
+            zp.extractall(extract_to)
+
+        tmp_dir = os.path.join(extract_to, file.name.replace('.zip', ''))
+        files = ['manifest.yml', 'icon.png', 'i18n.yml']
+        for name in files:
+            path = os.path.join(tmp_dir, name)
+            if not os.path.exists(path):
+                raise ValidationError({'error': 'Missing file {}'.format(name)})
+
+        with open(os.path.join(tmp_dir, 'manifest.yml')) as f:
+            manifest = yaml.safe_load(f)
+
+        if not manifest.get('name', ''):
+            raise ValidationError({'error': 'Missing name in manifest.yml'})
+        return manifest, tmp_dir
+
+    @action(detail=False, methods=['post'], serializer_class=AppletUploadSerializer)
+    def upload(self, request, *args, **kwargs):
+        manifest, tmp_dir = self.extract_and_check_file(request)
+        name = manifest['name']
+        update = request.query_params.get('update')
+
+        instance = models.Applet.objects.filter(name=name).first()
+        if instance and not update:
+            return Response({'error': 'Applet already exists: {}'.format(name)}, status=400)
+
+        serializer = serializers.AppletSerializer(data=manifest, instance=instance)
+        serializer.is_valid(raise_exception=True)
+        save_to = default_storage.path('applets/{}'.format(name))
+        if os.path.exists(save_to):
+            shutil.rmtree(save_to)
+        shutil.move(tmp_dir, save_to)
+        serializer.save()
+        return Response(serializer.data, status=201)
+
+
+class AppletPublicationViewSet(viewsets.ModelViewSet):
+    queryset = models.AppletPublication.objects.all()
+    serializer_class = serializers.AppletPublicationSerializer

apps/terminal/api/applet/host.py

@@ -0,0 +1,29 @@
+from rest_framework import viewsets
+from rest_framework.decorators import action
+
+from orgs.utils import tmp_to_root_org
+from orgs.models import Organization
+from assets.models import Host
+from terminal import serializers, models
+
+__all__ = ['AppletHostViewSet', 'AppletHostDeploymentViewSet']
+
+
+class AppletHostViewSet(viewsets.ModelViewSet):
+    queryset = models.AppletHost.objects.all()
+    serializer_class = serializers.AppletHostSerializer
+
+    @action(methods=['get'], detail=False)
+    def hosts(self, request):
+        with tmp_to_root_org():
+            kwargs = {
+                'platform__name': 'RemoteAppHost',
+                'org_id': Organization.SYSTEM_ID
+            }
+            return Host.objects.filter(**kwargs)
+
+
+class AppletHostDeploymentViewSet(viewsets.ModelViewSet):
+    queryset = models.AppletHostDeployment.objects.all()
+    serializer_class = serializers.AppletHostDeploymentSerializer
+

apps/terminal/api/component/__init__.py

@@ -0,0 +1,4 @@
+from .terminal import *
+from .storage import *
+from .status import *
+from .endpoint import *

apps/terminal/api/component/endpoint.py

@@ -2,15 +2,15 @@ from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework import status
 from rest_framework.request import Request
+
 from common.drf.api import JMSBulkModelViewSet
+from common.permissions import IsValidUserOrConnectionToken
 from django.utils.translation import ugettext_lazy as _
 from django.shortcuts import get_object_or_404
 from assets.models import Asset
 from orgs.utils import tmp_to_root_org
-from terminal.models import Session
+from terminal.models import Session, Endpoint, EndpointRule
-from ..models import Endpoint, EndpointRule
+from terminal import serializers
-from .. import serializers
-from common.permissions import IsValidUserOrConnectionToken
 
 
 __all__ = ['EndpointViewSet', 'EndpointRuleViewSet']

apps/terminal/api/component/status.py

@@ -9,9 +9,9 @@ from rest_framework import viewsets, generics
 from rest_framework.views import Response
 from rest_framework import status
 
-from ..models import Terminal, Status, Session
+from terminal.models import Terminal, Status, Session
-from .. import serializers
+from terminal import serializers
-from ..utils import TypedComponentsStatusMetricsUtil
+from terminal.utils import TypedComponentsStatusMetricsUtil
 
 logger = logging.getLogger(__file__)
 

apps/terminal/api/component/storage.py

@@ -11,8 +11,8 @@ from django_filters import utils
 from terminal import const
 from common.const.http import GET
 from terminal.filters import CommandStorageFilter, CommandFilter, CommandFilterForStorageTree
-from ..models import CommandStorage, ReplayStorage
+from terminal.models import CommandStorage, ReplayStorage
-from ..serializers import CommandStorageSerializer, ReplayStorageSerializer
+from terminal.serializers import CommandStorageSerializer, ReplayStorageSerializer
 
 __all__ = [
     'CommandStorageViewSet', 'CommandStorageTestConnectiveApi',

apps/terminal/api/component/terminal.py

@@ -14,9 +14,9 @@ from common.exceptions import JMSException
 from common.drf.api import JMSBulkModelViewSet
 from common.utils import get_object_or_none, get_request_ip
 from common.permissions import WithBootstrapToken
-from ..models import Terminal
+from terminal.models import Terminal
-from .. import serializers
+from terminal import serializers
-from .. import exceptions
+from terminal import exceptions
 
 __all__ = [
     'TerminalViewSet',  'TerminalConfig',

apps/terminal/api/session/__init__.py

@@ -0,0 +1,4 @@
+from .session import *
+from .sharing import *
+from .command import *
+from .task import *

apps/terminal/api/session/command.py

@@ -13,11 +13,11 @@ from common.drf.api import JMSBulkModelViewSet
 from common.utils import get_logger
 from terminal.backends.command.serializers import InsecureCommandAlertSerializer
 from terminal.exceptions import StorageInvalid
-from ..backends import (
+from terminal.backends import (
     get_command_storage, get_multi_command_storage,
     SessionCommandSerializer,
 )
-from ..notifications import CommandAlertMessage
+from terminal.notifications import CommandAlertMessage
 
 logger = get_logger(__name__)
 __all__ = ['CommandViewSet', 'InsecureCommandAlertAPI']

apps/terminal/api/session/session.py

@@ -24,15 +24,16 @@ from common.drf.renders import PassthroughRenderer
 from orgs.mixins.api import OrgBulkModelViewSet
 from orgs.utils import tmp_to_root_org, tmp_to_org
 from users.models import User
-from .. import utils
+from terminal.utils import (
-from ..utils import find_session_replay_local, download_session_replay
+    find_session_replay_local, download_session_replay,
-from ..models import Session
+    is_session_approver, get_session_replay_url
-from .. import serializers
+)
-from terminal.utils import is_session_approver
+from terminal.models import Session
+from terminal import serializers
 
 __all__ = [
-    'SessionViewSet', 'SessionReplayViewSet', 'SessionJoinValidateAPI',
+    'SessionViewSet', 'SessionReplayViewSet',
-    'MySessionAPIView',
+    'SessionJoinValidateAPI', 'MySessionAPIView',
 ]
 
 logger = get_logger(__name__)
@@ -93,7 +94,7 @@ class SessionViewSet(OrgBulkModelViewSet):
             url_name='replay-download')
     def download(self, request, *args, **kwargs):
         session = self.get_object()
-        local_path, url = utils.get_session_replay_url(session)
+        local_path, url = get_session_replay_url(session)
         if local_path is None:
             return Response({"error": url}, status=404)
         file = self.prepare_offline_file(session, local_path)

apps/terminal/api/session/sharing.py

@@ -5,9 +5,8 @@ from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
 from common.const.http import PATCH
-from common.permissions import IsValidUser
 from orgs.mixins.api import OrgModelViewSet
-from .. import serializers, models
+from terminal import serializers, models
 
 __all__ = ['SessionSharingViewSet', 'SessionJoinRecordsViewSet']
 

apps/terminal/api/session/task.py

@@ -7,10 +7,10 @@ from rest_framework import status
 from rest_framework.permissions import IsAuthenticated
 
 from common.utils import get_object_or_none
-from ..models import Session, Task
-from .. import serializers
-from terminal.utils import is_session_approver
 from orgs.utils import tmp_to_root_org
+from terminal.models import Session, Task
+from terminal import serializers
+from terminal.utils import is_session_approver
 
 __all__ = ['TaskViewSet', 'KillSessionAPI', 'KillSessionForTicketAPI']
 logger = logging.getLogger(__file__)

apps/terminal/migrations/0054_auto_20221024_1452.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.2.14 on 2022-10-21 06:33
+# Generated by Django 3.2.14 on 2022-10-24 06:52
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -8,6 +8,7 @@ import uuid
 class Migration(migrations.Migration):
 
     dependencies = [
+        ('assets', '0110_auto_20221021_1506'),
         ('terminal', '0053_auto_20220830_1244'),
     ]
 
@@ -21,11 +22,14 @@ class Migration(migrations.Migration):
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
                 ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
                 ('name', models.CharField(max_length=128, unique=True, verbose_name='Name')),
+                ('display_name', models.CharField(max_length=128, unique=True, verbose_name='Display name')),
                 ('version', models.CharField(max_length=16, verbose_name='Version')),
-                ('type', models.CharField(choices=[('app', 'App'), ('web', 'Web')], max_length=16, verbose_name='Type')),
-                ('icon', models.ImageField(upload_to='applet/icon', verbose_name='Icon')),
                 ('author', models.CharField(max_length=128, verbose_name='Author')),
+                ('type', models.CharField(choices=[('general', 'General'), ('web', 'Web')], default='general', max_length=16, verbose_name='Type')),
+                ('vcs_type', models.CharField(max_length=16, null=True, verbose_name='VCS type')),
+                ('vcs_url', models.CharField(max_length=256, null=True, verbose_name='URL')),
                 ('protocols', models.JSONField(default=list, verbose_name='Protocol')),
+                ('tags', models.JSONField(default=list, verbose_name='Tags')),
                 ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
             ],
             options={
@@ -33,14 +37,13 @@ class Migration(migrations.Migration):
             },
         ),
         migrations.CreateModel(
-            name='AppletProvider',
+            name='AppletHost',
             fields=[
                 ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
                 ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')),
                 ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
                 ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
                 ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('name', models.CharField(max_length=128, unique=True, verbose_name='Name')),
                 ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
                 ('account_automation', models.BooleanField(default=False, verbose_name='Account automation')),
                 ('date_synced', models.DateTimeField(blank=True, null=True, verbose_name='Date synced')),
@@ -50,22 +53,6 @@ class Migration(migrations.Migration):
                 'abstract': False,
             },
         ),
-        migrations.CreateModel(
-            name='ProviderDeployment',
-            fields=[
-                ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
-                ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')),
-                ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
-                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
-                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('status', models.CharField(max_length=16, verbose_name='Status')),
-                ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
-                ('provider', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='terminal.appletprovider', verbose_name='Provider')),
-            ],
-            options={
-                'abstract': False,
-            },
-        ),
         migrations.CreateModel(
             name='AppletPublication',
             fields=[
@@ -77,20 +64,36 @@ class Migration(migrations.Migration):
                 ('status', models.CharField(max_length=16, verbose_name='Status')),
                 ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
                 ('applet', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.applet', verbose_name='Applet')),
-                ('provider', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.appletprovider', verbose_name='Provider')),
+                ('host', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='terminal.applethost', verbose_name='Host')),
             ],
             options={
-                'unique_together': {('applet', 'provider')},
+                'unique_together': {('applet', 'host')},
+            },
+        ),
+        migrations.CreateModel(
+            name='AppletHostDeployment',
+            fields=[
+                ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')),
+                ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')),
+                ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')),
+                ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('status', models.CharField(max_length=16, verbose_name='Status')),
+                ('comment', models.TextField(blank=True, default='', verbose_name='Comment')),
+                ('host', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='terminal.applethost', verbose_name='Hosting')),
+            ],
+            options={
+                'abstract': False,
             },
         ),
         migrations.AddField(
-            model_name='appletprovider',
+            model_name='applethost',
             name='applets',
             field=models.ManyToManyField(through='terminal.AppletPublication', to='terminal.Applet', verbose_name='Applet'),
         ),
         migrations.AddField(
-            model_name='appletprovider',
+            model_name='applethost',
-            name='asset',
+            name='host',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='assets.asset', verbose_name='Asset'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='assets.host', verbose_name='Host'),
         ),
     ]

apps/terminal/models/applet/__init__.py

@@ -1,2 +1,2 @@
 from .applet import *
-from .provider import *
+from .host import *

apps/terminal/models/applet/applet.py

@@ -1,3 +1,8 @@
+import yaml
+import os.path
+
+from django.conf import settings
+from django.core.files.storage import default_storage
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
@@ -9,25 +14,53 @@ __all__ = ['Applet', 'AppletPublication']
 
 class Applet(JMSBaseModel):
     class Type(models.TextChoices):
-        app = 'app', _('App')
+        general = 'general', _('General')
         web = 'web', _('Web')
+
+    class VCSType(models.TextChoices):
+        manual = 'manual', _('Manual')
+        git = 'git', _('Git')
+        archive = 'archive', _('Remote gzip')
+
     name = models.CharField(max_length=128, verbose_name=_('Name'), unique=True)
+    display_name = models.CharField(max_length=128, verbose_name=_('Display name'))
     version = models.CharField(max_length=16, verbose_name=_('Version'))
-    type = models.CharField(max_length=16, choices=Type.choices, verbose_name=_('Type'))
-    icon = models.ImageField(upload_to='applet/icon', verbose_name=_('Icon'))
     author = models.CharField(max_length=128, verbose_name=_('Author'))
+    type = models.CharField(max_length=16, verbose_name=_('Type'), default='general', choices=Type.choices)
+    vcs_type = models.CharField(max_length=16, verbose_name=_('VCS type'), null=True)
+    vcs_url = models.CharField(max_length=256, verbose_name=_('URL'), null=True)
     protocols = models.JSONField(default=list, verbose_name=_('Protocol'))
+    tags = models.JSONField(default=list, verbose_name=_('Tags'))
     comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
 
     def __str__(self):
         return self.name
 
+    @property
+    def path(self):
+        return default_storage.path('applets/{}'.format(self.name))
+
+    @property
+    def manifest(self):
+        path = os.path.join(self.path, 'manifest.yml')
+        if not os.path.exists(path):
+            return None
+        with open(path, 'r') as f:
+            return yaml.safe_load(f)
+
+    @property
+    def icon(self):
+        path = os.path.join(self.path, 'icon.png')
+        if not os.path.exists(path):
+            return None
+        return os.path.join(settings.MEDIA_URL, 'applets', self.name, 'icon.png')
+
 
 class AppletPublication(JMSBaseModel):
     applet = models.ForeignKey('Applet', on_delete=models.PROTECT, verbose_name=_('Applet'))
-    provider = models.ForeignKey('AppletProvider', on_delete=models.PROTECT, verbose_name=_('Provider'))
+    host = models.ForeignKey('AppletHost', on_delete=models.PROTECT, verbose_name=_('Host'))
     status = models.CharField(max_length=16, verbose_name=_('Status'))
     comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
 
     class Meta:
-        unique_together = ('applet', 'provider')
+        unique_together = ('applet', 'host')

apps/terminal/models/applet/host.py

@@ -1,31 +1,34 @@
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-from celery import current_app
-
 from common.db.models import JMSBaseModel
 
 
-__all__ = ['AppletProvider', 'ProviderDeployment']
+__all__ = ['AppletHost', 'AppletHostDeployment']
 
 
-class AppletProvider(JMSBaseModel):
+class AppletHost(JMSBaseModel):
-    name = models.CharField(max_length=128, verbose_name=_('Name'), unique=True)
+    host = models.ForeignKey('assets.Host', on_delete=models.PROTECT, verbose_name=_('Host'))
-    asset = models.ForeignKey('assets.Asset', on_delete=models.PROTECT, verbose_name=_('Asset'))
     comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
     account_automation = models.BooleanField(default=False, verbose_name=_('Account automation'))
     date_synced = models.DateTimeField(null=True, blank=True, verbose_name=_('Date synced'))
     status = models.CharField(max_length=16, verbose_name=_('Status'))
     applets = models.ManyToManyField(
         'Applet', verbose_name=_('Applet'),
-        through='AppletPublication',  through_fields=('provider', 'applet'),
+        through='AppletPublication',  through_fields=('host', 'applet'),
     )
 
+    def __str__(self):
+        return self.host.name
 
-class ProviderDeployment(JMSBaseModel):
+
-    provider = models.ForeignKey('AppletProvider', on_delete=models.CASCADE, verbose_name=_('Provider'))
+class AppletHostDeployment(JMSBaseModel):
+    host = models.ForeignKey('AppletHost', on_delete=models.CASCADE, verbose_name=_('Hosting'))
     status = models.CharField(max_length=16, verbose_name=_('Status'))
     comment = models.TextField(default='', blank=True, verbose_name=_('Comment'))
 
-    def install(self):
+    def __str__(self):
+        return self.host
+
+    def start(self):
         pass

apps/terminal/serializers/__init__.py

@@ -5,3 +5,4 @@ from .session import *
 from .storage import *
 from .sharing import *
 from .endpoint import *
+from .applet import *

apps/terminal/serializers/applet.py

@@ -0,0 +1,97 @@
+from rest_framework import serializers
+from django.utils.translation import gettext_lazy as _
+
+from common.drf.fields import ObjectRelatedField, LabeledChoiceField
+from assets.models import Host, Platform
+from assets.serializers import HostSerializer
+from orgs.utils import tmp_to_builtin_org
+from ..models import Applet, AppletPublication, AppletHost, AppletHostDeployment
+
+
+__all__ = [
+    'AppletSerializer', 'AppletPublicationSerializer',
+    'AppletHostSerializer', 'AppletHostDeploymentSerializer',
+    'AppletUploadSerializer'
+]
+
+
+class AppletSerializer(serializers.ModelSerializer):
+    icon = serializers.ReadOnlyField(label=_("Icon"))
+    type = LabeledChoiceField(choices=Applet.Type.choices, label=_("Type"))
+
+    class Meta:
+        model = Applet
+        fields_mini = ['id', 'name', 'display_name']
+        read_only_fields = [
+            'icon', 'date_created', 'date_updated'
+        ]
+        fields = fields_mini + [
+            'version', 'author', 'type', 'protocols',
+            'tags', 'comment'
+        ] + read_only_fields
+
+
+class AppletUploadSerializer(serializers.Serializer):
+    file = serializers.FileField()
+
+
+class AppletPublicationSerializer(serializers.ModelSerializer):
+    applet = ObjectRelatedField(queryset=Applet.objects.all())
+    host = ObjectRelatedField(queryset=AppletHost.objects.all())
+
+    class Meta:
+        model = AppletPublication
+        fields_mini = ['id', 'applet', 'host']
+        read_only_fields = ['date_created', 'date_updated']
+        fields = fields_mini + [
+            'status', 'comment',
+        ] + read_only_fields
+
+
+class AppletHostSerializer(serializers.ModelSerializer):
+    host = HostSerializer(allow_null=True, required=False)
+
+    class Meta:
+        model = AppletHost
+        fields_mini = ['id', 'host']
+        read_only_fields = ['date_synced', 'status', 'date_created', 'date_updated']
+        fields = fields_mini + ['comment', 'account_automation'] + read_only_fields
+
+    def __init__(self, *args, **kwargs):
+        self.host_data = kwargs.get('data', {}).pop('host', {})
+        super().__init__(*args, **kwargs)
+
+    def _create_host(self):
+        platform = Platform.objects.get(name='RemoteAppHost')
+        data = {
+            **self.host_data,
+            'platform': platform.id,
+            'nodes_display': [
+                'RemoteAppHosts'
+            ]
+        }
+        serializer = HostSerializer(data=data)
+        try:
+            serializer.is_valid(raise_exception=True)
+        except serializers.ValidationError:
+            raise serializers.ValidationError({'host': serializer.errors})
+        host = serializer.save()
+        return host
+
+    def create(self, validated_data):
+        with tmp_to_builtin_org(system=1):
+            host = self._create_host()
+        instance = super().create({**validated_data, 'host': host})
+        return instance
+
+
+class AppletHostDeploymentSerializer(serializers.ModelSerializer):
+    host = ObjectRelatedField(queryset=AppletHost.objects.all())
+
+    class Meta:
+        model = AppletHostDeployment
+        fields_mini = ['id', 'host']
+        read_only_fields = ['date_created', 'date_updated']
+        fields = fields_mini + [
+            'status', 'comment',
+        ] + read_only_fields

apps/terminal/urls/api_urls.py

@@ -24,6 +24,11 @@ router.register(r'session-sharings', api.SessionSharingViewSet, 'session-sharing
 router.register(r'session-join-records', api.SessionJoinRecordsViewSet, 'session-sharing-record')
 router.register(r'endpoints', api.EndpointViewSet, 'endpoint')
 router.register(r'endpoint-rules', api.EndpointRuleViewSet, 'endpoint-rule')
+router.register(r'applets', api.AppletViewSet, 'applet')
+router.register(r'applet-hosts', api.AppletHostViewSet, 'applet-host')
+router.register(r'applet-publication', api.AppletPublicationViewSet, 'applet-publication')
+router.register(r'applet-host-deployment', api.AppletHostDeploymentViewSet, 'applet-host-deployment')
+
 
 urlpatterns = [
     path('my-sessions/', api.MySessionAPIView.as_view(), name='my-session'),