merge: with remote

Dockerfile

@@ -57,10 +57,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \
     && apt-get -y install --no-install-recommends ${TOOLS} \
     && mkdir -p /root/.ssh/ \
     && echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \
-    && sed -i "s@# alias l@alias l@g" ~/.bashrc \
     && echo "set mouse-=a" > ~/.vimrc \
     && echo "no" | dpkg-reconfigure dash \
     && echo "zh_CN.UTF-8" | dpkg-reconfigure locales \
+    && sed -i "s@# export @export @g" ~/.bashrc \
+    && sed -i "s@# alias @alias @g" ~/.bashrc \
     && rm -rf /var/lib/apt/lists/*
 
 ARG DOWNLOAD_URL=https://download.jumpserver.org

Dockerfile.loong64

@@ -0,0 +1,96 @@
+FROM python:3.8-slim as stage-build
+ARG TARGETARCH
+
+ARG VERSION
+ENV VERSION=$VERSION
+
+WORKDIR /opt/jumpserver
+ADD . .
+RUN cd utils && bash -ixeu build.sh
+
+FROM python:3.8-slim
+ARG TARGETARCH
+MAINTAINER JumpServer Team <ibuler@qq.com>
+
+ARG BUILD_DEPENDENCIES="              \
+        g++                           \
+        make                          \
+        pkg-config"
+
+ARG DEPENDENCIES="                    \
+        default-libmysqlclient-dev    \
+        freetds-dev                   \
+        libpq-dev                     \
+        libffi-dev                    \
+        libjpeg-dev                   \
+        libldap2-dev                  \
+        libsasl2-dev                  \
+        libxml2-dev                   \
+        libxmlsec1-dev                \
+        libxmlsec1-openssl            \
+        libaio-dev                    \
+        openssh-client                \
+        sshpass"
+
+ARG TOOLS="                           \
+        ca-certificates               \
+        curl                          \
+        default-mysql-client          \
+        iputils-ping                  \
+        locales                       \
+        netcat                        \
+        redis-server                  \
+        telnet                        \
+        vim                           \
+        unzip                         \
+        wget"
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \
+    set -ex \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && apt-get update \
+    && apt-get -y install --no-install-recommends ${BUILD_DEPENDENCIES} \
+    && apt-get -y install --no-install-recommends ${DEPENDENCIES} \
+    && apt-get -y install --no-install-recommends ${TOOLS} \
+    && mkdir -p /root/.ssh/ \
+    && echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \
+    && echo "set mouse-=a" > ~/.vimrc \
+    && echo "no" | dpkg-reconfigure dash \
+    && echo "zh_CN.UTF-8" | dpkg-reconfigure locales \
+    && sed -i "s@# export @export @g" ~/.bashrc \
+    && sed -i "s@# alias @alias @g" ~/.bashrc \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp/build
+COPY ./requirements ./requirements
+
+ARG PIP_MIRROR=https://pypi.douban.com/simple
+ENV PIP_MIRROR=$PIP_MIRROR
+ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple
+ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    set -ex \
+    && pip config set global.index-url ${PIP_MIRROR} \
+    && pip install --upgrade pip \
+    && pip install --upgrade setuptools wheel \
+    && pip install https://download.jumpserver.org/pypi/simple/cryptography/cryptography-36.0.1-cp38-cp38-linux_loongarch64.whl \
+    && pip install https://download.jumpserver.org/pypi/simple/greenlet/greenlet-1.1.2-cp38-cp38-linux_loongarch64.whl \
+    && pip install $(grep 'PyNaCl' requirements/requirements.txt) \
+    && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=true pip install grpcio \
+    && pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \
+    && pip install -r requirements/requirements.txt
+
+COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver
+RUN echo > /opt/jumpserver/config.yml \
+    && rm -rf /tmp/build
+
+WORKDIR /opt/jumpserver
+VOLUME /opt/jumpserver/data
+VOLUME /opt/jumpserver/logs
+
+ENV LANG=zh_CN.UTF-8
+
+EXPOSE 8070
+EXPOSE 8080
+ENTRYPOINT ["./entrypoint.sh"]

apps/assets/api/asset/asset.py

@@ -6,13 +6,11 @@ from rest_framework.decorators import action
 from rest_framework.response import Response
 
 from assets import serializers
+from assets.models import Asset
 from assets.filters import IpInFilterBackend, LabelFilterBackend, NodeFilterBackend
-from assets.models import Asset, Gateway
 from assets.tasks import (
-    push_accounts_to_assets,
+    push_accounts_to_assets, test_assets_connectivity_manual,
-    test_assets_connectivity_manual,
+    update_assets_hardware_info_manual, verify_accounts_connectivity,
-    update_assets_hardware_info_manual,
-    verify_accounts_connectivity,
 )
 from common.drf.filters import BaseFilterSet
 from common.mixins.api import SuggestionMixin
@@ -74,7 +72,7 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet):
     def gateways(self, *args, **kwargs):
         asset = self.get_object()
         if not asset.domain:
-            gateways = Gateway.objects.none()
+            gateways = Asset.objects.none()
         else:
             gateways = asset.domain.gateways.filter(protocol="ssh")
         return self.get_paginated_response_from_queryset(gateways)

apps/assets/api/asset/host.py

@@ -1,4 +1,3 @@
-
 from assets.models import Host
 from assets.serializers import HostSerializer
 from .asset import AssetViewSet

apps/assets/api/domain.py

@@ -7,21 +7,20 @@ from rest_framework.serializers import ValidationError
 
 from common.utils import get_logger
 from orgs.mixins.api import OrgBulkModelViewSet
-from ..models import Domain, Gateway
+from ..models import Domain, Host
 from .. import serializers
 
-
 logger = get_logger(__file__)
 __all__ = ['DomainViewSet', 'GatewayViewSet', "GatewayTestConnectionApi"]
 
 
 class DomainViewSet(OrgBulkModelViewSet):
     model = Domain
-    filterset_fields = ("name", )
+    filterset_fields = ("name",)
     search_fields = filterset_fields
     serializer_class = serializers.DomainSerializer
     ordering_fields = ('name',)
-    ordering = ('name', )
+    ordering = ('name',)
 
     def get_serializer_class(self):
         if self.request.query_params.get('gateway'):
@@ -30,21 +29,26 @@ class DomainViewSet(OrgBulkModelViewSet):
 
 
 class GatewayViewSet(OrgBulkModelViewSet):
-    model = Gateway
+    filterset_fields = ("domain__name", "name", "domain")
-    filterset_fields = ("domain__name", "name", "username", "domain")
+    search_fields = ("domain__name",)
-    search_fields = ("domain__name", "name", "username", )
     serializer_class = serializers.GatewaySerializer
 
+    def get_queryset(self):
+        queryset = Host.get_gateway_queryset()
+        return queryset
+
 
 class GatewayTestConnectionApi(SingleObjectMixin, APIView):
-    queryset = Gateway.objects.all()
-    object = None
     rbac_perms = {
         'POST': 'assets.test_gateway'
     }
 
+    def get_queryset(self):
+        queryset = Host.get_gateway_queryset()
+        return queryset
+
     def post(self, request, *args, **kwargs):
-        self.object = self.get_object(Gateway.objects.all())
+        self.object = self.get_object()
         local_port = self.request.data.get('port') or self.object.port
         try:
             local_port = int(local_port)

apps/assets/const/__init__.py

@@ -1,3 +1,5 @@
+from .base import *
+from .host import *
 from .types import *
 from .account import *
 from .protocol import *

apps/assets/const/host.py

@@ -1,5 +1,7 @@
 from .base import BaseType
 
+GATEWAY_NAME = 'Gateway'
+
 
 class HostTypes(BaseType):
     LINUX = 'linux', 'Linux'
@@ -67,7 +69,7 @@ class HostTypes(BaseType):
         return {
             cls.LINUX: [
                 {'name': 'Linux'},
-                {'name': 'Gateway'}
+                {'name': GATEWAY_NAME}
             ],
             cls.UNIX: [
                 {'name': 'Unix'},

apps/assets/migrations/0112_gateway_to_asset.py

@@ -0,0 +1,73 @@
+# Generated by Django 3.2.13 on 2022-09-29 11:03
+
+from django.db import migrations
+from assets.const.host import GATEWAY_NAME
+
+
+def _create_account_obj(secret, secret_type, gateway, asset, account_model):
+    return account_model(
+        asset=asset,
+        secret=secret,
+        org_id=gateway.org_id,
+        secret_type=secret_type,
+        username=gateway.username,
+        name=f'{gateway.name}-{secret_type}-{GATEWAY_NAME.lower()}',
+    )
+
+
+def migrate_gateway_to_asset(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    gateway_model = apps.get_model('assets', 'Gateway')
+    platform_model = apps.get_model('assets', 'Platform')
+    gateway_platform = platform_model.objects.using(db_alias).get(name=GATEWAY_NAME)
+
+    print('>>> migrate gateway to asset')
+    asset_dict = {}
+    host_model = apps.get_model('assets', 'Host')
+    asset_model = apps.get_model('assets', 'Asset')
+    protocol_model = apps.get_model('assets', 'Protocol')
+    gateways = gateway_model.objects.all()
+    for gateway in gateways:
+        comment = gateway.comment if gateway.comment else ''
+        data = {
+            'comment': comment,
+            'name': f'{gateway.name}-{GATEWAY_NAME.lower()}',
+            'address': gateway.ip,
+            'domain': gateway.domain,
+            'org_id': gateway.org_id,
+            'is_active': gateway.is_active,
+            'platform': gateway_platform,
+        }
+        asset = asset_model.objects.using(db_alias).create(**data)
+        asset_dict[gateway.id] = asset
+        protocol_model.objects.using(db_alias).create(name='ssh', port=gateway.port, asset=asset)
+    hosts = [host_model(asset_ptr=asset) for asset in asset_dict.values()]
+    host_model.objects.using(db_alias).bulk_create(hosts, ignore_conflicts=True)
+
+    print('>>> migrate gateway to account')
+    accounts = []
+    account_model = apps.get_model('assets', 'Account')
+    for gateway in gateways:
+        password = gateway.password
+        private_key = gateway.private_key
+        asset = asset_dict[gateway.id]
+        if password:
+            accounts.append(_create_account_obj(
+                password, 'password', gateway, asset, account_model
+            ))
+
+        if private_key:
+            accounts.append(_create_account_obj(
+                private_key, 'ssh_key', gateway, asset, account_model
+            ))
+    account_model.objects.using(db_alias).bulk_create(accounts)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('assets', '0111_alter_automationexecution_status'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_gateway_to_asset),
+    ]

apps/assets/models/asset/common.py

@@ -2,8 +2,8 @@
 # -*- coding: utf-8 -*-
 #
 
-import logging
 import uuid
+import logging
 from collections import defaultdict
 
 from django.db import models

apps/assets/models/asset/host.py

@@ -1,6 +1,13 @@
-from assets.const import Category
+from assets.const import GATEWAY_NAME
 from .common import Asset
 
 
 class Host(Asset):
     pass
+
+    @classmethod
+    def get_gateway_queryset(cls):
+        queryset = cls.objects.filter(
+            platform__name=GATEWAY_NAME
+        )
+        return queryset

apps/assets/models/automations/change_secret.py

@@ -65,3 +65,9 @@ class ChangeSecretRecord(JMSBaseModel):
 
     def __str__(self):
         return self.account.__str__()
+
+    @property
+    def timedelta(self):
+        if self.date_started and self.date_finished:
+            return self.date_finished - self.date_started
+        return None

apps/assets/models/base.py

@@ -6,10 +6,10 @@ import sshpubkeys
 from hashlib import md5
 
 from django.db import models
-from django.utils import timezone
-from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
+from django.utils import timezone
 from django.db.models import QuerySet
+from django.utils.translation import ugettext_lazy as _
 
 from common.utils import (
     ssh_key_string_to_obj, ssh_key_gen, get_logger,

apps/assets/models/domain.py

@@ -1,22 +1,24 @@
 # -*- coding: utf-8 -*-
 #
-import socket
 import uuid
+import socket
 import random
-
-from django.core.cache import cache
 import paramiko
+
 from django.db import models
+from django.core.cache import cache
+from django.db.models.query import QuerySet
 from django.utils.translation import ugettext_lazy as _
 
-from common.utils import get_logger, lazyproperty
 from common.db import fields
+from common.utils import get_logger, lazyproperty
 from orgs.mixins.models import OrgModelMixin
 from .base import BaseAccount
+from ..const import SecretType, GATEWAY_NAME
 
 logger = get_logger(__file__)
 
-__all__ = ['Domain', 'Gateway']
+__all__ = ['Domain', 'GatewayMixin']
 
 
 class Domain(OrgModelMixin):
@@ -33,12 +35,9 @@ class Domain(OrgModelMixin):
     def __str__(self):
         return self.name
 
-    def has_gateway(self):
-        return self.gateway_set.filter(is_active=True).exists()
-
     @lazyproperty
     def gateways(self):
-        return self.gateway_set.filter(is_active=True)
+        return self.assets.filter(platform__name=GATEWAY_NAME, is_active=True)
 
     def select_gateway(self):
         return self.random_gateway()
@@ -53,18 +52,141 @@ class Domain(OrgModelMixin):
             return random.choice(self.gateways)
 
 
-class Gateway(BaseAccount):
+class GatewayMixin:
-    UNCONNECTIVE_KEY_TMPL = 'asset_unconnective_gateway_{}'
+    id: uuid.UUID
-    UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL = 'asset_unconnective_gateway_silence_period_{}'
+    port: int
-    UNCONNECTIVE_SILENCE_PERIOD_BEGIN_VALUE = 60 * 5
+    address: str
+    accounts: QuerySet
+    private_key_path: str
+    private_key_obj: paramiko.RSAKey
+    UNCONNECTED_KEY_TMPL = 'asset_unconnective_gateway_{}'
+    UNCONNECTED_SILENCE_PERIOD_KEY_TMPL = 'asset_unconnective_gateway_silence_period_{}'
+    UNCONNECTED_SILENCE_PERIOD_BEGIN_VALUE = 60 * 5
 
+    def set_unconnected(self):
+        unconnected_key = self.UNCONNECTED_KEY_TMPL.format(self.id)
+        unconnected_silence_period_key = self.UNCONNECTED_SILENCE_PERIOD_KEY_TMPL.format(self.id)
+        unconnected_silence_period = cache.get(
+            unconnected_silence_period_key, self.UNCONNECTED_SILENCE_PERIOD_BEGIN_VALUE
+        )
+        cache.set(unconnected_silence_period_key, unconnected_silence_period * 2)
+        cache.set(unconnected_key, unconnected_silence_period, unconnected_silence_period)
+
+    def set_connective(self):
+        unconnected_key = self.UNCONNECTED_KEY_TMPL.format(self.id)
+        unconnected_silence_period_key = self.UNCONNECTED_SILENCE_PERIOD_KEY_TMPL.format(self.id)
+
+        cache.delete(unconnected_key)
+        cache.delete(unconnected_silence_period_key)
+
+    def get_is_unconnected(self):
+        unconnected_key = self.UNCONNECTED_KEY_TMPL.format(self.id)
+        return cache.get(unconnected_key, False)
+
+    @property
+    def is_connective(self):
+        return not self.get_is_unconnected()
+
+    @is_connective.setter
+    def is_connective(self, value):
+        if value:
+            self.set_connective()
+        else:
+            self.set_unconnected()
+
+    def test_connective(self, local_port=None):
+        # TODO 走ansible runner
+        if local_port is None:
+            local_port = self.port
+
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        proxy = paramiko.SSHClient()
+        proxy.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        try:
+            proxy.connect(self.address, port=self.port,
+                          username=self.username,
+                          password=self.password,
+                          pkey=self.private_key_obj)
+        except(paramiko.AuthenticationException,
+               paramiko.BadAuthenticationType,
+               paramiko.SSHException,
+               paramiko.ChannelException,
+               paramiko.ssh_exception.NoValidConnectionsError,
+               socket.gaierror) as e:
+            err = str(e)
+            if err.startswith('[Errno None] Unable to connect to port'):
+                err = _('Unable to connect to port {port} on {address}')
+                err = err.format(port=self.port, ip=self.address)
+            elif err == 'Authentication failed.':
+                err = _('Authentication failed')
+            elif err == 'Connect failed':
+                err = _('Connect failed')
+            self.is_connective = False
+            return False, err
+
+        try:
+            sock = proxy.get_transport().open_channel(
+                'direct-tcpip', ('127.0.0.1', local_port), ('127.0.0.1', 0)
+            )
+            client.connect("127.0.0.1", port=local_port,
+                           username=self.username,
+                           password=self.password,
+                           key_filename=self.private_key_path,
+                           sock=sock,
+                           timeout=5)
+        except (paramiko.SSHException,
+                paramiko.ssh_exception.SSHException,
+                paramiko.ChannelException,
+                paramiko.AuthenticationException,
+                TimeoutError) as e:
+
+            err = getattr(e, 'text', str(e))
+            if err == 'Connect failed':
+                err = _('Connect failed')
+            self.is_connective = False
+            return False, err
+        finally:
+            client.close()
+        self.is_connective = True
+        return True, None
+
+    @lazyproperty
+    def username(self):
+        account = self.accounts.all().first()
+        if account:
+            return account.username
+        logger.error(f'Gateway {self} has no account')
+        return ''
+
+    def get_secret(self, secret_type):
+        account = self.accounts.filter(secret_type=secret_type).first()
+        if account:
+            return account.secret
+        logger.error(f'Gateway {self} has no {secret_type} account')
+
+    @lazyproperty
+    def password(self):
+        secret_type = SecretType.PASSWORD
+        return self.get_secret(secret_type)
+
+    @lazyproperty
+    def private_key(self):
+        secret_type = SecretType.SSH_KEY
+        return self.get_secret(secret_type)
+
+
+class Gateway(BaseAccount):
     class Protocol(models.TextChoices):
         ssh = 'ssh', 'SSH'
 
     name = models.CharField(max_length=128, verbose_name='Name')
     ip = models.CharField(max_length=128, verbose_name=_('IP'), db_index=True)
     port = models.IntegerField(default=22, verbose_name=_('Port'))
-    protocol = models.CharField(choices=Protocol.choices, max_length=16, default=Protocol.ssh, verbose_name=_("Protocol"))
+    protocol = models.CharField(
+        choices=Protocol.choices, max_length=16, default=Protocol.ssh, verbose_name=_("Protocol")
+    )
     domain = models.ForeignKey(Domain, on_delete=models.CASCADE, verbose_name=_("Domain"))
     comment = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Comment"))
     is_active = models.BooleanField(default=True, verbose_name=_("Is active"))
@@ -85,91 +207,3 @@ class Gateway(BaseAccount):
         permissions = [
             ('test_gateway', _('Test gateway'))
         ]
-
-    def set_unconnective(self):
-        unconnective_key = self.UNCONNECTIVE_KEY_TMPL.format(self.id)
-        unconnective_silence_period_key = self.UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL.format(self.id)
-
-        unconnective_silence_period = cache.get(unconnective_silence_period_key,
-                                                self.UNCONNECTIVE_SILENCE_PERIOD_BEGIN_VALUE)
-        cache.set(unconnective_silence_period_key, unconnective_silence_period * 2)
-        cache.set(unconnective_key, unconnective_silence_period, unconnective_silence_period)
-
-    def set_connective(self):
-        unconnective_key = self.UNCONNECTIVE_KEY_TMPL.format(self.id)
-        unconnective_silence_period_key = self.UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL.format(self.id)
-
-        cache.delete(unconnective_key)
-        cache.delete(unconnective_silence_period_key)
-
-    def get_is_unconnective(self):
-        unconnective_key = self.UNCONNECTIVE_KEY_TMPL.format(self.id)
-        return cache.get(unconnective_key, False)
-
-    @property
-    def is_connective(self):
-        return not self.get_is_unconnective()
-
-    @is_connective.setter
-    def is_connective(self, value):
-        if value:
-            self.set_connective()
-        else:
-            self.set_unconnective()
-
-    def test_connective(self, local_port=None):
-        if local_port is None:
-            local_port = self.port
-
-        client = paramiko.SSHClient()
-        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        proxy = paramiko.SSHClient()
-        proxy.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-        try:
-            proxy.connect(self.ip, port=self.port,
-                          username=self.username,
-                          password=self.password,
-                          pkey=self.private_key_obj)
-        except(paramiko.AuthenticationException,
-               paramiko.BadAuthenticationType,
-               paramiko.SSHException,
-               paramiko.ChannelException,
-               paramiko.ssh_exception.NoValidConnectionsError,
-               socket.gaierror) as e:
-            err = str(e)
-            if err.startswith('[Errno None] Unable to connect to port'):
-                err = _('Unable to connect to port {port} on {address}')
-                err = err.format(port=self.port, ip=self.ip)
-            elif err == 'Authentication failed.':
-                err = _('Authentication failed')
-            elif err == 'Connect failed':
-                err = _('Connect failed')
-            self.is_connective = False
-            return False, err
-
-        try:
-            sock = proxy.get_transport().open_channel(
-                'direct-tcpip', ('127.0.0.1', local_port), ('127.0.0.1', 0)
-            )
-            client.connect("127.0.0.1", port=local_port,
-                           username=self.username,
-                           password=self.password,
-                           key_filename=self.private_key_file,
-                           sock=sock,
-                           timeout=5)
-        except (paramiko.SSHException,
-                paramiko.ssh_exception.SSHException,
-                paramiko.ChannelException,
-                paramiko.AuthenticationException,
-                TimeoutError) as e:
-
-            err = getattr(e, 'text', str(e))
-            if err == 'Connect failed':
-                err = _('Connect failed')
-            self.is_connective = False
-            return False, err
-        finally:
-            client.close()
-        self.is_connective = True
-        return True, None

apps/assets/serializers/automations/change_secret.py

@@ -42,6 +42,19 @@ class ChangeSecretAutomationSerializer(AuthValidateMixin, BaseAutomationSerializ
             )},
         }}
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.set_secret_type_choices()
+
+    def set_secret_type_choices(self):
+        secret_type = self.fields.get('secret_type')
+        if not secret_type:
+            return
+        choices = secret_type._choices
+        choices.pop(SecretType.ACCESS_KEY, None)
+        choices.pop(SecretType.TOKEN, None)
+        secret_type._choices = choices
+
     def validate_password_rules(self, password_rules):
         secret_type = self.initial_secret_type
         if secret_type != SecretType.PASSWORD:
@@ -93,8 +106,8 @@ class ChangeSecretRecordSerializer(serializers.ModelSerializer):
     class Meta:
         model = ChangeSecretRecord
         fields = [
-            'id', 'asset', 'account', 'date_started',
+            'id', 'asset', 'account', 'date_started', 'date_finished',
-            'date_finished', 'is_success', 'error', 'execution',
+            'timedelta', 'is_success', 'error', 'execution',
         ]
         read_only_fields = fields
 

apps/assets/serializers/domain.py

@@ -3,11 +3,9 @@
 from rest_framework import serializers
 from django.utils.translation import ugettext_lazy as _
 
-from common.validators import alphanumeric
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 from common.drf.serializers import SecretReadableMixin
-from ..models import Domain, Gateway
+from ..models import Domain, Asset
-from .base import AuthValidateMixin
 
 
 class DomainSerializer(BulkOrgResourceModelSerializer):
@@ -35,32 +33,23 @@ class DomainSerializer(BulkOrgResourceModelSerializer):
 
     @staticmethod
     def get_gateway_count(obj):
-        return obj.gateway_set.all().count()
+        return obj.gateways.count()
 
 
-class GatewaySerializer(AuthValidateMixin, BulkOrgResourceModelSerializer):
+class GatewaySerializer(BulkOrgResourceModelSerializer):
     is_connective = serializers.BooleanField(required=False, label=_('Connectivity'))
 
     class Meta:
-        model = Gateway
+        model = Asset
-        fields_mini = ['id', 'username']
+        fields_mini = ['id']
-        fields_write_only = [
+        fields_small = fields_mini + [
-            'password', 'private_key', 'public_key', 'passphrase'
+            'address', 'port', 'protocol',
-        ]
-        fields_small = fields_mini + fields_write_only + [
-            'ip', 'port', 'protocol',
             'is_active', 'is_connective',
             'date_created', 'date_updated',
             'created_by', 'comment',
         ]
         fields_fk = ['domain']
         fields = fields_small + fields_fk
-        extra_kwargs = {
-            'username': {"validators": [alphanumeric]},
-            'password': {'write_only': True},
-            'private_key': {"write_only": True},
-            'public_key': {"write_only": True},
-        }
 
 
 class GatewayWithAuthSerializer(SecretReadableMixin, GatewaySerializer):

apps/authentication/serializers/connection_token.py

@@ -1,8 +1,8 @@
 from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 
-from assets.models import Asset, Gateway, Domain, CommandFilterRule, Account, Platform
 from assets.serializers import PlatformSerializer
+from assets.models import Asset, Domain, CommandFilterRule, Account, Platform
 from authentication.models import ConnectionToken
 from orgs.mixins.serializers import OrgResourceModelSerializerMixin
 from perms.serializers.permission import ActionChoicesField
@@ -106,8 +106,8 @@ class ConnectionTokenGatewaySerializer(serializers.ModelSerializer):
     """ Gateway """
 
     class Meta:
-        model = Gateway
+        model = Asset
-        fields = ['id', 'ip', 'port', 'username', 'password', 'private_key']
+        fields = ['id', 'address', 'port', 'username', 'password', 'private_key']
 
 
 class ConnectionTokenDomainSerializer(serializers.ModelSerializer):

apps/ops/signal_handlers.py

@@ -1,14 +1,16 @@
 import ast
+from celery import signals
 
 from django.db import transaction
+from django.core.cache import cache
 from django.dispatch import receiver
+from django.db.utils import ProgrammingError
 from django.utils import translation, timezone
 from django.utils.translation import gettext as _
-from django.core.cache import cache
-from celery import signals, current_app
 
-from common.db.utils import close_old_connections, get_logger
 from common.signals import django_ready
+from common.db.utils import close_old_connections, get_logger
+
 from .celery import app
 from .models import CeleryTaskExecution, CeleryTask
 
@@ -23,15 +25,15 @@ def sync_registered_tasks(*args, **kwargs):
     with transaction.atomic():
         try:
             db_tasks = CeleryTask.objects.all()
-        except Exception as e:
+            celery_task_names = [key for key in app.tasks]
-            return
+            db_task_names = db_tasks.values_list('name', flat=True)
-        celery_task_names = [key for key in app.tasks]
-        db_task_names = db_tasks.values_list('name', flat=True)
 
-        db_tasks.exclude(name__in=celery_task_names).delete()
+            db_tasks.exclude(name__in=celery_task_names).delete()
-        not_in_db_tasks = set(celery_task_names) - set(db_task_names)
+            not_in_db_tasks = set(celery_task_names) - set(db_task_names)
-        tasks_to_create = [CeleryTask(name=name) for name in not_in_db_tasks]
+            tasks_to_create = [CeleryTask(name=name) for name in not_in_db_tasks]
-        CeleryTask.objects.bulk_create(tasks_to_create)
+            CeleryTask.objects.bulk_create(tasks_to_create)
+        except ProgrammingError:
+            pass
 
 
 @signals.before_task_publish.connect
@@ -45,7 +47,7 @@ def before_task_publish(headers=None, **kwargs):
 @signals.task_prerun.connect
 def on_celery_task_pre_run(task_id='', **kwargs):
     # 更新状态
-    CeleryTaskExecution.objects.filter(id=task_id)\
+    CeleryTaskExecution.objects.filter(id=task_id) \
         .update(state='RUNNING', date_start=timezone.now())
     # 关闭之前的数据库连接
     close_old_connections()

apps/orgs/api.py

@@ -14,7 +14,7 @@ from .serializers import (
 )
 from users.models import User, UserGroup
 from assets.models import (
-    Asset, Domain, Label, Node, Gateway,
+    Asset, Domain, Label, Node,
     CommandFilter, CommandFilterRule, GatheredUser
 )
 from perms.models import AssetPermission
@@ -27,7 +27,7 @@ logger = get_logger(__file__)
 
 # 部分 org 相关的 model，需要清空这些数据之后才能删除该组织
 org_related_models = [
-    User, UserGroup, Asset, Label, Domain, Gateway, Node, Label,
+    User, UserGroup, Asset, Label, Domain, Node, Label,
     CommandFilter, CommandFilterRule, GatheredUser,
     AssetPermission,
 ]

apps/orgs/caches.py

@@ -6,7 +6,7 @@ from orgs.utils import current_org, tmp_to_org
 from common.cache import Cache, IntegerField
 from common.utils import get_logger
 from users.models import UserGroup, User
-from assets.models import Node, Domain, Gateway, Asset, Account
+from assets.models import Node, Domain, Asset, Account
 from terminal.models import Session
 from perms.models import AssetPermission
 
@@ -54,7 +54,7 @@ class OrgResourceStatisticsCache(OrgRelatedCache):
     nodes_amount = IntegerField(queryset=Node.objects)
     accounts_amount = IntegerField(queryset=Account.objects)
     domains_amount = IntegerField(queryset=Domain.objects)
-    gateways_amount = IntegerField(queryset=Gateway.objects)
+    # gateways_amount = IntegerField(queryset=Gateway.objects)
     asset_perms_amount = IntegerField(queryset=AssetPermission.objects)
 
     total_count_online_users = IntegerField()

apps/orgs/signal_handlers/cache.py

@@ -8,7 +8,7 @@ from users.models import UserGroup, User
 from users.signals import pre_user_leave_org
 from terminal.models import Session
 from rbac.models import OrgRoleBinding, SystemRoleBinding, RoleBinding
-from assets.models import Asset, Domain, Gateway
+from assets.models import Asset, Domain
 from orgs.caches import OrgResourceStatisticsCache
 from orgs.utils import current_org
 from common.utils import get_logger
@@ -75,7 +75,6 @@ def on_user_delete_refresh_cache(sender, instance, **kwargs):
 class OrgResourceStatisticsRefreshUtil:
     model_cache_field_mapper = {
         AssetPermission: ['asset_perms_amount'],
-        Gateway: ['gateways_amount'],
         Domain: ['domains_amount'],
         Node: ['nodes_amount'],
         Asset: ['assets_amount'],

apps/orgs/signal_handlers/common.py

@@ -7,6 +7,7 @@ from functools import partial
 import django.db.utils
 from django.dispatch import receiver
 from django.conf import settings
+from django.db.utils import ProgrammingError, OperationalError
 from django.utils.functional import LazyObject
 from django.db.models.signals import post_save, pre_delete, m2m_changed
 
@@ -48,7 +49,7 @@ def subscribe_orgs_mapping_expire(sender, **kwargs):
     if settings.DEBUG:
         try:
             set_to_default_org()
-        except django.db.utils.OperationalError:
+        except (ProgrammingError, OperationalError):
             pass
 
     def keep_subscribe_org_mapping():

apps/perms/serializers/user_permission.py

@@ -30,7 +30,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer):
             'domain', 'platform',
             "comment", "org_id", "is_active",
         ]
-        fields = only_fields + ['category', 'type'] + ['org_name']
+        fields = only_fields + ['protocols', 'category', 'type'] + ['org_name']
         read_only_fields = fields
 
 

apps/tickets/migrations/0023_alter_applyassetticket_apply_actions.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('tickets', '0022_alter_applyassetticket_apply_actions'),
     ]
@@ -15,4 +14,73 @@ class Migration(migrations.Migration):
             name='apply_actions',
             field=models.IntegerField(default=31, verbose_name='Actions'),
         ),
+        migrations.AlterField(
+            model_name='approvalrule',
+            name='strategy',
+            field=models.CharField(
+                choices=[
+                    ('org_admin', 'Org admin'), ('custom_user', 'Custom user'),
+                    ('super_admin', 'Super admin'), ('super_org_admin', 'Super admin and org admin')
+                ], default='super_admin', max_length=64,
+                verbose_name='Approve strategy'),
+        ),
+        migrations.AlterField(
+            model_name='ticket',
+            name='state',
+            field=models.CharField(
+                choices=[
+                    ('pending', 'Open'), ('closed', 'Cancel'),
+                    ('reopen', 'Reopen'), ('approved', 'Approved'),
+                    ('rejected', 'Rejected')
+                ], default='pending', max_length=16, verbose_name='State'),
+        ),
+        migrations.AlterField(
+            model_name='ticket',
+            name='type',
+            field=models.CharField(
+                choices=[
+                    ('general', 'General'), ('apply_asset', 'Apply for asset'),
+                    ('login_confirm', 'Login confirm'), ('command_confirm', 'Command confirm'),
+                    ('login_asset_confirm', 'Login asset confirm')
+                ],
+                default='general', max_length=64, verbose_name='Type'),
+        ),
+        migrations.AlterField(
+            model_name='ticketassignee',
+            name='state',
+            field=models.CharField(
+                choices=[
+                    ('pending', 'Open'), ('closed', 'Cancel'),
+                    ('reopen', 'Reopen'), ('approved', 'Approved'),
+                    ('rejected', 'Rejected')
+                ], default='pending', max_length=64),
+        ),
+        migrations.AlterField(
+            model_name='ticketflow',
+            name='type',
+            field=models.CharField(
+                choices=[
+                    ('general', 'General'), ('apply_asset', 'Apply for asset'),
+                    ('login_confirm', 'Login confirm'), ('command_confirm', 'Command confirm'),
+                    ('login_asset_confirm', 'Login asset confirm')
+                ],
+                default='general', max_length=64, verbose_name='Type'),
+        ),
+        migrations.AlterField(
+            model_name='ticketstep',
+            name='state',
+            field=models.CharField(
+                choices=[
+                    ('pending', 'Pending'), ('closed', 'Closed'),
+                    ('reopen', 'Reopen'), ('approved', 'Approved'),
+                    ('rejected', 'Rejected')
+                ], default='pending', max_length=64, verbose_name='State'),
+        ),
+        migrations.AlterField(
+            model_name='ticketstep',
+            name='status',
+            field=models.CharField(
+                choices=[('active', 'Active'), ('closed', 'Closed'), ('pending', 'Pending')],
+                default='pending', max_length=16),
+        ),
     ]