From 94526e44f1de7d6d73c8c2220ddb21f6c49ae7d9 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Mon, 21 Nov 2022 15:18:09 +0800 Subject: [PATCH 1/7] perf: change secret timedelta --- apps/assets/models/automations/change_secret.py | 6 ++++++ apps/assets/serializers/automations/change_secret.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/assets/models/automations/change_secret.py b/apps/assets/models/automations/change_secret.py index c22b64f51..ecc0e98d4 100644 --- a/apps/assets/models/automations/change_secret.py +++ b/apps/assets/models/automations/change_secret.py @@ -65,3 +65,9 @@ class ChangeSecretRecord(JMSBaseModel): def __str__(self): return self.account.__str__() + + @property + def timedelta(self): + if self.date_started and self.date_finished: + return self.date_finished - self.date_started + return None diff --git a/apps/assets/serializers/automations/change_secret.py b/apps/assets/serializers/automations/change_secret.py index 3b9137bc4..85fad5855 100644 --- a/apps/assets/serializers/automations/change_secret.py +++ b/apps/assets/serializers/automations/change_secret.py @@ -93,8 +93,8 @@ class ChangeSecretRecordSerializer(serializers.ModelSerializer): class Meta: model = ChangeSecretRecord fields = [ - 'id', 'asset', 'account', 'date_started', - 'date_finished', 'is_success', 'error', 'execution', + 'id', 'asset', 'account', 'date_started', 'date_finished', + 'timedelta', 'is_success', 'error', 'execution', ] read_only_fields = fields From f39a3a34e40b9cc7fa72ed7036d6d00a454fb34c Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Mon, 21 Nov 2022 16:23:32 +0800 Subject: [PATCH 2/7] perf: change secret ignore secret type --- .../assets/serializers/automations/change_secret.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/assets/serializers/automations/change_secret.py b/apps/assets/serializers/automations/change_secret.py index 85fad5855..b0149334d 100644 --- a/apps/assets/serializers/automations/change_secret.py +++ b/apps/assets/serializers/automations/change_secret.py @@ -42,6 +42,19 @@ class ChangeSecretAutomationSerializer(AuthValidateMixin, BaseAutomationSerializ )}, }} + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_secret_type_choices() + + def set_secret_type_choices(self): + secret_type = self.fields.get('secret_type') + if not secret_type: + return + choices = secret_type._choices + choices.pop(SecretType.ACCESS_KEY, None) + choices.pop(SecretType.TOKEN, None) + secret_type._choices = choices + def validate_password_rules(self, password_rules): secret_type = self.initial_secret_type if secret_type != SecretType.PASSWORD: From 4b26fb3e6e1f5934edd58e94102b39ab9e6b7e91 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Mon, 21 Nov 2022 19:54:00 +0800 Subject: [PATCH 3/7] fix: jms upgrade_db bug --- apps/ops/signal_handlers.py | 26 ++++++++++++++------------ apps/orgs/signal_handlers/common.py | 3 ++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/apps/ops/signal_handlers.py b/apps/ops/signal_handlers.py index dd49a4d94..965bd494c 100644 --- a/apps/ops/signal_handlers.py +++ b/apps/ops/signal_handlers.py @@ -1,14 +1,16 @@ import ast +from celery import signals from django.db import transaction +from django.core.cache import cache from django.dispatch import receiver +from django.db.utils import ProgrammingError from django.utils import translation, timezone from django.utils.translation import gettext as _ -from django.core.cache import cache -from celery import signals, current_app -from common.db.utils import close_old_connections, get_logger from common.signals import django_ready +from common.db.utils import close_old_connections, get_logger + from .celery import app from .models import CeleryTaskExecution, CeleryTask @@ -23,15 +25,15 @@ def sync_registered_tasks(*args, **kwargs): with transaction.atomic(): try: db_tasks = CeleryTask.objects.all() - except Exception as e: - return - celery_task_names = [key for key in app.tasks] - db_task_names = db_tasks.values_list('name', flat=True) + celery_task_names = [key for key in app.tasks] + db_task_names = db_tasks.values_list('name', flat=True) - db_tasks.exclude(name__in=celery_task_names).delete() - not_in_db_tasks = set(celery_task_names) - set(db_task_names) - tasks_to_create = [CeleryTask(name=name) for name in not_in_db_tasks] - CeleryTask.objects.bulk_create(tasks_to_create) + db_tasks.exclude(name__in=celery_task_names).delete() + not_in_db_tasks = set(celery_task_names) - set(db_task_names) + tasks_to_create = [CeleryTask(name=name) for name in not_in_db_tasks] + CeleryTask.objects.bulk_create(tasks_to_create) + except ProgrammingError: + pass @signals.before_task_publish.connect @@ -45,7 +47,7 @@ def before_task_publish(headers=None, **kwargs): @signals.task_prerun.connect def on_celery_task_pre_run(task_id='', **kwargs): # 更新状态 - CeleryTaskExecution.objects.filter(id=task_id)\ + CeleryTaskExecution.objects.filter(id=task_id) \ .update(state='RUNNING', date_start=timezone.now()) # 关闭之前的数据库连接 close_old_connections() diff --git a/apps/orgs/signal_handlers/common.py b/apps/orgs/signal_handlers/common.py index 2136b28a0..666f68fa1 100644 --- a/apps/orgs/signal_handlers/common.py +++ b/apps/orgs/signal_handlers/common.py @@ -7,6 +7,7 @@ from functools import partial import django.db.utils from django.dispatch import receiver from django.conf import settings +from django.db.utils import ProgrammingError, OperationalError from django.utils.functional import LazyObject from django.db.models.signals import post_save, pre_delete, m2m_changed @@ -48,7 +49,7 @@ def subscribe_orgs_mapping_expire(sender, **kwargs): if settings.DEBUG: try: set_to_default_org() - except django.db.utils.OperationalError: + except (ProgrammingError, OperationalError): pass def keep_subscribe_org_mapping(): From 436cb7b6e142304e5b1435753af8948b6de694c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=B0=8F=E7=99=BD?= <296015668@qq.com> Date: Tue, 22 Nov 2022 10:15:55 +0800 Subject: [PATCH 4/7] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20Dockerfile.loo?= =?UTF-8?q?ng64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 +- Dockerfile.loong64 | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.loong64 diff --git a/Dockerfile b/Dockerfile index 2f65985b8..28dedbba9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,10 +57,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \ && apt-get -y install --no-install-recommends ${TOOLS} \ && mkdir -p /root/.ssh/ \ && echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \ - && sed -i "s@# alias l@alias l@g" ~/.bashrc \ && echo "set mouse-=a" > ~/.vimrc \ && echo "no" | dpkg-reconfigure dash \ && echo "zh_CN.UTF-8" | dpkg-reconfigure locales \ + && sed -i "s@# export @export @g" ~/.bashrc \ + && sed -i "s@# alias @alias @g" ~/.bashrc \ && rm -rf /var/lib/apt/lists/* ARG DOWNLOAD_URL=https://download.jumpserver.org diff --git a/Dockerfile.loong64 b/Dockerfile.loong64 new file mode 100644 index 000000000..580792776 --- /dev/null +++ b/Dockerfile.loong64 @@ -0,0 +1,96 @@ +FROM python:3.8-slim as stage-build +ARG TARGETARCH + +ARG VERSION +ENV VERSION=$VERSION + +WORKDIR /opt/jumpserver +ADD . . +RUN cd utils && bash -ixeu build.sh + +FROM python:3.8-slim +ARG TARGETARCH +MAINTAINER JumpServer Team + +ARG BUILD_DEPENDENCIES=" \ + g++ \ + make \ + pkg-config" + +ARG DEPENDENCIES=" \ + default-libmysqlclient-dev \ + freetds-dev \ + libpq-dev \ + libffi-dev \ + libjpeg-dev \ + libldap2-dev \ + libsasl2-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libxmlsec1-openssl \ + libaio-dev \ + openssh-client \ + sshpass" + +ARG TOOLS=" \ + ca-certificates \ + curl \ + default-mysql-client \ + iputils-ping \ + locales \ + netcat \ + redis-server \ + telnet \ + vim \ + unzip \ + wget" + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=core \ + set -ex \ + && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && apt-get update \ + && apt-get -y install --no-install-recommends ${BUILD_DEPENDENCIES} \ + && apt-get -y install --no-install-recommends ${DEPENDENCIES} \ + && apt-get -y install --no-install-recommends ${TOOLS} \ + && mkdir -p /root/.ssh/ \ + && echo "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config \ + && echo "set mouse-=a" > ~/.vimrc \ + && echo "no" | dpkg-reconfigure dash \ + && echo "zh_CN.UTF-8" | dpkg-reconfigure locales \ + && sed -i "s@# export @export @g" ~/.bashrc \ + && sed -i "s@# alias @alias @g" ~/.bashrc \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tmp/build +COPY ./requirements ./requirements + +ARG PIP_MIRROR=https://pypi.douban.com/simple +ENV PIP_MIRROR=$PIP_MIRROR +ARG PIP_JMS_MIRROR=https://pypi.douban.com/simple +ENV PIP_JMS_MIRROR=$PIP_JMS_MIRROR + +RUN --mount=type=cache,target=/root/.cache/pip \ + set -ex \ + && pip config set global.index-url ${PIP_MIRROR} \ + && pip install --upgrade pip \ + && pip install --upgrade setuptools wheel \ + && pip install https://download.jumpserver.org/pypi/simple/cryptography/cryptography-36.0.1-cp38-cp38-linux_loongarch64.whl \ + && pip install https://download.jumpserver.org/pypi/simple/greenlet/greenlet-1.1.2-cp38-cp38-linux_loongarch64.whl \ + && pip install $(grep 'PyNaCl' requirements/requirements.txt) \ + && GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=true pip install grpcio \ + && pip install $(grep -E 'jms|jumpserver' requirements/requirements.txt) -i ${PIP_JMS_MIRROR} \ + && pip install -r requirements/requirements.txt + +COPY --from=stage-build /opt/jumpserver/release/jumpserver /opt/jumpserver +RUN echo > /opt/jumpserver/config.yml \ + && rm -rf /tmp/build + +WORKDIR /opt/jumpserver +VOLUME /opt/jumpserver/data +VOLUME /opt/jumpserver/logs + +ENV LANG=zh_CN.UTF-8 + +EXPOSE 8070 +EXPOSE 8080 +ENTRYPOINT ["./entrypoint.sh"] From d543c3efe7c9beedff88083233a4691a37b5ccb1 Mon Sep 17 00:00:00 2001 From: "Jiangjie.Bai" Date: Tue, 22 Nov 2022 11:05:52 +0800 Subject: [PATCH 5/7] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20favorite-assets?= =?UTF-8?q?=20Serializer=20=E7=BC=BA=E5=B0=91=20protocols=20=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/perms/serializers/user_permission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/perms/serializers/user_permission.py b/apps/perms/serializers/user_permission.py index d95a14f99..1d795d650 100644 --- a/apps/perms/serializers/user_permission.py +++ b/apps/perms/serializers/user_permission.py @@ -30,7 +30,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer): 'domain', 'platform', "comment", "org_id", "is_active", ] - fields = only_fields + ['category', 'type'] + ['org_name'] + fields = only_fields + ['protocols', 'category', 'type'] + ['org_name'] read_only_fields = fields From 873b81e639dc98d7c55ba608f600341b8c7f6a50 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Tue, 22 Nov 2022 11:36:48 +0800 Subject: [PATCH 6/7] perf: ticket migrate --- ...23_alter_applyassetticket_apply_actions.py | 70 ++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/apps/tickets/migrations/0023_alter_applyassetticket_apply_actions.py b/apps/tickets/migrations/0023_alter_applyassetticket_apply_actions.py index 2c4224c80..f401fe298 100644 --- a/apps/tickets/migrations/0023_alter_applyassetticket_apply_actions.py +++ b/apps/tickets/migrations/0023_alter_applyassetticket_apply_actions.py @@ -4,7 +4,6 @@ from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('tickets', '0022_alter_applyassetticket_apply_actions'), ] @@ -15,4 +14,73 @@ class Migration(migrations.Migration): name='apply_actions', field=models.IntegerField(default=31, verbose_name='Actions'), ), + migrations.AlterField( + model_name='approvalrule', + name='strategy', + field=models.CharField( + choices=[ + ('org_admin', 'Org admin'), ('custom_user', 'Custom user'), + ('super_admin', 'Super admin'), ('super_org_admin', 'Super admin and org admin') + ], default='super_admin', max_length=64, + verbose_name='Approve strategy'), + ), + migrations.AlterField( + model_name='ticket', + name='state', + field=models.CharField( + choices=[ + ('pending', 'Open'), ('closed', 'Cancel'), + ('reopen', 'Reopen'), ('approved', 'Approved'), + ('rejected', 'Rejected') + ], default='pending', max_length=16, verbose_name='State'), + ), + migrations.AlterField( + model_name='ticket', + name='type', + field=models.CharField( + choices=[ + ('general', 'General'), ('apply_asset', 'Apply for asset'), + ('login_confirm', 'Login confirm'), ('command_confirm', 'Command confirm'), + ('login_asset_confirm', 'Login asset confirm') + ], + default='general', max_length=64, verbose_name='Type'), + ), + migrations.AlterField( + model_name='ticketassignee', + name='state', + field=models.CharField( + choices=[ + ('pending', 'Open'), ('closed', 'Cancel'), + ('reopen', 'Reopen'), ('approved', 'Approved'), + ('rejected', 'Rejected') + ], default='pending', max_length=64), + ), + migrations.AlterField( + model_name='ticketflow', + name='type', + field=models.CharField( + choices=[ + ('general', 'General'), ('apply_asset', 'Apply for asset'), + ('login_confirm', 'Login confirm'), ('command_confirm', 'Command confirm'), + ('login_asset_confirm', 'Login asset confirm') + ], + default='general', max_length=64, verbose_name='Type'), + ), + migrations.AlterField( + model_name='ticketstep', + name='state', + field=models.CharField( + choices=[ + ('pending', 'Pending'), ('closed', 'Closed'), + ('reopen', 'Reopen'), ('approved', 'Approved'), + ('rejected', 'Rejected') + ], default='pending', max_length=64, verbose_name='State'), + ), + migrations.AlterField( + model_name='ticketstep', + name='status', + field=models.CharField( + choices=[('active', 'Active'), ('closed', 'Closed'), ('pending', 'Pending')], + default='pending', max_length=16), + ), ] From 1a204618f71ca8d2aaceb7f1341cd8959be86789 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 22 Nov 2022 17:33:09 +0800 Subject: [PATCH 7/7] [v3] perf: migrate gateway to asset (#8928) * perf: migrate gateway to asset * perf: asset discriminate gateway Co-authored-by: feng626 <1304903146@qq.com> --- apps/assets/api/asset/asset.py | 10 +- apps/assets/api/asset/host.py | 1 - apps/assets/api/domain.py | 24 +- apps/assets/const/__init__.py | 2 + apps/assets/const/host.py | 4 +- .../migrations/0112_gateway_to_asset.py | 73 ++++++ apps/assets/models/asset/common.py | 2 +- apps/assets/models/asset/host.py | 9 +- apps/assets/models/base.py | 4 +- apps/assets/models/domain.py | 238 ++++++++++-------- apps/assets/serializers/domain.py | 25 +- .../serializers/connection_token.py | 6 +- apps/orgs/api.py | 4 +- apps/orgs/caches.py | 4 +- apps/orgs/signal_handlers/cache.py | 3 +- 15 files changed, 258 insertions(+), 151 deletions(-) create mode 100644 apps/assets/migrations/0112_gateway_to_asset.py diff --git a/apps/assets/api/asset/asset.py b/apps/assets/api/asset/asset.py index ad04966e3..f6cc509b3 100644 --- a/apps/assets/api/asset/asset.py +++ b/apps/assets/api/asset/asset.py @@ -6,13 +6,11 @@ from rest_framework.decorators import action from rest_framework.response import Response from assets import serializers +from assets.models import Asset from assets.filters import IpInFilterBackend, LabelFilterBackend, NodeFilterBackend -from assets.models import Asset, Gateway from assets.tasks import ( - push_accounts_to_assets, - test_assets_connectivity_manual, - update_assets_hardware_info_manual, - verify_accounts_connectivity, + push_accounts_to_assets, test_assets_connectivity_manual, + update_assets_hardware_info_manual, verify_accounts_connectivity, ) from common.drf.filters import BaseFilterSet from common.mixins.api import SuggestionMixin @@ -74,7 +72,7 @@ class AssetViewSet(SuggestionMixin, NodeFilterMixin, OrgBulkModelViewSet): def gateways(self, *args, **kwargs): asset = self.get_object() if not asset.domain: - gateways = Gateway.objects.none() + gateways = Asset.objects.none() else: gateways = asset.domain.gateways.filter(protocol="ssh") return self.get_paginated_response_from_queryset(gateways) diff --git a/apps/assets/api/asset/host.py b/apps/assets/api/asset/host.py index 3094e15a8..fbc2e997c 100644 --- a/apps/assets/api/asset/host.py +++ b/apps/assets/api/asset/host.py @@ -1,4 +1,3 @@ - from assets.models import Host from assets.serializers import HostSerializer from .asset import AssetViewSet diff --git a/apps/assets/api/domain.py b/apps/assets/api/domain.py index 39f2b44f9..bb705322c 100644 --- a/apps/assets/api/domain.py +++ b/apps/assets/api/domain.py @@ -7,21 +7,20 @@ from rest_framework.serializers import ValidationError from common.utils import get_logger from orgs.mixins.api import OrgBulkModelViewSet -from ..models import Domain, Gateway +from ..models import Domain, Host from .. import serializers - logger = get_logger(__file__) __all__ = ['DomainViewSet', 'GatewayViewSet', "GatewayTestConnectionApi"] class DomainViewSet(OrgBulkModelViewSet): model = Domain - filterset_fields = ("name", ) + filterset_fields = ("name",) search_fields = filterset_fields serializer_class = serializers.DomainSerializer ordering_fields = ('name',) - ordering = ('name', ) + ordering = ('name',) def get_serializer_class(self): if self.request.query_params.get('gateway'): @@ -30,21 +29,26 @@ class DomainViewSet(OrgBulkModelViewSet): class GatewayViewSet(OrgBulkModelViewSet): - model = Gateway - filterset_fields = ("domain__name", "name", "username", "domain") - search_fields = ("domain__name", "name", "username", ) + filterset_fields = ("domain__name", "name", "domain") + search_fields = ("domain__name",) serializer_class = serializers.GatewaySerializer + def get_queryset(self): + queryset = Host.get_gateway_queryset() + return queryset + class GatewayTestConnectionApi(SingleObjectMixin, APIView): - queryset = Gateway.objects.all() - object = None rbac_perms = { 'POST': 'assets.test_gateway' } + def get_queryset(self): + queryset = Host.get_gateway_queryset() + return queryset + def post(self, request, *args, **kwargs): - self.object = self.get_object(Gateway.objects.all()) + self.object = self.get_object() local_port = self.request.data.get('port') or self.object.port try: local_port = int(local_port) diff --git a/apps/assets/const/__init__.py b/apps/assets/const/__init__.py index 81115b412..bc30f388d 100644 --- a/apps/assets/const/__init__.py +++ b/apps/assets/const/__init__.py @@ -1,3 +1,5 @@ +from .base import * +from .host import * from .types import * from .account import * from .protocol import * diff --git a/apps/assets/const/host.py b/apps/assets/const/host.py index 371ab3688..8be44db6f 100644 --- a/apps/assets/const/host.py +++ b/apps/assets/const/host.py @@ -1,5 +1,7 @@ from .base import BaseType +GATEWAY_NAME = 'Gateway' + class HostTypes(BaseType): LINUX = 'linux', 'Linux' @@ -67,7 +69,7 @@ class HostTypes(BaseType): return { cls.LINUX: [ {'name': 'Linux'}, - {'name': 'Gateway'} + {'name': GATEWAY_NAME} ], cls.UNIX: [ {'name': 'Unix'}, diff --git a/apps/assets/migrations/0112_gateway_to_asset.py b/apps/assets/migrations/0112_gateway_to_asset.py new file mode 100644 index 000000000..da43b3a84 --- /dev/null +++ b/apps/assets/migrations/0112_gateway_to_asset.py @@ -0,0 +1,73 @@ +# Generated by Django 3.2.13 on 2022-09-29 11:03 + +from django.db import migrations +from assets.const.host import GATEWAY_NAME + + +def _create_account_obj(secret, secret_type, gateway, asset, account_model): + return account_model( + asset=asset, + secret=secret, + org_id=gateway.org_id, + secret_type=secret_type, + username=gateway.username, + name=f'{gateway.name}-{secret_type}-{GATEWAY_NAME.lower()}', + ) + + +def migrate_gateway_to_asset(apps, schema_editor): + db_alias = schema_editor.connection.alias + gateway_model = apps.get_model('assets', 'Gateway') + platform_model = apps.get_model('assets', 'Platform') + gateway_platform = platform_model.objects.using(db_alias).get(name=GATEWAY_NAME) + + print('>>> migrate gateway to asset') + asset_dict = {} + host_model = apps.get_model('assets', 'Host') + asset_model = apps.get_model('assets', 'Asset') + protocol_model = apps.get_model('assets', 'Protocol') + gateways = gateway_model.objects.all() + for gateway in gateways: + comment = gateway.comment if gateway.comment else '' + data = { + 'comment': comment, + 'name': f'{gateway.name}-{GATEWAY_NAME.lower()}', + 'address': gateway.ip, + 'domain': gateway.domain, + 'org_id': gateway.org_id, + 'is_active': gateway.is_active, + 'platform': gateway_platform, + } + asset = asset_model.objects.using(db_alias).create(**data) + asset_dict[gateway.id] = asset + protocol_model.objects.using(db_alias).create(name='ssh', port=gateway.port, asset=asset) + hosts = [host_model(asset_ptr=asset) for asset in asset_dict.values()] + host_model.objects.using(db_alias).bulk_create(hosts, ignore_conflicts=True) + + print('>>> migrate gateway to account') + accounts = [] + account_model = apps.get_model('assets', 'Account') + for gateway in gateways: + password = gateway.password + private_key = gateway.private_key + asset = asset_dict[gateway.id] + if password: + accounts.append(_create_account_obj( + password, 'password', gateway, asset, account_model + )) + + if private_key: + accounts.append(_create_account_obj( + private_key, 'ssh_key', gateway, asset, account_model + )) + account_model.objects.using(db_alias).bulk_create(accounts) + + +class Migration(migrations.Migration): + dependencies = [ + ('assets', '0111_alter_automationexecution_status'), + ] + + operations = [ + migrations.RunPython(migrate_gateway_to_asset), + ] diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index 8ea75bc2a..c9baf8818 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- # -import logging import uuid +import logging from collections import defaultdict from django.db import models diff --git a/apps/assets/models/asset/host.py b/apps/assets/models/asset/host.py index 4ce4be5c9..46aeed4f3 100644 --- a/apps/assets/models/asset/host.py +++ b/apps/assets/models/asset/host.py @@ -1,6 +1,13 @@ -from assets.const import Category +from assets.const import GATEWAY_NAME from .common import Asset class Host(Asset): pass + + @classmethod + def get_gateway_queryset(cls): + queryset = cls.objects.filter( + platform__name=GATEWAY_NAME + ) + return queryset diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 7920d3798..90fb384e6 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -6,10 +6,10 @@ import sshpubkeys from hashlib import md5 from django.db import models -from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ from django.conf import settings +from django.utils import timezone from django.db.models import QuerySet +from django.utils.translation import ugettext_lazy as _ from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, diff --git a/apps/assets/models/domain.py b/apps/assets/models/domain.py index 4abe8aa68..bf33caed6 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/domain.py @@ -1,22 +1,24 @@ # -*- coding: utf-8 -*- # -import socket import uuid +import socket import random - -from django.core.cache import cache import paramiko + from django.db import models +from django.core.cache import cache +from django.db.models.query import QuerySet from django.utils.translation import ugettext_lazy as _ -from common.utils import get_logger, lazyproperty from common.db import fields +from common.utils import get_logger, lazyproperty from orgs.mixins.models import OrgModelMixin from .base import BaseAccount +from ..const import SecretType, GATEWAY_NAME logger = get_logger(__file__) -__all__ = ['Domain', 'Gateway'] +__all__ = ['Domain', 'GatewayMixin'] class Domain(OrgModelMixin): @@ -33,12 +35,9 @@ class Domain(OrgModelMixin): def __str__(self): return self.name - def has_gateway(self): - return self.gateway_set.filter(is_active=True).exists() - @lazyproperty def gateways(self): - return self.gateway_set.filter(is_active=True) + return self.assets.filter(platform__name=GATEWAY_NAME, is_active=True) def select_gateway(self): return self.random_gateway() @@ -53,18 +52,141 @@ class Domain(OrgModelMixin): return random.choice(self.gateways) -class Gateway(BaseAccount): - UNCONNECTIVE_KEY_TMPL = 'asset_unconnective_gateway_{}' - UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL = 'asset_unconnective_gateway_silence_period_{}' - UNCONNECTIVE_SILENCE_PERIOD_BEGIN_VALUE = 60 * 5 +class GatewayMixin: + id: uuid.UUID + port: int + address: str + accounts: QuerySet + private_key_path: str + private_key_obj: paramiko.RSAKey + UNCONNECTED_KEY_TMPL = 'asset_unconnective_gateway_{}' + UNCONNECTED_SILENCE_PERIOD_KEY_TMPL = 'asset_unconnective_gateway_silence_period_{}' + UNCONNECTED_SILENCE_PERIOD_BEGIN_VALUE = 60 * 5 + def set_unconnected(self): + unconnected_key = self.UNCONNECTED_KEY_TMPL.format(self.id) + unconnected_silence_period_key = self.UNCONNECTED_SILENCE_PERIOD_KEY_TMPL.format(self.id) + unconnected_silence_period = cache.get( + unconnected_silence_period_key, self.UNCONNECTED_SILENCE_PERIOD_BEGIN_VALUE + ) + cache.set(unconnected_silence_period_key, unconnected_silence_period * 2) + cache.set(unconnected_key, unconnected_silence_period, unconnected_silence_period) + + def set_connective(self): + unconnected_key = self.UNCONNECTED_KEY_TMPL.format(self.id) + unconnected_silence_period_key = self.UNCONNECTED_SILENCE_PERIOD_KEY_TMPL.format(self.id) + + cache.delete(unconnected_key) + cache.delete(unconnected_silence_period_key) + + def get_is_unconnected(self): + unconnected_key = self.UNCONNECTED_KEY_TMPL.format(self.id) + return cache.get(unconnected_key, False) + + @property + def is_connective(self): + return not self.get_is_unconnected() + + @is_connective.setter + def is_connective(self, value): + if value: + self.set_connective() + else: + self.set_unconnected() + + def test_connective(self, local_port=None): + # TODO 走ansible runner + if local_port is None: + local_port = self.port + + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + proxy = paramiko.SSHClient() + proxy.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + proxy.connect(self.address, port=self.port, + username=self.username, + password=self.password, + pkey=self.private_key_obj) + except(paramiko.AuthenticationException, + paramiko.BadAuthenticationType, + paramiko.SSHException, + paramiko.ChannelException, + paramiko.ssh_exception.NoValidConnectionsError, + socket.gaierror) as e: + err = str(e) + if err.startswith('[Errno None] Unable to connect to port'): + err = _('Unable to connect to port {port} on {address}') + err = err.format(port=self.port, ip=self.address) + elif err == 'Authentication failed.': + err = _('Authentication failed') + elif err == 'Connect failed': + err = _('Connect failed') + self.is_connective = False + return False, err + + try: + sock = proxy.get_transport().open_channel( + 'direct-tcpip', ('127.0.0.1', local_port), ('127.0.0.1', 0) + ) + client.connect("127.0.0.1", port=local_port, + username=self.username, + password=self.password, + key_filename=self.private_key_path, + sock=sock, + timeout=5) + except (paramiko.SSHException, + paramiko.ssh_exception.SSHException, + paramiko.ChannelException, + paramiko.AuthenticationException, + TimeoutError) as e: + + err = getattr(e, 'text', str(e)) + if err == 'Connect failed': + err = _('Connect failed') + self.is_connective = False + return False, err + finally: + client.close() + self.is_connective = True + return True, None + + @lazyproperty + def username(self): + account = self.accounts.all().first() + if account: + return account.username + logger.error(f'Gateway {self} has no account') + return '' + + def get_secret(self, secret_type): + account = self.accounts.filter(secret_type=secret_type).first() + if account: + return account.secret + logger.error(f'Gateway {self} has no {secret_type} account') + + @lazyproperty + def password(self): + secret_type = SecretType.PASSWORD + return self.get_secret(secret_type) + + @lazyproperty + def private_key(self): + secret_type = SecretType.SSH_KEY + return self.get_secret(secret_type) + + +class Gateway(BaseAccount): class Protocol(models.TextChoices): ssh = 'ssh', 'SSH' name = models.CharField(max_length=128, verbose_name='Name') ip = models.CharField(max_length=128, verbose_name=_('IP'), db_index=True) port = models.IntegerField(default=22, verbose_name=_('Port')) - protocol = models.CharField(choices=Protocol.choices, max_length=16, default=Protocol.ssh, verbose_name=_("Protocol")) + protocol = models.CharField( + choices=Protocol.choices, max_length=16, default=Protocol.ssh, verbose_name=_("Protocol") + ) domain = models.ForeignKey(Domain, on_delete=models.CASCADE, verbose_name=_("Domain")) comment = models.CharField(max_length=128, blank=True, null=True, verbose_name=_("Comment")) is_active = models.BooleanField(default=True, verbose_name=_("Is active")) @@ -85,91 +207,3 @@ class Gateway(BaseAccount): permissions = [ ('test_gateway', _('Test gateway')) ] - - def set_unconnective(self): - unconnective_key = self.UNCONNECTIVE_KEY_TMPL.format(self.id) - unconnective_silence_period_key = self.UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL.format(self.id) - - unconnective_silence_period = cache.get(unconnective_silence_period_key, - self.UNCONNECTIVE_SILENCE_PERIOD_BEGIN_VALUE) - cache.set(unconnective_silence_period_key, unconnective_silence_period * 2) - cache.set(unconnective_key, unconnective_silence_period, unconnective_silence_period) - - def set_connective(self): - unconnective_key = self.UNCONNECTIVE_KEY_TMPL.format(self.id) - unconnective_silence_period_key = self.UNCONNECTIVE_SILENCE_PERIOD_KEY_TMPL.format(self.id) - - cache.delete(unconnective_key) - cache.delete(unconnective_silence_period_key) - - def get_is_unconnective(self): - unconnective_key = self.UNCONNECTIVE_KEY_TMPL.format(self.id) - return cache.get(unconnective_key, False) - - @property - def is_connective(self): - return not self.get_is_unconnective() - - @is_connective.setter - def is_connective(self, value): - if value: - self.set_connective() - else: - self.set_unconnective() - - def test_connective(self, local_port=None): - if local_port is None: - local_port = self.port - - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - proxy = paramiko.SSHClient() - proxy.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - try: - proxy.connect(self.ip, port=self.port, - username=self.username, - password=self.password, - pkey=self.private_key_obj) - except(paramiko.AuthenticationException, - paramiko.BadAuthenticationType, - paramiko.SSHException, - paramiko.ChannelException, - paramiko.ssh_exception.NoValidConnectionsError, - socket.gaierror) as e: - err = str(e) - if err.startswith('[Errno None] Unable to connect to port'): - err = _('Unable to connect to port {port} on {address}') - err = err.format(port=self.port, ip=self.ip) - elif err == 'Authentication failed.': - err = _('Authentication failed') - elif err == 'Connect failed': - err = _('Connect failed') - self.is_connective = False - return False, err - - try: - sock = proxy.get_transport().open_channel( - 'direct-tcpip', ('127.0.0.1', local_port), ('127.0.0.1', 0) - ) - client.connect("127.0.0.1", port=local_port, - username=self.username, - password=self.password, - key_filename=self.private_key_file, - sock=sock, - timeout=5) - except (paramiko.SSHException, - paramiko.ssh_exception.SSHException, - paramiko.ChannelException, - paramiko.AuthenticationException, - TimeoutError) as e: - - err = getattr(e, 'text', str(e)) - if err == 'Connect failed': - err = _('Connect failed') - self.is_connective = False - return False, err - finally: - client.close() - self.is_connective = True - return True, None diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 82fd9433f..37d17c814 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -3,11 +3,9 @@ from rest_framework import serializers from django.utils.translation import ugettext_lazy as _ -from common.validators import alphanumeric from orgs.mixins.serializers import BulkOrgResourceModelSerializer from common.drf.serializers import SecretReadableMixin -from ..models import Domain, Gateway -from .base import AuthValidateMixin +from ..models import Domain, Asset class DomainSerializer(BulkOrgResourceModelSerializer): @@ -35,32 +33,23 @@ class DomainSerializer(BulkOrgResourceModelSerializer): @staticmethod def get_gateway_count(obj): - return obj.gateway_set.all().count() + return obj.gateways.count() -class GatewaySerializer(AuthValidateMixin, BulkOrgResourceModelSerializer): +class GatewaySerializer(BulkOrgResourceModelSerializer): is_connective = serializers.BooleanField(required=False, label=_('Connectivity')) class Meta: - model = Gateway - fields_mini = ['id', 'username'] - fields_write_only = [ - 'password', 'private_key', 'public_key', 'passphrase' - ] - fields_small = fields_mini + fields_write_only + [ - 'ip', 'port', 'protocol', + model = Asset + fields_mini = ['id'] + fields_small = fields_mini + [ + 'address', 'port', 'protocol', 'is_active', 'is_connective', 'date_created', 'date_updated', 'created_by', 'comment', ] fields_fk = ['domain'] fields = fields_small + fields_fk - extra_kwargs = { - 'username': {"validators": [alphanumeric]}, - 'password': {'write_only': True}, - 'private_key': {"write_only": True}, - 'public_key': {"write_only": True}, - } class GatewayWithAuthSerializer(SecretReadableMixin, GatewaySerializer): diff --git a/apps/authentication/serializers/connection_token.py b/apps/authentication/serializers/connection_token.py index 5011d73a6..fd3895921 100644 --- a/apps/authentication/serializers/connection_token.py +++ b/apps/authentication/serializers/connection_token.py @@ -1,7 +1,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from assets.models import Asset, Gateway, Domain, CommandFilterRule, Account, Platform +from assets.models import Asset, Domain, CommandFilterRule, Account, Platform from authentication.models import ConnectionToken from common.utils import pretty_string from common.utils.random import random_string @@ -130,8 +130,8 @@ class ConnectionTokenGatewaySerializer(serializers.ModelSerializer): """ Gateway """ class Meta: - model = Gateway - fields = ['id', 'ip', 'port', 'username', 'password', 'private_key'] + model = Asset + fields = ['id', 'address', 'port', 'username', 'password', 'private_key'] class ConnectionTokenDomainSerializer(serializers.ModelSerializer): diff --git a/apps/orgs/api.py b/apps/orgs/api.py index 16fde4d69..da9b1530f 100644 --- a/apps/orgs/api.py +++ b/apps/orgs/api.py @@ -14,7 +14,7 @@ from .serializers import ( ) from users.models import User, UserGroup from assets.models import ( - Asset, Domain, Label, Node, Gateway, + Asset, Domain, Label, Node, CommandFilter, CommandFilterRule, GatheredUser ) from perms.models import AssetPermission @@ -27,7 +27,7 @@ logger = get_logger(__file__) # 部分 org 相关的 model，需要清空这些数据之后才能删除该组织 org_related_models = [ - User, UserGroup, Asset, Label, Domain, Gateway, Node, Label, + User, UserGroup, Asset, Label, Domain, Node, Label, CommandFilter, CommandFilterRule, GatheredUser, AssetPermission, ] diff --git a/apps/orgs/caches.py b/apps/orgs/caches.py index 5df387c91..a17cd832b 100644 --- a/apps/orgs/caches.py +++ b/apps/orgs/caches.py @@ -6,7 +6,7 @@ from orgs.utils import current_org, tmp_to_org from common.cache import Cache, IntegerField from common.utils import get_logger from users.models import UserGroup, User -from assets.models import Node, Domain, Gateway, Asset, Account +from assets.models import Node, Domain, Asset, Account from terminal.models import Session from perms.models import AssetPermission @@ -54,7 +54,7 @@ class OrgResourceStatisticsCache(OrgRelatedCache): nodes_amount = IntegerField(queryset=Node.objects) accounts_amount = IntegerField(queryset=Account.objects) domains_amount = IntegerField(queryset=Domain.objects) - gateways_amount = IntegerField(queryset=Gateway.objects) + # gateways_amount = IntegerField(queryset=Gateway.objects) asset_perms_amount = IntegerField(queryset=AssetPermission.objects) total_count_online_users = IntegerField() diff --git a/apps/orgs/signal_handlers/cache.py b/apps/orgs/signal_handlers/cache.py index 1e3343931..b3e06362d 100644 --- a/apps/orgs/signal_handlers/cache.py +++ b/apps/orgs/signal_handlers/cache.py @@ -8,7 +8,7 @@ from users.models import UserGroup, User from users.signals import pre_user_leave_org from terminal.models import Session from rbac.models import OrgRoleBinding, SystemRoleBinding, RoleBinding -from assets.models import Asset, Domain, Gateway +from assets.models import Asset, Domain from orgs.caches import OrgResourceStatisticsCache from orgs.utils import current_org from common.utils import get_logger @@ -75,7 +75,6 @@ def on_user_delete_refresh_cache(sender, instance, **kwargs): class OrgResourceStatisticsRefreshUtil: model_cache_field_mapper = { AssetPermission: ['asset_perms_amount'], - Gateway: ['gateways_amount'], Domain: ['domains_amount'], Node: ['nodes_amount'], Asset: ['assets_amount'],