diff --git a/.dockerignore b/.dockerignore index 9537605a4..1269698fe 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,6 @@ celerybeat.pid .vagrant/ apps/xpack/.git .history/ -.idea \ No newline at end of file +.idea +.venv/ +.env \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index da82b5fac..e69de29bb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +0,0 @@ -*.mmdb filter=lfs diff=lfs merge=lfs -text -*.mo filter=lfs diff=lfs merge=lfs -text -*.ipdb filter=lfs diff=lfs merge=lfs -text -leak_passwords.db filter=lfs diff=lfs merge=lfs -text diff --git a/.github/dependabot.yml b/.github/dependabot.yml.bak similarity index 54% rename from .github/dependabot.yml rename to .github/dependabot.yml.bak index 1e4a9e3c3..6a1311ced 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml.bak @@ -1,10 +1,14 @@ version: 2 updates: - - package-ecosystem: "pip" + - package-ecosystem: "uv" directory: "/" schedule: interval: "weekly" day: "monday" time: "09:30" timezone: "Asia/Shanghai" - target-branch: dev \ No newline at end of file + target-branch: dev + groups: + python-dependencies: + patterns: + - "*" diff --git a/.gitignore b/.gitignore index 107a42d46..70ecb4d8c 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ test.py .test/ *.mo apps.iml +*.db +*.mmdb +*.ipdb diff --git a/Dockerfile b/Dockerfile index ff7a40e79..e897fc84d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM jumpserver/core-base:20250415_032719 AS stage-build +FROM jumpserver/core-base:20250509_094529 AS stage-build ARG VERSION diff --git a/Dockerfile-base b/Dockerfile-base index 34219bf65..9f893a1d7 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -1,6 +1,6 @@ FROM python:3.11-slim-bullseye ARG TARGETARCH - +COPY --from=ghcr.io/astral-sh/uv:0.6.14 /uv /uvx /usr/local/bin/ # Install APT dependencies ARG DEPENDENCIES=" \ ca-certificates \ @@ -43,18 +43,19 @@ WORKDIR /opt/jumpserver ARG PIP_MIRROR=https://pypi.org/simple ENV POETRY_PYPI_MIRROR_URL=${PIP_MIRROR} ENV ANSIBLE_COLLECTIONS_PATHS=/opt/py3/lib/python3.11/site-packages/ansible_collections +ENV LANG=en_US.UTF-8 \ + PATH=/opt/py3/bin:$PATH + +ENV UV_LINK_MODE=copy RUN --mount=type=cache,target=/root/.cache \ - --mount=type=bind,source=poetry.lock,target=poetry.lock \ --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=utils/clean_site_packages.sh,target=clean_site_packages.sh \ + --mount=type=bind,source=requirements/clean_site_packages.sh,target=clean_site_packages.sh \ --mount=type=bind,source=requirements/collections.yml,target=collections.yml \ + --mount=type=bind,source=requirements/static_files.sh,target=utils/static_files.sh \ set -ex \ - && python3 -m venv /opt/py3 \ - && pip install poetry poetry-plugin-pypi-mirror -i ${PIP_MIRROR} \ - && . /opt/py3/bin/activate \ - && poetry config virtualenvs.create false \ - && poetry install --no-cache --only main \ - && ansible-galaxy collection install -r collections.yml --force --ignore-certs \ - && bash clean_site_packages.sh \ - && poetry cache clear pypi --all + && uv venv \ + && uv pip install -i${PIP_MIRROR} -r pyproject.toml \ + && ln -sf $(pwd)/.venv /opt/py3 \ + && bash utils/static_files.sh \ + && bash clean_site_packages.sh diff --git a/Dockerfile-ee b/Dockerfile-ee index 7fbf5ccda..565b6c579 100644 --- a/Dockerfile-ee +++ b/Dockerfile-ee @@ -24,11 +24,7 @@ RUN set -ex \ WORKDIR /opt/jumpserver ARG PIP_MIRROR=https://pypi.org/simple -ENV POETRY_PYPI_MIRROR_URL=${PIP_MIRROR} -COPY poetry.lock pyproject.toml ./ -RUN set -ex \ - && . /opt/py3/bin/activate \ - && pip install poetry poetry-plugin-pypi-mirror -i ${PIP_MIRROR} \ - && poetry install --only xpack \ - && poetry cache clear pypi --all + +RUN set -ex \ + && uv pip install -i${PIP_MIRROR} --group xpack diff --git a/README.md b/README.md index 7c659b2ae..b3a7c778e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ ## An open-source PAM tool (Bastion Host) [![][license-shield]][license-link] +[![][docs-shield]][docs-link] +[![][deepwiki-shield]][deepwiki-link] [![][discord-shield]][discord-link] [![][docker-shield]][docker-link] [![][github-release-shield]][github-release-link] @@ -101,6 +103,7 @@ Unless required by applicable law or agreed to in writing, software distributed [docs-link]: https://jumpserver.com/docs [discord-link]: https://discord.com/invite/W6vYXmAQG2 +[deepwiki-link]: https://deepwiki.com/jumpserver/jumpserver/ [contributing-link]: https://github.com/jumpserver/jumpserver/blob/dev/CONTRIBUTING.md @@ -111,8 +114,10 @@ Unless required by applicable law or agreed to in writing, software distributed [github-issues-link]: https://github.com/jumpserver/jumpserver/issues +[docs-shield]: https://img.shields.io/badge/documentation-148F76 [github-release-shield]: https://img.shields.io/github/v/release/jumpserver/jumpserver -[github-stars-shield]: https://img.shields.io/github/stars/jumpserver/jumpserver?color=%231890FF&style=flat-square +[github-stars-shield]: https://img.shields.io/github/stars/jumpserver/jumpserver?color=%231890FF&style=flat-square    [docker-shield]: https://img.shields.io/docker/pulls/jumpserver/jms_all.svg [license-shield]: https://img.shields.io/github/license/jumpserver/jumpserver +[deepwiki-shield]: https://img.shields.io/badge/deepwiki-devin?color=blue [discord-shield]: https://img.shields.io/discord/1194233267294052363?style=flat&logo=discord&logoColor=%23f5f5f5&labelColor=%235462eb&color=%235462eb diff --git a/apps/accounts/api/account/application.py b/apps/accounts/api/account/application.py index a395414b4..898a432a5 100644 --- a/apps/accounts/api/account/application.py +++ b/apps/accounts/api/account/application.py @@ -62,8 +62,7 @@ class IntegrationApplicationViewSet(OrgBulkModelViewSet): ) def get_once_secret(self, request, *args, **kwargs): instance = self.get_object() - secret = instance.get_secret() - return Response(data={'id': instance.id, 'secret': secret}) + return Response(data={'id': instance.id, 'secret': instance.secret}) @action(['GET'], detail=False, url_path='account-secret', permission_classes=[RBACPermission]) diff --git a/apps/accounts/automations/base/manager.py b/apps/accounts/automations/base/manager.py index bb94a2213..4c8263e5d 100644 --- a/apps/accounts/automations/base/manager.py +++ b/apps/accounts/automations/base/manager.py @@ -10,7 +10,7 @@ from accounts.models import BaseAccountQuerySet from accounts.utils import SecretGenerator from assets.automations.base.manager import BasePlaybookManager from assets.const import HostTypes -from common.db.utils import safe_db_connection +from common.db.utils import safe_atomic_db_connection from common.utils import get_logger logger = get_logger(__name__) @@ -170,7 +170,7 @@ class BaseChangeSecretPushManager(AccountBasePlaybookManager): ) super().on_host_success(host, result) - with safe_db_connection(): + with safe_atomic_db_connection(): account.save(update_fields=['secret', 'date_updated', 'date_change_secret', 'change_secret_status']) self.save_record(recorder) @@ -198,6 +198,6 @@ class BaseChangeSecretPushManager(AccountBasePlaybookManager): ) super().on_host_error(host, error, result) - with safe_db_connection(): + with safe_atomic_db_connection(): account.save(update_fields=['change_secret_status', 'date_change_secret', 'date_updated']) self.save_record(recorder) diff --git a/apps/accounts/automations/check_account/leak_passwords.db b/apps/accounts/automations/check_account/leak_passwords.db deleted file mode 100644 index 7f2c72ae5..000000000 --- a/apps/accounts/automations/check_account/leak_passwords.db +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a2805a0264fc07ae597704841ab060edef8bf74654f525bc778cb9195d8cad0e -size 2547712 diff --git a/apps/accounts/automations/check_account/manager.py b/apps/accounts/automations/check_account/manager.py index f4e646dd6..c62b7e5d7 100644 --- a/apps/accounts/automations/check_account/manager.py +++ b/apps/accounts/automations/check_account/manager.py @@ -12,6 +12,7 @@ from accounts.models import Account, AccountRisk, RiskChoice from assets.automations.base.manager import BaseManager from common.const import ConfirmOrIgnore from common.decorators import bulk_create_decorator, bulk_update_decorator +from settings.models import LeakPasswords @bulk_create_decorator(AccountRisk) @@ -157,10 +158,8 @@ class CheckLeakHandler(BaseCheckHandler): if not account.secret: return False - sql = 'SELECT 1 FROM passwords WHERE password = ? LIMIT 1' - self.cursor.execute(sql, (account.secret,)) - leak = self.cursor.fetchone() is not None - return leak + is_exist = LeakPasswords.objects.using('sqlite').filter(password=account.secret).exists() + return is_exist def clean(self): self.cursor.close() diff --git a/apps/accounts/automations/verify_account/manager.py b/apps/accounts/automations/verify_account/manager.py index 4135d742a..1cb44e5e1 100644 --- a/apps/accounts/automations/verify_account/manager.py +++ b/apps/accounts/automations/verify_account/manager.py @@ -85,6 +85,7 @@ class VerifyAccountManager(AccountBasePlaybookManager): def on_host_error(self, host, error, result): account = self.host_account_mapper.get(host) try: - account.set_connectivity(Connectivity.ERR) + error_tp = account.get_err_connectivity(error) + account.set_connectivity(error_tp) except Exception as e: print(f'\033[31m Update account {account.name} connectivity failed: {e} \033[0m\n') diff --git a/apps/accounts/migrations/0005_accountrisk_backupaccountautomation_and_more.py b/apps/accounts/migrations/0005_accountrisk_backupaccountautomation_and_more.py index 4eeaecd10..eba3068b4 100644 --- a/apps/accounts/migrations/0005_accountrisk_backupaccountautomation_and_more.py +++ b/apps/accounts/migrations/0005_accountrisk_backupaccountautomation_and_more.py @@ -629,10 +629,15 @@ class Migration(migrations.Migration): name="connectivity", field=models.CharField( choices=[ - ("-", "Unknown"), - ("na", "N/A"), - ("ok", "OK"), - ("err", "Error"), + ('-', 'Unknown'), + ('na', 'N/A'), + ('ok', 'OK'), + ('err', 'Error'), + ('auth_err', 'Authentication error'), + ('password_err', 'Invalid password error'), + ('openssh_key_err', 'OpenSSH key error'), + ('ntlm_err', 'NTLM credentials rejected error'), + ('create_temp_err', 'Create temporary error') ], default="-", max_length=16, diff --git a/apps/accounts/migrations/0007_alter_account_connectivity.py b/apps/accounts/migrations/0007_alter_account_connectivity.py new file mode 100644 index 000000000..d51b95651 --- /dev/null +++ b/apps/accounts/migrations/0007_alter_account_connectivity.py @@ -0,0 +1,29 @@ +# Generated by Django 4.1.13 on 2025-05-06 10:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('accounts', '0006_alter_accountrisk_username_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='account', + name='connectivity', + field=models.CharField(choices=[ + ('-', 'Unknown'), + ('na', 'N/A'), + ('ok', 'OK'), + ('err', 'Error'), + ('rdp_err', 'RDP error'), + ('auth_err', 'Authentication error'), + ('password_err', 'Invalid password error'), + ('openssh_key_err', 'OpenSSH key error'), + ('ntlm_err', 'NTLM credentials rejected error'), + ('create_temp_err', 'Create temporary error') + ], + default='-', max_length=16, verbose_name='Connectivity'), + ), + ] diff --git a/apps/accounts/models/account.py b/apps/accounts/models/account.py index 060948fe1..2624e921a 100644 --- a/apps/accounts/models/account.py +++ b/apps/accounts/models/account.py @@ -166,9 +166,12 @@ class Account(AbsConnectivity, LabeledMixin, BaseAccount, JSONFilterMixin): return self.ds.domain_name return '' + def username_has_domain(self): + return '@' in self.username or '\\' in self.username + @property def full_username(self): - if self.ds_domain: + if not self.username_has_domain() and self.ds_domain: return '{}@{}'.format(self.username, self.ds_domain) return self.username diff --git a/apps/accounts/serializers/account/service.py b/apps/accounts/serializers/account/service.py index d94d2417e..4502e81a8 100644 --- a/apps/accounts/serializers/account/service.py +++ b/apps/accounts/serializers/account/service.py @@ -5,6 +5,7 @@ from rest_framework import serializers from accounts.models import IntegrationApplication from acls.serializers.rules import ip_group_child_validator, ip_group_help_text from common.serializers.fields import JSONManyToManyField +from common.utils import random_string from orgs.mixins.serializers import BulkOrgResourceModelSerializer @@ -37,6 +38,10 @@ class IntegrationApplicationSerializer(BulkOrgResourceModelSerializer): data['logo'] = static('img/logo.png') return data + def validate(self, attrs): + attrs['secret'] = random_string(36) + return attrs + class IntegrationAccountSecretSerializer(serializers.Serializer): asset = serializers.CharField(required=False, allow_blank=True) diff --git a/apps/accounts/tasks/automation.py b/apps/accounts/tasks/automation.py index 3d1eb0883..552e51f34 100644 --- a/apps/accounts/tasks/automation.py +++ b/apps/accounts/tasks/automation.py @@ -107,16 +107,18 @@ def execute_automation_record_task(record_ids, tp): ) @register_as_period_task(crontab=CRONTAB_AT_AM_THREE) def clean_change_secret_and_push_record_period(): - from accounts.models import ChangeSecretRecord + from accounts.models import ChangeSecretRecord, PushSecretRecord print('Start clean change secret and push record period') with tmp_to_root_org(): now = timezone.now() days = get_log_keep_day('ACCOUNT_CHANGE_SECRET_RECORD_KEEP_DAYS') - expired_day = now - datetime.timedelta(days=days) - records = ChangeSecretRecord.objects.filter( - date_updated__lt=expired_day - ).filter( - Q(execution__isnull=True) | Q(asset__isnull=True) | Q(account__isnull=True) - ) + expired_time = now - datetime.timedelta(days=days) - records.delete() + null_related_q = Q(execution__isnull=True) | Q(asset__isnull=True) | Q(account__isnull=True) + expired_q = Q(date_updated__lt=expired_time) + + ChangeSecretRecord.objects.filter(null_related_q).delete() + ChangeSecretRecord.objects.filter(expired_q).delete() + + PushSecretRecord.objects.filter(null_related_q).delete() + PushSecretRecord.objects.filter(expired_q).delete() diff --git a/apps/acls/serializers/command_acl.py b/apps/acls/serializers/command_acl.py index 13a90129d..11df8914c 100644 --- a/apps/acls/serializers/command_acl.py +++ b/apps/acls/serializers/command_acl.py @@ -32,9 +32,9 @@ class CommandFilterACLSerializer(BaseSerializer, BulkOrgResourceModelSerializer) class Meta(BaseSerializer.Meta): model = CommandFilterACL fields = BaseSerializer.Meta.fields + ['command_groups'] - action_choices_exclude = [ActionChoices.notice, - ActionChoices.face_verify, - ActionChoices.face_online] + action_choices_exclude = [ + ActionChoices.notice, ActionChoices.face_verify, ActionChoices.face_online + ] class CommandReviewSerializer(serializers.Serializer): diff --git a/apps/acls/serializers/connect_method.py b/apps/acls/serializers/connect_method.py index 917933dcb..40f03e216 100644 --- a/apps/acls/serializers/connect_method.py +++ b/apps/acls/serializers/connect_method.py @@ -14,5 +14,6 @@ class ConnectMethodACLSerializer(BaseSerializer, BulkOrgResourceModelSerializer) if i not in ['assets', 'accounts'] ] action_choices_exclude = BaseSerializer.Meta.action_choices_exclude + [ - ActionChoices.review, ActionChoices.accept, ActionChoices.notice + ActionChoices.review, ActionChoices.accept, ActionChoices.notice, + ActionChoices.face_verify, ActionChoices.face_online ] diff --git a/apps/assets/api/__init__.py b/apps/assets/api/__init__.py index 3f49c2b62..947383513 100644 --- a/apps/assets/api/__init__.py +++ b/apps/assets/api/__init__.py @@ -1,10 +1,10 @@ from .asset import * from .category import * -from .domain import * from .favorite_asset import * from .mixin import * +from .my_asset import * from .node import * from .platform import * from .protocol import * from .tree import * -from .my_asset import * +from .zone import * diff --git a/apps/assets/api/asset/asset.py b/apps/assets/api/asset/asset.py index 0bc4df0f3..06818a0f3 100644 --- a/apps/assets/api/asset/asset.py +++ b/apps/assets/api/asset/asset.py @@ -37,12 +37,12 @@ class AssetFilterSet(BaseFilterSet): platform = drf_filters.CharFilter(method='filter_platform') is_gateway = drf_filters.BooleanFilter(method='filter_is_gateway') exclude_platform = drf_filters.CharFilter(field_name="platform__name", lookup_expr='exact', exclude=True) - domain = drf_filters.CharFilter(method='filter_domain') + zone = drf_filters.CharFilter(method='filter_zone') type = drf_filters.CharFilter(field_name="platform__type", lookup_expr="exact") category = drf_filters.CharFilter(field_name="platform__category", lookup_expr="exact") protocols = drf_filters.CharFilter(method='filter_protocols') - domain_enabled = drf_filters.BooleanFilter( - field_name="platform__domain_enabled", lookup_expr="exact" + gateway_enabled = drf_filters.BooleanFilter( + field_name="platform__gateway_enabled", lookup_expr="exact" ) ping_enabled = drf_filters.BooleanFilter( field_name="platform__automation__ping_enabled", lookup_expr="exact" @@ -85,11 +85,11 @@ class AssetFilterSet(BaseFilterSet): return queryset @staticmethod - def filter_domain(queryset, name, value): + def filter_zone(queryset, name, value): if is_uuid(value): - return queryset.filter(domain_id=value) + return queryset.filter(zone_id=value) else: - return queryset.filter(domain__name__contains=value) + return queryset.filter(zone__name__contains=value) @staticmethod def filter_protocols(queryset, name, value): @@ -171,10 +171,10 @@ class AssetViewSet(SuggestionMixin, BaseAssetViewSet): @action(methods=["GET"], detail=True, url_path="gateways") def gateways(self, *args, **kwargs): asset = self.get_object() - if not asset.domain: + if not asset.zone: gateways = Gateway.objects.none() else: - gateways = asset.domain.gateways + gateways = asset.zone.gateways return self.get_paginated_response_from_queryset(gateways) @action(methods=['post'], detail=False, url_path='sync-platform-protocols') diff --git a/apps/assets/api/domain.py b/apps/assets/api/zone.py similarity index 77% rename from apps/assets/api/domain.py rename to apps/assets/api/zone.py index 21cb3b2c7..6e0be3e7f 100644 --- a/apps/assets/api/domain.py +++ b/apps/assets/api/zone.py @@ -9,24 +9,24 @@ from common.utils import get_logger from orgs.mixins.api import OrgBulkModelViewSet from .asset import HostViewSet from .. import serializers -from ..models import Domain, Gateway +from ..models import Zone, Gateway logger = get_logger(__file__) -__all__ = ['DomainViewSet', 'GatewayViewSet', "GatewayTestConnectionApi"] +__all__ = ['ZoneViewSet', 'GatewayViewSet', "GatewayTestConnectionApi"] -class DomainViewSet(OrgBulkModelViewSet): - model = Domain +class ZoneViewSet(OrgBulkModelViewSet): + model = Zone filterset_fields = ("name",) search_fields = filterset_fields serializer_classes = { - 'default': serializers.DomainSerializer, - 'list': serializers.DomainListSerializer, + 'default': serializers.ZoneSerializer, + 'list': serializers.ZoneListSerializer, } def get_serializer_class(self): if self.request.query_params.get('gateway'): - return serializers.DomainWithGatewaySerializer + return serializers.ZoneWithGatewaySerializer return super().get_serializer_class() def partial_update(self, request, *args, **kwargs): @@ -36,8 +36,8 @@ class DomainViewSet(OrgBulkModelViewSet): class GatewayViewSet(HostViewSet): perm_model = Gateway - filterset_fields = ("domain__name", "name", "domain") - search_fields = ("domain__name",) + filterset_fields = ("zone__name", "name", "zone") + search_fields = ("zone__name",) def get_serializer_classes(self): serializer_classes = super().get_serializer_classes() @@ -45,7 +45,7 @@ class GatewayViewSet(HostViewSet): return serializer_classes def get_queryset(self): - queryset = Domain.get_gateway_queryset() + queryset = Zone.get_gateway_queryset() return queryset @@ -55,7 +55,7 @@ class GatewayTestConnectionApi(SingleObjectMixin, APIView): } def get_queryset(self): - queryset = Domain.get_gateway_queryset() + queryset = Zone.get_gateway_queryset() return queryset def post(self, request, *args, **kwargs): diff --git a/apps/assets/automations/base/manager.py b/apps/assets/automations/base/manager.py index 8446dbeb4..e4999d363 100644 --- a/apps/assets/automations/base/manager.py +++ b/apps/assets/automations/base/manager.py @@ -17,7 +17,7 @@ from sshtunnel import SSHTunnelForwarder from assets.automations.methods import platform_automation_methods from common.const import Status -from common.db.utils import safe_db_connection +from common.db.utils import safe_atomic_db_connection from common.tasks import send_mail_async from common.utils import get_logger, lazyproperty, is_openssh_format_key, ssh_pubkey_gen from ops.ansible import JMSInventory, DefaultCallback, SuperPlaybookRunner @@ -123,7 +123,7 @@ class BaseManager: self.execution.result = self.result self.execution.status = self.status - with safe_db_connection(): + with safe_atomic_db_connection(): self.execution.save() def print_summary(self): diff --git a/apps/assets/automations/gather_facts/format_asset_info.py b/apps/assets/automations/gather_facts/format_asset_info.py index d3184bf59..0aed98b0e 100644 --- a/apps/assets/automations/gather_facts/format_asset_info.py +++ b/apps/assets/automations/gather_facts/format_asset_info.py @@ -1,3 +1,5 @@ +from collections import Counter + __all__ = ['FormatAssetInfo'] @@ -7,13 +9,37 @@ class FormatAssetInfo: self.tp = tp @staticmethod - def posix_format(info): - for cpu_model in info.get('cpu_model', []): - if cpu_model.endswith('GHz') or cpu_model.startswith("Intel"): - break - else: - cpu_model = '' - info['cpu_model'] = cpu_model[:48] + def get_cpu_model_count(cpus): + try: + models = [cpus[i + 1] + " " + cpus[i + 2] for i in range(0, len(cpus), 3)] + + model_counts = Counter(models) + + result = ', '.join([f"{model} x{count}" for model, count in model_counts.items()]) + except Exception as e: + print(f"Error processing CPU model list: {e}") + result = '' + + return result + + @staticmethod + def get_gpu_model_count(gpus): + try: + model_counts = Counter(gpus) + + result = ', '.join([f"{model} x{count}" for model, count in model_counts.items()]) + except Exception as e: + print(f"Error processing GPU model list: {e}") + result = '' + + return result + + def posix_format(self, info): + cpus = self.get_cpu_model_count(info.get('cpu_model', [])) + gpus = self.get_gpu_model_count(info.get('gpu_model', [])) + + info['gpu_model'] = gpus + info['cpu_model'] = cpus info['cpu_count'] = info.get('cpu_count', 0) return info diff --git a/apps/assets/automations/gather_facts/host/posix/main.yml b/apps/assets/automations/gather_facts/host/posix/main.yml index 0b083c94b..9acfc7e7e 100644 --- a/apps/assets/automations/gather_facts/host/posix/main.yml +++ b/apps/assets/automations/gather_facts/host/posix/main.yml @@ -23,5 +23,16 @@ arch: "{{ ansible_architecture }}" kernel: "{{ ansible_kernel }}" + + - name: Get GPU info with nvidia-smi + shell: | + nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader,nounits + register: gpu_info + ignore_errors: yes + + - name: Merge GPU info into final info + set_fact: + info: "{{ info | combine({'gpu_model': gpu_info.stdout_lines | default([])}) }}" + - debug: var: info diff --git a/apps/assets/automations/ping/manager.py b/apps/assets/automations/ping/manager.py index 6249a40c7..2641d8b6c 100644 --- a/apps/assets/automations/ping/manager.py +++ b/apps/assets/automations/ping/manager.py @@ -37,10 +37,11 @@ class PingManager(BasePlaybookManager): def on_host_error(self, host, error, result): asset, account = self.host_asset_and_account_mapper.get(host) try: - asset.set_connectivity(Connectivity.ERR) + error_tp = asset.get_err_connectivity(error) + asset.set_connectivity(error_tp) if not account: return - account.set_connectivity(Connectivity.ERR) + account.set_connectivity(error_tp) except Exception as e: print(f'\033[31m Update account {account.name} or ' f'update asset {asset.name} connectivity failed: {e} \033[0m\n') diff --git a/apps/assets/const/automation.py b/apps/assets/const/automation.py index e70387cfe..1d48575a3 100644 --- a/apps/assets/const/automation.py +++ b/apps/assets/const/automation.py @@ -7,6 +7,12 @@ class Connectivity(TextChoices): NA = 'na', _('N/A') OK = 'ok', _('OK') ERR = 'err', _('Error') + RDP_ERR = 'rdp_err', _('RDP error') + AUTH_ERR = 'auth_err', _('Authentication error') + PASSWORD_ERR = 'password_err', _('Invalid password error') + OPENSSH_KEY_ERR = 'openssh_key_err', _('OpenSSH key error') + NTLM_ERR = 'ntlm_err', _('NTLM credentials rejected error') + CREATE_TEMPORARY_ERR = 'create_temp_err', _('Create temporary error') class AutomationTypes(TextChoices): diff --git a/apps/assets/const/base.py b/apps/assets/const/base.py index 7df231eb5..a98d661c9 100644 --- a/apps/assets/const/base.py +++ b/apps/assets/const/base.py @@ -37,7 +37,7 @@ class FillType(models.TextChoices): class BaseType(TextChoices): """ 约束应该考虑代是对平台对限制，避免多余对选项，如: mysql 开启 ssh, - 或者开启了也没有作用, 比如 k8s 开启了 domain，目前还不支持 + 或者开启了也没有作用, 比如 k8s 开启了 gateway 目前还不支持 """ @classmethod diff --git a/apps/assets/const/cloud.py b/apps/assets/const/cloud.py index 02410a6a9..51d615ef1 100644 --- a/apps/assets/const/cloud.py +++ b/apps/assets/const/cloud.py @@ -13,11 +13,11 @@ class CloudTypes(BaseType): return { '*': { 'charset_enabled': False, - 'domain_enabled': False, + 'gateway_enabled': False, 'su_enabled': False, }, cls.K8S: { - 'domain_enabled': True, + 'gateway_enabled': True, } } diff --git a/apps/assets/const/custom.py b/apps/assets/const/custom.py index 878207f26..f6f54c9ec 100644 --- a/apps/assets/const/custom.py +++ b/apps/assets/const/custom.py @@ -20,7 +20,7 @@ class CustomTypes(BaseType): return { '*': { 'charset_enabled': False, - 'domain_enabled': False, + 'gateway_enabled': False, 'su_enabled': False, }, } diff --git a/apps/assets/const/database.py b/apps/assets/const/database.py index 8acbdbb43..5123c5291 100644 --- a/apps/assets/const/database.py +++ b/apps/assets/const/database.py @@ -20,7 +20,7 @@ class DatabaseTypes(BaseType): return { '*': { 'charset_enabled': False, - 'domain_enabled': True, + 'gateway_enabled': True, 'su_enabled': False, } } diff --git a/apps/assets/const/device.py b/apps/assets/const/device.py index 9336d129d..6454cc3a7 100644 --- a/apps/assets/const/device.py +++ b/apps/assets/const/device.py @@ -19,8 +19,8 @@ class DeviceTypes(BaseType): return { '*': { 'charset_enabled': False, - 'domain_enabled': True, - 'ds_enabled': False, + 'gateway_enabled': True, + 'ds_enabled': True, 'su_enabled': True, 'su_methods': ['enable', 'super', 'super_level'] } diff --git a/apps/assets/const/ds.py b/apps/assets/const/ds.py index c99809646..5883a83d2 100644 --- a/apps/assets/const/ds.py +++ b/apps/assets/const/ds.py @@ -16,7 +16,7 @@ class DirectoryTypes(BaseType): return { '*': { 'charset_enabled': True, - 'domain_enabled': True, + 'gateway_enabled': True, 'ds_enabled': False, 'su_enabled': True, }, diff --git a/apps/assets/const/gpt.py b/apps/assets/const/gpt.py index 65d01ee97..b3b078031 100644 --- a/apps/assets/const/gpt.py +++ b/apps/assets/const/gpt.py @@ -11,7 +11,7 @@ class GPTTypes(BaseType): return { '*': { 'charset_enabled': False, - 'domain_enabled': False, + 'gateway_enabled': False, 'su_enabled': False, } } diff --git a/apps/assets/const/host.py b/apps/assets/const/host.py index cd0103f8c..1a9f7e485 100644 --- a/apps/assets/const/host.py +++ b/apps/assets/const/host.py @@ -18,7 +18,7 @@ class HostTypes(BaseType): '*': { 'charset_enabled': True, 'charset': 'utf-8', # default - 'domain_enabled': True, + 'gateway_enabled': True, 'su_enabled': True, 'ds_enabled': True, 'su_methods': ['sudo', 'su', 'only_sudo', 'only_su'], @@ -81,7 +81,7 @@ class HostTypes(BaseType): {'name': 'Linux'}, { 'name': GATEWAY_NAME, - 'domain_enabled': True, + 'gateway_enabled': True, } ], cls.UNIX: [ diff --git a/apps/assets/const/protocol.py b/apps/assets/const/protocol.py index dde7ccabe..2e3b751fd 100644 --- a/apps/assets/const/protocol.py +++ b/apps/assets/const/protocol.py @@ -344,6 +344,20 @@ class Protocol(ChoicesMixin, models.TextChoices): if not xpack_enabled and config.get('xpack', False): continue protocols.append(protocol) + + from assets.models.platform import PlatformProtocol + custom_protocols = ( + PlatformProtocol.objects + .filter(platform__category='custom') + .values_list('name', flat=True) + .distinct() + ) + for protocol in custom_protocols: + if protocol not in protocols: + if not protocol: + continue + label = protocol[0].upper() + protocol[1:] + protocols.append({'label': label, 'value': protocol}) return protocols @classmethod diff --git a/apps/assets/const/types.py b/apps/assets/const/types.py index dfa8ba399..0600daab8 100644 --- a/apps/assets/const/types.py +++ b/apps/assets/const/types.py @@ -312,7 +312,7 @@ class AllTypes(ChoicesMixin): 'category': category, 'type': tp, 'internal': True, 'charset': constraints.get('charset', 'utf-8'), - 'domain_enabled': constraints.get('domain_enabled', False), + 'gateway_enabled': constraints.get('gateway_enabled', False), 'su_enabled': constraints.get('su_enabled', False), } if data['su_enabled'] and data.get('su_methods'): diff --git a/apps/assets/const/web.py b/apps/assets/const/web.py index 42ea995ac..ab481e609 100644 --- a/apps/assets/const/web.py +++ b/apps/assets/const/web.py @@ -11,7 +11,7 @@ class WebTypes(BaseType): return { '*': { 'charset_enabled': False, - 'domain_enabled': False, + 'gateway_enabled': False, 'su_enabled': False, } } diff --git a/apps/assets/migrations/0001_initial.py b/apps/assets/migrations/0001_initial.py index d62a3cca9..6d057c9cf 100644 --- a/apps/assets/migrations/0001_initial.py +++ b/apps/assets/migrations/0001_initial.py @@ -29,8 +29,19 @@ class Migration(migrations.Migration): ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), ('connectivity', - models.CharField(choices=[('-', 'Unknown'), ('na', 'N/A'), ('ok', 'OK'), ('err', 'Error')], - default='-', max_length=16, verbose_name='Connectivity')), + models.CharField( + choices=[ + ('-', 'Unknown'), + ('na', 'N/A'), + ('ok', 'OK'), + ('err', 'Error'), + ('auth_err', 'Authentication error'), + ('password_err', 'Invalid password error'), + ('openssh_key_err', 'OpenSSH key error'), + ('ntlm_err', 'NTLM credentials rejected error'), + ('create_temp_err', 'Create temporary error') + ], + default='-', max_length=16, verbose_name='Connectivity')), ('date_verified', models.DateTimeField(null=True, verbose_name='Date verified')), ('name', models.CharField(max_length=128, verbose_name='Name')), ('address', models.CharField(db_index=True, max_length=767, verbose_name='Address')), @@ -46,7 +57,8 @@ class Migration(migrations.Migration): ('match_asset', 'Can match asset'), ('change_assetnodes', 'Can change asset nodes')], }, bases=( - assets.models.asset.common.NodesRelationMixin, assets.models.asset.common.JSONFilterMixin, models.Model), + assets.models.asset.common.NodesRelationMixin, assets.models.asset.common.JSONFilterMixin, + models.Model), ), migrations.CreateModel( name='AutomationExecution', diff --git a/apps/assets/migrations/0002_auto_20180105_1807.py b/apps/assets/migrations/0002_auto_20180105_1807.py index 22c114cf8..b1daeaac9 100644 --- a/apps/assets/migrations/0002_auto_20180105_1807.py +++ b/apps/assets/migrations/0002_auto_20180105_1807.py @@ -1,11 +1,11 @@ # Generated by Django 4.1.13 on 2024-05-09 03:16 -import json -import assets.models.asset.common -from django.db.models import F, Q +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion +from django.db.models import F + +import assets.models.asset.common class Migration(migrations.Migration): @@ -39,22 +39,26 @@ class Migration(migrations.Migration): migrations.AddField( model_name='automationexecution', name='automation', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='executions', to='assets.baseautomation', verbose_name='Automation task'), + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='executions', + to='assets.baseautomation', verbose_name='Automation task'), ), migrations.AddField( model_name='asset', name='domain', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='assets', to='assets.domain', verbose_name='Zone'), + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, + related_name='assets', to='assets.domain', verbose_name='Zone'), ), migrations.AddField( model_name='asset', name='nodes', - field=models.ManyToManyField(default=assets.models.asset.common.default_node, related_name='assets', to='assets.node', verbose_name='Nodes'), + field=models.ManyToManyField(default=assets.models.asset.common.default_node, related_name='assets', + to='assets.node', verbose_name='Nodes'), ), migrations.AddField( model_name='asset', name='platform', - field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='assets', to='assets.platform', verbose_name='Platform'), + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='assets', + to='assets.platform', verbose_name='Platform'), ), migrations.CreateModel( name='AssetBaseAutomation', @@ -71,7 +75,9 @@ class Migration(migrations.Migration): migrations.CreateModel( name='GatherFactsAutomation', fields=[ - ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), + ('baseautomation_ptr', + models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, + primary_key=True, serialize=False, to='assets.baseautomation')), ], options={ 'verbose_name': 'Gather asset facts', @@ -81,7 +87,9 @@ class Migration(migrations.Migration): migrations.CreateModel( name='PingAutomation', fields=[ - ('baseautomation_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='assets.baseautomation')), + ('baseautomation_ptr', + models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, + primary_key=True, serialize=False, to='assets.baseautomation')), ], options={ 'verbose_name': 'Ping asset', diff --git a/apps/assets/migrations/0016_directory_service.py b/apps/assets/migrations/0016_directory_service.py index 39de22fff..334a1b51d 100644 --- a/apps/assets/migrations/0016_directory_service.py +++ b/apps/assets/migrations/0016_directory_service.py @@ -51,7 +51,7 @@ class Migration(migrations.Migration): field=models.ManyToManyField( related_name="assets", to="assets.directoryservice", - verbose_name="Directory services", - ), + verbose_name="Directory service", + ) ), ] diff --git a/apps/assets/migrations/0018_rename_domain_zone.py b/apps/assets/migrations/0018_rename_domain_zone.py new file mode 100644 index 000000000..f63b3d266 --- /dev/null +++ b/apps/assets/migrations/0018_rename_domain_zone.py @@ -0,0 +1,26 @@ +# Generated by Django 4.1.13 on 2025-04-18 08:05 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("assets", "0017_auto_20250407_1124"), + ] + + operations = [ + migrations.RenameField( + model_name="platform", + old_name="domain_enabled", + new_name="gateway_enabled", + ), + migrations.RenameModel( + old_name="Domain", + new_name="Zone", + ), + migrations.RenameField( + model_name="asset", + old_name="domain", + new_name="zone", + ), + ] diff --git a/apps/assets/migrations/0019_alter_asset_connectivity.py b/apps/assets/migrations/0019_alter_asset_connectivity.py new file mode 100644 index 000000000..279cbf01d --- /dev/null +++ b/apps/assets/migrations/0019_alter_asset_connectivity.py @@ -0,0 +1,29 @@ +# Generated by Django 4.1.13 on 2025-05-06 10:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('assets', '0018_rename_domain_zone'), + ] + + operations = [ + migrations.AlterField( + model_name='asset', + name='connectivity', + field=models.CharField( + choices=[ + ('-', 'Unknown'), + ('na', 'N/A'), + ('ok', 'OK'), + ('err', 'Error'), + ('rdp_err', 'RDP error'), + ('auth_err', 'Authentication error'), + ('password_err', 'Invalid password error'), + ('openssh_key_err', 'OpenSSH key error'), + ('ntlm_err', 'NTLM credentials rejected error'), + ('create_temp_err', 'Create temporary error') + ], default='-', max_length=16, verbose_name='Connectivity'), + ), + ] diff --git a/apps/assets/models/__init__.py b/apps/assets/models/__init__.py index dc6ca8e97..3b9c390b7 100644 --- a/apps/assets/models/__init__.py +++ b/apps/assets/models/__init__.py @@ -1,9 +1,10 @@ +# noqa from .base import * from .platform import * from .asset import * from .label import Label from .gateway import * -from .domain import * +from .zone import * # noqa from .node import * from .favorite_asset import * from .automations import * diff --git a/apps/assets/models/asset/common.py b/apps/assets/models/asset/common.py index 93e883045..efd50a3ec 100644 --- a/apps/assets/models/asset/common.py +++ b/apps/assets/models/asset/common.py @@ -168,8 +168,8 @@ class Asset(NodesRelationMixin, LabeledMixin, AbsConnectivity, JSONFilterMixin, platform = models.ForeignKey( Platform, on_delete=models.PROTECT, verbose_name=_("Platform"), related_name='assets' ) - domain = models.ForeignKey( - "assets.Domain", null=True, blank=True, related_name='assets', + zone = models.ForeignKey( + "assets.Zone", null=True, blank=True, related_name='assets', verbose_name=_("Zone"), on_delete=models.SET_NULL ) nodes = models.ManyToManyField( @@ -244,7 +244,7 @@ class Asset(NodesRelationMixin, LabeledMixin, AbsConnectivity, JSONFilterMixin, platform = self.platform auto_config = { 'su_enabled': platform.su_enabled, - 'domain_enabled': platform.domain_enabled, + 'gateway_enabled': platform.gateway_enabled, 'ansible_enabled': False } automation = getattr(self.platform, 'automation', None) @@ -362,11 +362,11 @@ class Asset(NodesRelationMixin, LabeledMixin, AbsConnectivity, JSONFilterMixin, @lazyproperty def gateway(self): - if not self.domain_id: + if not self.zone_id: return - if not self.platform.domain_enabled: + if not self.platform.gateway_enabled: return - return self.domain.select_gateway() + return self.zone.select_gateway() def as_node(self): from assets.models import Node diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 031335b72..2ad3cae8d 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -23,6 +23,27 @@ class AbsConnectivity(models.Model): self.date_verified = timezone.now() self.save(update_fields=['connectivity', 'date_verified']) + @staticmethod + def get_err_connectivity(msg=None): + msg = (msg or '').strip().lower() + + error_map = { + 'rdp connection failed': Connectivity.RDP_ERR, + 'expected openssh key': Connectivity.OPENSSH_KEY_ERR, + 'invalid/incorrect password': Connectivity.PASSWORD_ERR, + 'failed to create temporary': Connectivity.CREATE_TEMPORARY_ERR, + 'ntlm: the specified credentials were rejected by the server': Connectivity.NTLM_ERR, + 'permission denied': Connectivity.AUTH_ERR, + 'authentication failed': Connectivity.AUTH_ERR, + 'authentication failure': Connectivity.AUTH_ERR, + } + + for key, value in error_map.items(): + if key in msg: + return value + + return Connectivity.ERR + @property def is_connective(self): if self.connectivity == Connectivity.OK: diff --git a/apps/assets/models/platform.py b/apps/assets/models/platform.py index ed7816afa..b3a7e845f 100644 --- a/apps/assets/models/platform.py +++ b/apps/assets/models/platform.py @@ -101,7 +101,7 @@ class Platform(LabeledMixin, JMSBaseModel): default=CharsetChoices.utf8, choices=CharsetChoices.choices, max_length=8, verbose_name=_("Charset") ) - domain_enabled = models.BooleanField(default=True, verbose_name=_("Gateway enabled")) + gateway_enabled = models.BooleanField(default=True, verbose_name=_("Gateway enabled")) ds_enabled = models.BooleanField(default=False, verbose_name=_("DS enabled")) # 账号有关的 su_enabled = models.BooleanField(default=False, verbose_name=_("Su enabled")) diff --git a/apps/assets/models/domain.py b/apps/assets/models/zone.py similarity index 86% rename from apps/assets/models/domain.py rename to apps/assets/models/zone.py index 4a9311aed..601edcf74 100644 --- a/apps/assets/models/domain.py +++ b/apps/assets/models/zone.py @@ -12,10 +12,10 @@ from .gateway import Gateway logger = get_logger(__file__) -__all__ = ['Domain'] +__all__ = ['Zone'] -class Domain(LabeledMixin, JMSOrgBaseModel): +class Zone(LabeledMixin, JMSOrgBaseModel): name = models.CharField(max_length=128, verbose_name=_('Name')) class Meta: @@ -39,7 +39,7 @@ class Domain(LabeledMixin, JMSOrgBaseModel): if not gateways: gateways = self.active_gateways if not gateways: - logger.warn(f'Not active gateway, domain={self}, pass') + logger.warning(f'Not active gateway, domain={self}, pass') return None return random.choice(gateways) @@ -49,7 +49,7 @@ class Domain(LabeledMixin, JMSOrgBaseModel): @property def gateways(self): - queryset = self.get_gateway_queryset().filter(domain=self) + queryset = self.get_gateway_queryset().filter(zone=self) return queryset @classmethod diff --git a/apps/assets/pagination.py b/apps/assets/pagination.py index 316029301..d98fdc168 100644 --- a/apps/assets/pagination.py +++ b/apps/assets/pagination.py @@ -32,7 +32,7 @@ class AssetPaginationBase(LimitOffsetPagination): } for k, v in self._request.query_params.items(): if k not in exclude_query_params and v is not None: - logger.warn(f'Not hit node.assets_amount because find a unknown query_param ' + logger.warning(f'Not hit node.assets_amount because find a unknown query_param ' f'`{k}` -> {self._request.get_full_path()}') return super().get_count(queryset) node_assets_count = self.get_count_from_nodes(queryset) diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index b61e2e286..fa639eb48 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -154,7 +154,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer, ResourceLabelsMixin, Writa class Meta: model = Asset - fields_fk = ['domain', 'platform'] + fields_fk = ['zone', 'platform'] fields_mini = ['id', 'name', 'address'] + fields_fk fields_small = fields_mini + ['is_active', 'comment'] fields_m2m = [ @@ -233,7 +233,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer, ResourceLabelsMixin, Writa @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('domain', 'nodes', 'protocols', 'directory_services') \ + queryset = queryset.prefetch_related('zone', 'nodes', 'protocols', 'directory_services') \ .prefetch_related('platform', 'platform__automation') \ .annotate(category=F("platform__category")) \ .annotate(type=F("platform__type")) \ @@ -271,9 +271,9 @@ class AssetSerializer(BulkOrgResourceModelSerializer, ResourceLabelsMixin, Writa raise serializers.ValidationError({'platform': _("Platform not exist")}) return platform - def validate_domain(self, value): + def validate_zone(self, value): platform = self._asset_platform - if platform.domain_enabled: + if platform.gateway_enabled: return value else: return None diff --git a/apps/assets/serializers/asset/info/gathered.py b/apps/assets/serializers/asset/info/gathered.py index 71722c072..6474d08fc 100644 --- a/apps/assets/serializers/asset/info/gathered.py +++ b/apps/assets/serializers/asset/info/gathered.py @@ -6,7 +6,7 @@ class HostGatheredInfoSerializer(serializers.Serializer): vendor = serializers.CharField(max_length=64, required=False, allow_blank=True, label=_('Vendor')) model = serializers.CharField(max_length=54, required=False, allow_blank=True, label=_('Model')) sn = serializers.CharField(max_length=128, required=False, allow_blank=True, label=_('Serial number')) - cpu_model = serializers.CharField(max_length=64, allow_blank=True, required=False, label=_('CPU model')) + cpu_model = serializers.CharField(allow_blank=True, required=False, label=_('CPU model')) cpu_count = serializers.CharField(max_length=64, required=False, allow_blank=True, label=_('CPU count')) cpu_cores = serializers.CharField(max_length=64, required=False, allow_blank=True, label=_('CPU cores')) cpu_vcpus = serializers.CharField(max_length=64, required=False, allow_blank=True, label=_('CPU vcpus')) @@ -17,6 +17,8 @@ class HostGatheredInfoSerializer(serializers.Serializer): distribution_version = serializers.CharField(max_length=16, allow_blank=True, required=False, label=_('OS version')) arch = serializers.CharField(max_length=16, allow_blank=True, required=False, label=_('OS arch')) + gpu_model = serializers.CharField(allow_blank=True, required=False, label=_('GPU model')) + category_gathered_serializer_map = { 'host': HostGatheredInfoSerializer, diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 9181db18c..4dee96a68 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -8,12 +8,12 @@ from common.serializers import ResourceLabelsMixin from common.serializers.fields import ObjectRelatedField from orgs.mixins.serializers import BulkOrgResourceModelSerializer from .gateway import GatewayWithAccountSecretSerializer -from ..models import Domain, Gateway +from ..models import Zone, Gateway -__all__ = ['DomainSerializer', 'DomainWithGatewaySerializer', 'DomainListSerializer'] +__all__ = ['ZoneSerializer', 'ZoneWithGatewaySerializer', 'ZoneListSerializer'] -class DomainSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer): +class ZoneSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer): gateways = ObjectRelatedField( many=True, required=False, label=_('Gateway'), queryset=Gateway.objects, help_text=_( @@ -23,7 +23,7 @@ class DomainSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer): assets_amount = serializers.IntegerField(label=_('Assets amount'), read_only=True) class Meta: - model = Domain + model = Zone fields_mini = ['id', 'name'] fields_small = fields_mini + ['comment'] fields_m2m = ['assets', 'gateways', 'labels', 'assets_amount'] @@ -55,9 +55,9 @@ class DomainSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer): return super().update(instance, validated_data) -class DomainListSerializer(DomainSerializer): - class Meta(DomainSerializer.Meta): - fields = list(set(DomainSerializer.Meta.fields + ['assets_amount']) - {'assets'}) +class ZoneListSerializer(ZoneSerializer): + class Meta(ZoneSerializer.Meta): + fields = list(set(ZoneSerializer.Meta.fields + ['assets_amount']) - {'assets'}) @classmethod def setup_eager_loading(cls, queryset): @@ -67,9 +67,9 @@ class DomainListSerializer(DomainSerializer): return queryset -class DomainWithGatewaySerializer(serializers.ModelSerializer): +class ZoneWithGatewaySerializer(serializers.ModelSerializer): gateways = GatewayWithAccountSecretSerializer(many=True, read_only=True) class Meta: - model = Domain + model = Zone fields = '__all__' diff --git a/apps/assets/serializers/platform.py b/apps/assets/serializers/platform.py index ebe28ca2a..03f5314a7 100644 --- a/apps/assets/serializers/platform.py +++ b/apps/assets/serializers/platform.py @@ -194,7 +194,7 @@ class PlatformSerializer(ResourceLabelsMixin, CommonSerializerMixin, WritableNes ] fields_m2m = ['assets', 'assets_amount'] fields = fields_small + fields_m2m + [ - "protocols", "domain_enabled", "su_enabled", "su_method", + "protocols", "gateway_enabled", "su_enabled", "su_method", "ds_enabled", "automation", "comment", "custom_fields", "labels" ] + read_only_fields extra_kwargs = { @@ -205,11 +205,11 @@ class PlatformSerializer(ResourceLabelsMixin, CommonSerializerMixin, WritableNes "similar to logging in with a regular account and then switching to root" ) }, - "domain_enabled": { + "gateway_enabled": { "label": _('Gateway enabled'), "help_text": _("Assets can be connected using a zone gateway") }, - "domain_default": {"label": _('Default Domain')}, + "zone_default": {"label": _('Default zone')}, 'assets': {'required': False, 'label': _('Assets')}, } @@ -222,7 +222,7 @@ class PlatformSerializer(ResourceLabelsMixin, CommonSerializerMixin, WritableNes return name = self.initial_data.get('name') - if ' ' in name: + if name is not None and ' ' in name: self.initial_data['name'] = name.replace(' ', '-') if self.instance: @@ -262,8 +262,8 @@ class PlatformSerializer(ResourceLabelsMixin, CommonSerializerMixin, WritableNes def validate_su_enabled(self, su_enabled): return su_enabled and self.constraints.get('su_enabled', False) - def validate_domain_enabled(self, domain_enabled): - return domain_enabled and self.constraints.get('domain_enabled', False) + def validate_gateway_enabled(self, gateway_enabled): + return gateway_enabled and self.constraints.get('gateway_enabled', False) def validate_automation(self, automation): automation = automation or {} diff --git a/apps/assets/tasks/utils.py b/apps/assets/tasks/utils.py index d65351eaa..3e435edb9 100644 --- a/apps/assets/tasks/utils.py +++ b/apps/assets/tasks/utils.py @@ -25,10 +25,10 @@ def check_asset_can_run_ansible(asset): def check_system_user_can_run_ansible(system_user): if not system_user.auto_push_account: - logger.warn(f'Push system user task skip, auto push not enable: system_user={system_user.name}') + logger.warning(f'Push system user task skip, auto push not enable: system_user={system_user.name}') return False if not system_user.is_protocol_support_push: - logger.warn(f'Push system user task skip, protocol not support: ' + logger.warning(f'Push system user task skip, protocol not support: ' f'system_user={system_user.name} protocol={system_user.protocol} ' f'support_protocol={system_user.SUPPORT_PUSH_PROTOCOLS}') return False diff --git a/apps/assets/urls/api_urls.py b/apps/assets/urls/api_urls.py index 0ff108802..7527be88e 100644 --- a/apps/assets/urls/api_urls.py +++ b/apps/assets/urls/api_urls.py @@ -20,7 +20,7 @@ router.register(r'directories', api.DSViewSet, 'ds') router.register(r'customs', api.CustomViewSet, 'custom') router.register(r'platforms', api.AssetPlatformViewSet, 'platform') router.register(r'nodes', api.NodeViewSet, 'node') -router.register(r'domains', api.DomainViewSet, 'domain') +router.register(r'zones', api.ZoneViewSet, 'zone') router.register(r'gateways', api.GatewayViewSet, 'gateway') router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset') router.register(r'protocol-settings', api.PlatformProtocolViewSet, 'protocol-setting') diff --git a/apps/audits/migrations/0006_alter_ftplog_account_alter_ftplog_asset_and_more.py b/apps/audits/migrations/0006_alter_ftplog_account_alter_ftplog_asset_and_more.py new file mode 100644 index 000000000..a3a4a0466 --- /dev/null +++ b/apps/audits/migrations/0006_alter_ftplog_account_alter_ftplog_asset_and_more.py @@ -0,0 +1,32 @@ +# Generated by Django 4.1.13 on 2025-04-21 06:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('audits', '0005_rename_serviceaccesslog'), + ] + + operations = [ + migrations.AlterField( + model_name='ftplog', + name='account', + field=models.CharField(db_index=True, max_length=128, verbose_name='Account'), + ), + migrations.AlterField( + model_name='ftplog', + name='asset', + field=models.CharField(db_index=True, max_length=1024, verbose_name='Asset'), + ), + migrations.AlterField( + model_name='ftplog', + name='date_start', + field=models.DateTimeField(auto_now_add=True, verbose_name='Date start'), + ), + migrations.AddIndex( + model_name='ftplog', + index=models.Index(fields=['date_start', 'org_id'], name='idx_date_start_org'), + ), + ] diff --git a/apps/audits/models.py b/apps/audits/models.py index 1ad4b02ef..45cfd67fa 100644 --- a/apps/audits/models.py +++ b/apps/audits/models.py @@ -56,19 +56,22 @@ class FTPLog(OrgModelMixin): remote_addr = models.CharField( max_length=128, verbose_name=_("Remote addr"), blank=True, null=True ) - asset = models.CharField(max_length=1024, verbose_name=_("Asset")) - account = models.CharField(max_length=128, verbose_name=_("Account")) + asset = models.CharField(max_length=1024, verbose_name=_("Asset"), db_index=True) + account = models.CharField(max_length=128, verbose_name=_("Account"), db_index=True) operate = models.CharField( max_length=16, verbose_name=_("Operate"), choices=OperateChoices.choices ) filename = models.CharField(max_length=1024, verbose_name=_("Filename")) is_success = models.BooleanField(default=True, verbose_name=_("Success")) - date_start = models.DateTimeField(auto_now_add=True, verbose_name=_("Date start"), db_index=True) + date_start = models.DateTimeField(auto_now_add=True, verbose_name=_("Date start")) has_file = models.BooleanField(default=False, verbose_name=_("Can Download")) session = models.CharField(max_length=36, verbose_name=_("Session"), default=uuid.uuid4) class Meta: verbose_name = _("File transfer log") + indexes = [ + models.Index(fields=['date_start', 'org_id'], name='idx_date_start_org'), + ] @property def filepath(self): diff --git a/apps/audits/signal_handlers/operate_log.py b/apps/audits/signal_handlers/operate_log.py index 2586c34eb..71c911edd 100644 --- a/apps/audits/signal_handlers/operate_log.py +++ b/apps/audits/signal_handlers/operate_log.py @@ -183,11 +183,11 @@ def on_django_start_set_operate_log_monitor_models(sender, **kwargs): 'ConnectionToken', 'SessionJoinRecord', 'HistoricalJob', 'Status', 'TicketStep', 'Ticket', 'UserAssetGrantedTreeNodeRelation', 'TicketAssignee', - 'SuperTicket', 'SuperConnectionToken', 'PermNode', + 'SuperTicket', 'SuperConnectionToken', 'AdminConnectionToken', 'PermNode', 'PermedAsset', 'PermedAccount', 'MenuPermission', 'Permission', 'TicketSession', 'ApplyLoginTicket', 'ApplyCommandTicket', 'ApplyLoginAssetTicket', - 'FavoriteAsset', 'ChangeSecretRecord', 'AppProvider', 'Variable' + 'FavoriteAsset', 'ChangeSecretRecord', 'AppProvider', 'Variable', 'LeakPasswords' } include_models = {'UserSession'} for i, app in enumerate(apps.get_models(), 1): diff --git a/apps/audits/tasks.py b/apps/audits/tasks.py index 013141f82..b9d2c617e 100644 --- a/apps/audits/tasks.py +++ b/apps/audits/tasks.py @@ -96,17 +96,20 @@ def batch_delete(queryset, batch_size=3000): def remove_files_by_days(root_path, days, file_types=None): if file_types is None: file_types = ['.json', '.tar', '.gz', '.mp4'] - need_rm_files = [] expire_date = timezone.now() - timezone.timedelta(days=days) timestamp = expire_date.timestamp() for root, dirs, files in os.walk(root_path): + rm_files = [] for file in files: if any(file.endswith(file_type) for file_type in file_types): file_path = os.path.join(root, file) if os.path.getmtime(file_path) <= timestamp: - need_rm_files.append(file_path) - for file in need_rm_files: - os.remove(file) + rm_files.append(file_path) + for file in rm_files: + try: + os.remove(file) + except Exception as e: + logger.error(f"Remove file {file} error: {e}") def clean_expired_session_period(): diff --git a/apps/authentication/backends/passkey/api.py b/apps/authentication/backends/passkey/api.py index ace882a92..86c4ebb3d 100644 --- a/apps/authentication/backends/passkey/api.py +++ b/apps/authentication/backends/passkey/api.py @@ -1,9 +1,12 @@ +import time + from django.conf import settings from django.http import JsonResponse from django.shortcuts import render from django.utils.translation import gettext as _ from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated, AllowAny +from rest_framework.response import Response from authentication.mixins import AuthMixin from common.api import JMSModelViewSet @@ -44,6 +47,9 @@ class PasskeyViewSet(AuthMixin, FlashMessageMixin, JMSModelViewSet): @action(methods=['get'], detail=False, url_path='login', permission_classes=[AllowAny]) def login(self, request): + confirm_mfa = request.GET.get('mfa') + if confirm_mfa: + request.session['passkey_confirm_mfa'] = '1' return render(request, 'authentication/passkey.html', {}) def redirect_to_error(self, error): @@ -64,8 +70,16 @@ class PasskeyViewSet(AuthMixin, FlashMessageMixin, JMSModelViewSet): if not user: return self.redirect_to_error(_('Auth failed')) + confirm_mfa = request.session.get('passkey_confirm_mfa') + if confirm_mfa: + request.session['CONFIRM_LEVEL'] = ConfirmType.values.index('mfa') + 1 + request.session['CONFIRM_TIME'] = int(time.time()) + request.session['passkey_confirm_mfa'] = '' + return Response('ok') + try: self.check_oauth2_auth(user, settings.AUTH_BACKEND_PASSKEY) + self.mark_mfa_ok('passkey', user) return self.redirect_to_guard_view() except Exception as e: msg = getattr(e, 'msg', '') or str(e) diff --git a/apps/authentication/const.py b/apps/authentication/const.py index e95d97b11..ad38eebf6 100644 --- a/apps/authentication/const.py +++ b/apps/authentication/const.py @@ -34,6 +34,7 @@ class MFAType(TextChoices): Email = 'email', _('Email') Face = 'face', _('Face Recognition') Radius = 'otp_radius', _('Radius') + Passkey = 'passkey', _('Passkey') Custom = 'mfa_custom', _('Custom') diff --git a/apps/authentication/mfa/base.py b/apps/authentication/mfa/base.py index ace5a1424..b7f7ae4ee 100644 --- a/apps/authentication/mfa/base.py +++ b/apps/authentication/mfa/base.py @@ -7,6 +7,7 @@ from django.utils.translation import gettext_lazy as _ class BaseMFA(abc.ABC): placeholder = _('Please input security code') skip_cache_check = False + has_code = True def __init__(self, user): """ diff --git a/apps/authentication/mfa/face.py b/apps/authentication/mfa/face.py index 494a44277..424ee3f80 100644 --- a/apps/authentication/mfa/face.py +++ b/apps/authentication/mfa/face.py @@ -11,6 +11,7 @@ class MFAFace(BaseMFA, AuthFaceMixin): display_name = MFAType.Face.name placeholder = 'Face Recognition' skip_cache_check = True + has_code = False def _check_code(self, code): assert self.is_authenticated() diff --git a/apps/authentication/mfa/otp.py b/apps/authentication/mfa/otp.py index 6e56f8a14..ac866b882 100644 --- a/apps/authentication/mfa/otp.py +++ b/apps/authentication/mfa/otp.py @@ -49,4 +49,3 @@ class MFAOtp(BaseMFA): def help_text_of_disable(self): return '' - diff --git a/apps/authentication/mfa/passkey.py b/apps/authentication/mfa/passkey.py new file mode 100644 index 000000000..572d3d859 --- /dev/null +++ b/apps/authentication/mfa/passkey.py @@ -0,0 +1,46 @@ +from django.conf import settings +from django.utils.translation import gettext_lazy as _ + +from authentication.mfa.base import BaseMFA +from ..const import MFAType + + +class MFAPasskey(BaseMFA): + name = MFAType.Passkey.value + display_name = MFAType.Passkey.name + placeholder = 'Passkey' + has_code = False + + def _check_code(self, code): + assert self.is_authenticated() + + return False, '' + + def is_active(self): + if not self.is_authenticated(): + return True + return self.user.passkey_set.count() + + @staticmethod + def global_enabled(): + return settings.AUTH_PASSKEY + + def get_enable_url(self) -> str: + return '/ui/#/profile/passkeys' + + def get_disable_url(self) -> str: + return '/ui/#/profile/passkeys' + + def disable(self): + pass + + def can_disable(self) -> bool: + return False + + @staticmethod + def help_text_of_enable(): + return _("Using passkey as MFA") + + @staticmethod + def help_text_of_disable(): + return _("Using passkey as MFA") diff --git a/apps/authentication/mixins.py b/apps/authentication/mixins.py index c346ab1bb..b8c7d1858 100644 --- a/apps/authentication/mixins.py +++ b/apps/authentication/mixins.py @@ -174,7 +174,7 @@ class AuthPreCheckMixin: is_block = LoginBlockUtil(username, ip).is_block() if not is_block: return - logger.warn('Ip was blocked' + ': ' + username + ':' + ip) + logger.warning('Ip was blocked' + ': ' + username + ':' + ip) exception = errors.BlockLoginError(username=username, ip=ip) if raise_exception: raise errors.BlockLoginError(username=username, ip=ip) @@ -253,7 +253,7 @@ class MFAMixin: blocked = MFABlockUtils(username, ip).is_block() if not blocked: return - logger.warn('Ip was blocked' + ': ' + username + ':' + ip) + logger.warning('Ip was blocked' + ': ' + username + ':' + ip) exception = errors.BlockMFAError(username=username, request=self.request, ip=ip) if raise_exception: raise exception @@ -323,7 +323,7 @@ class AuthPostCheckMixin: def _check_passwd_is_too_simple(cls, user: User, password): if not user.is_auth_backend_model(): return - if user.check_passwd_too_simple(password): + if user.check_passwd_too_simple(password) or user.check_leak_password(password): message = _('Your password is too simple, please change it for security') url = cls.generate_reset_password_url_with_flash_msg(user, message=message) raise errors.PasswordTooSimple(url) diff --git a/apps/authentication/models/connection_token.py b/apps/authentication/models/connection_token.py index 9c8fe5f6a..95e0c3d7e 100644 --- a/apps/authentication/models/connection_token.py +++ b/apps/authentication/models/connection_token.py @@ -251,7 +251,7 @@ class ConnectionToken(JMSOrgBaseModel): raise JMSException({'error': 'No host account available, please check the applet, host and account'}) host, account, lock_key = bulk_get(host_account, ('host', 'account', 'lock_key')) - gateway = host.domain.select_gateway() if host.domain else None + gateway = host.zone.select_gateway() if host.zone else None platform = host.platform data = { @@ -305,17 +305,17 @@ class ConnectionToken(JMSOrgBaseModel): return account @lazyproperty - def domain(self): - if not self.asset.platform.domain_enabled: + def zone(self): + if not self.asset.platform.gateway_enabled: return if self.asset.platform.name == GATEWAY_NAME: return - domain = self.asset.domain if self.asset.domain else None - return domain + zone = self.asset.zone if self.asset.zone else None + return zone @lazyproperty def gateway(self): - if not self.asset or not self.domain: + if not self.asset or not self.zone: return return self.asset.gateway diff --git a/apps/authentication/serializers/connect_token_secret.py b/apps/authentication/serializers/connect_token_secret.py index e38cbea18..8f3c60936 100644 --- a/apps/authentication/serializers/connect_token_secret.py +++ b/apps/authentication/serializers/connect_token_secret.py @@ -4,7 +4,7 @@ from rest_framework import serializers from accounts.const import SecretType from accounts.models import Account from acls.models import CommandGroup, CommandFilterACL -from assets.models import Asset, Platform, Gateway, Domain +from assets.models import Asset, Platform, Gateway, Zone from assets.serializers.asset import AssetProtocolsSerializer from assets.serializers.platform import PlatformSerializer from common.serializers.fields import LabeledChoiceField @@ -135,7 +135,7 @@ class ConnectionTokenSecretSerializer(OrgResourceModelSerializerMixin): account = _ConnectionTokenAccountSerializer(read_only=True, source='account_object') gateway = _ConnectionTokenGatewaySerializer(read_only=True) platform = _ConnectionTokenPlatformSerializer(read_only=True) - domain = ObjectRelatedField(queryset=Domain.objects, required=False, label=_('Domain')) + zone = ObjectRelatedField(queryset=Zone.objects, required=False, label=_('Domain')) command_filter_acls = _ConnectionTokenCommandFilterACLSerializer(read_only=True, many=True) expire_now = serializers.BooleanField(label=_('Expired now'), write_only=True, default=True) connect_method = _ConnectTokenConnectMethodSerializer(read_only=True, source='connect_method_object') @@ -148,7 +148,7 @@ class ConnectionTokenSecretSerializer(OrgResourceModelSerializerMixin): fields = [ 'id', 'value', 'user', 'asset', 'account', 'platform', 'command_filter_acls', 'protocol', - 'domain', 'gateway', 'actions', 'expire_at', + 'zone', 'gateway', 'actions', 'expire_at', 'from_ticket', 'expire_now', 'connect_method', 'connect_options', 'face_monitor_token' ] diff --git a/apps/authentication/templates/authentication/login.html b/apps/authentication/templates/authentication/login.html index 88899c044..c5f1394f5 100644 --- a/apps/authentication/templates/authentication/login.html +++ b/apps/authentication/templates/authentication/login.html @@ -330,7 +330,7 @@