diff --git a/apps/accounts/serializers/account/account.py b/apps/accounts/serializers/account/account.py index cc13aced8..36dff55a9 100644 --- a/apps/accounts/serializers/account/account.py +++ b/apps/accounts/serializers/account/account.py @@ -17,8 +17,8 @@ class AccountSerializerCreateValidateMixin: replace_attrs: callable def to_internal_value(self, data): + self.id = data.pop('id', None) ret = super().to_internal_value(data) - self.id = ret.pop('id', None) self.push_now = ret.pop('push_now', False) self.template = ret.pop('template', False) return ret diff --git a/apps/acls/migrations/0008_commandgroup_comment.py b/apps/acls/migrations/0008_commandgroup_comment.py index 82e44912c..8937d2a93 100644 --- a/apps/acls/migrations/0008_commandgroup_comment.py +++ b/apps/acls/migrations/0008_commandgroup_comment.py @@ -20,14 +20,14 @@ class Migration(migrations.Migration): ), migrations.AlterModelOptions( name='commandfilteracl', - options={'ordering': ('priority', 'name'), 'verbose_name': 'Command acl'}, + options={'ordering': ('priority', 'date_updated', 'name'), 'verbose_name': 'Command acl'}, ), migrations.AlterModelOptions( name='loginacl', - options={'ordering': ('priority', 'name'), 'verbose_name': 'Login acl'}, + options={'ordering': ('priority', 'date_updated', 'name'), 'verbose_name': 'Login acl'}, ), migrations.AlterModelOptions( name='loginassetacl', - options={'ordering': ('priority', 'name'), 'verbose_name': 'Login asset acl'}, + options={'ordering': ('priority', 'date_updated', 'name'), 'verbose_name': 'Login asset acl'}, ), ] diff --git a/apps/acls/models/base.py b/apps/acls/models/base.py index 7ba48b3c6..53a566048 100644 --- a/apps/acls/models/base.py +++ b/apps/acls/models/base.py @@ -82,7 +82,7 @@ class BaseACL(JMSBaseModel): objects = ACLManager.from_queryset(BaseACLQuerySet)() class Meta: - ordering = ('priority', 'name') + ordering = ('priority', 'date_updated', 'name') abstract = True def is_action(self, action): diff --git a/apps/acls/serializers/login_acl.py b/apps/acls/serializers/login_acl.py index 99b1f1077..59b9cf0ad 100644 --- a/apps/acls/serializers/login_acl.py +++ b/apps/acls/serializers/login_acl.py @@ -52,10 +52,10 @@ class LoginACLSerializer(BulkModelSerializer): action = self.fields.get("action") if not action: return - choices = action._choices + choices = action.choices if not has_valid_xpack_license(): choices.pop(LoginACL.ActionChoices.review, None) - action._choices = choices + action.choices = choices def get_rules_serializer(self): return RuleSerializer() diff --git a/apps/assets/migrations/0100_auto_20220711_1413.py b/apps/assets/migrations/0100_auto_20220711_1413.py index 16e75cbae..2412c6df0 100644 --- a/apps/assets/migrations/0100_auto_20220711_1413.py +++ b/apps/assets/migrations/0100_auto_20220711_1413.py @@ -1,8 +1,7 @@ # Generated by Django 3.2.12 on 2022-07-11 06:13 import time -from django.db import migrations, models -from assets.models import Platform +from django.db import migrations def migrate_asset_accounts(apps, schema_editor): diff --git a/apps/assets/migrations/0101_auto_20220811_1511.py b/apps/assets/migrations/0101_auto_20220811_1511.py index 9ca39774e..0287a8414 100644 --- a/apps/assets/migrations/0101_auto_20220811_1511.py +++ b/apps/assets/migrations/0101_auto_20220811_1511.py @@ -1,5 +1,4 @@ # Generated by Django 3.2.14 on 2022-08-11 07:11 -import assets.models.platform import django.db.models from django.db import migrations, models diff --git a/apps/assets/migrations/0103_auto_20220902_1021.py b/apps/assets/migrations/0103_auto_20220902_1021.py index 8d324fa12..dfc731736 100644 --- a/apps/assets/migrations/0103_auto_20220902_1021.py +++ b/apps/assets/migrations/0103_auto_20220902_1021.py @@ -18,6 +18,8 @@ def _create_account_obj(secret, secret_type, gateway, asset, account_model): def migrate_gateway_to_asset(apps, schema_editor): db_alias = schema_editor.connection.alias + node_model = apps.get_model('assets', 'Node') + org_model = apps.get_model('orgs', 'Organization') gateway_model = apps.get_model('assets', 'Gateway') platform_model = apps.get_model('assets', 'Platform') gateway_platform = platform_model.objects.using(db_alias).get(name=GATEWAY_NAME) @@ -28,6 +30,16 @@ def migrate_gateway_to_asset(apps, schema_editor): asset_model = apps.get_model('assets', 'Asset') protocol_model = apps.get_model('assets', 'Protocol') gateways = gateway_model.objects.all() + + org_ids = gateways.order_by('org_id').values_list('org_id', flat=True).distinct() + node_dict = {} + for org_id in org_ids: + org = org_model.objects.using(db_alias).filter(id=org_id).first() + node = node_model.objects.using(db_alias).filter( + org_id=org_id, value=org.name, full_value=f'/{org.name}' + ).first() + node_dict[org_id] = node + for gateway in gateways: comment = gateway.comment if gateway.comment else '' data = { @@ -40,6 +52,8 @@ def migrate_gateway_to_asset(apps, schema_editor): 'platform': gateway_platform, } asset = asset_model.objects.using(db_alias).create(**data) + node = node_dict.get(str(gateway.org_id)) + asset.nodes.set([node]) asset_dict[gateway.id] = asset protocol_model.objects.using(db_alias).create(name='ssh', port=gateway.port, asset=asset) hosts = [host_model(asset_ptr=asset) for asset in asset_dict.values()] diff --git a/apps/assets/serializers/asset/common.py b/apps/assets/serializers/asset/common.py index 82a163d84..e51c4558f 100644 --- a/apps/assets/serializers/asset/common.py +++ b/apps/assets/serializers/asset/common.py @@ -139,9 +139,9 @@ class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSeriali return category = request.path.strip('/').split('/')[-1].rstrip('s') field_category = self.fields.get('category') - field_category._choices = Category.filter_choices(category) + field_category.choices = Category.filter_choices(category) field_type = self.fields.get('type') - field_type._choices = AllTypes.filter_choices(category) + field_type.choices = AllTypes.filter_choices(category) @classmethod def setup_eager_loading(cls, queryset): diff --git a/apps/audits/tasks.py b/apps/audits/tasks.py index 5b58c5fd2..1dc507c25 100644 --- a/apps/audits/tasks.py +++ b/apps/audits/tasks.py @@ -9,6 +9,7 @@ from ops.celery.decorator import ( ) from .models import UserLoginLog, OperateLog, FTPLog from common.utils import get_log_keep_day +from django.utils.translation import gettext_lazy as _ def clean_login_log_period(): @@ -32,8 +33,8 @@ def clean_ftp_log_period(): FTPLog.objects.filter(date_start__lt=expired_day).delete() -@register_as_period_task(interval=3600*24) -@shared_task +@register_as_period_task(interval=3600 * 24) +@shared_task(verbose_name=_('Clean audits log')) def clean_audits_log_period(): clean_login_log_period() clean_operation_log_period() diff --git a/apps/authentication/tasks.py b/apps/authentication/tasks.py index 08472e931..978e4ace0 100644 --- a/apps/authentication/tasks.py +++ b/apps/authentication/tasks.py @@ -5,9 +5,10 @@ from celery import shared_task from ops.celery.decorator import register_as_period_task from django.contrib.sessions.models import Session from django.utils import timezone +from django.utils.translation import gettext_lazy as _ -@register_as_period_task(interval=3600*24) -@shared_task +@register_as_period_task(interval=3600 * 24) +@shared_task(verbose_name=_('Clean expired session')) def clean_django_sessions(): Session.objects.filter(expire_date__lt=timezone.now()).delete() diff --git a/apps/common/serializers/fields.py b/apps/common/serializers/fields.py index 96ff7c103..84d4d5927 100644 --- a/apps/common/serializers/fields.py +++ b/apps/common/serializers/fields.py @@ -66,7 +66,7 @@ class LabeledChoiceField(ChoiceField): def to_internal_value(self, data): if isinstance(data, dict): - return data.get("value") + data = data.get("value") return super(LabeledChoiceField, self).to_internal_value(data) diff --git a/apps/common/utils/verify_code.py b/apps/common/utils/verify_code.py index 85589de69..0dbf810fd 100644 --- a/apps/common/utils/verify_code.py +++ b/apps/common/utils/verify_code.py @@ -8,12 +8,12 @@ from common.sdk.sms.endpoint import SMS from common.exceptions import JMSException from common.utils.random import random_string from common.utils import get_logger - +from django.utils.translation import gettext_lazy as _ logger = get_logger(__file__) -@shared_task +@shared_task(verbose_name=_('Send email')) def send_async(sender): sender.gen_and_send() diff --git a/apps/ops/api/playbook.py b/apps/ops/api/playbook.py index 36c1c4994..322129b8b 100644 --- a/apps/ops/api/playbook.py +++ b/apps/ops/api/playbook.py @@ -193,7 +193,7 @@ class PlaybookFileBrowserAPIView(APIView): "id": os.path.join(relative_path, d) if not os.path.join(relative_path, d).startswith( '.') else d, "isParent": True, - "open": False, + "open": True, "pId": relative_path if not relative_path.startswith('.') else 'root', "temp": False } diff --git a/apps/ops/migrations/0025_auto_20230117_1130.py b/apps/ops/migrations/0025_auto_20230117_1130.py index b6d3881e7..7a5aa33b8 100644 --- a/apps/ops/migrations/0025_auto_20230117_1130.py +++ b/apps/ops/migrations/0025_auto_20230117_1130.py @@ -13,7 +13,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='playbook', name='create_method', - field=models.CharField(choices=[('blank', 'Blank'), ('upload', 'Upload'), ('vcs', 'VCS')], default='blank', max_length=128, verbose_name='CreateMethod'), + field=models.CharField(choices=[('blank', 'Blank'), ('vcs', 'VCS')], default='blank', max_length=128, verbose_name='CreateMethod'), ), migrations.AddField( model_name='playbook', diff --git a/apps/perms/serializers/permission.py b/apps/perms/serializers/permission.py index 63ba3a765..fef9929b8 100644 --- a/apps/perms/serializers/permission.py +++ b/apps/perms/serializers/permission.py @@ -70,9 +70,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer): actions = self.fields.get("actions") if not actions: return - choices = actions._choices - actions._choices = choices - actions.default = list(choices.keys()) + actions.default = list(actions.choices.keys()) @classmethod def setup_eager_loading(cls, queryset): diff --git a/apps/perms/tasks.py b/apps/perms/tasks.py index 57b9f48bd..cd4e7bd86 100644 --- a/apps/perms/tasks.py +++ b/apps/perms/tasks.py @@ -19,12 +19,13 @@ from perms.notifications import ( PermedAssetsWillExpireUserMsg, AssetPermsWillExpireForOrgAdminMsg, ) +from django.utils.translation import gettext_lazy as _ logger = get_logger(__file__) @register_as_period_task(interval=settings.PERM_EXPIRED_CHECK_PERIODIC) -@shared_task() +@shared_task(verbose_name=_('Check asset permission expired')) @atomic() @tmp_to_root_org() def check_asset_permission_expired(): @@ -36,7 +37,7 @@ def check_asset_permission_expired(): @register_as_period_task(crontab=CRONTAB_AT_AM_TEN) -@shared_task() +@shared_task(verbose_name=_('Send asset permission expired notification')) @atomic() @tmp_to_root_org() def check_asset_permission_will_expired(): diff --git a/apps/rbac/api/role.py b/apps/rbac/api/role.py index 0a4d52706..dd0220692 100644 --- a/apps/rbac/api/role.py +++ b/apps/rbac/api/role.py @@ -4,7 +4,6 @@ from rest_framework.exceptions import PermissionDenied from rest_framework.decorators import action from common.api import JMSModelViewSet -from common.api import PaginatedResponseMixin from ..filters import RoleFilter from ..serializers import RoleSerializer, RoleUserSerializer from ..models import Role, SystemRole, OrgRole @@ -18,6 +17,7 @@ __all__ = [ class RoleViewSet(JMSModelViewSet): queryset = Role.objects.all() + ordering = ('-builtin', 'scope', 'name') serializer_classes = { 'default': RoleSerializer, 'users': RoleUserSerializer, @@ -62,8 +62,7 @@ class RoleViewSet(JMSModelViewSet): return super().perform_update(serializer) def get_queryset(self): - queryset = super().get_queryset() \ - .annotate(permissions_amount=Count('permissions')) + queryset = super().get_queryset().annotate(permissions_amount=Count('permissions')) return queryset @action(methods=['GET'], detail=True) diff --git a/apps/settings/serializers/auth/ldap.py b/apps/settings/serializers/auth/ldap.py index a6cc22455..472627e8a 100644 --- a/apps/settings/serializers/auth/ldap.py +++ b/apps/settings/serializers/auth/ldap.py @@ -15,7 +15,7 @@ class LDAPTestConfigSerializer(serializers.Serializer): AUTH_LDAP_BIND_PASSWORD = EncryptedField(required=False, allow_blank=True) AUTH_LDAP_SEARCH_OU = serializers.CharField() AUTH_LDAP_SEARCH_FILTER = serializers.CharField() - AUTH_LDAP_USER_ATTR_MAP = serializers.CharField() + AUTH_LDAP_USER_ATTR_MAP = serializers.JSONField() AUTH_LDAP_START_TLS = serializers.BooleanField(required=False) AUTH_LDAP = serializers.BooleanField(required=False) diff --git a/apps/terminal/automations/deploy_applet_host/__init__.py b/apps/terminal/automations/deploy_applet_host/__init__.py index c5e903f35..164340098 100644 --- a/apps/terminal/automations/deploy_applet_host/__init__.py +++ b/apps/terminal/automations/deploy_applet_host/__init__.py @@ -50,17 +50,18 @@ class DeployAppletHostManager: host_id = str(self.deployment.host.id) if not site_url: site_url = "http://localhost:8080" - if not download_host: - download_host = site_url options = self.deployment.host.deploy_options - site_url = site_url.rstrip("/") + core_host = options.get("CORE_HOST", site_url) + core_host = core_host.rstrip("/") + if not download_host: + download_host = core_host download_host = download_host.rstrip("/") def handler(plays): for play in plays: play["vars"].update(options) play["vars"]["APPLET_DOWNLOAD_HOST"] = download_host - play["vars"]["CORE_HOST"] = site_url + play["vars"]["CORE_HOST"] = core_host play["vars"]["BOOTSTRAP_TOKEN"] = bootstrap_token play["vars"]["HOST_ID"] = host_id play["vars"]["HOST_NAME"] = self.deployment.host.name diff --git a/apps/terminal/serializers/applet_host.py b/apps/terminal/serializers/applet_host.py index 9ec4b8104..cf7895a71 100644 --- a/apps/terminal/serializers/applet_host.py +++ b/apps/terminal/serializers/applet_host.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.utils.translation import gettext_lazy as _ from rest_framework import serializers @@ -27,6 +28,8 @@ class DeployOptionsSerializer(serializers.Serializer): (1, _("Disabled")), (0, _("Enabled")), ) + + CORE_HOST = serializers.CharField(default=settings.SITE_URL, label=_('API Server'), max_length=1024) RDS_Licensing = serializers.BooleanField(default=False, label=_("RDS Licensing")) RDS_LicenseServer = serializers.CharField(default='127.0.0.1', label=_('RDS License Server'), max_length=1024) RDS_LicensingMode = serializers.ChoiceField(choices=LICENSE_MODE_CHOICES, default=4, label=_('RDS Licensing Mode')) diff --git a/apps/terminal/tasks.py b/apps/terminal/tasks.py index 4e67d1fc7..63b449320 100644 --- a/apps/terminal/tasks.py +++ b/apps/terminal/tasks.py @@ -21,13 +21,14 @@ from .models import ( Status, Session, Command, Task, AppletHostDeployment ) from .utils import find_session_replay_local +from django.utils.translation import gettext_lazy as _ CACHE_REFRESH_INTERVAL = 10 RUNNING = False logger = get_task_logger(__name__) -@shared_task +@shared_task(verbose_name=_('Periodic delete terminal status')) @register_as_period_task(interval=3600) @after_app_ready_start @after_app_shutdown_clean_periodic @@ -36,7 +37,7 @@ def delete_terminal_status_period(): Status.objects.filter(date_created__lt=yesterday).delete() -@shared_task +@shared_task(verbose_name=_('Clean orphan session')) @register_as_period_task(interval=600) @after_app_ready_start @after_app_shutdown_clean_periodic @@ -55,7 +56,7 @@ def clean_orphan_session(): session.save() -@shared_task +@shared_task(verbose_name=_('Periodic clean expired session')) @register_as_period_task(interval=3600 * 24) @after_app_ready_start @after_app_shutdown_clean_periodic @@ -81,7 +82,7 @@ def clean_expired_session_period(): logger.info("Clean session replay done") -@shared_task +@shared_task(verbose_name=_('Upload session replay to external storage')) def upload_session_replay_to_external_storage(session_id): logger.info(f'Start upload session to external storage: {session_id}') session = Session.objects.filter(id=session_id).first() @@ -108,14 +109,14 @@ def upload_session_replay_to_external_storage(session_id): return -@shared_task +@shared_task(verbose_name=_('Run applet host deployment')) def run_applet_host_deployment(did): with tmp_to_builtin_org(system=1): deployment = AppletHostDeployment.objects.get(id=did) deployment.start() -@shared_task +@shared_task(verbose_name=_('Install applet')) def run_applet_host_deployment_install_applet(did, applet_id): with tmp_to_builtin_org(system=1): deployment = AppletHostDeployment.objects.get(id=did) diff --git a/apps/tickets/serializers/ticket/ticket.py b/apps/tickets/serializers/ticket/ticket.py index 334031e5e..4a36ccf4f 100644 --- a/apps/tickets/serializers/ticket/ticket.py +++ b/apps/tickets/serializers/ticket/ticket.py @@ -39,9 +39,9 @@ class TicketSerializer(OrgResourceModelSerializerMixin): tp = self.fields.get('type') if not tp: return - choices = tp._choices + choices = tp.choices choices.pop(TicketType.general, None) - tp._choices = choices + tp.choices = choices @classmethod def setup_eager_loading(cls, queryset): diff --git a/apps/users/filters.py b/apps/users/filters.py index e72523d00..1256dde16 100644 --- a/apps/users/filters.py +++ b/apps/users/filters.py @@ -36,7 +36,7 @@ class UserFilter(BaseFilterSet): if not role: return queryset.none() queryset = queryset.prefetch_related('role_bindings') \ - .filter(role_bindings__role_id=role.id) \ + .filter(role_bindings__role_id=role.id, role_bindings__role__scope='system') \ .distinct() return queryset @@ -45,6 +45,6 @@ class UserFilter(BaseFilterSet): if not role: return queryset.none() queryset = queryset.prefetch_related('role_bindings') \ - .filter(role_bindings__role_id=role.id) \ + .filter(role_bindings__role_id=role.id, role_bindings__role__scope='org') \ .distinct() return queryset diff --git a/apps/users/tasks.py b/apps/users/tasks.py index cddf0d2ec..2e19678ad 100644 --- a/apps/users/tasks.py +++ b/apps/users/tasks.py @@ -16,12 +16,12 @@ from .models import User from users.notifications import UserExpirationReminderMsg from settings.utils import LDAPServerUtil, LDAPImportUtil from common.const.crontab import CRONTAB_AT_AM_TEN, CRONTAB_AT_PM_TWO - +from django.utils.translation import gettext_lazy as _ logger = get_logger(__file__) -@shared_task +@shared_task(verbose_name=_('Check password expired')) def check_password_expired(): users = User.get_nature_users().filter(source=User.Source.local) for user in users: @@ -35,7 +35,7 @@ def check_password_expired(): PasswordExpirationReminderMsg(user).publish_async() -@shared_task +@shared_task(verbose_name=_('Periodic check password expired')) @after_app_ready_start def check_password_expired_periodic(): tasks = { @@ -49,11 +49,11 @@ def check_password_expired_periodic(): create_or_update_celery_periodic_tasks(tasks) -@shared_task +@shared_task(verbose_name=_('Check user expired')) def check_user_expired(): date_expired_lt = timezone.now() + timezone.timedelta(days=User.DATE_EXPIRED_WARNING_DAYS) - users = User.get_nature_users()\ - .filter(source=User.Source.local)\ + users = User.get_nature_users() \ + .filter(source=User.Source.local) \ .filter(date_expired__lt=date_expired_lt) for user in users: @@ -66,7 +66,7 @@ def check_user_expired(): UserExpirationReminderMsg(user).publish_async() -@shared_task +@shared_task(verbose_name=_('Periodic check user expired')) @after_app_ready_start def check_user_expired_periodic(): tasks = { @@ -80,7 +80,7 @@ def check_user_expired_periodic(): create_or_update_celery_periodic_tasks(tasks) -@shared_task +@shared_task(verbose_name=_('Import ldap user')) def import_ldap_user(): logger.info("Start import ldap user task") util_server = LDAPServerUtil() @@ -101,7 +101,7 @@ def import_ldap_user(): logger.info('Imported {} users successfully'.format(len(users))) -@shared_task +@shared_task(verbose_name=_('Periodic import ldap user')) @after_app_ready_start def import_ldap_user_periodic(): if not settings.AUTH_LDAP: