merge: with dev

apps/accounts/serializers/account/account.py

@@ -17,8 +17,8 @@ class AccountSerializerCreateValidateMixin:
     replace_attrs: callable
 
     def to_internal_value(self, data):
+        self.id = data.pop('id', None)
         ret = super().to_internal_value(data)
-        self.id = ret.pop('id', None)
         self.push_now = ret.pop('push_now', False)
         self.template = ret.pop('template', False)
         return ret

apps/acls/migrations/0008_commandgroup_comment.py

@@ -20,14 +20,14 @@ class Migration(migrations.Migration):
         ),
         migrations.AlterModelOptions(
             name='commandfilteracl',
-            options={'ordering': ('priority', 'name'), 'verbose_name': 'Command acl'},
+            options={'ordering': ('priority', 'date_updated', 'name'), 'verbose_name': 'Command acl'},
         ),
         migrations.AlterModelOptions(
             name='loginacl',
-            options={'ordering': ('priority', 'name'), 'verbose_name': 'Login acl'},
+            options={'ordering': ('priority', 'date_updated', 'name'), 'verbose_name': 'Login acl'},
         ),
         migrations.AlterModelOptions(
             name='loginassetacl',
-            options={'ordering': ('priority', 'name'), 'verbose_name': 'Login asset acl'},
+            options={'ordering': ('priority', 'date_updated', 'name'), 'verbose_name': 'Login asset acl'},
         ),
     ]

apps/acls/models/base.py

@@ -82,7 +82,7 @@ class BaseACL(JMSBaseModel):
     objects = ACLManager.from_queryset(BaseACLQuerySet)()
 
     class Meta:
-        ordering = ('priority', 'name')
+        ordering = ('priority', 'date_updated', 'name')
         abstract = True
 
     def is_action(self, action):

apps/acls/serializers/login_acl.py

@@ -52,10 +52,10 @@ class LoginACLSerializer(BulkModelSerializer):
         action = self.fields.get("action")
         if not action:
             return
-        choices = action._choices
+        choices = action.choices
         if not has_valid_xpack_license():
             choices.pop(LoginACL.ActionChoices.review, None)
-        action._choices = choices
+        action.choices = choices
 
     def get_rules_serializer(self):
         return RuleSerializer()

apps/assets/migrations/0100_auto_20220711_1413.py

@@ -1,8 +1,7 @@
 # Generated by Django 3.2.12 on 2022-07-11 06:13
 
 import time
-from django.db import migrations, models
+from django.db import migrations
-from assets.models import Platform
 
 
 def migrate_asset_accounts(apps, schema_editor):

apps/assets/migrations/0101_auto_20220811_1511.py

@@ -1,5 +1,4 @@
 # Generated by Django 3.2.14 on 2022-08-11 07:11
-import assets.models.platform
 import django.db.models
 from django.db import migrations, models
 

apps/assets/migrations/0103_auto_20220902_1021.py

@@ -18,6 +18,8 @@ def _create_account_obj(secret, secret_type, gateway, asset, account_model):
 
 def migrate_gateway_to_asset(apps, schema_editor):
     db_alias = schema_editor.connection.alias
+    node_model = apps.get_model('assets', 'Node')
+    org_model = apps.get_model('orgs', 'Organization')
     gateway_model = apps.get_model('assets', 'Gateway')
     platform_model = apps.get_model('assets', 'Platform')
     gateway_platform = platform_model.objects.using(db_alias).get(name=GATEWAY_NAME)
@@ -28,6 +30,16 @@ def migrate_gateway_to_asset(apps, schema_editor):
     asset_model = apps.get_model('assets', 'Asset')
     protocol_model = apps.get_model('assets', 'Protocol')
     gateways = gateway_model.objects.all()
+
+    org_ids = gateways.order_by('org_id').values_list('org_id', flat=True).distinct()
+    node_dict = {}
+    for org_id in org_ids:
+        org = org_model.objects.using(db_alias).filter(id=org_id).first()
+        node = node_model.objects.using(db_alias).filter(
+            org_id=org_id, value=org.name, full_value=f'/{org.name}'
+        ).first()
+        node_dict[org_id] = node
+
     for gateway in gateways:
         comment = gateway.comment if gateway.comment else ''
         data = {
@@ -40,6 +52,8 @@ def migrate_gateway_to_asset(apps, schema_editor):
             'platform': gateway_platform,
         }
         asset = asset_model.objects.using(db_alias).create(**data)
+        node = node_dict.get(str(gateway.org_id))
+        asset.nodes.set([node])
         asset_dict[gateway.id] = asset
         protocol_model.objects.using(db_alias).create(name='ssh', port=gateway.port, asset=asset)
     hosts = [host_model(asset_ptr=asset) for asset in asset_dict.values()]

apps/assets/serializers/asset/common.py

@@ -139,9 +139,9 @@ class AssetSerializer(BulkOrgResourceModelSerializer, WritableNestedModelSeriali
             return
         category = request.path.strip('/').split('/')[-1].rstrip('s')
         field_category = self.fields.get('category')
-        field_category._choices = Category.filter_choices(category)
+        field_category.choices = Category.filter_choices(category)
         field_type = self.fields.get('type')
-        field_type._choices = AllTypes.filter_choices(category)
+        field_type.choices = AllTypes.filter_choices(category)
 
     @classmethod
     def setup_eager_loading(cls, queryset):

apps/audits/tasks.py

@@ -9,6 +9,7 @@ from ops.celery.decorator import (
 )
 from .models import UserLoginLog, OperateLog, FTPLog
 from common.utils import get_log_keep_day
+from django.utils.translation import gettext_lazy as _
 
 
 def clean_login_log_period():
@@ -32,8 +33,8 @@ def clean_ftp_log_period():
     FTPLog.objects.filter(date_start__lt=expired_day).delete()
 
 
-@register_as_period_task(interval=3600*24)
+@register_as_period_task(interval=3600 * 24)
-@shared_task
+@shared_task(verbose_name=_('Clean audits log'))
 def clean_audits_log_period():
     clean_login_log_period()
     clean_operation_log_period()

apps/authentication/tasks.py

@@ -5,9 +5,10 @@ from celery import shared_task
 from ops.celery.decorator import register_as_period_task
 from django.contrib.sessions.models import Session
 from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
 
 
-@register_as_period_task(interval=3600*24)
+@register_as_period_task(interval=3600 * 24)
-@shared_task
+@shared_task(verbose_name=_('Clean expired session'))
 def clean_django_sessions():
     Session.objects.filter(expire_date__lt=timezone.now()).delete()

apps/common/serializers/fields.py

@@ -66,7 +66,7 @@ class LabeledChoiceField(ChoiceField):
 
     def to_internal_value(self, data):
         if isinstance(data, dict):
-            return data.get("value")
+            data = data.get("value")
         return super(LabeledChoiceField, self).to_internal_value(data)
 
 

apps/common/utils/verify_code.py

@@ -8,12 +8,12 @@ from common.sdk.sms.endpoint import SMS
 from common.exceptions import JMSException
 from common.utils.random import random_string
 from common.utils import get_logger
-
+from django.utils.translation import gettext_lazy as _
 
 logger = get_logger(__file__)
 
 
-@shared_task
+@shared_task(verbose_name=_('Send email'))
 def send_async(sender):
     sender.gen_and_send()
 

apps/ops/api/playbook.py

@@ -193,7 +193,7 @@ class PlaybookFileBrowserAPIView(APIView):
                     "id": os.path.join(relative_path, d) if not os.path.join(relative_path, d).startswith(
                         '.') else d,
                     "isParent": True,
-                    "open": False,
+                    "open": True,
                     "pId": relative_path if not relative_path.startswith('.') else 'root',
                     "temp": False
                 }

apps/ops/migrations/0025_auto_20230117_1130.py

@@ -13,7 +13,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='playbook',
             name='create_method',
-            field=models.CharField(choices=[('blank', 'Blank'), ('upload', 'Upload'), ('vcs', 'VCS')], default='blank', max_length=128, verbose_name='CreateMethod'),
+            field=models.CharField(choices=[('blank', 'Blank'), ('vcs', 'VCS')], default='blank', max_length=128, verbose_name='CreateMethod'),
         ),
         migrations.AddField(
             model_name='playbook',

apps/perms/serializers/permission.py

@@ -70,9 +70,7 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer):
         actions = self.fields.get("actions")
         if not actions:
             return
-        choices = actions._choices
+        actions.default = list(actions.choices.keys())
-        actions._choices = choices
-        actions.default = list(choices.keys())
 
     @classmethod
     def setup_eager_loading(cls, queryset):

apps/perms/tasks.py

@@ -19,12 +19,13 @@ from perms.notifications import (
     PermedAssetsWillExpireUserMsg,
     AssetPermsWillExpireForOrgAdminMsg,
 )
+from django.utils.translation import gettext_lazy as _
 
 logger = get_logger(__file__)
 
 
 @register_as_period_task(interval=settings.PERM_EXPIRED_CHECK_PERIODIC)
-@shared_task()
+@shared_task(verbose_name=_('Check asset permission expired'))
 @atomic()
 @tmp_to_root_org()
 def check_asset_permission_expired():
@@ -36,7 +37,7 @@ def check_asset_permission_expired():
 
 
 @register_as_period_task(crontab=CRONTAB_AT_AM_TEN)
-@shared_task()
+@shared_task(verbose_name=_('Send asset permission expired notification'))
 @atomic()
 @tmp_to_root_org()
 def check_asset_permission_will_expired():

apps/rbac/api/role.py

@@ -4,7 +4,6 @@ from rest_framework.exceptions import PermissionDenied
 from rest_framework.decorators import action
 
 from common.api import JMSModelViewSet
-from common.api import PaginatedResponseMixin
 from ..filters import RoleFilter
 from ..serializers import RoleSerializer, RoleUserSerializer
 from ..models import Role, SystemRole, OrgRole
@@ -18,6 +17,7 @@ __all__ = [
 
 class RoleViewSet(JMSModelViewSet):
     queryset = Role.objects.all()
+    ordering = ('-builtin', 'scope', 'name')
     serializer_classes = {
         'default': RoleSerializer,
         'users': RoleUserSerializer,
@@ -62,8 +62,7 @@ class RoleViewSet(JMSModelViewSet):
         return super().perform_update(serializer)
 
     def get_queryset(self):
-        queryset = super().get_queryset() \
+        queryset = super().get_queryset().annotate(permissions_amount=Count('permissions'))
-            .annotate(permissions_amount=Count('permissions'))
         return queryset
 
     @action(methods=['GET'], detail=True)

apps/settings/serializers/auth/ldap.py

@@ -15,7 +15,7 @@ class LDAPTestConfigSerializer(serializers.Serializer):
     AUTH_LDAP_BIND_PASSWORD = EncryptedField(required=False, allow_blank=True)
     AUTH_LDAP_SEARCH_OU = serializers.CharField()
     AUTH_LDAP_SEARCH_FILTER = serializers.CharField()
-    AUTH_LDAP_USER_ATTR_MAP = serializers.CharField()
+    AUTH_LDAP_USER_ATTR_MAP = serializers.JSONField()
     AUTH_LDAP_START_TLS = serializers.BooleanField(required=False)
     AUTH_LDAP = serializers.BooleanField(required=False)
 

apps/terminal/automations/deploy_applet_host/__init__.py

@@ -50,17 +50,18 @@ class DeployAppletHostManager:
         host_id = str(self.deployment.host.id)
         if not site_url:
             site_url = "http://localhost:8080"
-        if not download_host:
-            download_host = site_url
         options = self.deployment.host.deploy_options
-        site_url = site_url.rstrip("/")
+        core_host = options.get("CORE_HOST", site_url)
+        core_host = core_host.rstrip("/")
+        if not download_host:
+            download_host = core_host
         download_host = download_host.rstrip("/")
 
         def handler(plays):
             for play in plays:
                 play["vars"].update(options)
                 play["vars"]["APPLET_DOWNLOAD_HOST"] = download_host
-                play["vars"]["CORE_HOST"] = site_url
+                play["vars"]["CORE_HOST"] = core_host
                 play["vars"]["BOOTSTRAP_TOKEN"] = bootstrap_token
                 play["vars"]["HOST_ID"] = host_id
                 play["vars"]["HOST_NAME"] = self.deployment.host.name

apps/terminal/serializers/applet_host.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
@@ -27,6 +28,8 @@ class DeployOptionsSerializer(serializers.Serializer):
         (1, _("Disabled")),
         (0, _("Enabled")),
     )
+
+    CORE_HOST = serializers.CharField(default=settings.SITE_URL, label=_('API Server'), max_length=1024)
     RDS_Licensing = serializers.BooleanField(default=False, label=_("RDS Licensing"))
     RDS_LicenseServer = serializers.CharField(default='127.0.0.1', label=_('RDS License Server'), max_length=1024)
     RDS_LicensingMode = serializers.ChoiceField(choices=LICENSE_MODE_CHOICES, default=4, label=_('RDS Licensing Mode'))

apps/terminal/tasks.py

@@ -21,13 +21,14 @@ from .models import (
     Status, Session, Command, Task, AppletHostDeployment
 )
 from .utils import find_session_replay_local
+from django.utils.translation import gettext_lazy as _
 
 CACHE_REFRESH_INTERVAL = 10
 RUNNING = False
 logger = get_task_logger(__name__)
 
 
-@shared_task
+@shared_task(verbose_name=_('Periodic delete terminal status'))
 @register_as_period_task(interval=3600)
 @after_app_ready_start
 @after_app_shutdown_clean_periodic
@@ -36,7 +37,7 @@ def delete_terminal_status_period():
     Status.objects.filter(date_created__lt=yesterday).delete()
 
 
-@shared_task
+@shared_task(verbose_name=_('Clean orphan session'))
 @register_as_period_task(interval=600)
 @after_app_ready_start
 @after_app_shutdown_clean_periodic
@@ -55,7 +56,7 @@ def clean_orphan_session():
         session.save()
 
 
-@shared_task
+@shared_task(verbose_name=_('Periodic clean expired session'))
 @register_as_period_task(interval=3600 * 24)
 @after_app_ready_start
 @after_app_shutdown_clean_periodic
@@ -81,7 +82,7 @@ def clean_expired_session_period():
     logger.info("Clean session replay done")
 
 
-@shared_task
+@shared_task(verbose_name=_('Upload session replay to external storage'))
 def upload_session_replay_to_external_storage(session_id):
     logger.info(f'Start upload session to external storage: {session_id}')
     session = Session.objects.filter(id=session_id).first()
@@ -108,14 +109,14 @@ def upload_session_replay_to_external_storage(session_id):
     return
 
 
-@shared_task
+@shared_task(verbose_name=_('Run applet host deployment'))
 def run_applet_host_deployment(did):
     with tmp_to_builtin_org(system=1):
         deployment = AppletHostDeployment.objects.get(id=did)
         deployment.start()
 
 
-@shared_task
+@shared_task(verbose_name=_('Install applet'))
 def run_applet_host_deployment_install_applet(did, applet_id):
     with tmp_to_builtin_org(system=1):
         deployment = AppletHostDeployment.objects.get(id=did)

apps/tickets/serializers/ticket/ticket.py

@@ -39,9 +39,9 @@ class TicketSerializer(OrgResourceModelSerializerMixin):
         tp = self.fields.get('type')
         if not tp:
             return
-        choices = tp._choices
+        choices = tp.choices
         choices.pop(TicketType.general, None)
-        tp._choices = choices
+        tp.choices = choices
 
     @classmethod
     def setup_eager_loading(cls, queryset):

apps/users/filters.py

@@ -36,7 +36,7 @@ class UserFilter(BaseFilterSet):
         if not role:
             return queryset.none()
         queryset = queryset.prefetch_related('role_bindings') \
-            .filter(role_bindings__role_id=role.id) \
+            .filter(role_bindings__role_id=role.id, role_bindings__role__scope='system') \
             .distinct()
         return queryset
 
@@ -45,6 +45,6 @@ class UserFilter(BaseFilterSet):
         if not role:
             return queryset.none()
         queryset = queryset.prefetch_related('role_bindings') \
-            .filter(role_bindings__role_id=role.id) \
+            .filter(role_bindings__role_id=role.id, role_bindings__role__scope='org') \
             .distinct()
         return queryset

apps/users/tasks.py

@@ -16,12 +16,12 @@ from .models import User
 from users.notifications import UserExpirationReminderMsg
 from settings.utils import LDAPServerUtil, LDAPImportUtil
 from common.const.crontab import CRONTAB_AT_AM_TEN, CRONTAB_AT_PM_TWO
-
+from django.utils.translation import gettext_lazy as _
 
 logger = get_logger(__file__)
 
 
-@shared_task
+@shared_task(verbose_name=_('Check password expired'))
 def check_password_expired():
     users = User.get_nature_users().filter(source=User.Source.local)
     for user in users:
@@ -35,7 +35,7 @@ def check_password_expired():
         PasswordExpirationReminderMsg(user).publish_async()
 
 
-@shared_task
+@shared_task(verbose_name=_('Periodic check password expired'))
 @after_app_ready_start
 def check_password_expired_periodic():
     tasks = {
@@ -49,11 +49,11 @@ def check_password_expired_periodic():
     create_or_update_celery_periodic_tasks(tasks)
 
 
-@shared_task
+@shared_task(verbose_name=_('Check user expired'))
 def check_user_expired():
     date_expired_lt = timezone.now() + timezone.timedelta(days=User.DATE_EXPIRED_WARNING_DAYS)
-    users = User.get_nature_users()\
+    users = User.get_nature_users() \
-        .filter(source=User.Source.local)\
+        .filter(source=User.Source.local) \
         .filter(date_expired__lt=date_expired_lt)
 
     for user in users:
@@ -66,7 +66,7 @@ def check_user_expired():
         UserExpirationReminderMsg(user).publish_async()
 
 
-@shared_task
+@shared_task(verbose_name=_('Periodic check user expired'))
 @after_app_ready_start
 def check_user_expired_periodic():
     tasks = {
@@ -80,7 +80,7 @@ def check_user_expired_periodic():
     create_or_update_celery_periodic_tasks(tasks)
 
 
-@shared_task
+@shared_task(verbose_name=_('Import ldap user'))
 def import_ldap_user():
     logger.info("Start import ldap user task")
     util_server = LDAPServerUtil()
@@ -101,7 +101,7 @@ def import_ldap_user():
         logger.info('Imported {} users successfully'.format(len(users)))
 
 
-@shared_task
+@shared_task(verbose_name=_('Periodic import ldap user'))
 @after_app_ready_start
 def import_ldap_user_periodic():
     if not settings.AUTH_LDAP: