From 6ff852e225681960f3925dacbe0cbbf67fc26ae6 Mon Sep 17 00:00:00 2001 From: Bai Date: Thu, 18 Jan 2024 19:43:55 +0800 Subject: [PATCH 1/4] =?UTF-8?q?perf:=20=E4=BF=AE=E5=A4=8D=20Count=20?= =?UTF-8?q?=E6=97=B6=E6=B2=A1=E6=9C=89=E5=8E=BB=E9=87=8D=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/accounts/tasks/remove_account.py | 2 +- apps/assets/serializers/domain.py | 2 +- apps/common/drf/filters.py | 2 +- apps/jumpserver/api.py | 4 ++-- apps/labels/serializers.py | 2 +- apps/ops/api/job.py | 2 +- apps/perms/serializers/permission.py | 8 ++++---- apps/rbac/api/role.py | 6 ++++-- apps/users/models/user.py | 2 +- apps/users/serializers/group.py | 2 +- 10 files changed, 17 insertions(+), 15 deletions(-) diff --git a/apps/accounts/tasks/remove_account.py b/apps/accounts/tasks/remove_account.py index 44bb9a840..6b637ed96 100644 --- a/apps/accounts/tasks/remove_account.py +++ b/apps/accounts/tasks/remove_account.py @@ -55,7 +55,7 @@ def clean_historical_accounts(): history_model = Account.history.model history_id_mapper = defaultdict(list) - ids = history_model.objects.values('id').annotate(count=Count('id')) \ + ids = history_model.objects.values('id').annotate(count=Count('id', distinct=True)) \ .filter(count__gte=limit).values_list('id', flat=True) if not ids: diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 9ea603ac3..4ada8fc2f 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -58,7 +58,7 @@ class DomainListSerializer(DomainSerializer): @classmethod def setup_eager_loading(cls, queryset): queryset = queryset.annotate( - assets_amount=Count('assets'), + assets_amount=Count('assets', distinct=True), ) return queryset diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index d84692dfe..85475f479 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -206,7 +206,7 @@ class LabelFilterBackend(filters.BaseFilterBackend): resources = resources.filter(q) \ .values('res_id') \ .order_by('res_id') \ - .annotate(count=Count('res_id')) \ + .annotate(count=Count('res_id', distinct=True)) \ .values('res_id', 'count') \ .filter(count=len(args)) return resources diff --git a/apps/jumpserver/api.py b/apps/jumpserver/api.py index cfa6b771d..5b672d5f9 100644 --- a/apps/jumpserver/api.py +++ b/apps/jumpserver/api.py @@ -182,14 +182,14 @@ class DatesLoginMetricMixin: def get_dates_login_times_assets(self): assets = self.sessions_queryset.values("asset") \ - .annotate(total=Count("asset")) \ + .annotate(total=Count("asset", distinct=True)) \ .annotate(last=Cast(Max("date_start"), output_field=CharField())) \ .order_by("-total") return list(assets[:10]) def get_dates_login_times_users(self): users = self.sessions_queryset.values("user_id") \ - .annotate(total=Count("user_id")) \ + .annotate(total=Count("user_id", distinct=True)) \ .annotate(user=Max('user')) \ .annotate(last=Cast(Max("date_start"), output_field=CharField())) \ .order_by("-total") diff --git a/apps/labels/serializers.py b/apps/labels/serializers.py index 803843ad8..62abc1bc8 100644 --- a/apps/labels/serializers.py +++ b/apps/labels/serializers.py @@ -34,7 +34,7 @@ class LabelSerializer(BulkOrgResourceModelSerializer): @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.annotate(res_count=Count('labeled_resources')) + queryset = queryset.annotate(res_count=Count('labeled_resources', distinct=True)) return queryset diff --git a/apps/ops/api/job.py b/apps/ops/api/job.py index 61768f5d9..eb908d04c 100644 --- a/apps/ops/api/job.py +++ b/apps/ops/api/job.py @@ -246,6 +246,6 @@ class UsernameHintsAPI(APIView): .filter(username__icontains=query) \ .filter(asset__in=assets) \ .values('username') \ - .annotate(total=Count('username')) \ + .annotate(total=Count('username', distinct=True)) \ .order_by('total', '-username')[:10] return Response(data=top_accounts) diff --git a/apps/perms/serializers/permission.py b/apps/perms/serializers/permission.py index 3e32b40e1..35d6ecacd 100644 --- a/apps/perms/serializers/permission.py +++ b/apps/perms/serializers/permission.py @@ -198,9 +198,9 @@ class AssetPermissionListSerializer(AssetPermissionSerializer): """Perform necessary eager loading of data.""" queryset = queryset \ .prefetch_related('labels', 'labels__label') \ - .annotate(users_amount=Count("users"), - user_groups_amount=Count("user_groups"), - assets_amount=Count("assets"), - nodes_amount=Count("nodes"), + .annotate(users_amount=Count("users", distinct=True), + user_groups_amount=Count("user_groups", distinct=True), + assets_amount=Count("assets", distinct=True), + nodes_amount=Count("nodes", distinct=True), ) return queryset diff --git a/apps/rbac/api/role.py b/apps/rbac/api/role.py index b9133c5c5..022a2ac05 100644 --- a/apps/rbac/api/role.py +++ b/apps/rbac/api/role.py @@ -80,9 +80,11 @@ class RoleViewSet(JMSModelViewSet): queryset = Role.objects.filter(id__in=ids).order_by(*self.ordering) org_id = current_org.id q = Q(role__scope=Role.Scope.system) | Q(role__scope=Role.Scope.org, org_id=org_id) - role_bindings = RoleBinding.objects.filter(q).values_list('role_id').annotate(user_count=Count('user_id')) + role_bindings = RoleBinding.objects.filter(q).values_list('role_id').annotate( + user_count=Count('user_id', distinct=True) + ) role_user_amount_mapper = {role_id: user_count for role_id, user_count in role_bindings} - queryset = queryset.annotate(permissions_amount=Count('permissions')) + queryset = queryset.annotate(permissions_amount=Count('permissions', distinct=True)) queryset = list(queryset) for role in queryset: role.users_amount = role_user_amount_mapper.get(role.id, 0) diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 290c0c1f1..f31ec6690 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -729,7 +729,7 @@ class JSONFilterMixin: bindings = RoleBinding.objects.filter(**kwargs, role__in=value) if match == 'm2m_all': - user_id = bindings.values('user_id').annotate(count=Count('user_id')) \ + user_id = bindings.values('user_id').annotate(count=Count('user_id', distinct=True)) \ .filter(count=len(value)).values_list('user_id', flat=True) else: user_id = bindings.values_list('user_id', flat=True) diff --git a/apps/users/serializers/group.py b/apps/users/serializers/group.py index 4c0de20f6..bbfc04918 100644 --- a/apps/users/serializers/group.py +++ b/apps/users/serializers/group.py @@ -46,7 +46,7 @@ class UserGroupSerializer(ResourceLabelsMixin, BulkOrgResourceModelSerializer): def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ queryset = queryset.prefetch_related('labels', 'labels__label') \ - .annotate(users_amount=Count('users', filter=Q(users__is_service_account=False))) + .annotate(users_amount=Count('users', distinct=True, filter=Q(users__is_service_account=False))) return queryset From 6c9d271ae145cb2f8c8eda610a9af7426fb70546 Mon Sep 17 00:00:00 2001 From: feng <1304903146@qq.com> Date: Thu, 18 Jan 2024 18:05:21 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20redis=20=E5=AF=86=E7=A0=81=E6=9C=89?= =?UTF-8?q?=E7=89=B9=E6=AE=8A=E5=AD=97=E7=AC=A6celery=20beat=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/start_celery_beat.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/utils/start_celery_beat.py b/utils/start_celery_beat.py index 19d578e70..2f089a4af 100644 --- a/utils/start_celery_beat.py +++ b/utils/start_celery_beat.py @@ -1,13 +1,12 @@ #!/usr/bin/env python # import os -import sys import signal import subprocess +import sys import redis_lock - -from redis import Redis, Sentinel +from redis import Redis, Sentinel, ConnectionPool BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) APPS_DIR = os.path.join(BASE_DIR, 'apps') @@ -50,9 +49,15 @@ if REDIS_SENTINEL_SERVICE_NAME and REDIS_SENTINELS: ) redis_client = sentinel_client.master_for(REDIS_SENTINEL_SERVICE_NAME) else: - connection_params['host'] = settings.REDIS_HOST - connection_params['port'] = settings.REDIS_PORT - redis_client = Redis(**connection_params) + REDIS_PROTOCOL = 'rediss' if settings.REDIS_USE_SSL else 'redis' + REDIS_LOCATION_NO_DB = '%(protocol)s://:%(password)s@%(host)s:%(port)s' % { + 'protocol': REDIS_PROTOCOL, + 'password': settings.REDIS_PASSWORD, + 'host': settings.REDIS_HOST, + 'port': settings.REDIS_PORT, + } + pool = ConnectionPool.from_url(REDIS_LOCATION_NO_DB, **connection_params) + redis_client = Redis(connection_pool=pool) scheduler = "ops.celery.beat.schedulers:DatabaseScheduler" processes = [] From 1f1c1a9157477f07ebfe97ca25b31084da3a67ea Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 23 Jan 2024 17:15:34 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=E7=94=A8=E6=88=B7=E6=98=AF=E5=90=A6=E6=B4=BB?= =?UTF-8?q?=E8=B7=83=E4=BB=BB=E5=8A=A1=E6=97=A0=E6=B3=95=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/users/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/users/tasks.py b/apps/users/tasks.py index 638e3bdec..7fd707ac7 100644 --- a/apps/users/tasks.py +++ b/apps/users/tasks.py @@ -86,7 +86,7 @@ def check_user_expired_periodic(): @tmp_to_root_org() def check_unused_users(): uncommon_users_ttl = settings.SECURITY_UNCOMMON_USERS_TTL - if not uncommon_users_ttl or not uncommon_users_ttl.isdigit(): + if not uncommon_users_ttl: return uncommon_users_ttl = int(uncommon_users_ttl) From 0edad24d5db08f5459f6a755f54d3718f8d0de0d Mon Sep 17 00:00:00 2001 From: wangruidong <940853815@qq.com> Date: Fri, 2 Feb 2024 14:54:19 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20=E8=B5=84=E4=BA=A7=E8=BF=87=E6=9C=9F?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E6=8F=90=E7=A4=BA=E5=8F=91=E9=80=81=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/perms/notifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/perms/notifications.py b/apps/perms/notifications.py index 5c82c2589..f168a0517 100644 --- a/apps/perms/notifications.py +++ b/apps/perms/notifications.py @@ -9,7 +9,7 @@ class PermedAssetsWillExpireUserMsg(UserMessage): def __init__(self, user, assets, day_count=0): super().__init__(user) self.assets = assets - self.day_count = _('today') if day_count == 0 else day_count + _('day') + self.day_count = _('today') if day_count == 0 else str(day_count) + _('day') def get_html_msg(self) -> dict: subject = _("You permed assets is about to expire") @@ -41,7 +41,7 @@ class AssetPermsWillExpireForOrgAdminMsg(UserMessage): super().__init__(user) self.perms = perms self.org = org - self.day_count = _('today') if day_count == 0 else day_count + _('day') + self.day_count = _('today') if day_count == 0 else str(day_count) + _('day') def get_items_with_url(self): items_with_url = []