From 068b39d922e23340d18c47cd777686061668de3a Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 15 Mar 2021 05:15:48 +0800 Subject: [PATCH 1/8] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20jms=20?= =?UTF-8?q?=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jms | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jms b/jms index 8ba32460f..24b71f4e4 100755 --- a/jms +++ b/jms @@ -28,7 +28,7 @@ try: except ImportError as e: print("Not found __version__: {}".format(e)) print("Python is: ") - logging.info(subprocess.call('which python', shell=True)) + logging.info(sys.executable) __version__ = 'Unknown' sys.exit(1) From 5546719712af06014ed4758148e2a52efe5d9a4b Mon Sep 17 00:00:00 2001 From: Bai Date: Mon, 15 Mar 2021 11:38:08 +0800 Subject: [PATCH 2/8] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9get=5Finstance?= =?UTF-8?q?=E9=80=BB=E8=BE=91;=E4=BA=8C=E6=AC=A1=E6=9E=84=E5=BB=BAorg=5Fma?= =?UTF-8?q?pping;=E8=AE=A2=E9=98=85=E5=A4=B1=E6=95=88=E9=80=9F=E5=BA=A6?= =?UTF-8?q?=E6=85=A2=E4=BA=8E=E8=AF=BB=E5=8F=96=E9=80=9F=E5=BA=A6;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/models.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 65e7bd886..5c0e59538 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -51,7 +51,14 @@ class Organization(models.Model): def get_instance_from_memory(cls, id_or_name): if not isinstance(cls.orgs_mapping, dict): cls.orgs_mapping = cls.construct_orgs_mapping() - return cls.orgs_mapping.get(str(id_or_name)) + + org = cls.orgs_mapping.get(str(id_or_name)) + if not org: + # 内存失效速度慢于读取速度(on_org_create_or_update) + cls.orgs_mapping = cls.construct_orgs_mapping() + + org = cls.orgs_mapping.get(str(id_or_name)) + return org @classmethod def construct_orgs_mapping(cls): From bd8a1a7d0e0a2f44e5e7ab20e154e9e4e4433065 Mon Sep 17 00:00:00 2001 From: Bai Date: Mon, 15 Mar 2021 17:03:43 +0800 Subject: [PATCH 3/8] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8DMFA=E7=BB=91?= =?UTF-8?q?=E5=AE=9A=E5=A4=B1=E8=B4=A5=E7=9A=84=E9=97=AE=E9=A2=98(?= =?UTF-8?q?=E9=80=9A=E8=BF=87=E4=BF=AE=E6=94=B9session=E4=B8=ADauth=5Fback?= =?UTF-8?q?end=E7=9A=84key=E5=AE=9E=E7=8E=B0;django.auth.get=5Fuser?= =?UTF-8?q?=E6=97=B6=E6=A0=A1=E9=AA=8Cbackends=E8=B7=AF=E5=BE=84=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E8=BF=94=E5=9B=9EAnonymousUser)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/signals_handlers.py | 7 +++---- apps/users/views/profile/otp.py | 3 --- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/apps/authentication/signals_handlers.py b/apps/authentication/signals_handlers.py index ca06a0433..8e353ddf6 100644 --- a/apps/authentication/signals_handlers.py +++ b/apps/authentication/signals_handlers.py @@ -1,6 +1,5 @@ from importlib import import_module -from django.contrib.auth import BACKEND_SESSION_KEY from django.conf import settings from django.contrib.auth import user_logged_in from django.core.cache import cache @@ -25,17 +24,17 @@ def on_user_auth_login_success(sender, user, request, **kwargs): @receiver(openid_user_login_success) def on_oidc_user_login_success(sender, request, user, create=False, **kwargs): - request.session[BACKEND_SESSION_KEY] = 'OIDCAuthCodeBackend' + request.session['auth_backend'] = settings.AUTH_BACKEND_OIDC_CODE post_auth_success.send(sender, user=user, request=request) @receiver(openid_user_login_failed) def on_oidc_user_login_failed(sender, username, request, reason, **kwargs): - request.session[BACKEND_SESSION_KEY] = 'OIDCAuthCodeBackend' + request.session['auth_backend'] = settings.AUTH_BACKEND_OIDC_CODE post_auth_failed.send(sender, username=username, request=request, reason=reason) @receiver(cas_user_authenticated) def on_cas_user_login_success(sender, request, user, **kwargs): - request.session[BACKEND_SESSION_KEY] = 'CASBackend' + request.session['auth_backend'] = settings.AUTH_BACKEND_CAS post_auth_success.send(sender, user=user, request=request) diff --git a/apps/users/views/profile/otp.py b/apps/users/views/profile/otp.py index f0e938170..caed50532 100644 --- a/apps/users/views/profile/otp.py +++ b/apps/users/views/profile/otp.py @@ -51,9 +51,6 @@ class UserOtpEnableInstallAppView(TemplateView): return super().get_context_data(**kwargs) - - - class UserOtpEnableBindView(AuthMixin, TemplateView, FormView): template_name = 'users/user_otp_enable_bind.html' form_class = forms.UserCheckOtpCodeForm From a50dfe9c18f99654cf5d745a58685928930ce4c6 Mon Sep 17 00:00:00 2001 From: Bai Date: Mon, 15 Mar 2021 18:51:21 +0800 Subject: [PATCH 4/8] =?UTF-8?q?perf:=20=E7=AE=A1=E7=90=86=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E5=88=9B=E5=BB=BA/=E6=9B=B4=E6=96=B0username=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E8=AE=BE=E7=BD=AE=E4=B8=BArequired?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/serializers/admin_user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/assets/serializers/admin_user.py b/apps/assets/serializers/admin_user.py index 21eca51d0..0211d88cc 100644 --- a/apps/assets/serializers/admin_user.py +++ b/apps/assets/serializers/admin_user.py @@ -23,6 +23,7 @@ class AdminUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): read_only_fields = ['date_created', 'date_updated', 'created_by', 'assets_amount'] extra_kwargs = { + 'username': {"required": True}, 'password': {"write_only": True}, 'private_key': {"write_only": True}, 'public_key': {"write_only": True}, From 41f375a4f7ac3b74c21a8748b5e482218f62d8f6 Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 15 Mar 2021 15:11:50 +0800 Subject: [PATCH 5/8] =?UTF-8?q?fix:=20=E4=BF=AE=E6=AD=A3=E5=A4=9A=E4=B8=AA?= =?UTF-8?q?=20DEFAULT=20=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/node.py | 46 +++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index dfd527dba..d2a8c3ccf 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -22,7 +22,7 @@ from common.utils.common import timeit from common.db.models import output_as_string from common.utils import get_logger from orgs.mixins.models import OrgModelMixin, OrgManager -from orgs.utils import get_current_org, tmp_to_org +from orgs.utils import get_current_org, tmp_to_org, tmp_to_root_org from orgs.models import Organization @@ -462,25 +462,45 @@ class NodeAssetsMixin(NodeAllAssetsMappingMixin): class SomeNodesMixin: key = '' default_key = '1' - default_value = 'Default' empty_key = '-11' empty_value = _("empty") + @classmethod + def correct_default_node_if_need(cls): + with tmp_to_root_org(): + wrong_default_org = cls.objects.filter(key='1', value='Default').first() + if not wrong_default_org: + return + + if wrong_default_org.has_children_or_has_assets(): + return + + default_org = Organization.default() + right_default_org = cls.objects.filter(value=default_org.name).first() + if not right_default_org: + return + + if right_default_org.date_create > wrong_default_org.date_create: + return + + with atomic(): + logger.warn(f'Correct default node: ' + f'old={wrong_default_org.value}-{wrong_default_org.key} ' + f'new={right_default_org.value}-{right_default_org.key}') + wrong_default_org.delete() + right_default_org.key = '1' + right_default_org.save() + @classmethod def default_node(cls): + cls.correct_default_node_if_need() + default_org = Organization.default() with tmp_to_org(default_org): defaults = {'value': default_org.name} - try: - obj, created = cls.objects.get_or_create( - defaults=defaults, key=cls.default_key, - ) - except IntegrityError as e: - logger.error("Create default node failed: {}".format(e)) - cls.modify_other_org_root_node_key() - obj, created = cls.objects.get_or_create( - defaults=defaults, key=cls.default_key, - ) + obj, created = cls.objects.get_or_create( + defaults=defaults, key=cls.default_key, + ) return obj def is_default_node(self): @@ -500,7 +520,7 @@ class SomeNodesMixin: if not org_nodes_roots_keys: org_nodes_roots_keys = ['1'] max_key = max([int(k) for k in org_nodes_roots_keys]) - key = str(max_key + 1) if max_key != 0 else '2' + key = str(max_key + 1) if max_key > 0 else '2' return key @classmethod From 7dfd0ee8feed076feff7c8e2265f26f61d99ab1d Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 15 Mar 2021 14:53:19 +0800 Subject: [PATCH 6/8] =?UTF-8?q?fix(orgs):=20=E4=BF=AE=E5=A4=8D=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=20current=20org=20api=20=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf(users): 优化用户删除和移除行为 perf: 优化组织权限判断 --- apps/common/permissions.py | 4 ++-- apps/orgs/api.py | 4 ++-- apps/orgs/models.py | 9 +++++++++ apps/settings/api/common.py | 9 +++------ apps/users/api/user.py | 30 +++++++++++++++++++++--------- apps/users/serializers/profile.py | 2 ++ 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/apps/common/permissions.py b/apps/common/permissions.py index 32ea8ca94..65a57827d 100644 --- a/apps/common/permissions.py +++ b/apps/common/permissions.py @@ -109,9 +109,9 @@ class PermissionsMixin(UserPassesTestMixin): return True -class UserCanUseCurrentOrg(permissions.BasePermission): +class UserCanAnyPermCurrentOrg(permissions.BasePermission): def has_permission(self, request, view): - return current_org.can_use_by(request.user) + return current_org.can_any_by(request.user) class UserCanUpdatePassword(permissions.BasePermission): diff --git a/apps/orgs/api.py b/apps/orgs/api.py index 618ebe9ed..17b01a092 100644 --- a/apps/orgs/api.py +++ b/apps/orgs/api.py @@ -8,7 +8,7 @@ from rest_framework_bulk import BulkModelViewSet from rest_framework.generics import RetrieveAPIView from rest_framework.exceptions import PermissionDenied -from common.permissions import IsSuperUserOrAppUser, IsValidUser, UserCanUseCurrentOrg +from common.permissions import IsSuperUserOrAppUser, IsValidUser, UserCanAnyPermCurrentOrg from common.drf.api import JMSBulkRelationModelViewSet from .models import Organization, ROLE from .serializers import ( @@ -136,7 +136,7 @@ class OrgMemberUserRelationBulkViewSet(JMSBulkRelationModelViewSet): class CurrentOrgDetailApi(RetrieveAPIView): serializer_class = CurrentOrgSerializer - permission_classes = (IsValidUser, UserCanUseCurrentOrg) + permission_classes = (IsValidUser, UserCanAnyPermCurrentOrg) def get_object(self): return current_org diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 5c0e59538..72231f3c3 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -118,6 +118,8 @@ class Organization(models.Model): def can_audit_by(self, user): if user.is_superuser or user.is_super_auditor: return True + if self.can_admin_by(user): + return True if self.auditors.filter(id=user.id).exists(): return True return False @@ -125,10 +127,17 @@ class Organization(models.Model): def can_use_by(self, user): if user.is_superuser or user.is_super_auditor: return True + if self.can_audit_by(user): + return True if self.users.filter(id=user.id).exists(): return True return False + def can_any_by(self, user): + if user.is_superuser or user.is_super_auditor: + return True + return self.members.filter(id=user.id).exists() + @classmethod def get_user_orgs_by_role(cls, user, role): if not isinstance(role, (tuple, list)): diff --git a/apps/settings/api/common.py b/apps/settings/api/common.py index 2f01687f9..04674d25b 100644 --- a/apps/settings/api/common.py +++ b/apps/settings/api/common.py @@ -98,12 +98,9 @@ class PublicSettingApi(generics.RetrieveAPIView): def get_xpack_license_is_valid(): if not settings.XPACK_ENABLED: return False - try: - from xpack.plugins.license.models import License - return License.has_valid_license() - except Exception as e: - logger.error(e) - return False + + from xpack.plugins.license.models import License + return License.has_valid_license() @staticmethod def get_login_title(): diff --git a/apps/users/api/user.py b/apps/users/api/user.py index 7f7d91deb..5973be849 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -2,7 +2,7 @@ from django.core.cache import cache from django.utils.translation import ugettext as _ from rest_framework.decorators import action - +from django.conf import settings from rest_framework import generics from rest_framework.response import Response from rest_framework_bulk import BulkModelViewSet @@ -88,17 +88,14 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): def get_permissions(self): if self.action in ["retrieve", "list"]: - self.permission_classes = (IsOrgAdminOrAppUser,) - if self.request.query_params.get('all'): + if self.request.query_params.get('all'): + self.permission_classes = (IsSuperUser,) + else: + self.permission_classes = (IsOrgAdminOrAppUser,) + elif self.action in ['destroy']: self.permission_classes = (IsSuperUser,) return super().get_permissions() - def perform_destroy(self, instance): - if not current_org.is_root(): - instance.remove() - else: - return super().perform_destroy(instance) - def perform_bulk_destroy(self, objects): for obj in objects: self.check_object_permissions(self.request, obj) @@ -164,6 +161,21 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): OrganizationMember.objects.bulk_create(relations, ignore_conflicts=True) return Response(serializer.data, status=201) + @action(methods=['post'], detail=True, permission_classes=(IsOrgAdmin,)) + def remove(self, request, *args, **kwargs): + instance = self.get_object() + instance.remove() + return Response(status=204) + + @action(methods=['post'], detail=False, permission_classes=(IsOrgAdmin,), url_path='remove') + def bulk_remove(self, request, *args, **kwargs): + qs = self.get_queryset() + filtered = self.filter_queryset(qs) + + for instance in filtered: + instance.remove() + return Response(status=204) + class UserChangePasswordApi(UserQuerysetMixin, generics.RetrieveUpdateAPIView): permission_classes = (IsOrgAdmin,) diff --git a/apps/users/serializers/profile.py b/apps/users/serializers/profile.py index b1611307a..c89b173ff 100644 --- a/apps/users/serializers/profile.py +++ b/apps/users/serializers/profile.py @@ -11,6 +11,8 @@ from .user import UserSerializer class UserOrgSerializer(serializers.Serializer): id = serializers.CharField() name = serializers.CharField() + is_default = serializers.BooleanField(read_only=True) + is_root = serializers.BooleanField(read_only=True) class UserOrgLabelSerializer(serializers.Serializer): From 7f4377b0e80729b6d0af5bb770136be29a376214 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 15 Mar 2021 14:34:40 +0800 Subject: [PATCH 7/8] =?UTF-8?q?fix(auditor):=20=E4=BF=AE=E5=A4=8D=E5=AE=A1?= =?UTF-8?q?=E8=AE=A1=E5=91=98=E6=97=A0=E6=B3=95=E8=AE=BF=E9=97=AE=E5=91=BD?= =?UTF-8?q?=E4=BB=A4=E5=88=97=E8=A1=A8=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/api/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/terminal/api/storage.py b/apps/terminal/api/storage.py index b2bd69d28..a7c05adca 100644 --- a/apps/terminal/api/storage.py +++ b/apps/terminal/api/storage.py @@ -10,7 +10,7 @@ from django_filters import utils from terminal import const from common.const.http import GET -from common.permissions import IsSuperUser +from common.permissions import IsSuperUser, IsOrgAuditor from terminal.filters import CommandStorageFilter, CommandFilter, CommandFilterForStorageTree from ..models import CommandStorage, ReplayStorage from ..serializers import CommandStorageSerializer, ReplayStorageSerializer @@ -42,7 +42,7 @@ class CommandStorageViewSet(BaseStorageViewSetMixin, viewsets.ModelViewSet): permission_classes = (IsSuperUser,) filterset_class = CommandStorageFilter - @action(methods=[GET], detail=False, filterset_class=CommandFilterForStorageTree) + @action(methods=[GET], detail=False, permission_classes=(IsOrgAuditor, ), filterset_class=CommandFilterForStorageTree) def tree(self, request: Request): storage_qs = self.get_queryset().exclude(name='null') storages_with_count = [] From 049f6dca678848080c0dc03151233802fc698f9e Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 15 Mar 2021 17:46:43 +0800 Subject: [PATCH 8/8] =?UTF-8?q?fix:=20=E5=88=A0=E9=99=A4=E7=BB=84=E7=BB=87?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E7=A1=AE=E4=BF=9D=E6=B2=A1=E6=9C=89=E8=B7=9F?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E4=B9=8B=E5=A4=96=E7=9A=84=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E3=80=82=E4=BB=A5=E5=8F=8A=E7=BB=84=E7=BB=87?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=90=8E=EF=BC=8C=E5=B0=86=E8=B7=9F=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/api.py | 32 ++++++++++++++++++++--------- apps/orgs/signals_handler/common.py | 13 +++++++++++- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/apps/orgs/api.py b/apps/orgs/api.py index 17b01a092..4f72fc792 100644 --- a/apps/orgs/api.py +++ b/apps/orgs/api.py @@ -18,9 +18,13 @@ from .serializers import ( CurrentOrgSerializer ) from users.models import User, UserGroup -from assets.models import Asset, Domain, AdminUser, SystemUser, Label -from perms.models import AssetPermission -from orgs.utils import current_org +from assets.models import ( + Asset, Domain, AdminUser, SystemUser, Label, Node, Gateway, + CommandFilter, CommandFilterRule, GatheredUser +) +from applications.models import Application +from perms.models import AssetPermission, ApplicationPermission +from orgs.utils import current_org, tmp_to_root_org from common.utils import get_logger from .filters import OrgMemberRelationFilterSet from .models import OrganizationMember @@ -29,6 +33,15 @@ from .models import OrganizationMember logger = get_logger(__file__) +# 部分 org 相关的 model，需要清空这些数据之后才能删除该组织 +org_related_models = [ + User, UserGroup, Asset, Label, Domain, Gateway, Node, AdminUser, SystemUser, Label, + CommandFilter, CommandFilterRule, GatheredUser, + AssetPermission, ApplicationPermission, + Application, +] + + class OrgViewSet(BulkModelViewSet): filterset_fields = ('name',) search_fields = ('name', 'comment') @@ -44,24 +57,23 @@ class OrgViewSet(BulkModelViewSet): } return mapper.get(self.action, super().get_serializer_class()) + @tmp_to_root_org() def get_data_from_model(self, model): if model == User: data = model.objects.filter(orgs__id=self.org.id, m2m_org_members__role=ROLE.USER) + elif model == Node: + # 跟节点不能手动删除，所以排除检查 + data = model.objects.filter(org_id=self.org.id).exclude(parent_key='', key__regex=r'^[0-9]+$') else: data = model.objects.filter(org_id=self.org.id) return data def destroy(self, request, *args, **kwargs): self.org = self.get_object() - models = [ - User, UserGroup, - Asset, Domain, AdminUser, SystemUser, Label, - AssetPermission, - ] - for model in models: + for model in org_related_models: data = self.get_data_from_model(model) if data: - msg = _('Organization contains undeleted resources') + msg = _(f'Have `{model._meta.verbose_name}` exists, Please delete') return Response(data={'error': msg}, status=status.HTTP_403_FORBIDDEN) else: if str(current_org) == str(self.org): diff --git a/apps/orgs/signals_handler/common.py b/apps/orgs/signals_handler/common.py index 419bccdeb..59dc2806f 100644 --- a/apps/orgs/signals_handler/common.py +++ b/apps/orgs/signals_handler/common.py @@ -7,7 +7,8 @@ from functools import partial from django.dispatch import receiver from django.utils.functional import LazyObject from django.db.models.signals import m2m_changed -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save, post_delete, pre_delete +from django.utils.translation import ugettext as _ from orgs.utils import tmp_to_org from orgs.models import Organization, OrganizationMember @@ -18,6 +19,7 @@ from common.const.signals import PRE_REMOVE, POST_REMOVE from common.signals import django_ready from common.utils import get_logger from common.utils.connection import RedisPubSub +from common.exceptions import JMSException logger = get_logger(__file__) @@ -75,6 +77,15 @@ def on_org_delete(sender, **kwargs): expire_orgs_mapping_for_memory() +@receiver(pre_delete, sender=Organization) +def on_org_delete(sender, instance, **kwargs): + # 删除该组织下所有 节点 + with tmp_to_org(instance): + root_node = Node.org_root() + if root_node: + root_node.delete() + + def _remove_users(model, users, org): with tmp_to_org(org): if not isinstance(users, (tuple, list, set)):