From dd2413edd888d64e9a3157f42ef24cfd69154ec6 Mon Sep 17 00:00:00 2001 From: xinwen Date: Sat, 10 Oct 2020 16:27:29 +0800 Subject: [PATCH 01/12] =?UTF-8?q?fix(perms):=20=E6=8E=88=E6=9D=83=E6=A0=91?= =?UTF-8?q?=E4=B8=8E=E8=B5=84=E4=BA=A7=E5=88=97=E8=A1=A8=E7=9A=84=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/pagination.py | 2 +- .../api/user_permission/user_permission_nodes_with_assets.py | 2 ++ apps/perms/signals_handler.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/assets/pagination.py b/apps/assets/pagination.py index dd37600f8..4fd866e3d 100644 --- a/apps/assets/pagination.py +++ b/apps/assets/pagination.py @@ -18,7 +18,7 @@ class AssetLimitOffsetPagination(LimitOffsetPagination): self.limit_query_param, self.offset_query_param, 'node', 'all', 'show_current_asset', - 'node_id', 'display', 'draw', + 'node_id', 'display', 'draw', 'fields_size', } for k, v in self._request.query_params.items(): diff --git a/apps/perms/api/user_permission/user_permission_nodes_with_assets.py b/apps/perms/api/user_permission/user_permission_nodes_with_assets.py index e0591b7df..ae9f00536 100644 --- a/apps/perms/api/user_permission/user_permission_nodes_with_assets.py +++ b/apps/perms/api/user_permission/user_permission_nodes_with_assets.py @@ -3,6 +3,7 @@ from rest_framework.generics import ListAPIView from rest_framework.request import Request from rest_framework.response import Response +from django.db.models import F from common.permissions import IsValidUser from common.utils import get_logger, get_object_or_none @@ -41,6 +42,7 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): rebuild_user_tree_if_need(request, user) all_nodes = get_user_granted_nodes_list_via_mapping_node(user) all_assets = get_user_granted_all_assets(user) + all_assets = all_assets.annotate(parent_key=F('nodes__key')) data = [ *self.serialize_nodes(all_nodes, with_asset_amount=True), diff --git a/apps/perms/signals_handler.py b/apps/perms/signals_handler.py index 3882cd1f5..fe39804f6 100644 --- a/apps/perms/signals_handler.py +++ b/apps/perms/signals_handler.py @@ -35,7 +35,9 @@ def on_asset_permission_delete(instance, **kwargs): def create_rebuild_user_tree_task_by_asset_perm(asset_perm: AssetPermission): user_ids = set() user_ids.update( - UserGroup.objects.filter(assetpermissions=asset_perm).distinct().values_list('users__id', flat=True) + UserGroup.objects.filter( + assetpermissions=asset_perm, users__id__isnull=False + ).distinct().values_list('users__id', flat=True) ) user_ids.update( User.objects.filter(assetpermissions=asset_perm).distinct().values_list('id', flat=True) From 04393763266f56bdcad263987d4e49607164b604 Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 10 Oct 2020 19:05:22 +0800 Subject: [PATCH 02/12] =?UTF-8?q?feat(users):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E7=94=A8=E6=88=B7suggetion=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/users/api/user.py | 62 +++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/apps/users/api/user.py b/apps/users/api/user.py index 148378522..ebf652117 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -1,7 +1,7 @@ # ~*~ coding: utf-8 ~*~ from django.core.cache import cache -from django.db.models import CharField from django.utils.translation import ugettext as _ +from rest_framework.decorators import action from rest_framework import generics from rest_framework.response import Response @@ -17,7 +17,7 @@ from common.utils import get_logger from orgs.utils import current_org from orgs.models import ROLE as ORG_ROLE, OrganizationMember from .. import serializers -from ..serializers import UserSerializer, UserRetrieveSerializer +from ..serializers import UserSerializer, UserRetrieveSerializer, MiniUserSerializer from .mixins import UserQuerysetMixin from ..models import User from ..signals import post_user_create @@ -37,7 +37,8 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): permission_classes = (IsOrgAdmin, CanUpdateDeleteUser) serializer_classes = { 'default': UserSerializer, - 'retrieve': UserRetrieveSerializer + 'retrieve': UserRetrieveSerializer, + 'suggestion': MiniUserSerializer } extra_filter_backends = [OrgRoleUserFilterBackend] @@ -52,27 +53,24 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): for user in users: post_user_create.send(self.__class__, user=user) + @staticmethod + def set_users_to_org(users, org_roles): + # 只有真实存在的组织才真正关联用户 + if not current_org or not current_org.is_real(): + return + for user, roles in zip(users, org_roles): + if not roles: + # 当前组织创建的用户，至少是该组织的`User` + roles = [ORG_ROLE.USER] + OrganizationMember.objects.set_user_roles(current_org, user, roles) + def perform_create(self, serializer): - validated_data = serializer.validated_data - - # `org_roles` 先 `pop` - if isinstance(validated_data, list): - org_roles = [item.pop('org_roles', []) for item in validated_data] - else: - org_roles = [validated_data.pop('org_roles', [])] - + org_roles = self.get_serializer_org_roles(serializer) # 创建用户 users = serializer.save() if isinstance(users, User): users = [users] - - # 只有真实存在的组织才真正关联用户 - if current_org and current_org.is_real(): - for user, roles in zip(users, org_roles): - if not roles: - # 当前组织创建的用户，至少是该组织的`User` - roles.append(ORG_ROLE.USER) - OrganizationMember.objects.set_user_roles(current_org, user, roles) + self.set_users_to_org(users, org_roles) self.send_created_signal(users) def get_permissions(self): @@ -93,22 +91,22 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): self.check_object_permissions(self.request, obj) self.perform_destroy(obj) - def perform_update(self, serializer): + @staticmethod + def get_serializer_org_roles(serializer): validated_data = serializer.validated_data # `org_roles` 先 `pop` if isinstance(validated_data, list): org_roles = [item.pop('org_roles', None) for item in validated_data] else: org_roles = [validated_data.pop('org_roles', None)] + return org_roles + def perform_update(self, serializer): + org_roles = self.get_serializer_org_roles(serializer) users = serializer.save() if isinstance(users, User): users = [users] - if current_org and current_org.is_real(): - for user, roles in zip(users, org_roles): - if roles is not None: - # roles 是 `Node` 表明不需要更新 - OrganizationMember.objects.set_user_roles(current_org, user, roles) + self.set_users_to_org(users, org_roles) def perform_bulk_update(self, serializer): # TODO: 需要测试 @@ -120,6 +118,20 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): self.check_object_permissions(self.request, user) return super().perform_bulk_update(serializer) + @action(methods=['get'], detail=False, permission_classes=(IsOrgAdminOrAppUser,)) + def suggestion(self, request): + queryset = User.objects.exclude(role=User.ROLE.APP) + queryset = self.filter_queryset(queryset) + queryset = queryset[:3] + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + class UserChangePasswordApi(UserQuerysetMixin, generics.RetrieveUpdateAPIView): permission_classes = (IsOrgAdmin,) From f97685c788673020efaf550e4712c3fb56b28680 Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 10 Oct 2020 18:33:43 +0800 Subject: [PATCH 03/12] =?UTF-8?q?perf(orgs):=20=E4=BC=98=E5=8C=96=E7=BB=84?= =?UTF-8?q?=E7=BB=87=E7=94=A8=E6=88=B7=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/api.py | 54 +++------------------------------------- apps/orgs/filters.py | 2 +- apps/orgs/serializers.py | 5 ++++ 3 files changed, 10 insertions(+), 51 deletions(-) diff --git a/apps/orgs/api.py b/apps/orgs/api.py index ba8b06564..e7c83fa36 100644 --- a/apps/orgs/api.py +++ b/apps/orgs/api.py @@ -75,56 +75,10 @@ class OrgMemberRelationBulkViewSet(JMSBulkRelationModelViewSet): serializer_class = OrgMemberSerializer filterset_class = OrgMemberRelationFilterSet - @staticmethod - def clear_request_data(request): - data = request.data - - ignore_already_exist = request.query_params.get('ignore_already_exist') - if not ignore_already_exist: - return data - - query_params = Q() - for _data in data: - query_fields = {} - org = _data.get('org') - if org: - query_fields.update({'org': org}) - user = _data.get('user') - if user: - query_fields.update({'user': user}) - role = _data.get('role') - if role: - query_fields.update({'role': role}) - query_params |= Q(**query_fields) - - if not query_params: - return data - - members = OrganizationMember.objects.filter(query_params) - members = [ - {'org': str(member.org_id), 'user': str(member.user_id), 'role': member.role} - for member in members - ] - if not members: - return data - - for member in members: - if member in data: - data.remove(member) - return data - - def create(self, request, *args, **kwargs): - bulk = isinstance(request.data, list) - - if not bulk: - return CreateModelMixin.create(self, request, *args, **kwargs) - - else: - data = self.clear_request_data(request) - serializer = self.get_serializer(data=data, many=True) - serializer.is_valid(raise_exception=True) - self.perform_bulk_create(serializer) - return Response(serializer.data, status=status.HTTP_201_CREATED) + def perform_bulk_create(self, serializer): + data = serializer.validated_data + relations = [OrganizationMember(**i) for i in data] + OrganizationMember.objects.bulk_create(relations, ignore_conflicts=True) def perform_bulk_destroy(self, queryset): objs = list(queryset.all().prefetch_related('user', 'org')) diff --git a/apps/orgs/filters.py b/apps/orgs/filters.py index df68e468f..c6c026198 100644 --- a/apps/orgs/filters.py +++ b/apps/orgs/filters.py @@ -13,4 +13,4 @@ class OrgMemberRelationFilterSet(filterset.FilterSet): class Meta: model = OrganizationMember - fields = ('org_id', 'user_id', 'role', 'id') + fields = ('org_id', 'user_id', 'org', 'user', 'role', 'id') diff --git a/apps/orgs/serializers.py b/apps/orgs/serializers.py index 21b695996..2a732a03f 100644 --- a/apps/orgs/serializers.py +++ b/apps/orgs/serializers.py @@ -60,6 +60,11 @@ class OrgMemberSerializer(BulkModelSerializer): model = OrganizationMember fields = ('id', 'org', 'user', 'role', 'org_display', 'user_display', 'role_display') + def get_unique_together_validators(self): + if self.parent: + return [] + return super().get_unique_together_validators() + @classmethod def setup_eager_loading(cls, queryset): return queryset.annotate( From f4799d90c04c08b469dbdf510414aa15fb8d3082 Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 10 Oct 2020 15:39:13 +0800 Subject: [PATCH 04/12] =?UTF-8?q?fix(assets):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=82=B9=E5=87=BB=E8=8A=82=E7=82=B9=E6=9B=B4=E6=96=B0=E7=A1=AC?= =?UTF-8?q?=E4=BB=B6=E4=BF=A1=E6=81=AF=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/tasks/gather_asset_hardware_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/assets/tasks/gather_asset_hardware_info.py b/apps/assets/tasks/gather_asset_hardware_info.py index e473366a5..aa79e1655 100644 --- a/apps/assets/tasks/gather_asset_hardware_info.py +++ b/apps/assets/tasks/gather_asset_hardware_info.py @@ -129,5 +129,5 @@ def update_assets_hardware_info_period(): def update_node_assets_hardware_info_manual(node): task_name = _("Update node asset hardware information: {}").format(node.name) assets = node.get_all_assets() - result = update_assets_hardware_info_util.delay(assets, task_name=task_name) + result = update_assets_hardware_info_util(assets, task_name=task_name) return result From c4caeb92ee6a6e582f96ba6c265dcb342be3dbeb Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Mon, 12 Oct 2020 12:44:30 +0800 Subject: [PATCH 05/12] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E7=94=9F?= =?UTF-8?q?=E6=88=90=E5=81=87=E6=95=B0=E6=8D=AE=20(#4759)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 优化生成假数据 --- apps/assets/models/asset.py | 33 ------- apps/assets/models/cluster.py | 24 ----- apps/assets/models/group.py | 18 ---- apps/assets/models/node.py | 21 +--- apps/assets/models/user.py | 40 -------- apps/perms/models/asset_permission.py | 47 --------- apps/perms/utils/user_asset_permission.py | 7 ++ apps/users/models/group.py | 17 ---- apps/users/models/user.py | 25 ----- requirements/requirements.txt | 2 +- utils/generate_fake_data/__init__.py | 0 utils/generate_fake_data/generate.py | 49 ++++++++++ .../generate_fake_data/resources/__init__.py | 0 utils/generate_fake_data/resources/assets.py | 96 +++++++++++++++++++ utils/generate_fake_data/resources/base.py | 45 +++++++++ utils/generate_fake_data/resources/perms.py | 82 ++++++++++++++++ utils/generate_fake_data/resources/users.py | 62 ++++++++++++ 17 files changed, 345 insertions(+), 223 deletions(-) create mode 100644 utils/generate_fake_data/__init__.py create mode 100644 utils/generate_fake_data/generate.py create mode 100644 utils/generate_fake_data/resources/__init__.py create mode 100644 utils/generate_fake_data/resources/assets.py create mode 100644 utils/generate_fake_data/resources/base.py create mode 100644 utils/generate_fake_data/resources/perms.py create mode 100644 utils/generate_fake_data/resources/users.py diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 65414e86d..900a87060 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -355,36 +355,3 @@ class Asset(ProtocolsMixin, NodesRelationMixin, OrgModelMixin): class Meta: unique_together = [('org_id', 'hostname')] verbose_name = _("Asset") - - @classmethod - def generate_fake(cls, count=100): - from .user import AdminUser, SystemUser - from random import seed, choice - from django.db import IntegrityError - from .node import Node - from orgs.utils import get_current_org - from orgs.models import Organization - org = get_current_org() - if not org or not org.is_real(): - Organization.default().change_to() - - nodes = list(Node.objects.all()) - seed() - for i in range(count): - ip = [str(i) for i in random.sample(range(255), 4)] - asset = cls(ip='.'.join(ip), - hostname='.'.join(ip), - admin_user=choice(AdminUser.objects.all()), - created_by='Fake') - try: - asset.save() - asset.protocols = 'ssh/22' - if nodes and len(nodes) > 3: - _nodes = random.sample(nodes, 3) - else: - _nodes = [Node.default_node()] - asset.nodes.set(_nodes) - logger.debug('Generate fake asset : %s' % asset.ip) - except IntegrityError: - print('Error continue') - continue diff --git a/apps/assets/models/cluster.py b/apps/assets/models/cluster.py index 84f8f2374..6c0692ab9 100644 --- a/apps/assets/models/cluster.py +++ b/apps/assets/models/cluster.py @@ -38,27 +38,3 @@ class Cluster(models.Model): class Meta: ordering = ['name'] verbose_name = _("Cluster") - - @classmethod - def generate_fake(cls, count=5): - from random import seed, choice - import forgery_py - from django.db import IntegrityError - - seed() - for i in range(count): - cluster = cls(name=forgery_py.name.full_name(), - bandwidth='200M', - contact=forgery_py.name.full_name(), - phone=forgery_py.address.phone(), - address=forgery_py.address.city() + forgery_py.address.street_address(), - # operator=choice(['北京联通', '北京电信', 'BGP全网通']), - operator=choice([_('Beijing unicom'), _('Beijing telecom'), _('BGP full netcom')]), - comment=forgery_py.lorem_ipsum.sentence(), - created_by='Fake') - try: - cluster.save() - logger.debug('Generate fake asset group: %s' % cluster.name) - except IntegrityError: - print('Error continue') - continue diff --git a/apps/assets/models/group.py b/apps/assets/models/group.py index b9bf16f18..ee5472622 100644 --- a/apps/assets/models/group.py +++ b/apps/assets/models/group.py @@ -33,21 +33,3 @@ class AssetGroup(models.Model): def initial(cls): asset_group = cls(name=_('Default'), comment=_('Default asset group')) asset_group.save() - - @classmethod - def generate_fake(cls, count=100): - from random import seed - import forgery_py - from django.db import IntegrityError - - seed() - for i in range(count): - group = cls(name=forgery_py.name.full_name(), - comment=forgery_py.lorem_ipsum.sentence(), - created_by='Fake') - try: - group.save() - logger.debug('Generate fake asset group: %s' % group.name) - except IntegrityError: - print('Error continue') - continue diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 5b4dd7259..7c1c81cde 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -97,9 +97,11 @@ class FamilyMixin: def all_children(self): return self.get_all_children(with_self=False) - def create_child(self, value, _id=None): + def create_child(self, value=None, _id=None): with atomic(savepoint=False): child_key = self.get_next_child_key() + if value is None: + value = child_key child = self.__class__.objects.create( id=_id, key=child_key, value=value, parent_key=self.key, ) @@ -456,20 +458,3 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): if self.has_children_or_has_assets(): return return super().delete(using=using, keep_parents=keep_parents) - - @classmethod - def generate_fake(cls, count=100): - import random - org = get_current_org() - if not org or not org.is_real(): - Organization.default().change_to() - nodes = list(cls.objects.all()) - if count > 100: - length = 100 - else: - length = count - - for i in range(length): - node = random.choice(nodes) - child = node.create_child('Node {}'.format(i)) - print("{}. {}".format(i, child)) diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 3fb862760..087a05bef 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -65,26 +65,6 @@ class AdminUser(BaseUser): unique_together = [('name', 'org_id')] verbose_name = _("Admin user") - @classmethod - def generate_fake(cls, count=10): - from random import seed - import forgery_py - from django.db import IntegrityError - - seed() - for i in range(count): - obj = cls(name=forgery_py.name.full_name(), - username=forgery_py.internet.user_name(), - password=forgery_py.lorem_ipsum.word(), - comment=forgery_py.lorem_ipsum.sentence(), - created_by='Fake') - try: - obj.save() - logger.debug('Generate fake asset group: %s' % obj.name) - except IntegrityError: - print('Error continue') - continue - class SystemUser(BaseUser): PROTOCOL_SSH = 'ssh' @@ -199,23 +179,3 @@ class SystemUser(BaseUser): ordering = ['name'] unique_together = [('name', 'org_id')] verbose_name = _("System user") - - @classmethod - def generate_fake(cls, count=10): - from random import seed - import forgery_py - from django.db import IntegrityError - - seed() - for i in range(count): - obj = cls(name=forgery_py.name.full_name(), - username=forgery_py.internet.user_name(), - password=forgery_py.lorem_ipsum.word(), - comment=forgery_py.lorem_ipsum.sentence(), - created_by='Fake') - try: - obj.save() - logger.debug('Generate fake asset group: %s' % obj.name) - except IntegrityError: - print('Error continue') - continue diff --git a/apps/perms/models/asset_permission.py b/apps/perms/models/asset_permission.py index 5611f6ad5..9fc00f4a7 100644 --- a/apps/perms/models/asset_permission.py +++ b/apps/perms/models/asset_permission.py @@ -136,53 +136,6 @@ class AssetPermission(BasePermission): assets = Asset.objects.filter(id__in=assets_ids) return assets - @classmethod - def generate_fake(cls, count=100): - from ..hands import User, Node, SystemUser - import random - - org = get_current_org() - if not org or not org.is_real(): - Organization.default().change_to() - - nodes = list(Node.objects.all()) - assets = list(Asset.objects.all()) - system_users = list(SystemUser.objects.all()) - users = User.objects.filter(username='admin') - - for i in range(count): - name = "fake_perm_to_admin_{}".format(str(uuid.uuid4())[:6]) - perm = cls(name=name) - try: - perm.save() - perm.users.set(users) - if system_users and len(system_users) > 3: - _system_users = random.sample(system_users, 3) - elif system_users: - _system_users = [system_users[0]] - else: - _system_users = [] - perm.system_users.set(_system_users) - - if nodes and len(nodes) > 3: - _nodes = random.sample(nodes, 3) - else: - _nodes = [Node.default_node()] - perm.nodes.set(_nodes) - - if assets and len(assets) > 3: - _assets = random.sample(assets, 3) - elif assets: - _assets = [assets[0]] - else: - _assets = [] - perm.assets.set(_assets) - - logger.debug('Generate fake perm: %s' % perm.name) - - except Exception as e: - print('Error continue') - continue class UserGrantedMappingNode(FamilyMixin, models.JMSBaseModel): diff --git a/apps/perms/utils/user_asset_permission.py b/apps/perms/utils/user_asset_permission.py index 01a558ae2..3fb1ed9d0 100644 --- a/apps/perms/utils/user_asset_permission.py +++ b/apps/perms/utils/user_asset_permission.py @@ -256,6 +256,13 @@ def rebuild_user_mapping_nodes(user): logger.info(f'>>> {dt_formater(now())} end rebuild {user} mapping nodes') +def rebuild_all_user_mapping_nodes(): + from users.models import User + users = User.objects.all() + for user in users: + rebuild_user_mapping_nodes(user) + + def get_user_granted_nodes_list_via_mapping_node(user): """ 这里的 granted nodes, 是整棵树需要的node，推算出来的也算 diff --git a/apps/users/models/group.py b/apps/users/models/group.py index 0ae636f76..d06fce989 100644 --- a/apps/users/models/group.py +++ b/apps/users/models/group.py @@ -39,20 +39,3 @@ class UserGroup(OrgModelMixin): else: group = default_group[0] return group - - @classmethod - def generate_fake(cls, count=100): - from random import seed, choice - import forgery_py - from . import User - - seed() - for i in range(count): - group = cls(name=forgery_py.name.full_name(), - comment=forgery_py.lorem_ipsum.sentence(), - created_by=choice(User.objects.all()).username) - try: - group.save() - except IntegrityError: - print('Error continue') - continue diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 22adab238..81f0f593b 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -670,28 +670,3 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser): if self.email and self.source == self.SOURCE_LOCAL: return True return False - - @classmethod - def generate_fake(cls, count=100): - from random import seed, choice - import forgery_py - from django.db import IntegrityError - from .group import UserGroup - - seed() - for i in range(count): - user = cls(username=forgery_py.internet.user_name(True), - email=forgery_py.internet.email_address(), - name=forgery_py.name.full_name(), - password=make_password(forgery_py.lorem_ipsum.word()), - role=choice(list(dict(User.ROLE.choices).keys())), - wechat=forgery_py.internet.user_name(True), - comment=forgery_py.lorem_ipsum.sentence(), - created_by=choice(cls.objects.all()).username) - try: - user.save() - except IntegrityError: - print('Duplicate Error, continue ...') - continue - user.groups.add(choice(UserGroup.objects.all())) - user.save() diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 018632ed3..0b71e5394 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -33,7 +33,7 @@ enum-compat==0.0.2 ephem==3.7.6.0 eventlet==0.24.1 future==0.16.0 -ForgeryPy==0.1 +ForgeryPy3==0.3.2 greenlet==0.4.14 gunicorn==19.9.0 idna==2.6 diff --git a/utils/generate_fake_data/__init__.py b/utils/generate_fake_data/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/generate_fake_data/generate.py b/utils/generate_fake_data/generate.py new file mode 100644 index 000000000..f504c8e55 --- /dev/null +++ b/utils/generate_fake_data/generate.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +import os +import sys +import django +import argparse + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +APPS_DIR = os.path.join(BASE_DIR, 'apps') +sys.path.insert(0, APPS_DIR) + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jumpserver.settings") +django.setup() + +from resources.assets import AssetsGenerator, NodesGenerator, SystemUsersGenerator, AdminUsersGenerator +from resources.users import UserGroupGenerator, UserGenerator +from resources.perms import AssetPermissionGenerator + + +resource_generator_mapper = { + 'asset': AssetsGenerator, + 'node': NodesGenerator, + 'system_user': SystemUsersGenerator, + 'admin_user': AdminUsersGenerator, + 'user': UserGenerator, + 'user_group': UserGroupGenerator, + 'asset_permission': AssetPermissionGenerator +} + + +def main(): + parser = argparse.ArgumentParser(description='Generate fake data') + parser.add_argument( + 'resource', type=str, + choices=resource_generator_mapper.keys(), + help="resource to generate" + ) + parser.add_argument('-c', '--count', type=int, default=100) + parser.add_argument('-b', '--batch_size', type=int, default=100) + parser.add_argument('-o', '--org', type=str, default='') + args = parser.parse_args() + resource, count, batch_size, org_id = args.resource, args.count, args.batch_size, args.org + generator_cls = resource_generator_mapper[resource] + generator = generator_cls(org_id=org_id, batch_size=batch_size) + generator.generate(count) + + +if __name__ == '__main__': + main() diff --git a/utils/generate_fake_data/resources/__init__.py b/utils/generate_fake_data/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/generate_fake_data/resources/assets.py b/utils/generate_fake_data/resources/assets.py new file mode 100644 index 000000000..a0edc9f08 --- /dev/null +++ b/utils/generate_fake_data/resources/assets.py @@ -0,0 +1,96 @@ +from random import choice +import random +import forgery_py + +from .base import FakeDataGenerator + +from assets.models import * +from assets.utils import check_node_assets_amount + + +class AdminUsersGenerator(FakeDataGenerator): + resource = 'admin_user' + + def do_generate(self, batch, batch_size): + admin_users = [] + for i in batch: + username = forgery_py.internet.user_name(True) + password = forgery_py.basic.password() + admin_users.append(AdminUser( + name=username.title(), + username=username, + password=password, + org_id=self.org.id, + created_by='Fake', + )) + AdminUser.objects.bulk_create(admin_users, ignore_conflicts=True) + + +class SystemUsersGenerator(FakeDataGenerator): + def do_generate(self, batch, batch_size): + system_users = [] + protocols = list(dict(SystemUser.PROTOCOL_CHOICES).keys()) + for i in batch: + username = forgery_py.internet.user_name(True) + protocol = random.choice(protocols) + name = username.title() + name = f'{name}-{protocol}' + system_users.append(SystemUser( + name=name, + username=username, + password=forgery_py.basic.password(), + protocol=protocol, + org_id=self.org.id, + created_by='Fake', + )) + SystemUser.objects.bulk_create(system_users, ignore_conflicts=True) + + +class NodesGenerator(FakeDataGenerator): + resource = 'node' + + def do_generate(self, batch, batch_size): + nodes_to_generate_children = list(Node.objects.all()) + for i in batch: + parent = random.choice(nodes_to_generate_children) + parent.create_child() + + +class AssetsGenerator(FakeDataGenerator): + resource = 'asset' + admin_users_id: list + nodes_id: list + + def pre_generate(self): + self.admin_users_id = list(AdminUser.objects.all().values_list('id', flat=True)) + self.nodes_id = list(Node.objects.all().values_list('id', flat=True)) + + def set_assets_nodes(self, assets): + assets_id = [asset.id for asset in assets] + objs = [] + for asset_id in assets_id: + nodes_id_add_to = random.sample(self.nodes_id, 3) + objs_add = [Asset.nodes.through(asset_id=asset_id, node_id=nid) for nid in nodes_id_add_to] + objs.extend(objs_add) + Asset.nodes.through.objects.bulk_create(objs, ignore_conflicts=True) + + def do_generate(self, batch, batch_size): + assets = [] + + for i in batch: + ip = forgery_py.internet.ip_v4() + hostname = forgery_py.email.address().replace('@', '.') + hostname = f'{hostname}-{ip}' + data = dict( + ip=ip, + hostname=hostname, + admin_user_id=choice(self.admin_users_id), + created_by='Fake', + org_id=self.org.id + ) + assets.append(Asset(**data)) + creates = Asset.objects.bulk_create(assets, ignore_conflicts=True) + self.set_assets_nodes(creates) + + def after_generate(self): + check_node_assets_amount() diff --git a/utils/generate_fake_data/resources/base.py b/utils/generate_fake_data/resources/base.py new file mode 100644 index 000000000..ba45f602b --- /dev/null +++ b/utils/generate_fake_data/resources/base.py @@ -0,0 +1,45 @@ +#!/usr/bin/python + +from random import seed +from itertools import islice + +from orgs.models import Organization + + +class FakeDataGenerator: + resource = 'Fake' + + def __init__(self, batch_size=100, org_id=None): + self.batch_size = batch_size + self.org = self.switch_org(org_id) + seed() + + def switch_org(self, org_id): + o = Organization.get_instance(org_id, default=True) + if o: + o.change_to() + print('Current org is: {}'.format(o)) + return o + + def do_generate(self, batch, batch_size): + raise NotImplementedError + + def pre_generate(self): + pass + + def after_generate(self): + pass + + def generate(self, count=100): + self.pre_generate() + counter = iter(range(count)) + created = 0 + while True: + batch = list(islice(counter, self.batch_size)) + if not batch: + break + self.do_generate(batch, self.batch_size) + from_size = created + created += len(batch) + print('Generate %s: %s-%s' % (self.resource, from_size, created)) + self.after_generate() \ No newline at end of file diff --git a/utils/generate_fake_data/resources/perms.py b/utils/generate_fake_data/resources/perms.py new file mode 100644 index 000000000..e76e3618f --- /dev/null +++ b/utils/generate_fake_data/resources/perms.py @@ -0,0 +1,82 @@ +from random import choice, sample +import forgery_py + +from .base import FakeDataGenerator + +from users.models import * +from assets.models import * +from perms.models import * + + +class AssetPermissionGenerator(FakeDataGenerator): + resource = 'asset_permission' + users_id: list + user_groups_id: list + assets_id: list + nodes_id: list + system_users_id: list + + def pre_generate(self): + self.nodes_id = list(Node.objects.all().values_list('id', flat=True)) + self.assets_id = list(Asset.objects.all().values_list('id', flat=True)) + self.system_users_id = list(SystemUser.objects.all().values_list('id', flat=True)) + self.users_id = list(User.objects.all().values_list('id', flat=True)) + self.user_groups_id = list(UserGroup.objects.all().values_list('id', flat=True)) + + def set_users(self, perms): + through = AssetPermission.users.through + choices = self.users_id + relation_name = 'user_id' + self.set_relations(perms, through, relation_name, choices) + + def set_user_groups(self, perms): + through = AssetPermission.user_groups.through + choices = self.user_groups_id + relation_name = 'usergroup_id' + self.set_relations(perms, through, relation_name, choices) + + def set_assets(self, perms): + through = AssetPermission.assets.through + choices = self.assets_id + relation_name = 'asset_id' + self.set_relations(perms, through, relation_name, choices) + + def set_nodes(self, perms): + through = AssetPermission.nodes.through + choices = self.nodes_id + relation_name = 'node_id' + self.set_relations(perms, through, relation_name, choices) + + def set_system_users(self, perms): + through = AssetPermission.system_users.through + choices = self.system_users_id + relation_name = 'systemuser_id' + self.set_relations(perms, through, relation_name, choices) + + def set_relations(self, perms, through, relation_name, choices, choice_count=None): + relations = [] + + for perm in perms: + if choice_count is None: + choice_count = choice(range(8)) + resources_id = sample(choices, choice_count) + for rid in resources_id: + data = {'assetpermission_id': perm.id} + data[relation_name] = rid + relations.append(through(**data)) + through.objects.bulk_create(relations, ignore_conflicts=True) + + def do_generate(self, batch, batch_size): + perms = [] + + for i in batch: + name = forgery_py.basic.text() + name = f'AssetPermission: {name}' + perm = AssetPermission(name=name, org_id=self.org.id) + perms.append(perm) + created = AssetPermission.objects.bulk_create(perms, ignore_conflicts=True) + self.set_users(created) + self.set_user_groups(created) + self.set_assets(created) + self.set_nodes(created) + self.set_system_users(created) diff --git a/utils/generate_fake_data/resources/users.py b/utils/generate_fake_data/resources/users.py new file mode 100644 index 000000000..05332e4e2 --- /dev/null +++ b/utils/generate_fake_data/resources/users.py @@ -0,0 +1,62 @@ +from random import choice, sample +import forgery_py + +from .base import FakeDataGenerator + +from users.models import * +from orgs.models import OrganizationMember + + +class UserGroupGenerator(FakeDataGenerator): + resource = 'usergroup' + + def do_generate(self, batch, batch_size): + groups = [] + for i in batch: + group_name = forgery_py.name.job_title() + groups.append(UserGroup(name=group_name, org_id=self.org.id)) + UserGroup.objects.bulk_create(groups, ignore_conflicts=True) + + +class UserGenerator(FakeDataGenerator): + resource = 'user' + roles: list + groups_id: list + + def pre_generate(self): + self.roles = list(dict(User.ROLE.choices).keys()) + self.groups_id = list(UserGroup.objects.all().values_list('id', flat=True)) + + def set_org(self, users): + relations = [] + for u in users: + relations.append(OrganizationMember( + org_id=self.org.id, + user_id=u.id, + )) + OrganizationMember.objects.bulk_create(relations, ignore_conflicts=True) + + def set_groups(self, users): + relations = [] + for i in users: + groups_to_join = sample(self.groups_id, 3) + _relations = [User.groups.through(user_id=i.id, usergroup_id=gid) for gid in groups_to_join] + relations.extend(_relations) + User.groups.through.objects.bulk_create(relations, ignore_conflicts=True) + + def do_generate(self, batch, batch_size): + users = [] + for i in batch: + username = forgery_py.internet.user_name(True) + email = forgery_py.internet.email_address() + u = User( + username=username, + email=email, + name=username.title(), + role=choice(self.roles), + created_by='Faker' + ) + users.append(u) + users = User.objects.bulk_create(users, ignore_conflicts=True) + self.set_org(users) + self.set_groups(users) From 5ac974a44cc96c7878ac6f6fa097397ca4ec3979 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 12 Oct 2020 18:12:00 +0800 Subject: [PATCH 06/12] =?UTF-8?q?fix(deps):=20=E4=BF=AE=E5=A4=8D=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0b71e5394..78961dfbf 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -33,7 +33,7 @@ enum-compat==0.0.2 ephem==3.7.6.0 eventlet==0.24.1 future==0.16.0 -ForgeryPy3==0.3.2 +ForgeryPy3==0.3.1 greenlet==0.4.14 gunicorn==19.9.0 idna==2.6 From 82dd1c35ead7e49353c68ff83d5b26a27bba3e37 Mon Sep 17 00:00:00 2001 From: clannon <34529891+clannon@users.noreply.github.com> Date: Mon, 12 Oct 2020 13:44:27 +0800 Subject: [PATCH 07/12] Update config_example.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SECRET_KEY 和 BOOTSTRAP_TOKEN 后面默认留个空格，遇到好几个新手对yaml格式不注意了，虽然是个小问题…… --- config_example.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config_example.yml b/config_example.yml index f23b5c7e5..c3e2ce0f5 100644 --- a/config_example.yml +++ b/config_example.yml @@ -1,11 +1,11 @@ # SECURITY WARNING: keep the secret key used in production secret! # 加密秘钥 生产环境中请修改为随机字符串，请勿外泄, 可使用命令生成 # $ cat /dev/urandom | tr -dc A-Za-z0-9 | head -c 49;echo -SECRET_KEY: +SECRET_KEY: # SECURITY WARNING: keep the bootstrap token used in production secret! # 预共享Token coco和guacamole用来注册服务账号，不在使用原来的注册接受机制 -BOOTSTRAP_TOKEN: +BOOTSTRAP_TOKEN: # Development env open this, when error occur display the full process track, Production disable it # DEBUG 模式 开启DEBUG后遇到错误时可以看到更多日志 From 84e42388486448118aa8b729ef23e5131bed38ad Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 12 Oct 2020 19:08:13 +0800 Subject: [PATCH 08/12] =?UTF-8?q?fix(ops):=20=E4=BF=AE=E5=A4=8D=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1schedule=E5=B1=9E=E6=80=A7=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/mixin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/ops/mixin.py b/apps/ops/mixin.py index fd6f9cf27..166c5db75 100644 --- a/apps/ops/mixin.py +++ b/apps/ops/mixin.py @@ -95,10 +95,8 @@ class PeriodTaskModelMixin(models.Model): @property def schedule(self): from django_celery_beat.models import PeriodicTask - try: - return PeriodicTask.objects.get(name=str(self)) - except PeriodicTask.DoesNotExist: - return None + name = self.get_register_task()[0] + return PeriodicTask.objects.filter(name=name).first() class Meta: abstract = True From a449d97f67d4d90fbe2c6d05e7b643d286ea696b Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 12 Oct 2020 19:29:22 +0800 Subject: [PATCH 09/12] =?UTF-8?q?fix(orgs):=20=E7=BB=84=E7=BB=87=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=88=90=E5=91=98bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 0b732506b..b244ddec4 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -231,7 +231,7 @@ def _none2list(*args): def _users2pks(users, admins, auditors): - return [user.pk for user in chain(users, admins, auditors)] + return [user.pk for user in chain(users, admins, auditors) if hasattr(user, 'pk')] class UserRoleMapper(dict): @@ -295,11 +295,11 @@ class OrgMemeberManager(models.Manager): ) oms_add = [] - for users, role in add_mapper: - for user in users: - if isinstance(user, models.Model): - user = user.id - oms_add.append(self.model(org_id=org.id, user_id=user, role=role)) + for _users, _role in add_mapper: + for _user in _users: + if isinstance(_user, models.Model): + _user = _user.id + oms_add.append(self.model(org_id=org.id, user_id=_user, role=_role)) send = partial(signals.m2m_changed.send, sender=self.model, instance=org, reverse=False, model=User, pk_set=_users2pks(users, admins, auditors), using=self.db) From a57ce482ddfb2ad43fe2c6cb967bbd8682e1e510 Mon Sep 17 00:00:00 2001 From: xinwen Date: Tue, 13 Oct 2020 16:42:41 +0800 Subject: [PATCH 10/12] =?UTF-8?q?fix(assets):=20=E8=B5=84=E4=BA=A7?= =?UTF-8?q?=E6=A0=91=E6=89=B9=E9=87=8F=E5=88=A0=E9=99=A4=E8=B5=84=E4=BA=A7?= =?UTF-8?q?=E6=95=B0=E9=87=8F=E4=B8=8D=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/signals_handler.py | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 41b60d30a..a90636161 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -9,9 +9,8 @@ from django.db.models.signals import ( from django.db.models import Q, F from django.dispatch import receiver -from common.local import thread_local from common.exceptions import M2MReverseNotAllowed -from common.const.signals import PRE_ADD, POST_ADD, POST_REMOVE, PRE_CLEAR +from common.const.signals import PRE_ADD, POST_ADD, POST_REMOVE, PRE_CLEAR, PRE_REMOVE from common.utils import get_logger from common.decorator import on_transaction_commit from .models import Asset, SystemUser, Node, compute_parent_key @@ -321,26 +320,26 @@ def update_nodes_assets_amount(action, instance, reverse, pk_set, **kwargs): _update_nodes_asset_amount(node_keys, asset_pk, operator) -ASSET_DELETE_SIGNAL_FOR_NODE_TREE_PARAMS = 'asset_delete_signal_for_node_tree_params' +RELATED_NODE_IDS = '_related_node_ids' @receiver(pre_delete, sender=Asset) -def on_asset_delete(instance: Asset, **kwargs): - node_keys = Node.objects.filter( +def on_asset_delete(instance: Asset, using, **kwargs): + node_ids = set(Node.objects.filter( assets=instance - ).distinct().values_list('key', flat=True) - - params = { - 'node_keys': set(node_keys), - 'asset_pk': instance.id, - 'operator': sub - } - - setattr(thread_local, ASSET_DELETE_SIGNAL_FOR_NODE_TREE_PARAMS, params) + ).distinct().values_list('id', flat=True)) + setattr(instance, RELATED_NODE_IDS, node_ids) + m2m_changed.send( + sender=Asset.nodes.through, instance=instance, reverse=False, + model=Node, pk_set=node_ids, using=using, action=PRE_REMOVE + ) @receiver(post_delete, sender=Asset) -def on_asset_post_delete(instance: Asset, **kwargs): - params = getattr(thread_local, ASSET_DELETE_SIGNAL_FOR_NODE_TREE_PARAMS, None) - if params and params.get('asset_pk') == instance.id: - _update_nodes_asset_amount(**params) +def on_asset_post_delete(instance: Asset, using, **kwargs): + node_ids = getattr(instance, RELATED_NODE_IDS, None) + if node_ids: + m2m_changed.send( + sender=Asset.nodes.through, instance=instance, reverse=False, + model=Node, pk_set=node_ids, using=using, action=POST_REMOVE + ) From f3d052554d51f47a37c3c1943b68de10009a7c87 Mon Sep 17 00:00:00 2001 From: xinwen Date: Tue, 13 Oct 2020 19:04:26 +0800 Subject: [PATCH 11/12] =?UTF-8?q?fix(perms):=20=E4=BF=AE=E5=A4=8D=E5=A4=B1?= =?UTF-8?q?=E6=95=88=E8=B5=84=E4=BA=A7=E6=8E=88=E6=9D=83action=20=E8=BF=98?= =?UTF-8?q?=E5=9C=A8=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/perms/utils/asset_permission.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/perms/utils/asset_permission.py b/apps/perms/utils/asset_permission.py index 6474b4334..09218de04 100644 --- a/apps/perms/utils/asset_permission.py +++ b/apps/perms/utils/asset_permission.py @@ -37,12 +37,12 @@ def get_asset_system_users_id_with_actions(asset_perm_queryset: BasePermissionQu def get_asset_system_users_id_with_actions_by_user(user: User, asset: Asset): queryset = AssetPermission.objects.filter( Q(users=user) | Q(user_groups__users=user) - ) + ).valid() return get_asset_system_users_id_with_actions(queryset, asset) def get_asset_system_users_id_with_actions_by_group(group: UserGroup, asset: Asset): queryset = AssetPermission.objects.filter( user_groups=group - ) + ).valid() return get_asset_system_users_id_with_actions(queryset, asset) From 373c6c77e09e3f224f47e946cc1f916fd7b929d5 Mon Sep 17 00:00:00 2001 From: xinwen Date: Tue, 13 Oct 2020 15:50:11 +0800 Subject: [PATCH 12/12] =?UTF-8?q?fix(perms):=20=E6=9C=AA=E6=BF=80=E6=B4=BB?= =?UTF-8?q?=E8=B5=84=E4=BA=A7=E4=B8=8D=E8=83=BD=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/mixin.py | 1 + apps/perms/serializers/user_permission.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/assets/api/mixin.py b/apps/assets/api/mixin.py index 89555324b..4374f399a 100644 --- a/apps/assets/api/mixin.py +++ b/apps/assets/api/mixin.py @@ -60,6 +60,7 @@ class SerializeToTreeNodeMixin: 'isParent': False, 'open': False, 'iconSkin': self.get_platform(asset), + 'chkDisabled': not asset.is_active, 'meta': { 'type': 'asset', 'asset': { diff --git a/apps/perms/serializers/user_permission.py b/apps/perms/serializers/user_permission.py index 40334786f..bfc273775 100644 --- a/apps/perms/serializers/user_permission.py +++ b/apps/perms/serializers/user_permission.py @@ -75,7 +75,7 @@ class AssetGrantedSerializer(serializers.ModelSerializer): model = Asset only_fields = [ "id", "hostname", "ip", "protocols", "os", 'domain', - "platform", "comment", "org_id", + "platform", "comment", "org_id", "is_active" ] fields = only_fields + ['org_name'] read_only_fields = fields