From 22f362aab36725e27a6bd434414395d4698e2085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AB=E5=8D=83=E6=B5=81?= <40739051+jym503558564@users.noreply.github.com> Date: Tue, 21 May 2019 16:24:01 +0800 Subject: [PATCH] Dev csv (#2640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] 封装JMSCSVRender和JMSCSVParser * [Update] 更改JMSCSVRender,根据请求参数控制导出csv的字段和下载csv模板的字段 * [Update] 导入空数据,提示错误消息 * [Update] 修改用户导入和导出功能代码 * [Update] 修改导入路由为动态反向解析 * [Update] 修改JMSCSVRender和JMSCSVParser以及用户导入导出代码 * [Update] 优化parsers逻辑 * [Update] 优化parsers csv代码结构 * [Update] 优化renders csv代码逻辑 * [Update] 删除parsers csv多余代码 * [Update] 删除parsers csv多余变量 * [Update] 优化renders csv代码结构 * [Update] 优化renders csv代码结构2 * [Update] 优化renders csv获取header逻辑 * [Update] 优化Cache Resources ID View逻辑 * [Update] 优化ViewSet IDCacheFilterMixin逻辑 * [Update] csv: parser render 添加异常捕获逻辑 * [Update] 删除多余代码 * [Update] 优化前端代码 * [Update] 修改小问题 * [Update] 修改前端导出用户的问题 * [Update] 前端 - 优化数据导出逻辑 APIExportData * [Update] 修复批量创建用户时发送created信号的bug * [Update] 优化导入时错误信息展示 * [Update] 优化parser、render时,对于多对多字段的处理 * [Update] 修改前端上传空文件问题 * [Update] 添加IDExportFilter,控制下载模版时的queryset * [Update] 修改判断导出模版时参数变量名 action => template * [Update] 修复导入用户数据时,用户组不生效的bug * [Update] 修改前端导入信息展示 * [Update] 抽象资源导入模版 * [Update] 优化资源导入模版 * [Update] 修改js设置url的params逻辑 * [Update] 修改users序列类控制read_only字段方式 * [Update] 资产列表采用新的导入/导出csv文件逻辑 * [Update] 修改导入资产时设置资产所在节点逻辑 * [Update] 添加用户组导入/导出功能 * [Update] 修改前端变量名 * [Update] 修改下载导入模版,不包含org字段 * [Update] 增加管理用户导入/导出功能 * [Update] 导入模版提供id字段(为了资源备份后导入直接使用); 修复资源导入时联合唯一字段不校验导致创建时报错的bug * [Update] 增加系统用户导入/导出功能 * [Update] 排序资源导入/导出字段 * [Update] 翻译导入/导出的字段和模版 * [Update] 更改csv导出和导出模版数据的控制在render实现 * [Update] 资产添加 更新导入 功能 * [Update] 用户/用户组/管理用户/系统用户/ 添加导入更新 * [Update] 翻译 * [Update] 优化资源序列化中的label * [Update] 去掉资源IDInFilterMixin过滤 * [Update] 翻译 --- apps/assets/api/admin_user.py | 4 +- apps/assets/api/asset.py | 22 +- apps/assets/api/system_user.py | 3 +- apps/assets/serializers/admin_user.py | 24 +- apps/assets/serializers/asset.py | 29 +- apps/assets/serializers/system_user.py | 34 +- .../assets/_admin_user_import_modal.html | 6 + .../assets/_admin_user_update_modal.html | 4 + .../templates/assets/_asset_import_modal.html | 33 +- .../templates/assets/_asset_update_modal.html | 4 + .../assets/_system_user_import_modal.html | 6 + .../assets/_system_user_update_modal.html | 4 + .../templates/assets/admin_user_list.html | 107 ++- apps/assets/templates/assets/asset_list.html | 237 ++++-- .../templates/assets/system_user_list.html | 99 +++ apps/assets/views/asset.py | 6 +- apps/common/api.py | 23 +- apps/common/const.py | 1 + apps/common/mixins.py | 30 +- apps/common/parsers/__init__.py | 1 + apps/common/parsers/csv.py | 101 +++ apps/common/renders/__init__.py | 1 + apps/common/renders/csv.py | 71 ++ apps/common/urls/__init__.py | 0 apps/common/urls/api_urls.py | 13 + apps/jumpserver/settings.py | 10 + apps/jumpserver/urls.py | 1 + apps/locale/zh/LC_MESSAGES/django.mo | Bin 74215 -> 72271 bytes apps/locale/zh/LC_MESSAGES/django.po | 792 ++++++++++++++---- apps/orgs/mixins.py | 16 +- apps/orgs/utils.py | 6 + apps/static/js/jumpserver.js | 85 +- apps/templates/_import_modal.html | 28 + apps/templates/_modal.html | 26 +- apps/templates/_update_modal.html | 28 + apps/users/api/group.py | 4 +- apps/users/api/user.py | 16 +- apps/users/serializers/v1.py | 36 +- apps/users/signals_handler.py | 1 - .../users/_user_groups_import_modal.html | 6 + .../users/_user_groups_update_modal.html | 4 + .../templates/users/_user_import_modal.html | 32 +- .../templates/users/_user_update_modal.html | 4 + .../templates/users/user_group_list.html | 100 ++- apps/users/templates/users/user_list.html | 219 +++-- apps/users/views/user.py | 13 +- 46 files changed, 1868 insertions(+), 422 deletions(-) create mode 100644 apps/assets/templates/assets/_admin_user_import_modal.html create mode 100644 apps/assets/templates/assets/_admin_user_update_modal.html create mode 100644 apps/assets/templates/assets/_asset_update_modal.html create mode 100644 apps/assets/templates/assets/_system_user_import_modal.html create mode 100644 apps/assets/templates/assets/_system_user_update_modal.html create mode 100644 apps/common/parsers/__init__.py create mode 100644 apps/common/parsers/csv.py create mode 100644 apps/common/renders/__init__.py create mode 100644 apps/common/renders/csv.py create mode 100644 apps/common/urls/__init__.py create mode 100644 apps/common/urls/api_urls.py create mode 100644 apps/templates/_import_modal.html create mode 100644 apps/templates/_update_modal.html create mode 100644 apps/users/templates/users/_user_groups_import_modal.html create mode 100644 apps/users/templates/users/_user_groups_update_modal.html create mode 100644 apps/users/templates/users/_user_update_modal.html diff --git a/apps/assets/api/admin_user.py b/apps/assets/api/admin_user.py index f2229022f..e84d9731a 100644 --- a/apps/assets/api/admin_user.py +++ b/apps/assets/api/admin_user.py @@ -20,7 +20,7 @@ from rest_framework.response import Response from rest_framework_bulk import BulkModelViewSet from rest_framework.pagination import LimitOffsetPagination -from common.mixins import IDInFilterMixin +from common.mixins import IDInCacheFilterMixin from common.utils import get_logger from ..hands import IsOrgAdmin from ..models import AdminUser, Asset @@ -36,7 +36,7 @@ __all__ = [ ] -class AdminUserViewSet(IDInFilterMixin, BulkModelViewSet): +class AdminUserViewSet(IDInCacheFilterMixin, BulkModelViewSet): """ Admin user api set, for add,delete,update,list,retrieve resource """ diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index a734806fb..9e21c81be 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -16,8 +16,9 @@ from django.urls import reverse_lazy from django.core.cache import cache from django.db.models import Q -from common.mixins import IDInFilterMixin -from common.utils import get_logger +from common.mixins import IDInCacheFilterMixin + +from common.utils import get_logger, get_object_or_none from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser from ..const import CACHE_KEY_ASSET_BULK_UPDATE_ID_PREFIX from ..models import Asset, AdminUser, Node @@ -35,7 +36,7 @@ __all__ = [ ] -class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet): +class AssetViewSet(IDInCacheFilterMixin, LabelFilter, BulkModelViewSet): """ API endpoint that allows Asset to be viewed or edited. """ @@ -47,6 +48,19 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet): pagination_class = LimitOffsetPagination permission_classes = (IsOrgAdminOrAppUser,) + def set_assets_node(self, assets): + if not isinstance(assets, list): + assets = [assets] + node = Node.objects.get(value='Default') + node_id = self.request.query_params.get('node_id') + if node_id: + node = get_object_or_none(Node, pk=node_id) + node.assets.add(*assets) + + def perform_create(self, serializer): + assets = serializer.save() + self.set_assets_node(assets) + def filter_node(self, queryset): node_id = self.request.query_params.get("node_id") if not node_id: @@ -89,7 +103,7 @@ class AssetViewSet(IDInFilterMixin, LabelFilter, BulkModelViewSet): return queryset -class AssetListUpdateApi(IDInFilterMixin, ListBulkCreateUpdateDestroyAPIView): +class AssetListUpdateApi(IDInCacheFilterMixin, ListBulkCreateUpdateDestroyAPIView): """ Asset bulk update api """ diff --git a/apps/assets/api/system_user.py b/apps/assets/api/system_user.py index 9805872e7..f6398e974 100644 --- a/apps/assets/api/system_user.py +++ b/apps/assets/api/system_user.py @@ -21,6 +21,7 @@ from rest_framework.pagination import LimitOffsetPagination from common.utils import get_logger from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser +from common.mixins import IDInCacheFilterMixin from ..models import SystemUser, Asset from .. import serializers from ..tasks import push_system_user_to_assets_manual, \ @@ -38,7 +39,7 @@ __all__ = [ ] -class SystemUserViewSet(BulkModelViewSet): +class SystemUserViewSet(IDInCacheFilterMixin, BulkModelViewSet): """ System user api set, for add,delete,update,list,retrieve resource """ diff --git a/apps/assets/serializers/admin_user.py b/apps/assets/serializers/admin_user.py index e44679995..66f25db87 100644 --- a/apps/assets/serializers/admin_user.py +++ b/apps/assets/serializers/admin_user.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # from django.core.cache import cache +from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from common.serializers import AdaptedBulkListSerializer @@ -15,14 +16,29 @@ class AdminUserSerializer(serializers.ModelSerializer): """ 管理用户 """ - assets_amount = serializers.SerializerMethodField() - unreachable_amount = serializers.SerializerMethodField() - reachable_amount = serializers.SerializerMethodField() + password = serializers.CharField( + required=False, write_only=True, label=_('Password') + ) + unreachable_amount = serializers.SerializerMethodField(label=_('Unreachable')) + assets_amount = serializers.SerializerMethodField(label=_('Asset')) + reachable_amount = serializers.SerializerMethodField(label=_('Reachable')) class Meta: list_serializer_class = AdaptedBulkListSerializer model = AdminUser - fields = '__all__' + fields = [ + 'id', 'org_id', 'name', 'username', 'assets_amount', + 'reachable_amount', 'unreachable_amount', 'password', 'comment', + 'date_created', 'date_updated', 'become', 'become_method', + 'become_user', 'created_by', + ] + + extra_kwargs = { + 'date_created': {'label': _('Date created')}, + 'date_updated': {'label': _('Date updated')}, + 'become': {'read_only': True}, 'become_method': {'read_only': True}, + 'become_user': {'read_only': True}, 'created_by': {'read_only': True} + } def get_field_names(self, declared_fields, info): fields = super().get_field_names(declared_fields, info) diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index c0f435adc..3e1cf39bb 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -2,6 +2,9 @@ # from rest_framework import serializers +from django.utils.translation import ugettext_lazy as _ + +from orgs.mixins import OrgResourceSerializerMixin from common.mixins import BulkSerializerMixin from common.serializers import AdaptedBulkListSerializer from ..models import Asset @@ -13,15 +16,35 @@ __all__ = [ ] -class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer): +class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer, OrgResourceSerializerMixin): """ 资产的数据结构 """ class Meta: model = Asset list_serializer_class = AdaptedBulkListSerializer - fields = '__all__' - validators = [] + # validators = [] # 解决批量导入时unique_together字段校验失败 + fields = [ + 'id', 'org_id', 'org_name', 'ip', 'hostname', 'protocol', 'port', + 'platform', 'is_active', 'public_ip', 'domain', 'admin_user', + 'nodes', 'labels', 'number', 'vendor', 'model', 'sn', + 'cpu_model', 'cpu_count', 'cpu_cores', 'cpu_vcpus', 'memory', + 'disk_total', 'disk_info', 'os', 'os_version', 'os_arch', + 'hostname_raw', 'comment', 'created_by', 'date_created', + 'hardware_info', 'connectivity' + ] + read_only_fields = ( + 'number', 'vendor', 'model', 'sn', 'cpu_model', 'cpu_count', + 'cpu_cores', 'cpu_vcpus', 'memory', 'disk_total', 'disk_info', + 'os', 'os_version', 'os_arch', 'hostname_raw', + 'created_by', 'date_created', + ) + extra_kwargs = { + 'hardware_info': {'label': _('Hardware info')}, + 'connectivity': {'label': _('Connectivity')}, + 'org_name': {'label': _('Org name')} + + } @classmethod def setup_eager_loading(cls, queryset): diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index c737f8cbe..88ae6eb38 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -1,5 +1,7 @@ from rest_framework import serializers +from django.utils.translation import ugettext_lazy as _ + from common.serializers import AdaptedBulkListSerializer from ..models import SystemUser, Asset @@ -10,16 +12,36 @@ class SystemUserSerializer(serializers.ModelSerializer): """ 系统用户 """ - unreachable_amount = serializers.SerializerMethodField() - reachable_amount = serializers.SerializerMethodField() - unreachable_assets = serializers.SerializerMethodField() - reachable_assets = serializers.SerializerMethodField() - assets_amount = serializers.SerializerMethodField() + password = serializers.CharField( + required=False, write_only=True, label=_('Password') + ) + unreachable_amount = serializers.SerializerMethodField( + label=_('Unreachable') + ) + unreachable_assets = serializers.SerializerMethodField( + label=_('Unreachable assets') + ) + reachable_assets = serializers.SerializerMethodField( + label=_('Reachable assets') + ) + reachable_amount = serializers.SerializerMethodField(label=_('Reachable')) + assets_amount = serializers.SerializerMethodField(label=_('Asset')) class Meta: model = SystemUser - exclude = ('_password', '_private_key', '_public_key') list_serializer_class = AdaptedBulkListSerializer + fields = [ + 'id', 'org_id', 'name', 'username', 'login_mode', + 'login_mode_display', 'priority', 'protocol', 'auto_push', + 'password', 'assets_amount', 'reachable_amount', 'reachable_assets', + 'unreachable_amount', 'unreachable_assets', 'cmd_filters', 'sudo', + 'shell', 'comment', 'nodes', 'assets' + ] + extra_kwargs = { + 'login_mode_display': {'label': _('Login mode display')}, + 'created_by': {'read_only': True}, 'nodes': {'read_only': True}, + 'assets': {'read_only': True} + } def get_field_names(self, declared_fields, info): fields = super(SystemUserSerializer, self).get_field_names(declared_fields, info) diff --git a/apps/assets/templates/assets/_admin_user_import_modal.html b/apps/assets/templates/assets/_admin_user_import_modal.html new file mode 100644 index 000000000..a4afc1a14 --- /dev/null +++ b/apps/assets/templates/assets/_admin_user_import_modal.html @@ -0,0 +1,6 @@ +{% extends '_import_modal.html' %} +{% load i18n %} + +{% block modal_title%}{% trans "Import admin user" %}{% endblock %} + +{% block import_modal_download_template_url %}{% url "api-assets:admin-user-list" %}{% endblock %} diff --git a/apps/assets/templates/assets/_admin_user_update_modal.html b/apps/assets/templates/assets/_admin_user_update_modal.html new file mode 100644 index 000000000..9af051dd2 --- /dev/null +++ b/apps/assets/templates/assets/_admin_user_update_modal.html @@ -0,0 +1,4 @@ +{% extends '_update_modal.html' %} +{% load i18n %} + +{% block modal_title%}{% trans "Update admin user" %}{% endblock %} \ No newline at end of file diff --git a/apps/assets/templates/assets/_asset_import_modal.html b/apps/assets/templates/assets/_asset_import_modal.html index ca7729e05..2460cb053 100644 --- a/apps/assets/templates/assets/_asset_import_modal.html +++ b/apps/assets/templates/assets/_asset_import_modal.html @@ -1,29 +1,6 @@ -{% extends '_modal.html' %} +{% extends '_import_modal.html' %} {% load i18n %} -{% block modal_id %}asset_import_modal{% endblock %} -{% block modal_title%}{% trans "Import asset" %}{% endblock %} -{% block modal_body %} -
--
- - - - - - -{% endblock %} -{% block modal_confirm_id %}btn_asset_import{% endblock %} + +{% block modal_title%}{% trans "Import assets" %}{% endblock %} + +{% block import_modal_download_template_url %}{% url "api-assets:asset-list" %}{% endblock %} diff --git a/apps/assets/templates/assets/_asset_update_modal.html b/apps/assets/templates/assets/_asset_update_modal.html new file mode 100644 index 000000000..68b2ff8db --- /dev/null +++ b/apps/assets/templates/assets/_asset_update_modal.html @@ -0,0 +1,4 @@ +{% extends '_update_modal.html' %} +{% load i18n %} + +{% block modal_title%}{% trans "Update assets" %}{% endblock %} diff --git a/apps/assets/templates/assets/_system_user_import_modal.html b/apps/assets/templates/assets/_system_user_import_modal.html new file mode 100644 index 000000000..b8687d696 --- /dev/null +++ b/apps/assets/templates/assets/_system_user_import_modal.html @@ -0,0 +1,6 @@ +{% extends '_import_modal.html' %} +{% load i18n %} + +{% block modal_title%}{% trans "Import system user" %}{% endblock %} + +{% block import_modal_download_template_url %}{% url "api-assets:system-user-list" %}{% endblock %} diff --git a/apps/assets/templates/assets/_system_user_update_modal.html b/apps/assets/templates/assets/_system_user_update_modal.html new file mode 100644 index 000000000..9e2920e6a --- /dev/null +++ b/apps/assets/templates/assets/_system_user_update_modal.html @@ -0,0 +1,4 @@ +{% extends '_update_modal.html' %} +{% load i18n %} + +{% block modal_title%}{% trans "Update system user" %}{% endblock %} \ No newline at end of file diff --git a/apps/assets/templates/assets/admin_user_list.html b/apps/assets/templates/assets/admin_user_list.html index 605e89060..c182ba4c4 100644 --- a/apps/assets/templates/assets/admin_user_list.html +++ b/apps/assets/templates/assets/admin_user_list.html @@ -1,8 +1,5 @@ {% extends '_base_list.html' %} {% load i18n static %} -{% block table_search %} -{% endblock %} - {% block help_message %} {% endblock %} +{% block table_search %} +{% trans "Download template or use export csv format" %}
- --
- - - - - - -{% endblock %} -{% block modal_confirm_id %}btn_user_import{% endblock %} + +{% block modal_title%}{% trans "Import users" %}{% endblock %} + +{% block import_modal_download_template_url %}{% url "api-users:user-list" %}{% endblock %} diff --git a/apps/users/templates/users/_user_update_modal.html b/apps/users/templates/users/_user_update_modal.html new file mode 100644 index 000000000..9dfe60c96 --- /dev/null +++ b/apps/users/templates/users/_user_update_modal.html @@ -0,0 +1,4 @@ +{% extends '_update_modal.html' %} +{% load i18n %} + +{% block modal_title%}{% trans "Update user" %}{% endblock %} \ No newline at end of file diff --git a/apps/users/templates/users/user_group_list.html b/apps/users/templates/users/user_group_list.html index 6f6c6fc72..d8fc92e87 100644 --- a/apps/users/templates/users/user_group_list.html +++ b/apps/users/templates/users/user_group_list.html @@ -1,6 +1,29 @@ {% extends '_base_list.html' %} {% load i18n static %} -{% block table_search %}{% endblock %} +{% block table_search %} +