From 6e86e3f1189ea3e5caa1fdff14b6d9cfc96a29a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AB=E5=8D=83=E6=B5=81?= <40739051+jym503558564@users.noreply.github.com> Date: Tue, 9 Jul 2019 17:30:53 +0800 Subject: [PATCH 1/3] =?UTF-8?q?[Update]=20=20=E5=88=9B=E5=BB=BA/=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=20=E7=BD=91=E5=85=B3=E4=BD=BF=E7=94=A8api=20(#2911)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] 创建/更新 网关使用api * [Update] 修改小问题 * [Update] 修改小问题 * [Update] 修改网关一些信息可读 --- apps/assets/serializers/domain.py | 9 ++++--- .../assets/gateway_create_update.html | 26 +++++++++++++++++++ apps/assets/views/domain.py | 2 ++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index cda208b9f..68145e7a9 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -6,6 +6,7 @@ from common.serializers import AdaptedBulkListSerializer from orgs.mixins import BulkOrgResourceModelSerializer from ..models import Domain, Gateway +from .base import AuthSerializerMixin class DomainSerializer(BulkOrgResourceModelSerializer): @@ -26,14 +27,14 @@ class DomainSerializer(BulkOrgResourceModelSerializer): return obj.gateway_set.all().count() -class GatewaySerializer(BulkOrgResourceModelSerializer): +class GatewaySerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): class Meta: model = Gateway list_serializer_class = AdaptedBulkListSerializer fields = [ - 'id', 'name', 'ip', 'port', 'protocol', 'username', - 'domain', 'is_active', 'date_created', 'date_updated', - 'created_by', 'comment', + 'id', 'name', 'ip', 'port', 'protocol', 'username', 'password', + 'private_key', 'public_key', 'domain', 'is_active', 'date_created', + 'date_updated', 'created_by', 'comment', ] diff --git a/apps/assets/templates/assets/gateway_create_update.html b/apps/assets/templates/assets/gateway_create_update.html index fea540385..a22428087 100644 --- a/apps/assets/templates/assets/gateway_create_update.html +++ b/apps/assets/templates/assets/gateway_create_update.html @@ -95,6 +95,32 @@ function protocolChange() { $(document).ready(function(){ protocolChange(); }) +.on("submit", "form", function (evt) { + evt.preventDefault(); + var form = $("form"); + var data = form.serializeObject(); + data["private_key"] = $("#id_private_key_file").data('file'); + var method = "POST"; + var the_url = '{% url "api-assets:gateway-list" %}'; + var redirect_to = '{% url "assets:domain-gateway-list" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", data.domain); + {% if type == "update" %} + the_url = '{% url 'api-assets:gateway-detail' pk=object.id %}'; + method = "PUT"; + {% endif %} + var props = { + url:the_url, + data:data, + method:method, + form:form, + redirect_to:redirect_to + }; + formSubmit(props); +}) +.on('change', '#id_private_key_file', function () { + readFile($(this)).on("onload", function (evt, data) { + $(this).attr("data-file", data) + }) +}) .on('change', protocol_id, function(){ protocolChange(); }); diff --git a/apps/assets/views/domain.py b/apps/assets/views/domain.py index 797bae1f4..6bbf09e25 100644 --- a/apps/assets/views/domain.py +++ b/apps/assets/views/domain.py @@ -132,6 +132,7 @@ class DomainGatewayCreateView(PermissionsMixin, CreateView): context = { 'app': _('Assets'), 'action': _('Create gateway'), + 'type': 'create' } kwargs.update(context) return super().get_context_data(**kwargs) @@ -152,6 +153,7 @@ class DomainGatewayUpdateView(PermissionsMixin, UpdateView): context = { 'app': _('Assets'), 'action': _('Update gateway'), + "type": "update" } kwargs.update(context) return super().get_context_data(**kwargs) From ff85e2ef570ede3b5cdea42d1cdb1da1305291ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=AB=E5=8D=83=E6=B5=81?= <40739051+jym503558564@users.noreply.github.com> Date: Thu, 11 Jul 2019 12:02:51 +0800 Subject: [PATCH 2/3] =?UTF-8?q?[Update]=20=E5=88=9B=E5=BB=BA/=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=20=E5=91=BD=E4=BB=A4=E8=BF=87=E6=BB=A4=E5=99=A8=20?= =?UTF-8?q?=E4=BD=BF=E7=94=A8api=20(#2926)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/serializers/cmd_filter.py | 12 ++++++--- .../assets/cmd_filter_create_update.html | 26 +++++++++++++++++++ apps/assets/views/cmd_filter.py | 2 ++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/apps/assets/serializers/cmd_filter.py b/apps/assets/serializers/cmd_filter.py index e5ab62037..dfdff2cdb 100644 --- a/apps/assets/serializers/cmd_filter.py +++ b/apps/assets/serializers/cmd_filter.py @@ -10,13 +10,19 @@ from orgs.mixins import BulkOrgResourceModelSerializer class CommandFilterSerializer(BulkOrgResourceModelSerializer): - rules = serializers.PrimaryKeyRelatedField(queryset=CommandFilterRule.objects.all(), many=True) - system_users = serializers.PrimaryKeyRelatedField(queryset=SystemUser.objects.all(), many=True) class Meta: model = CommandFilter list_serializer_class = AdaptedBulkListSerializer - fields = '__all__' + fields = [ + 'id', 'name', 'org_id', 'org_name', 'is_active', 'comment', + 'created_by', 'date_created', 'date_updated', 'rules', 'system_users' + ] + + extra_kwargs = { + 'rules': {'read_only': True}, + 'system_users': {'read_only': True} + } class CommandFilterRuleSerializer(BulkOrgResourceModelSerializer): diff --git a/apps/assets/templates/assets/cmd_filter_create_update.html b/apps/assets/templates/assets/cmd_filter_create_update.html index b1f7a57a8..678e1e3eb 100644 --- a/apps/assets/templates/assets/cmd_filter_create_update.html +++ b/apps/assets/templates/assets/cmd_filter_create_update.html @@ -18,3 +18,29 @@ {% endblock %} +{% block custom_foot_js %} + +{% endblock %} \ No newline at end of file diff --git a/apps/assets/views/cmd_filter.py b/apps/assets/views/cmd_filter.py index 354c1d852..7eef5980a 100644 --- a/apps/assets/views/cmd_filter.py +++ b/apps/assets/views/cmd_filter.py @@ -47,6 +47,7 @@ class CommandFilterCreateView(PermissionsMixin, CreateView): context = { 'app': _('Assets'), 'action': _('Create command filter'), + 'type': 'create' } kwargs.update(context) return super().get_context_data(**kwargs) @@ -64,6 +65,7 @@ class CommandFilterUpdateView(PermissionsMixin, UpdateView): context = { 'app': _('Assets'), 'action': _('Update command filter'), + 'type': 'update' } kwargs.update(context) return super().get_context_data(**kwargs) From 5f9f970abdf78f481a4c0d26145da46cb143637b Mon Sep 17 00:00:00 2001 From: BaiJiangJie <32935519+BaiJiangJie@users.noreply.github.com> Date: Thu, 11 Jul 2019 18:12:14 +0800 Subject: [PATCH 3/3] Perf (#2929) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [Update] 优化性能 * [Update] 修改assets * [Update] 优化处理 * [Update] Youhua * [Update] 修改ungroup * [Update] 修改perms的api地址,去掉sysuser adminuser的可连接性 * [Update] 修改perms urls兼容 * [Update] 修改分类 * [Update] 修改信号 * [Update] 优化获取授权资产 * [Update] 添加注释 (#2928) * [update] 去掉nodes 的部分字段 * [Update] 删除不用的代码 * [Update] 拆分users * [Update] 修改用户的属性 --- apps/assets/api/node.py | 2 +- apps/assets/models/asset.py | 65 ++- apps/assets/models/node.py | 4 +- apps/assets/models/user.py | 10 - apps/assets/serializers/admin_user.py | 3 +- apps/assets/serializers/node.py | 5 +- apps/assets/serializers/system_user.py | 3 +- apps/assets/signals_handler.py | 1 + apps/assets/templates/assets/_node_tree.html | 48 -- .../templates/assets/admin_user_list.html | 86 ++-- apps/assets/templates/assets/asset_list.html | 39 +- .../templates/assets/system_user_list.html | 84 +-- .../templates/assets/user_asset_list.html | 13 +- apps/assets/utils.py | 78 ++- apps/common/api.py | 10 + apps/common/http.py | 12 + apps/common/tree.py | 17 +- apps/common/utils/common.py | 16 + apps/jumpserver/views.py | 13 +- apps/locale/zh/LC_MESSAGES/django.mo | Bin 77611 -> 77406 bytes apps/locale/zh/LC_MESSAGES/django.po | 248 +++++---- apps/perms/api/remote_app_permission.py | 1 + apps/perms/api/user_group_permission.py | 137 +---- apps/perms/api/user_permission.py | 464 ++++++++++------- apps/perms/api/user_remote_app_permission.py | 21 +- apps/perms/const.py | 1 + apps/perms/models/asset_permission.py | 7 +- apps/perms/serializers/user_permission.py | 92 ++-- apps/perms/urls/api_urls.py | 69 ++- apps/perms/utils/asset_permission.py | 487 ++++++++++-------- apps/perms/utils/stack.py | 136 +++++ apps/users/models/user.py | 406 ++++++++------- .../templates/users/user_granted_asset.html | 12 +- 33 files changed, 1410 insertions(+), 1180 deletions(-) create mode 100644 apps/common/http.py create mode 100644 apps/perms/utils/stack.py diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 51f9a8739..2478303d0 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -131,7 +131,7 @@ class NodeChildrenAsTreeApi(generics.ListAPIView): if not include_assets: return queryset assets = self.node.get_assets().only( - "id", "hostname", "ip", 'platform', "os", "org_id", + "id", "hostname", "ip", 'platform', "os", "org_id", "protocols", ) for asset in assets: queryset.append(asset.as_tree_node(self.node)) diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index a37aa16b1..2161113e2 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -6,7 +6,8 @@ import uuid import logging import random from functools import reduce -from collections import OrderedDict +from collections import OrderedDict, defaultdict +from django.core.cache import cache from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -96,7 +97,53 @@ class ProtocolsMixin: return self.protocols_as_dict.get("ssh", 22) -class Asset(ProtocolsMixin, OrgModelMixin): +class NodesRelationMixin: + NODES_CACHE_KEY = 'ASSET_NODES_{}' + ALL_ASSET_NODES_CACHE_KEY = 'ALL_ASSETS_NODES' + CACHE_TIME = 3600 * 24 * 7 + id = "" + _all_nodes_keys = None + + @classmethod + def get_all_nodes_keys(cls): + """ + :return: {asset.id: [node.key, ]} + """ + from .node import Node + cache_key = cls.ALL_ASSET_NODES_CACHE_KEY + cached = cache.get(cache_key) + if cached: + return cached + assets = Asset.objects.all().only('id').prefetch_related( + models.Prefetch('nodes', queryset=Node.objects.all().only('key')) + ) + assets_nodes_keys = {} + for asset in assets: + assets_nodes_keys[asset.id] = [n.key for n in asset.nodes.all()] + cache.set(cache_key, assets_nodes_keys, cls.CACHE_TIME) + return assets_nodes_keys + + @classmethod + def expire_all_nodes_keys_cache(cls): + cache_key = cls.ALL_ASSET_NODES_CACHE_KEY + cache.delete(cache_key) + + def get_nodes(self): + from .node import Node + nodes = self.nodes.all() or [Node.root()] + return nodes + + def get_all_nodes(self, flat=False): + nodes = [] + for node in self.get_nodes(): + _nodes = node.get_ancestor(with_self=True) + nodes.append(_nodes) + if flat: + nodes = list(reduce(lambda x, y: set(x) | set(y), nodes)) + return nodes + + +class Asset(ProtocolsMixin, NodesRelationMixin, OrgModelMixin): # Important PLATFORM_CHOICES = ( ('Linux', 'Linux'), @@ -182,20 +229,6 @@ class Asset(ProtocolsMixin, OrgModelMixin): def is_support_ansible(self): return self.has_protocol('ssh') and self.platform not in ("Other",) - def get_nodes(self): - from .node import Node - nodes = self.nodes.all() or [Node.root()] - return nodes - - def get_all_nodes(self, flat=False): - nodes = [] - for node in self.get_nodes(): - _nodes = node.get_ancestor(with_self=True) - nodes.append(_nodes) - if flat: - nodes = list(reduce(lambda x, y: set(x) | set(y), nodes)) - return nodes - @property def cpu_info(self): info = "" diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index aab5ebaf4..d668cea51 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -212,14 +212,12 @@ class AssetsAmountMixin: if cached is not None: return cached assets_amount = self.get_all_assets().count() - self.assets_amount = assets_amount + cache.set(cache_key, assets_amount, self.cache_time) return assets_amount @assets_amount.setter def assets_amount(self, value): self._assets_amount = value - cache_key = self._assets_amount_cache_key.format(self.key) - cache.set(cache_key, value, self.cache_time) def expire_assets_amount(self): ancestor_keys = self.get_ancestor_keys(with_self=True) diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 32f569b80..bbd808b80 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -117,16 +117,6 @@ class SystemUser(AssetUser): def __str__(self): return '{0.name}({0.username})'.format(self) - def to_json(self): - return { - 'id': self.id, - 'name': self.name, - 'username': self.username, - 'protocol': self.protocol, - 'priority': self.priority, - 'auto_push': self.auto_push, - } - @property def login_mode_display(self): return self.get_login_mode_display() diff --git a/apps/assets/serializers/admin_user.py b/apps/assets/serializers/admin_user.py index b05de01f6..b8295457f 100644 --- a/apps/assets/serializers/admin_user.py +++ b/apps/assets/serializers/admin_user.py @@ -21,7 +21,7 @@ class AdminUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): model = AdminUser fields = [ 'id', 'name', 'username', 'password', 'private_key', 'public_key', - 'comment', 'connectivity_amount', 'assets_amount', + 'comment', 'assets_amount', 'date_created', 'date_updated', 'created_by', ] @@ -33,7 +33,6 @@ class AdminUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): 'date_updated': {'read_only': True}, 'created_by': {'read_only': True}, 'assets_amount': {'label': _('Asset')}, - 'connectivity_amount': {'label': _('Connectivity')}, } diff --git a/apps/assets/serializers/node.py b/apps/assets/serializers/node.py index b33f22116..ea4aa7355 100644 --- a/apps/assets/serializers/node.py +++ b/apps/assets/serializers/node.py @@ -17,9 +17,8 @@ class NodeSerializer(BulkOrgResourceModelSerializer): class Meta: model = Node - fields = [ - 'id', 'key', 'value', 'assets_amount', 'org_id', - ] + only_fields = ['id', 'key', 'value', 'org_id'] + fields = only_fields + ['assets_amount'] read_only_fields = [ 'key', 'assets_amount', 'org_id', ] diff --git a/apps/assets/serializers/system_user.py b/apps/assets/serializers/system_user.py index d853984e2..70855c9f7 100644 --- a/apps/assets/serializers/system_user.py +++ b/apps/assets/serializers/system_user.py @@ -21,14 +21,13 @@ class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): 'id', 'name', 'username', 'password', 'public_key', 'private_key', 'login_mode', 'login_mode_display', 'priority', 'protocol', 'auto_push', 'cmd_filters', 'sudo', 'shell', 'comment', 'nodes', - 'assets_amount', 'connectivity_amount', 'auto_generate_key' + 'assets_amount', 'auto_generate_key' ] extra_kwargs = { 'password': {"write_only": True}, 'public_key': {"write_only": True}, 'private_key': {"write_only": True}, 'assets_amount': {'label': _('Asset')}, - 'connectivity_amount': {'label': _('Connectivity')}, 'login_mode_display': {'label': _('Login mode display')}, 'created_by': {'read_only': True}, } diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler.py index 59d01c98a..b2316d7d0 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler.py @@ -78,6 +78,7 @@ def on_system_user_assets_change(sender, instance=None, **kwargs): @receiver(m2m_changed, sender=Asset.nodes.through) def on_asset_node_changed(sender, instance=None, **kwargs): logger.debug("Asset nodes change signal received") + Asset.expire_all_nodes_keys_cache() if isinstance(instance, Asset): if kwargs['action'] == 'pre_remove': nodes = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) diff --git a/apps/assets/templates/assets/_node_tree.html b/apps/assets/templates/assets/_node_tree.html index 93aed7d89..61737184c 100644 --- a/apps/assets/templates/assets/_node_tree.html +++ b/apps/assets/templates/assets/_node_tree.html @@ -291,42 +291,6 @@ function defaultCallback(action) { $(document).ready(function () { }) -.on('click', '.btn-refresh-hardware', function () { - var url = "{% url 'api-assets:node-refresh-hardware-info' pk=DEFAULT_PK %}"; - var the_url = url.replace("{{ DEFAULT_PK }}", current_node_id); - function success(data) { - rMenu.css({"visibility" : "hidden"}); - var task_id = data.task; - var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id); - window.open(url, '', 'width=800,height=600') - } - APIUpdateAttr({ - url: the_url, - method: "GET", - success: success, - flash_message: false - }); - -}) -.on('click', '.btn-test-connective', function () { - var url = "{% url 'api-assets:node-test-connective' pk=DEFAULT_PK %}"; - if (!current_node_id) { - return null; - } - var the_url = url.replace("{{ DEFAULT_PK }}", current_node_id); - function success(data) { - rMenu.css({"visibility" : "hidden"}); - var task_id = data.task; - var url = '{% url "ops:celery-task-log" pk=DEFAULT_PK %}'.replace("{{ DEFAULT_PK }}", task_id); - window.open(url, '', 'width=800,height=600') - } - APIUpdateAttr({ - url: the_url, - method: "GET", - success: success, - flash_message: false - }); -}) .on('click', '.btn-show-current-asset', function(){ hideRMenu(); $(this).css('display', 'none'); @@ -341,17 +305,5 @@ $(document).ready(function () { setCookie('show_current_asset', ''); location.reload(); }) -.on('click', '.btn-test-connective', function () { - hideRMenu(); -}) -.on('click', '#menu_refresh_assets_amount', function () { - hideRMenu(); - var url = "{% url 'api-assets:refresh-assets-amount' %}"; - APIUpdateAttr({ - 'url': url, - 'method': 'GET' - }); - window.location.reload(); -}) \ No newline at end of file diff --git a/apps/assets/templates/assets/admin_user_list.html b/apps/assets/templates/assets/admin_user_list.html index 0a17eccaa..61389ad08 100644 --- a/apps/assets/templates/assets/admin_user_list.html +++ b/apps/assets/templates/assets/admin_user_list.html @@ -2,8 +2,6 @@ {% load i18n static %} {% block help_message %}