From d0c9aa2c55b2af39592fa0067a31cd2f6bce693c Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 14 Dec 2020 10:19:18 +0800 Subject: [PATCH 1/6] =?UTF-8?q?fix(assets):=20=E4=BF=AE=E5=A4=8D=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E5=AD=A9=E5=AD=90=E8=8A=82=E7=82=B9=E6=97=B6=E7=9A=84?= =?UTF-8?q?log=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index f8d5f9691..841fd2487 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -493,7 +493,7 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): nodes_mapper = {n.key: n for n in nodes_sorted} for node in nodes_sorted: parent = nodes_mapper.get(node.parent_key) - if not parent: + if not parent and node.parent_key: logger.error(f'Node parent node in mapper: {node.parent_key} {node.value}') continue node.full_value = parent.full_value + '/' + node.value From ff428b84f9899e11455d1169bfb00d601a21d76f Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 14 Dec 2020 10:21:26 +0800 Subject: [PATCH 2/6] =?UTF-8?q?fix(assets):=20=E4=BF=AE=E5=A4=8Dasset?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AD=90=E8=8A=82=E7=82=B9=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/node.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 841fd2487..72000d9f1 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -493,8 +493,9 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): nodes_mapper = {n.key: n for n in nodes_sorted} for node in nodes_sorted: parent = nodes_mapper.get(node.parent_key) - if not parent and node.parent_key: - logger.error(f'Node parent node in mapper: {node.parent_key} {node.value}') + if not parent: + if node.parent_key: + logger.error(f'Node parent node in mapper: {node.parent_key} {node.value}') continue node.full_value = parent.full_value + '/' + node.value self.__class__.objects.bulk_update(nodes, ['full_value']) From 0813cff74f3f3260051b672ca32244312b759a6a Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 14 Dec 2020 11:25:34 +0800 Subject: [PATCH 3/6] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=20docs=20swagger?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E5=86=8D=E8=A6=81=E6=B1=82debug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/jumpserver/urls.py | 37 ++++++++++++++------------------ apps/jumpserver/views/swagger.py | 3 +-- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/apps/jumpserver/urls.py b/apps/jumpserver/urls.py index e34c3e3a3..713db5791 100644 --- a/apps/jumpserver/urls.py +++ b/apps/jumpserver/urls.py @@ -38,11 +38,7 @@ app_view_patterns = [ re_path(r'flower/(?P.*)', views.celery_flower_view, name='flower-view'), ] - if settings.XPACK_ENABLED: - app_view_patterns.append( - path('xpack/', include('xpack.urls.view_urls', namespace='xpack')) - ) api_v1.append( path('xpack/', include('xpack.urls.api_urls', namespace='api-xpack')) ) @@ -54,7 +50,6 @@ apps = [ 'flower', 'luna', 'koko', 'ws', 'docs', 'redocs', ] - urlpatterns = [ path('', views.IndexView.as_view(), name='index'), path('api/v1/', include(api_v1)), @@ -67,29 +62,27 @@ urlpatterns = [ path('ui/', views.UIView.as_view()), ] +# 静态文件处理路由 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \ + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) -js_i18n_patterns = [ +# js i18n 路由文件 +urlpatterns += [ path('core/jsi18n/', JavaScriptCatalog.as_view(), name='javascript-catalog'), ] -urlpatterns += js_i18n_patterns -handler404 = 'jumpserver.views.handler404' -handler500 = 'jumpserver.views.handler500' +# docs 路由 +urlpatterns += [ + re_path('^api/swagger(?P\.json|\.yaml)$', + views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'), + re_path('api/docs/?', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"), + re_path('api/redoc/?', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'), -if settings.DEBUG: - urlpatterns += [ - re_path('^api/swagger(?P\.json|\.yaml)$', - views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'), - re_path('api/docs/?', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"), - re_path('api/redoc/?', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'), - - re_path('^api/v2/swagger(?P\.json|\.yaml)$', - views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'), - path('api/docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"), - path('api/redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'), - ] + re_path('^api/v2/swagger(?P\.json|\.yaml)$', + views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'), + path('api/docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"), + path('api/redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'), +] # 兼容之前的 @@ -98,3 +91,5 @@ old_app_pattern = r'^{}'.format(old_app_pattern) urlpatterns += [re_path(old_app_pattern, views.redirect_old_apps_view)] +handler404 = 'jumpserver.views.handler404' +handler500 = 'jumpserver.views.handler500' diff --git a/apps/jumpserver/views/swagger.py b/apps/jumpserver/views/swagger.py index f6f195bed..34a5777c6 100644 --- a/apps/jumpserver/views/swagger.py +++ b/apps/jumpserver/views/swagger.py @@ -76,9 +76,8 @@ def get_swagger_view(version='v1'): patterns = api_v1_patterns schema_view = get_schema_view( api_info, - public=True, patterns=patterns, - permission_classes=(permissions.AllowAny,), + permission_classes=(permissions.IsAuthenticated,), ) return schema_view From 6e0fbd78e7e309cc483c8c95c7d4e9fe61906ab7 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 15 Dec 2020 13:00:59 +0800 Subject: [PATCH 4/6] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dprometheus=5Fmetri?= =?UTF-8?q?csAPI=E6=95=B0=E6=8D=AE=E8=8E=B7=E5=8F=96bug=EF=BC=9B=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E7=BB=84=E4=BB=B6=E6=B3=A8=E5=86=8Ctype=E4=B8=BA?= =?UTF-8?q?=E7=A9=BAbug=20(#5253)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 修复prometheus_metricsAPI数据获取bug；修复组件注册type为空bug * fix: 修改审计migrations/userloginlog/backend的verbose_name字段 Co-authored-by: Bai --- .../migrations/0011_userloginlog_backend.py | 2 +- apps/terminal/api/component.py | 6 +- .../terminal/migrations/0030_terminal_type.py | 3 +- apps/terminal/models/terminal.py | 5 +- apps/terminal/serializers_v2/terminal.py | 5 +- apps/terminal/utils.py | 100 ++++++++---------- 6 files changed, 59 insertions(+), 62 deletions(-) diff --git a/apps/audits/migrations/0011_userloginlog_backend.py b/apps/audits/migrations/0011_userloginlog_backend.py index 5a708b198..4e82d7dda 100644 --- a/apps/audits/migrations/0011_userloginlog_backend.py +++ b/apps/audits/migrations/0011_userloginlog_backend.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='userloginlog', name='backend', - field=models.CharField(default='', max_length=32, verbose_name='Login backend'), + field=models.CharField(default='', max_length=32, verbose_name='Authentication backend'), ), ] diff --git a/apps/terminal/api/component.py b/apps/terminal/api/component.py index aec404370..f881b5e98 100644 --- a/apps/terminal/api/component.py +++ b/apps/terminal/api/component.py @@ -28,7 +28,7 @@ class ComponentsMetricsAPIView(generics.GenericAPIView): permission_classes = (IsSuperUser,) def get(self, request, *args, **kwargs): - component_type = request.query_params.get('type') - util = ComponentsMetricsUtil(component_type) - metrics = util.get_metrics() + tp = request.query_params.get('type') + util = ComponentsMetricsUtil() + metrics = util.get_metrics(tp) return Response(metrics, status=status.HTTP_200_OK) diff --git a/apps/terminal/migrations/0030_terminal_type.py b/apps/terminal/migrations/0030_terminal_type.py index 4e4d871f8..1ede3ec94 100644 --- a/apps/terminal/migrations/0030_terminal_type.py +++ b/apps/terminal/migrations/0030_terminal_type.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1 on 2020-12-10 07:05 +# Generated by Django 3.1 on 2020-12-15 04:52 from django.db import migrations, models @@ -36,7 +36,6 @@ class Migration(migrations.Migration): model_name='terminal', name='type', field=models.CharField(choices=[('koko', 'KoKo'), ('guacamole', 'Guacamole'), ('omnidb', 'OmniDB')], default='koko', max_length=64, verbose_name='type'), - preserve_default=False, ), migrations.RunPython(migrate_terminal_type) ] diff --git a/apps/terminal/models/terminal.py b/apps/terminal/models/terminal.py index fb7a1dc24..b8b44dfbc 100644 --- a/apps/terminal/models/terminal.py +++ b/apps/terminal/models/terminal.py @@ -129,7 +129,10 @@ class TerminalStatusMixin(TerminalStateMixin): class Terminal(TerminalStatusMixin, models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True) name = models.CharField(max_length=128, verbose_name=_('Name')) - type = models.CharField(choices=const.TerminalTypeChoices.choices, max_length=64, verbose_name=_('type')) + type = models.CharField( + choices=const.TerminalTypeChoices.choices, default=const.TerminalTypeChoices.koko.value, + max_length=64, verbose_name=_('type') + ) remote_addr = models.CharField(max_length=128, blank=True, verbose_name=_('Remote Address')) ssh_port = models.IntegerField(verbose_name=_('SSH Port'), default=2222) http_port = models.IntegerField(verbose_name=_('HTTP Port'), default=5000) diff --git a/apps/terminal/serializers_v2/terminal.py b/apps/terminal/serializers_v2/terminal.py index f4e26b230..8121410e6 100644 --- a/apps/terminal/serializers_v2/terminal.py +++ b/apps/terminal/serializers_v2/terminal.py @@ -18,7 +18,7 @@ class TerminalSerializer(serializers.ModelSerializer): class Meta: model = Terminal fields = [ - 'id', 'name', 'remote_addr', 'command_storage', + 'id', 'name', 'type', 'remote_addr', 'command_storage', 'replay_storage', 'user', 'is_accepted', 'is_deleted', 'date_created', 'comment' ] @@ -55,5 +55,6 @@ class TerminalSerializer(serializers.ModelSerializer): class TerminalRegistrationSerializer(serializers.Serializer): name = serializers.CharField(max_length=128) - comment = serializers.CharField(max_length=128, ) + comment = serializers.CharField(max_length=128) + type = serializers.CharField(max_length=64) service_account = ServiceAccountSerializer(read_only=True) diff --git a/apps/terminal/utils.py b/apps/terminal/utils.py index 68456dfbe..d85e21deb 100644 --- a/apps/terminal/utils.py +++ b/apps/terminal/utils.py @@ -106,46 +106,43 @@ def send_command_alert_mail(command): class ComponentsMetricsUtil(object): - def __init__(self, component_type=None): - self.type = component_type - self.components = [] - self.initial_components() - - def initial_components(self): + @staticmethod + def get_components(tp=None): from .models import Terminal - terminals = Terminal.objects.all().order_by('type') - if self.type: - terminals = terminals.filter(type=self.type) - self.components = list(terminals) + components = Terminal.objects.all().order_by('type') + if tp: + components = components.filter(type=tp) + return components - def get_metrics(self): + def get_metrics(self, tp=None): + components = self.get_components(tp) total_count = normal_count = high_count = critical_count = session_active_total = 0 - for component in self.components: + for component in components: total_count += 1 - if not component.is_alive: - critical_count += 1 - continue - session_active_total += component.state.get('session_active_count', 0) - if component.is_normal: - normal_count += 1 - elif component.is_high: - high_count += 1 + if component.is_alive: + if component.is_normal: + normal_count += 1 + elif component.is_high: + high_count += 1 + else: + # critical + critical_count += 1 + session_active_total += component.state.get('session_active_count', 0) else: critical_count += 1 - metrics = { + return { 'total': total_count, 'normal': normal_count, 'high': high_count, 'critical': critical_count, 'session_active': session_active_total } - return metrics class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil): @staticmethod - def get_status_metrics(metrics): + def convert_status_metrics(metrics): return { 'any': metrics['total'], 'normal': metrics['normal'], @@ -154,50 +151,47 @@ class ComponentsPrometheusMetricsUtil(ComponentsMetricsUtil): } def get_prometheus_metrics_text(self): - prometheus_metrics = [] - prometheus_metrics.append('# JumpServer 各组件状态个数汇总') - base_status_metric_text = 'jumpserver_components_status_total{component_type="%s", status="%s"} %s' - for component in self.components: - component_type = component.type - base_metrics = self.get_metrics() + prometheus_metrics = list() - prometheus_metrics.append(f'## 组件: {component_type}') - status_metrics = self.get_status_metrics(base_metrics) + # 各组件状态个数汇总 + prometheus_metrics.append('# JumpServer 各组件状态个数汇总') + status_metric_text = 'jumpserver_components_status_total{component_type="%s", status="%s"} %s' + for tp in const.TerminalTypeChoices.types(): + prometheus_metrics.append(f'## 组件: {tp}') + metrics_tp = self.get_metrics(tp) + status_metrics = self.convert_status_metrics(metrics_tp) for status, value in status_metrics.items(): - metric_text = base_status_metric_text % (component_type, status, value) + metric_text = status_metric_text % (tp, status, value) prometheus_metrics.append(metric_text) prometheus_metrics.append('\n') + + # 各组件在线会话数汇总 prometheus_metrics.append('# JumpServer 各组件在线会话数汇总') - base_session_active_metric_text = 'jumpserver_components_session_active_total{component_type="%s"} %s' - for component in self.components: - component_type = component.type - prometheus_metrics.append(f'## 组件: {component_type}') - base_metrics = self.get_metrics() - metric_text = base_session_active_metric_text % ( - component_type, - base_metrics['session_active'] - ) + session_active_metric_text = 'jumpserver_components_session_active_total{component_type="%s"} %s' + for tp in const.TerminalTypeChoices.types(): + prometheus_metrics.append(f'## 组件: {tp}') + metrics_tp = self.get_metrics(tp) + metric_text = session_active_metric_text % (tp, metrics_tp['session_active']) prometheus_metrics.append(metric_text) prometheus_metrics.append('\n') - prometheus_metrics.append('# JumpServer 各组件节点一些指标') - base_system_state_metric_text = 'jumpserver_components_%s{component_type="%s", component="%s"} %s' - system_states_name = [ + + # 各组件节点指标 + prometheus_metrics.append('# JumpServer 各组件一些指标') + state_metric_text = 'jumpserver_components_%s{component_type="%s", component="%s"} %s' + states = [ 'system_cpu_load_1', 'system_memory_used_percent', 'system_disk_used_percent', 'session_active_count' ] - for system_state_name in system_states_name: - prometheus_metrics.append(f'## 指标: {system_state_name}') - for component in self.components: + for state in states: + prometheus_metrics.append(f'## 指标: {state}') + components = self.get_components() + for component in components: if not component.is_alive: continue - component_type = component.type - metric_text = base_system_state_metric_text % ( - system_state_name, - component_type, - component.name, - component.state.get(system_state_name) + metric_text = state_metric_text % ( + state, component.type, component.name, component.state.get(state) ) prometheus_metrics.append(metric_text) From ca883f1fb4bf30779e25fd221920d9c4b02c3f9f Mon Sep 17 00:00:00 2001 From: xinwen Date: Tue, 15 Dec 2020 12:21:17 +0800 Subject: [PATCH 5/6] =?UTF-8?q?fix:=20=E5=B7=A5=E5=8D=95=E7=94=B3=E8=AF=B7?= =?UTF-8?q?=E8=B5=84=E4=BA=A7=E5=AE=A1=E6=89=B9=E6=97=B6=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=B2=A1=E6=9C=89=E6=8E=A8=E8=8D=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tickets/serializers/request_asset_perm.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/tickets/serializers/request_asset_perm.py b/apps/tickets/serializers/request_asset_perm.py index 77757ff62..83c480e61 100644 --- a/apps/tickets/serializers/request_asset_perm.py +++ b/apps/tickets/serializers/request_asset_perm.py @@ -1,5 +1,3 @@ -from itertools import chain - from rest_framework import serializers from django.conf import settings from django.utils.translation import ugettext_lazy as _ @@ -9,7 +7,7 @@ from django.db.models import Q from common.utils.timezone import dt_parser, dt_formater from orgs.utils import tmp_to_root_org from orgs.models import Organization, ROLE as ORG_ROLE -from assets.models.asset import Asset +from assets.models import Asset, SystemUser from users.models.user import User from perms.serializers import ActionsField from perms.models import Action @@ -130,12 +128,23 @@ class RequestAssetPermTicketSerializer(serializers.ModelSerializer): if hostname: q |= Q(hostname__icontains=hostname) - data['confirmed_assets'] = list( - map(lambda x: str(x), chain(*Asset.objects.filter(q)[0: limit].values_list('id')))) + recomand_assets_id = Asset.objects.filter(q)[:limit].values_list('id', flat=True) + data['confirmed_assets'] = [str(i) for i in recomand_assets_id] + + def _recommend_system_users(self, data, instance): + confirmed_system_users = data.get('confirmed_system_users') + if not confirmed_system_users and self._is_assignee(instance): + system_user = data.get('system_user') + + recomand_system_users_id = SystemUser.objects.filter( + name__icontains=system_user + )[:3].values_list('id', flat=True) + data['confirmed_system_users'] = [str(i) for i in recomand_system_users_id] def to_representation(self, instance): data = super().to_representation(instance) self._recommend_assets(data, instance) + self._recommend_system_users(data, instance) return data def _create_body(self, validated_data): From 0d469ff95b0069dcf8d3d15bd9751683c84d41a3 Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 14 Dec 2020 20:31:12 +0800 Subject: [PATCH 6/6] =?UTF-8?q?fix(orgs):=20=E7=94=A8=E6=88=B7=E7=A6=BB?= =?UTF-8?q?=E5=BC=80=E7=BB=84=E7=BB=87=E5=90=8E=E6=8E=88=E6=9D=83=E7=9A=84?= =?UTF-8?q?=E8=B5=84=E4=BA=A7=E6=B2=A1=E4=B8=BB=E5=8A=A8=E5=88=B7=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/signals_handler.py | 50 ++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/apps/orgs/signals_handler.py b/apps/orgs/signals_handler.py index 8e4d969eb..adfd9d373 100644 --- a/apps/orgs/signals_handler.py +++ b/apps/orgs/signals_handler.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # +from collections import defaultdict +from functools import partial from django.db.models.signals import m2m_changed from django.db.models.signals import post_save @@ -7,10 +9,10 @@ from django.dispatch import receiver from orgs.utils import tmp_to_org from .models import Organization, OrganizationMember -from .hands import set_current_org, current_org, Node, get_current_org -from perms.models import (AssetPermission, DatabaseAppPermission, - K8sAppPermission, RemoteAppPermission) -from users.models import UserGroup +from .hands import set_current_org, Node, get_current_org +from perms.models import (AssetPermission, ApplicationPermission) +from users.models import UserGroup, User +from common.const.signals import PRE_REMOVE, POST_REMOVE @receiver(post_save, sender=Organization) @@ -34,11 +36,44 @@ def _remove_users(model, users, org): users = (users, ) m2m_model = model.users.through - if model.users.reverse: + reverse = model.users.reverse + if reverse: m2m_field_name = model.users.field.m2m_reverse_field_name() else: m2m_field_name = model.users.field.m2m_field_name() - m2m_model.objects.filter(**{'user__in': users, f'{m2m_field_name}__org_id': org.id}).delete() + relations = m2m_model.objects.filter(**{ + 'user__in': users, + f'{m2m_field_name}__org_id': org.id + }) + + object_id_users_id_map = defaultdict(set) + + m2m_field_attr_name = f'{m2m_field_name}_id' + for relation in relations: + object_id = getattr(relation, m2m_field_attr_name) + object_id_users_id_map[object_id].add(relation.user_id) + + objects = model.objects.filter(id__in=object_id_users_id_map.keys()) + send_m2m_change_signal = partial( + m2m_changed.send, + sender=m2m_model, reverse=reverse, model=User, using=model.objects.db + ) + + for obj in objects: + send_m2m_change_signal( + instance=obj, + pk_set=object_id_users_id_map[obj.id], + action=PRE_REMOVE + ) + + relations.delete() + + for obj in objects: + send_m2m_change_signal( + instance=obj, + pk_set=object_id_users_id_map[obj.id], + action=POST_REMOVE + ) def _clear_users_from_org(org, users): @@ -48,8 +83,7 @@ def _clear_users_from_org(org, users): if not users: return - models = (AssetPermission, DatabaseAppPermission, - RemoteAppPermission, K8sAppPermission, UserGroup) + models = (AssetPermission, ApplicationPermission, UserGroup) for m in models: _remove_users(m, users, org)