Config (#3502)

* [Update] 修改config

* [Update] 移动存储设置到到terminal中

* [Update] 修改permission 查看

* [Update] pre merge

* [Update] 录像存储

* [Update] 命令存储

* [Update] 添加存储测试可连接性

* [Update] 修改 meta 值的 key 为大写

* [Update] 修改 Terminal 相关 Storage 配置

* [Update] 删除之前获取录像/命令存储的代码

* [Update] 修改导入失败

* [Update] 迁移文件添加default存储

* [Update] 删除之前代码，添加help_text信息

* [Update] 删除之前代码

* [Update] 删除之前代码

* [Update] 抽象命令/录像存储 APIView

* [Update] 抽象命令/录像存储 APIView 1

* [Update] 抽象命令/录像存储 DictField

* [Update] 抽象命令/录像存储列表页面

* [Update] 修复CustomDictField的bug

* [Update] RemoteApp 页面添加 hidden

* [Update] 用户页面添加用户关联授权

* [Update] 修改存储测试可连接性 target

* [Update] 修改配置

* [Update] 修改存储前端 Form 渲染逻辑

* [Update] 修改存储细节

* [Update] 统一存储类型到 const 文件

* [Update] 修改迁移文件及Model，创建默认存储

* [Update] 修改迁移文件及Model初始化默认数据

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 修改迁移文件

* [Update] 限制删除默认存储配置，只允许创建扩展的存储类型

* [Update] 修改ip字段长度

* [Update] 修改ip字段长度

* [Update] 修改一些css

* [Update] 修改关联

* [Update] 添加操作日志定时清理

* [Update] 修改记录syslog的instance encoder

* [Update] 忽略登录产生的操作日志

* [Update] 限制更新存储时不覆盖原有AK SK 等字段

* [Update] 修改迁移文件添加comment字段

* [Update] 修改迁移文件

* [Update] 添加 comment 字段

* [Update] 修改默认存储no -> null

* [Update] 修改细节

* [Update] 更新翻译（存储配置

* [Update] 修改定时任务注册，修改系统用户资产、节点关系api

* [Update] 添加监控磁盘任务

* [Update] 修改session

* [Update] 拆分serializer

* [Update] 还原setting原来的manager

apps/applications/serializers/remote_app.py

@@ -5,6 +5,7 @@
 from rest_framework import serializers
 
 from common.serializers import AdaptedBulkListSerializer
+from common.fields.serializer import CustomMetaDictField
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
 
 from .. import const
@@ -16,54 +17,9 @@ __all__ = [
 ]
 
 
-class RemoteAppParamsDictField(serializers.DictField):
-    """
-    RemoteApp field => params
-    """
-    @staticmethod
-    def filter_attribute(attribute, instance):
-        """
-        过滤掉params字段值中write_only特性的key-value值
-        For example, the chrome_password field is not returned when serializing
-        {
-            'chrome_target': 'http://www.jumpserver.org/',
-            'chrome_username': 'admin',
-            'chrome_password': 'admin',
-        }
-        """
-        for field in const.REMOTE_APP_TYPE_MAP_FIELDS[instance.type]:
-            if field.get('write_only', False):
-                attribute.pop(field['name'], None)
-        return attribute
-
-    def get_attribute(self, instance):
-        """
-        序列化时调用
-        """
-        attribute = super().get_attribute(instance)
-        attribute = self.filter_attribute(attribute, instance)
-        return attribute
-
-    @staticmethod
-    def filter_value(dictionary, value):
-        """
-        过滤掉不属于当前app_type所包含的key-value值
-        """
-        app_type = dictionary.get('type', const.REMOTE_APP_TYPE_CHROME)
-        fields = const.REMOTE_APP_TYPE_MAP_FIELDS[app_type]
-        fields_names = [field['name'] for field in fields]
-        no_need_keys = [k for k in value.keys() if k not in fields_names]
-        for k in no_need_keys:
-            value.pop(k)
-        return value
-
-    def get_value(self, dictionary):
-        """
-        反序列化时调用
-        """
-        value = super().get_value(dictionary)
-        value = self.filter_value(dictionary, value)
-        return value
+class RemoteAppParamsDictField(CustomMetaDictField):
+    type_map_fields = const.REMOTE_APP_TYPE_MAP_FIELDS
+    default_type = const.REMOTE_APP_TYPE_CHROME
 
 
 class RemoteAppSerializer(BulkOrgResourceModelSerializer):

apps/applications/templates/applications/remote_app_create_update.html

@@ -19,14 +19,14 @@
         <div class="hr-line-dashed"></div>
 
         {# chrome #}
-        <div class="chrome-fields">
+        <div class="chrome-fields hidden">
         {% bootstrap_field form.chrome_target layout="horizontal" %}
         {% bootstrap_field form.chrome_username layout="horizontal" %}
         {% bootstrap_field form.chrome_password layout="horizontal" %}
         </div>
 
         {# mysql workbench #}
-        <div class="mysql_workbench-fields">
+        <div class="mysql_workbench-fields hidden">
             {% bootstrap_field form.mysql_workbench_ip layout="horizontal" %}
             {% bootstrap_field form.mysql_workbench_name layout="horizontal" %}
             {% bootstrap_field form.mysql_workbench_username layout="horizontal" %}
@@ -34,14 +34,14 @@
         </div>
 
         {# vmware #}
-        <div class="vmware_client-fields">
+        <div class="vmware_client-fields hidden">
         {% bootstrap_field form.vmware_target layout="horizontal" %}
         {% bootstrap_field form.vmware_username layout="horizontal" %}
         {% bootstrap_field form.vmware_password layout="horizontal" %}
         </div>
 
         {# custom #}
-        <div class="custom-fields">
+        <div class="custom-fields hidden">
         {% bootstrap_field form.custom_cmdline layout="horizontal" %}
         {% bootstrap_field form.custom_target layout="horizontal" %}
         {% bootstrap_field form.custom_username layout="horizontal" %}

apps/applications/templates/applications/remote_app_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">
@@ -102,4 +97,4 @@ $(document).ready(function () {
     objectDelete($this, name, the_url, redirect_url);
 })
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/api/__init__.py

@@ -2,6 +2,7 @@ from .admin_user import *
 from .asset import *
 from .label import *
 from .system_user import *
+from .system_user_relation import *
 from .node import *
 from .domain import *
 from .cmd_filter import *

apps/assets/api/admin_user.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from django.db import transaction
+from django.db.models import Count
 from django.shortcuts import get_object_or_404
 from rest_framework.response import Response
 from orgs.mixins.api import OrgBulkModelViewSet
@@ -44,6 +45,11 @@ class AdminUserViewSet(OrgBulkModelViewSet):
     serializer_class = serializers.AdminUserSerializer
     permission_classes = (IsOrgAdmin,)
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.annotate(_assets_amount=Count('assets'))
+        return queryset
+
 
 class AdminUserAuthApi(generics.UpdateAPIView):
     model = AdminUser

apps/assets/api/asset_user.py

@@ -114,7 +114,7 @@ class AssetUserExportViewSet(AssetUserViewSet):
     permission_classes = [IsOrgAdminOrAppUser]
 
     def get_permissions(self):
-        if settings.CONFIG.SECURITY_VIEW_AUTH_NEED_MFA:
+        if settings.SECURITY_VIEW_AUTH_NEED_MFA:
             self.permission_classes = [IsOrgAdminOrAppUser, NeedMFAVerify]
         return super().get_permissions()
 
@@ -124,7 +124,7 @@ class AssetUserAuthInfoApi(generics.RetrieveAPIView):
     permission_classes = [IsOrgAdminOrAppUser]
 
     def get_permissions(self):
-        if settings.CONFIG.SECURITY_VIEW_AUTH_NEED_MFA:
+        if settings.SECURITY_VIEW_AUTH_NEED_MFA:
             self.permission_classes = [IsOrgAdminOrAppUser, NeedMFAVerify]
         return super().get_permissions()
 

apps/assets/api/system_user.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 from django.shortcuts import get_object_or_404
-from django.conf import settings
 from rest_framework.response import Response
+from django.db.models import Count
 
 from common.serializers import CeleryTaskSerializer
 from common.utils import get_logger
@@ -50,6 +50,11 @@ class SystemUserViewSet(OrgBulkModelViewSet):
     serializer_class = serializers.SystemUserSerializer
     permission_classes = (IsOrgAdminOrAppUser,)
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.annotate(_assets_amount=Count('assets'))
+        return queryset
+
 
 class SystemUserAuthInfoApi(generics.RetrieveUpdateDestroyAPIView):
     """

apps/assets/api/system_user_relation.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+#
+from django.db.models import F, Value
+from django.db.models.functions import Concat
+
+from common.permissions import IsOrgAdmin
+from orgs.mixins.api import OrgBulkModelViewSet
+from orgs.utils import current_org
+from .. import models, serializers
+
+__all__ = ['SystemUserAssetRelationViewSet', 'SystemUserNodeRelationViewSet']
+
+
+class RelationMixin(OrgBulkModelViewSet):
+    def get_queryset(self):
+        queryset = self.model.objects.all()
+        org_id = current_org.org_id()
+        if org_id is not None:
+            queryset = queryset.filter(systemuser__org_id=org_id)
+        queryset = queryset.annotate(systemuser_display=Concat(
+            F('systemuser__name'), Value('('), F('systemuser__username'),
+            Value(')')
+        ))
+        return queryset
+
+
+class SystemUserAssetRelationViewSet(RelationMixin):
+    serializer_class = serializers.SystemUserAssetRelationSerializer
+    model = models.SystemUser.assets.through
+    permission_classes = (IsOrgAdmin,)
+    filterset_fields = [
+        'id', 'asset', 'systemuser',
+    ]
+    search_fields = [
+        "id", "asset__hostname", "asset__ip",
+        "systemuser__name", "systemuser__username"
+    ]
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.annotate(
+            asset_display=Concat(
+                F('asset__hostname'), Value('('),
+                F('asset__ip'), Value(')')
+            )
+        )
+        return queryset
+
+
+class SystemUserNodeRelationViewSet(RelationMixin):
+    serializer_class = serializers.SystemUserNodeRelationSerializer
+    model = models.SystemUser.nodes.through
+    permission_classes = (IsOrgAdmin,)
+    filterset_fields = [
+        'id', 'node', 'systemuser',
+    ]
+    search_fields = [
+        "node__value", "systemuser__name", "systemuser_username"
+    ]
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset \
+            .annotate(node_key=F('node__key'))
+        return queryset

apps/assets/models/base.py

@@ -41,6 +41,7 @@ class AssetUser(OrgModelMixin):
     ASSET_USER_CACHE_TIME = 3600 * 24
 
     _prefer = "system_user"
+    _assets_amount = None
 
     @property
     def private_key_obj(self):
@@ -143,6 +144,8 @@ class AssetUser(OrgModelMixin):
 
     @property
     def assets_amount(self):
+        if self._assets_amount is not None:
+            return self._assets_amount
         cache_key = self.ASSETS_AMOUNT_CACHE_KEY.format(self.id)
         cached = cache.get(cache_key)
         if not cached:

apps/assets/serializers/system_user.py

@@ -4,8 +4,10 @@ from rest_framework import serializers
 from django.utils.translation import ugettext_lazy as _
 
 from common.serializers import AdaptedBulkListSerializer
+from common.mixins.serializers import BulkSerializerMixin
 from common.utils import ssh_pubkey_gen
 from orgs.mixins.serializers import BulkOrgResourceModelSerializer
+from assets.models import Node
 from ..models import SystemUser
 from ..const import (
     GENERAL_FORBIDDEN_SPECIAL_CHARACTERS_PATTERN,
@@ -13,6 +15,12 @@ from ..const import (
 )
 from .base import AuthSerializer, AuthSerializerMixin
 
+__all__ = [
+    'SystemUserSerializer', 'SystemUserAuthSerializer',
+    'SystemUserSimpleSerializer', 'SystemUserAssetRelationSerializer',
+    'SystemUserNodeRelationSerializer',
+]
+
 
 class SystemUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer):
     """
@@ -143,4 +151,43 @@ class SystemUserSimpleSerializer(serializers.ModelSerializer):
         fields = ('id', 'name', 'username')
 
 
+class RelationMixin(BulkSerializerMixin, serializers.Serializer):
+    systemuser_display = serializers.ReadOnlyField()
 
+    def get_field_names(self, declared_fields, info):
+        fields = super().get_field_names(declared_fields, info)
+        fields.extend(['systemuser', "systemuser_display"])
+        return fields
+
+    class Meta:
+        list_serializer_class = AdaptedBulkListSerializer
+
+
+class SystemUserAssetRelationSerializer(RelationMixin, serializers.ModelSerializer):
+    asset_display = serializers.ReadOnlyField()
+
+    class Meta(RelationMixin.Meta):
+        model = SystemUser.assets.through
+        fields = [
+            'id', "asset", "asset_display",
+        ]
+
+
+class SystemUserNodeRelationSerializer(RelationMixin, serializers.ModelSerializer):
+    node_display = serializers.SerializerMethodField()
+
+    class Meta(RelationMixin.Meta):
+        model = SystemUser.nodes.through
+        fields = [
+            'id', 'node', "node_display",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.tree = Node.tree()
+
+    def get_node_display(self, obj):
+        if hasattr(obj, 'node_key'):
+            return self.tree.get_node_full_tag(obj.node_key)
+        else:
+            return obj.node.full_value

apps/assets/templates/assets/_asset_list_modal.html

@@ -190,6 +190,16 @@ function setAssetModalOptions(options) {
     assetModalOption = options;
 }
 
+function initAssetTreeModel(selector) {
+    $(selector).parent().find(".select2-selection").on('click', function (e) {
+        if ($(e.target).attr('class') !== 'select2-selection__choice__remove'){
+            e.preventDefault();
+            e.stopPropagation();
+            $("#asset_list_modal").modal();
+        }
+    })
+}
+
 
 $(document).ready(function(){
 

apps/assets/templates/assets/_system_user.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -247,4 +243,4 @@ $(document).ready(function () {
 })
 
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/admin_user_assets.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/admin_user_create_update.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -87,4 +83,4 @@ $(document).ready(function () {
     })
 })
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/admin_user_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/asset_bulk_update.html

@@ -32,13 +32,7 @@
 <script>
     $(document).ready(function () {
         $('.select2').select2();
-        $("#id_assets").parent().find(".select2-selection").on('click', function (e) {
-            if ($(e.target).attr('class') !== 'select2-selection__choice__remove'){
-                e.preventDefault();
-                e.stopPropagation();
-                $("#asset_list_modal").modal();
-            }
-        })
+        initAssetTreeModel("#id_assets");
     }).on('click', '.field-tag', function() {
         changeField(this);
     }).on('click', '#change_all', function () {

apps/assets/templates/assets/asset_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
     <link href='{% static "css/plugins/sweetalert/sweetalert.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
     <script src='{% static "js/plugins/sweetalert/sweetalert.min.js" %}'></script>
 {% endblock %}
 

apps/assets/templates/assets/asset_list.html

@@ -382,9 +382,6 @@ $(document).ready(function(){
     var data = {
         'resources': id_list
     };
-    function refreshPage() {
-        setTimeout( function () {window.location.reload();}, 300);
-    }
 
     function reloadTable() {
         asset_table.ajax.reload();

apps/assets/templates/assets/cmd_filter_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/cmd_filter_rule_create_update.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -91,4 +87,4 @@ $(document).ready(function(){
     formSubmit(props);
 })
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/cmd_filter_rule_list.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/domain_create_update.html

@@ -25,9 +25,7 @@
 <script type="text/javascript">
 $(document).ready(function () {
     $('.select2').select2().off("select2:open");
-}).on('click', '.select2-selection__rendered', function (e) {
-    e.preventDefault();
-    $("#asset_list_modal").modal();
+    initAssetTreeModel('#id_assets');
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();
@@ -51,4 +49,4 @@ $(document).ready(function () {
     formSubmit(props);
  })
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/domain_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">
@@ -138,4 +133,4 @@ $(document).ready(function(){
 })
 ;
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/domain_gateway_list.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/templates/assets/gateway_create_update.html

@@ -2,10 +2,6 @@
 {% load i18n %}
 {% load static %}
 {% load bootstrap3 %}
-{% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
-{% endblock %}
 
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
@@ -125,4 +121,4 @@ $(document).ready(function(){
     protocolChange();
 });
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/label_create_update.html

@@ -29,9 +29,7 @@ $(document).ready(function () {
     $('.select2').select2({
         closeOnSelect: false
     })
-}).on('click', '.select2-selection__rendered', function (e) {
-    e.preventDefault();
-    $("#asset_list_modal").modal();
+    initAssetTreeModel("#id_assets");
 })
 .on("submit", "form", function (evt) {
     evt.preventDefault();
@@ -55,4 +53,4 @@ $(document).ready(function () {
     formSubmit(props);
 })
 </script>
-{% endblock %}
+{% endblock %}

apps/assets/templates/assets/system_user_assets.html

@@ -4,9 +4,13 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
+    <style>
+    .table.node_edit {
+        margin-bottom: 0;
+    }
+    </style>
 {% endblock %}
+
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">
@@ -98,15 +102,8 @@
                                                 </td>
                                             </tr>
                                         </form>
-
-                                        {% for node in system_user.nodes.all|sort %}
-                                        <tr>
-                                          <td ><b class="bdg_node" data-gid={{ node.id }}>{{ node.full_value }}</b></td>
-                                          <td>
-                                              <button class="btn btn-danger pull-right btn-xs btn-remove-from-node" type="button"><i class="fa fa-minus"></i></button>
-                                          </td>
-                                        </tr>
-                                        {% endfor %}
+                                        <table class="table" id="node_list_table">
+                                        </table>
                                         </tbody>
                                     </table>
                                 </div>
@@ -120,91 +117,115 @@
 {% endblock %}
 {% block custom_foot_js %}
 <script>
+var assetsRelationUrl = "{% url 'api-assets:system-users-assets-relation-list' %}";
+var nodesRelationUrl = "{% url 'api-assets:system-users-nodes-relation-list' %}";
 
-function updateSystemUserNode(nodes) {
-    var the_url = "{% url 'api-assets:system-user-detail' pk=system_user.id %}";
-    var body = {
-        nodes: Object.assign([], nodes)
-    };
-    var success = function(data) {
-        // remove all the selected groups from select > option and rendered ul element;
-        $('.select2-selection__rendered').empty();
-        $('#node_selected').val('');
-        $.map(jumpserver.nodes_selected, function(node_name, index) {
-            $('#opt_' + index).remove();
-            // change tr html of user groups.
-            $('.node_edit tbody').append(
-                '<tr>' +
-                '<td><b class="bdg_node" data-gid="' + index + '">' + node_name + '</b></td>' +
-                '<td><button class="btn btn-danger btn-xs pull-right btn-remove-from-node" type="button"><i class="fa fa-minus"></i></button></td>' +
-                '</tr>'
-            )
-        });
-        // clear jumpserver.nodes_selected
-        jumpserver.nodes_selected = {};
-    };
+function getRelationUrl(type) {
+    var theUrl = "";
+    switch (type) {
+        case "asset":
+            theUrl = assetsRelationUrl;
+            break;
+        case "node":
+            theUrl = nodesRelationUrl;
+            break;
+    }
+    return theUrl;
+}
+
+function addObjects(objectsId, type) {
+    if (!objectsId || objectsId.length === 0) {
+        return
+    }
+    var theUrl = getRelationUrl(type);
+    var body = [];
+    objectsId.forEach(function (v) {
+        var data = {systemuser: "{{ object.id }}"};
+        data[type] = v;
+        body.push(data)
+    });
     requestApi({
-        url: the_url,
+        url: theUrl,
         body: JSON.stringify(body),
+        method: "POST",
+        success: reloadPage
+    });
+}
+
+function removeObject(objectId, type, success) {
+    if (!objectId)  {
+        return
+    }
+    var theUrl = getRelationUrl(type);
+    theUrl = setUrlParam(theUrl, 'systemuser', "{{ object.id }}");
+    theUrl = setUrlParam(theUrl, type, objectId);
+    if (!success) {
+        success = reloadPage
+    }
+    requestApi({
+        url: theUrl,
+        method: "DELETE",
         success: success
     });
 }
-jumpserver.nodes_selected = {};
+
+function initNodeTable() {
+    var theUrl = setUrlParam(nodesRelationUrl, "systemuser", "{{ object.id }}");
+    var options = {
+        ele: $('#node_list_table'),
+        toggle: true,
+        columnDefs: [
+             {targets: 1, createdCell: function (td, cellData) {
+                  var removeBtn = '<button class="btn btn-danger pull-right btn-xs btn-remove-from-node" data-uid="UID" type="button"><i class="fa fa-minus"></i></button>'
+                      .replace('UID', cellData);
+		  		  $(td).html(removeBtn);
+             }}
+        ],
+        ajax_url: theUrl,
+        dom: "trp",
+        hideDefaultDefs: true,
+        columns: [
+            {data: "node_display", orderable: false},
+            {data: "node", orderable: false},
+        ],
+    };
+    table = jumpserver.initServerSideDataTable(options);
+    return table
+}
+
 
 $(document).ready(function () {
-	$('.select2').select2()
-    nodesSelect2Init(".nodes-select2")
-	.on('select2:select', function(evt) {
-            var data = evt.params.data;
-            jumpserver.nodes_selected[data.id] = data.text;
-    })
-    .on('select2:unselect', function(evt) {
-        var data = evt.params.data;
-        delete jumpserver.nodes_selected[data.id];
-    });
+	$('.select2').select2();
+    nodesSelect2Init(".nodes-select2");
 	assetUserListUrl = setUrlParam(assetUserListUrl, "system_user_id", "{{ system_user.id }}");
     needPush = true;
     initAssetUserTable();
-
+    initNodeTable();
 })
 .on('click', '.btn-remove-from-node', function() {
     var $this = $(this);
-    var $tr = $this.closest('tr');
-    var $badge = $tr.find('.bdg_node');
-    var gid = $badge.data('gid');
-    var node_name = $badge.html() || $badge.text();
-    $('#groups_selected').append(
-        '<option value="' + gid + '" id="opt_' + gid + '">' + node_name + '</option>'
-    );
-    $tr.remove();
-    var nodes = $('.bdg_node').map(function () {
-        return $(this).data('gid');
-    }).get();
-    updateSystemUserNode(nodes);
+    var nodeId = $(this).data('uid');
+    var success = function() {
+        var $tr = $this.closest('tr');
+        $tr.remove();
+    };
+    removeObject(nodeId, "node", success)
 })
 .on('click', '#btn-add-to-node', function() {
-    if (Object.keys(jumpserver.nodes_selected).length === 0) {
-        return false;
-    }
-    var nodes = $('.bdg_node').map(function() {
-        return $(this).data('gid');
-    }).get();
-    $.map(jumpserver.nodes_selected, function(value, index) {
-        nodes.push(index);
-    });
-    updateSystemUserNode(nodes);
+    var nodes = $("#node_selected").val();
+    addObjects(nodes, "node");
 })
 .on('click', '.btn-push', function () {
-    var the_url = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
+    var theUrl = "{% url 'api-assets:system-user-push' pk=system_user.id %}";
     var error = function (data) {
         alert(data)
     };
     var success = function (data) {
-        var task_id = data.task;
-        showCeleryTaskLog(task_id);
+        var taskId = data.task;
+        showCeleryTaskLog(taskId);
     };
     requestApi({
-        url: the_url,
+        url: theUrl,
         error: error,
         method: 'GET',
         success: success
@@ -213,39 +234,37 @@ $(document).ready(function () {
 .on('click', '.btn-push-auth', function () {
     var $this = $(this);
     var asset_id = $this.data('asset');
-    var the_url = "{% url 'api-assets:system-user-push-to-asset' pk=object.id aid=DEFAULT_PK %}";
-    the_url = the_url.replace("{{ DEFAULT_PK }}", asset_id);
+    var theUrl = "{% url 'api-assets:system-user-push-to-asset' pk=object.id aid=DEFAULT_PK %}";
+    theUrl = theUrl.replace("{{ DEFAULT_PK }}", asset_id);
     var success = function (data) {
-        var task_id = data.task;
-        showCeleryTaskLog(task_id);
+        var taskId = data.task;
+        showCeleryTaskLog(taskId);
     };
     var error = function (data) {
         alert(data)
     };
     requestApi({
-        url: the_url,
+        url: theUrl,
         method: 'GET',
         success: success,
         error: error
     })
 })
 .on('click', '.btn-test-connective', function () {
-    var the_url = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
+    var theUrl = "{% url 'api-assets:system-user-connective' pk=system_user.id %}";
     var error = function (data) {
         alert(data)
     };
     var success = function (data) {
-        var task_id = data.task;
-        showCeleryTaskLog(task_id);
+        var taskId = data.task;
+        showCeleryTaskLog(taskId);
     };
     requestApi({
-        url: the_url,
+        url: theUrl,
         error: error,
         method: 'GET',
         success: success
     });
 })
-
-
 </script>
 {% endblock %}

apps/assets/templates/assets/system_user_detail.html

@@ -2,11 +2,6 @@
 {% load static %}
 {% load i18n %}
 
-{% block custom_head_css_js %}
-    <link href='{% static "css/plugins/select2/select2.min.css" %}' rel="stylesheet">
-    <script src='{% static "js/plugins/select2/select2.full.min.js" %}'></script>
-{% endblock %}
-
 {% block content %}
     <div class="wrapper wrapper-content animated fadeInRight">
         <div class="row">

apps/assets/urls/api_urls.py

@@ -23,6 +23,8 @@ router.register(r'asset-users', api.AssetUserViewSet, 'asset-user')
 router.register(r'asset-users-info', api.AssetUserExportViewSet, 'asset-user-info')
 router.register(r'gathered-users', api.GatheredUserViewSet, 'gathered-user')
 router.register(r'favorite-assets', api.FavoriteAssetViewSet, 'favorite-asset')
+router.register(r'system-users-assets-relations', api.SystemUserAssetRelationViewSet, 'system-users-assets-relation')
+router.register(r'system-users-nodes-relations', api.SystemUserNodeRelationViewSet, 'system-users-nodes-relation')
 
 cmd_filter_router = routers.NestedDefaultRouter(router, r'cmd-filters', lookup='filter')
 cmd_filter_router.register(r'rules', api.CommandFilterRuleViewSet, 'cmd-filter-rule')

apps/audits/api.py

@@ -11,4 +11,4 @@ class FTPLogViewSet(OrgModelViewSet):
     model = FTPLog
     serializer_class = FTPLogSerializer
     permission_classes = (IsOrgAdminOrAppUser | IsOrgAuditor,)
-
+    http_method_names = ['get', 'post', 'head', 'options']

apps/audits/migrations/0007_auto_20191202_1010.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.7 on 2019-12-02 02:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('audits', '0006_auto_20190726_1753'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='ftplog',
+            name='remote_addr',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Remote addr'),
+        ),
+        migrations.AlterField(
+            model_name='operatelog',
+            name='remote_addr',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Remote addr'),
+        ),
+        migrations.AlterField(
+            model_name='passwordchangelog',
+            name='remote_addr',
+            field=models.CharField(blank=True, max_length=128, null=True, verbose_name='Remote addr'),
+        ),
+    ]

apps/audits/models.py

@@ -16,7 +16,7 @@ __all__ = [
 class FTPLog(OrgModelMixin):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     user = models.CharField(max_length=128, verbose_name=_('User'))
-    remote_addr = models.CharField(max_length=15, verbose_name=_("Remote addr"), blank=True, null=True)
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     asset = models.CharField(max_length=1024, verbose_name=_("Asset"))
     system_user = models.CharField(max_length=128, verbose_name=_("System user"))
     operate = models.CharField(max_length=16, verbose_name=_("Operate"))
@@ -39,7 +39,7 @@ class OperateLog(OrgModelMixin):
     action = models.CharField(max_length=16, choices=ACTION_CHOICES, verbose_name=_("Action"))
     resource_type = models.CharField(max_length=64, verbose_name=_("Resource Type"))
     resource = models.CharField(max_length=128, verbose_name=_("Resource"))
-    remote_addr = models.CharField(max_length=15, verbose_name=_("Remote addr"), blank=True, null=True)
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     datetime = models.DateTimeField(auto_now=True)
 
     def __str__(self):
@@ -50,7 +50,7 @@ class PasswordChangeLog(models.Model):
     id = models.UUIDField(default=uuid.uuid4, primary_key=True)
     user = models.CharField(max_length=128, verbose_name=_('User'))
     change_by = models.CharField(max_length=128, verbose_name=_("Change by"))
-    remote_addr = models.CharField(max_length=15, verbose_name=_("Remote addr"), blank=True, null=True)
+    remote_addr = models.CharField(max_length=128, verbose_name=_("Remote addr"), blank=True, null=True)
     datetime = models.DateTimeField(auto_now=True)
 
     def __str__(self):

apps/audits/signals_handler.py

@@ -13,8 +13,8 @@ from common.utils import get_request_ip, get_logger, get_syslogger
 from users.models import User
 from authentication.signals import post_auth_failed, post_auth_success
 from terminal.models import Session, Command
-from terminal.backends.command.serializers import SessionCommandSerializer
-from . import models, serializers
+from common.utils.encode import model_to_json
+from . import models
 from .tasks import write_login_log_async
 
 logger = get_logger(__name__)
@@ -51,7 +51,10 @@ def create_operate_log(action, sender, resource):
 
 
 @receiver(post_save, dispatch_uid="my_unique_identifier")
-def on_object_created_or_update(sender, instance=None, created=False, **kwargs):
+def on_object_created_or_update(sender, instance=None, created=False, update_fields=None, **kwargs):
+    if instance._meta.object_name == 'User' and \
+            update_fields and 'last_login' in update_fields:
+        return
     if created:
         action = models.OperateLog.ACTION_CREATE
     else:
@@ -79,27 +82,20 @@ def on_user_change_password(sender, instance=None, **kwargs):
 def on_audits_log_create(sender, instance=None, **kwargs):
     if sender == models.UserLoginLog:
         category = "login_log"
-        serializer = serializers.LoginLogSerializer
     elif sender == models.FTPLog:
-        serializer = serializers.FTPLogSerializer
         category = "ftp_log"
     elif sender == models.OperateLog:
         category = "operation_log"
-        serializer = serializers.OperateLogSerializer
     elif sender == models.PasswordChangeLog:
         category = "password_change_log"
-        serializer = serializers.PasswordChangeLogSerializer
     elif sender == Session:
         category = "host_session_log"
-        serializer = serializers.SessionAuditSerializer
     elif sender == Command:
         category = "session_command_log"
-        serializer = SessionCommandSerializer
     else:
         return
 
-    s = serializer(instance=instance)
-    data = json_render.render(s.data).decode(errors='ignore')
+    data = model_to_json(instance)
     msg = "{} - {}".format(category, data)
     sys_logger.info(msg)
 

apps/audits/tasks.py

@@ -6,7 +6,7 @@ from django.conf import settings
 from celery import shared_task
 
 from ops.celery.decorator import register_as_period_task
-from .models import UserLoginLog
+from .models import UserLoginLog, OperateLog
 from .utils import write_login_log
 
 
@@ -22,6 +22,18 @@ def clean_login_log_period():
     UserLoginLog.objects.filter(datetime__lt=expired_day).delete()
 
 
+@register_as_period_task(interval=3600*24)
+@shared_task
+def clean_operation_log_period():
+    now = timezone.now()
+    try:
+        days = int(settings.LOGIN_LOG_KEEP_DAYS)
+    except ValueError:
+        days = 90
+    expired_day = now - datetime.timedelta(days=days)
+    OperateLog.objects.filter(datetime__lt=expired_day).delete()
+
+
 @shared_task
 def write_login_log_async(*args, **kwargs):
     write_login_log(*args, **kwargs)

apps/audits/templates/audits/ftp_log_list.html

@@ -5,8 +5,6 @@
 {% load common_tags %}
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         #search_btn {
             margin-bottom: 0;
@@ -14,6 +12,9 @@
         .form-control {
             height: 30px;
         }
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
     </style>
 {% endblock %}
 

apps/audits/templates/audits/operate_log_list.html

@@ -5,8 +5,6 @@
 {% load common_tags %}
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         #search_btn {
             margin-bottom: 0;

apps/audits/templates/audits/password_change_log_list.html

@@ -5,8 +5,6 @@
 {% load common_tags %}
 {% block custom_head_css_js %}
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         #search_btn {
             margin-bottom: 0;

apps/authentication/backends/api.py

@@ -109,7 +109,7 @@ class AccessKeyAuthentication(authentication.BaseAuthentication):
 
 class AccessTokenAuthentication(authentication.BaseAuthentication):
     keyword = 'Bearer'
-    expiration = settings.TOKEN_EXPIRATION or 3600
+    # expiration = settings.TOKEN_EXPIRATION or 3600
     model = get_user_model()
 
     def authenticate(self, request):

apps/authentication/backends/ldap.py

@@ -1,10 +1,11 @@
 # coding:utf-8
 #
 
+import warnings
 import ldap
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured, ObjectDoesNotExist
-from django_auth_ldap.backend import _LDAPUser, LDAPBackend
+from django_auth_ldap.backend import _LDAPUser, LDAPBackend, LDAPSettings
 from django_auth_ldap.config import _LDAPConfig, LDAPSearch, LDAPSearchUnion
 
 from users.utils import construct_user_email
@@ -17,7 +18,6 @@ class LDAPAuthorizationBackend(LDAPBackend):
     """
     Override this class to override _LDAPUser to LDAPUser
     """
-
     @staticmethod
     def user_can_authenticate(user):
         """
@@ -27,18 +27,26 @@ class LDAPAuthorizationBackend(LDAPBackend):
         is_valid = getattr(user, 'is_valid', None)
         return is_valid or is_valid is None
 
-    def authenticate(self, request=None, username=None, password=None, **kwargs):
+    def pre_check(self, username):
+        if not settings.AUTH_LDAP:
+            return False
         logger.info('Authentication LDAP backend')
         if not username:
             logger.info('Authenticate failed: username is None')
-            return None
+            return False
         if settings.AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS:
             user_model = self.get_user_model()
             exist = user_model.objects.filter(username=username).exists()
             if not exist:
                 msg = 'Authentication failed: user ({}) is not in the user list'
                 logger.info(msg.format(username))
-                return None
+                return False
+        return True
+
+    def authenticate(self, request=None, username=None, password=None, **kwargs):
+        match = self.pre_check(username)
+        if not match:
+            return None
         ldap_user = LDAPUser(self, username=username.strip(), request=request)
         user = self.authenticate_ldap_user(ldap_user, password)
         logger.info('Authenticate user: {}'.format(user))

apps/authentication/backends/openid/views.py

@@ -25,7 +25,8 @@ __all__ = ['OpenIDLoginView', 'OpenIDLoginCompleteView']
 
 class OpenIDLoginView(RedirectView):
     def get_redirect_url(self, *args, **kwargs):
-        redirect_uri = settings.BASE_SITE_URL + str(settings.LOGIN_COMPLETE_URL)
+        redirect_uri = settings.BASE_SITE_URL + \
+                       str(settings.AUTH_OPENID_LOGIN_COMPLETE_URL)
         nonce = Nonce(
             redirect_uri=redirect_uri,
             next_path=self.request.GET.get('next')

apps/authentication/mixins.py

@@ -141,7 +141,7 @@ class AuthMixin:
             )
 
     def check_user_login_confirm_if_need(self, user):
-        if not settings.CONFIG.LOGIN_CONFIRM_ENABLE:
+        if not settings.LOGIN_CONFIRM_ENABLE:
             return
         confirm_setting = user.get_login_confirm_setting()
         if self.request.session.get('auth_confirm') or not confirm_setting:

apps/authentication/views/login.py

@@ -60,7 +60,8 @@ class UserLoginView(mixins.AuthMixin, FormView):
         # show jumpserver login page if request http://{JUMP-SERVER}/?admin=1
         if settings.AUTH_OPENID and not self.request.GET.get('admin', 0):
             query_string = request.GET.urlencode()
-            login_url = "{}?{}".format(settings.LOGIN_URL, query_string)
+            openid_login_url = reverse_lazy("authentication:openid:openid-login")
+            login_url = "{}?{}".format(openid_login_url, query_string)
             return redirect(login_url)
         request.session.set_test_cookie()
         return super().get(request, *args, **kwargs)

apps/common/drf/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+#

apps/common/drf/filters.py

@@ -7,7 +7,7 @@ from rest_framework.serializers import ValidationError
 from django.core.cache import cache
 import logging
 
-from . import const
+from common import const
 
 __all__ = ["DatetimeRangeFilter", "IDSpmFilter", "CustomFilter"]
 

apps/common/drf/parsers/csv.py

@@ -9,7 +9,7 @@ import unicodecsv
 from rest_framework.parsers import BaseParser
 from rest_framework.exceptions import ParseError
 
-from ..utils import get_logger
+from common.utils import get_logger
 
 logger = get_logger(__file__)
 

apps/common/drf/renders/csv.py

@@ -9,7 +9,7 @@ from six import BytesIO
 from rest_framework.renderers import BaseRenderer
 from rest_framework.utils import encoders, json
 
-from ..utils import get_logger
+from common.utils import get_logger
 
 logger = get_logger(__file__)
 

apps/common/drf/routers.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+#
+from rest_framework_nested.routers import NestedMixin
+from rest_framework_bulk.routes import BulkRouter
+
+__all__ = ['BulkNestDefaultRouter']
+
+
+class BulkNestDefaultRouter(NestedMixin, BulkRouter):
+    pass

apps/common/fields/serializer.py

@@ -5,7 +5,10 @@ from rest_framework import serializers
 from django.utils import six
 
 
-__all__ = ['StringIDField', 'StringManyToManyField', 'ChoiceDisplayField']
+__all__ = [
+    'StringIDField', 'StringManyToManyField', 'ChoiceDisplayField',
+    'CustomMetaDictField'
+]
 
 
 class StringIDField(serializers.Field):
@@ -39,3 +42,63 @@ class DictField(serializers.DictField):
         if not value or not isinstance(value, dict):
             value = {}
         return super().to_representation(value)
+
+
+class CustomMetaDictField(serializers.DictField):
+    """
+    In use:
+    RemoteApp params field
+    CommandStorage meta field
+    ReplayStorage meta field
+    """
+    type_map_fields = {}
+    default_type = None
+    need_convert_key = False
+
+    def filter_attribute(self, attribute, instance):
+        fields = self.type_map_fields.get(instance.type, [])
+        for field in fields:
+            if field.get('write_only', False):
+                attribute.pop(field['name'], None)
+        return attribute
+
+    def get_attribute(self, instance):
+        """
+        序列化时调用
+        """
+        attribute = super().get_attribute(instance)
+        attribute = self.filter_attribute(attribute, instance)
+        return attribute
+
+    def convert_value_key(self, dictionary, value):
+        if not self.need_convert_key:
+            # remote app
+            return value
+        tp = dictionary.get('type')
+        _value = {}
+        for k, v in value.items():
+            prefix = '{}_'.format(tp)
+            _k = k
+            if k.lower().startswith(prefix):
+                _k = k.lower().split(prefix, 1)[1]
+            _k = _k.upper()
+            _value[_k] = value[k]
+        return _value
+
+    def filter_value_key(self, dictionary, value):
+        tp = dictionary.get('type', self.default_type)
+        fields = self.type_map_fields.get(tp, [])
+        fields_names = [field['name'] for field in fields]
+        no_need_keys = [k for k in value.keys() if k not in fields_names]
+        for k in no_need_keys:
+            value.pop(k)
+        return value
+
+    def get_value(self, dictionary):
+        """
+        反序列化时调用
+        """
+        value = super().get_value(dictionary)
+        value = self.convert_value_key(dictionary, value)
+        value = self.filter_value_key(dictionary, value)
+        return value

apps/common/mixins/api.py

@@ -3,11 +3,11 @@
 from django.http import JsonResponse
 from rest_framework.settings import api_settings
 
-from ..filters import IDSpmFilter, CustomFilter
+from common.drf.filters import IDSpmFilter, CustomFilter
 
 __all__ = [
     "JSONResponseMixin", "CommonApiMixin",
-    "IDSpmFilterMixin", "CommonApiMixin",
+    "IDSpmFilterMixin",
 ]
 
 

apps/common/signals_handlers.py

@@ -7,10 +7,13 @@ from django.conf import settings
 from django.dispatch import receiver
 from django.core.signals import request_finished
 from django.db import connection
+from django.conf import LazySettings
+from django.db.utils import ProgrammingError, OperationalError
 
 
 from common.utils import get_logger
 from .local import thread_local
+from .signals import django_ready
 
 pattern = re.compile(r'FROM `(\w+)`')
 logger = get_logger(__name__)
@@ -62,7 +65,15 @@ if settings.DEBUG and DEBUG_DB:
     request_finished.connect(on_request_finished_logging_db_query)
 
 
+@receiver(django_ready)
+def monkey_patch_settings(sender, **kwargs):
+    def monkey_patch_getattr(self, name):
+        val = getattr(self._wrapped, name)
+        if callable(val):
+            val = val()
+        return val
 
-
-
-
+    try:
+        LazySettings.__getattr__ = monkey_patch_getattr
+    except (ProgrammingError, OperationalError):
+        pass

apps/common/tasks.py

@@ -1,6 +1,7 @@
 from django.core.mail import send_mail
 from django.conf import settings
 from celery import shared_task
+
 from .utils import get_logger
 
 

apps/common/utils/common.py

@@ -9,6 +9,7 @@ import uuid
 from functools import wraps
 import time
 import ipaddress
+import psutil
 
 
 UUID_PATTERN = re.compile(r'\w{8}(-\w{4}){3}-\w{12}')
@@ -234,3 +235,10 @@ class lazyproperty:
             value = self.func(instance)
             setattr(instance, self.func.__name__, value)
             return value
+
+
+def get_disk_usage():
+    partitions = psutil.disk_partitions()
+    mount_points = [p.mountpoint for p in partitions]
+    usages = {p: psutil.disk_usage(p) for p in mount_points}
+    return usages

apps/common/utils/django.py

@@ -35,20 +35,3 @@ def date_expired_default():
         years = 70
     return timezone.now() + timezone.timedelta(days=365*years)
 
-
-def get_command_storage_setting():
-    default = settings.DEFAULT_TERMINAL_COMMAND_STORAGE
-    value = settings.TERMINAL_COMMAND_STORAGE
-    if not value:
-        return default
-    value.update(default)
-    return value
-
-
-def get_replay_storage_setting():
-    default = settings.DEFAULT_TERMINAL_REPLAY_STORAGE
-    value = settings.TERMINAL_REPLAY_STORAGE
-    if not value:
-        return default
-    value.update(default)
-    return value

apps/common/utils/encode.py

@@ -1,12 +1,14 @@
 # -*- coding: utf-8 -*-
 #
 import re
+import json
 from six import string_types
 import base64
 import os
 import time
 import hashlib
 from io import StringIO
+from itertools import chain
 
 import paramiko
 import sshpubkeys
@@ -15,6 +17,8 @@ from itsdangerous import (
     BadSignature, SignatureExpired
 )
 from django.conf import settings
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db.models.fields.files import FileField
 
 from .http import http_date
 
@@ -186,3 +190,35 @@ def get_signer():
 
 def ensure_last_char_is_ascii(data):
     remain = ''
+
+
+secret_pattern = re.compile(r'password|secret|key', re.IGNORECASE)
+
+
+def model_to_dict_pro(instance, fields=None, exclude=None):
+    from ..fields.model import EncryptMixin
+    opts = instance._meta
+    data = {}
+    for f in chain(opts.concrete_fields, opts.private_fields):
+        if not getattr(f, 'editable', False):
+            continue
+        if fields and f.name not in fields:
+            continue
+        if exclude and f.name in exclude:
+            continue
+        if isinstance(f, FileField):
+            continue
+        if isinstance(f, EncryptMixin):
+            continue
+        if secret_pattern.search(f.name):
+            continue
+        value = f.value_from_object(instance)
+        data[f.name] = value
+    return data
+
+
+def model_to_json(instance, sort_keys=True, indent=2, cls=None):
+    data = model_to_dict_pro(instance)
+    if cls is None:
+        cls = DjangoJSONEncoder
+    return json.dumps(data, sort_keys=sort_keys, indent=indent, cls=cls)

apps/common/utils/ipip/__init__.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 #
-from .ipdb import *
+from .utils import *

apps/common/utils/ipip/utils.py

@@ -1,18 +1,24 @@
 # -*- coding: utf-8 -*-
 #
 import os
+from django.utils.translation import ugettext as _
 
 import ipdb
 
+__all__ = ['get_ip_city']
 ipip_db = None
 
 
 def get_ip_city(ip):
     global ipip_db
+    if not ip or not isinstance(ip, str):
+        return _("Invalid ip")
+    if ':' in ip:
+        return 'IPv6'
     if ipip_db is None:
         ipip_db_path = os.path.join(os.path.dirname(__file__), 'ipipfree.ipdb')
         ipip_db = ipdb.City(ipip_db_path)
     info = list(set(ipip_db.find(ip, 'CN')))
     if '' in info:
         info.remove('')
-    return ' '.join(info)
+    return ' '.join(info)

apps/jumpserver/conf.py

@@ -1,6 +1,12 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
+"""
+配置分类：
+1. Django使用的配置文件，写到settings中
+2. 程序需要, 用户不需要更改的写到settings中
+3. 程序需要, 用户需要更改的写到本config中
+"""
 import os
 import sys
 import types
@@ -8,6 +14,7 @@ import errno
 import json
 import yaml
 from importlib import import_module
+from django.urls import reverse_lazy
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 PROJECT_DIR = os.path.dirname(BASE_DIR)
@@ -29,6 +36,10 @@ def import_string(dotted_path):
         ) from err
 
 
+class DoesNotExist(Exception):
+    pass
+
+
 class Config(dict):
     """Works exactly like a dict but provides ways to fill it from files
     or special dictionaries.  There are two common patterns to populate the
@@ -72,34 +83,233 @@ class Config(dict):
                       the application's :attr:`~flask.Flask.root_path`.
     :param defaults: an optional dictionary of default values
     """
+    defaults = {
+        # Django Config
+        'SECRET_KEY': '',
+        'BOOTSTRAP_TOKEN': '',
+        'DEBUG': True,
+        'SITE_URL': 'http://localhost:8080',
+        'LOG_LEVEL': 'DEBUG',
+        'LOG_DIR': os.path.join(PROJECT_DIR, 'logs'),
+        'DB_ENGINE': 'mysql',
+        'DB_NAME': 'jumpserver',
+        'DB_HOST': '127.0.0.1',
+        'DB_PORT': 3306,
+        'DB_USER': 'root',
+        'DB_PASSWORD': '',
+        'REDIS_HOST': '127.0.0.1',
+        'REDIS_PORT': 6379,
+        'REDIS_PASSWORD': '',
+        'REDIS_DB_CELERY': 3,
+        'REDIS_DB_CACHE': 4,
+        'REDIS_DB_SESSION': 5,
+        'REDIS_DB_WS': 6,
+        'CAPTCHA_TEST_MODE': None,
+        'TOKEN_EXPIRATION': 3600 * 24,
+        'DISPLAY_PER_PAGE': 25,
+        'DEFAULT_EXPIRED_YEARS': 70,
+        'SESSION_COOKIE_DOMAIN': None,
+        'CSRF_COOKIE_DOMAIN': None,
+        'SESSION_COOKIE_AGE': 3600 * 24,
+        'SESSION_EXPIRE_AT_BROWSER_CLOSE': False,
+        'LOGIN_URL': reverse_lazy('authentication:login'),
 
-    def __init__(self, root_path=None, defaults=None):
-        self.defaults = defaults or {}
-        self.root_path = root_path
-        super().__init__({})
+        # Custom Config
+        # Auth LDAP settings
+        'AUTH_LDAP': False,
+        'AUTH_LDAP_SERVER_URI': 'ldap://localhost:389',
+        'AUTH_LDAP_BIND_DN': 'cn=admin,dc=jumpserver,dc=org',
+        'AUTH_LDAP_BIND_PASSWORD': '',
+        'AUTH_LDAP_SEARCH_OU': 'ou=tech,dc=jumpserver,dc=org',
+        'AUTH_LDAP_SEARCH_FILTER': '(cn=%(user)s)',
+        'AUTH_LDAP_START_TLS': False,
+        'AUTH_LDAP_USER_ATTR_MAP': {"username": "cn", "name": "sn", "email": "mail"},
+        'AUTH_LDAP_CONNECT_TIMEOUT': 30,
+        'AUTH_LDAP_SEARCH_PAGED_SIZE': 1000,
+        'AUTH_LDAP_SYNC_IS_PERIODIC': False,
+        'AUTH_LDAP_SYNC_INTERVAL': None,
+        'AUTH_LDAP_SYNC_CRONTAB': None,
+        'AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS': False,
+        'AUTH_LDAP_OPTIONS_OPT_REFERRALS': -1,
 
-    def from_envvar(self, variable_name, silent=False):
-        """Loads a configuration from an environment variable pointing to
-        a configuration file.  This is basically just a shortcut with nicer
-        error messages for this line of code::
+        'AUTH_OPENID': False,
+        'BASE_SITE_URL': 'http://localhost:8080',
+        'AUTH_OPENID_SERVER_URL': 'http://openid',
+        'AUTH_OPENID_REALM_NAME': 'jumpserver',
+        'AUTH_OPENID_CLIENT_ID': 'jumpserver',
+        'AUTH_OPENID_CLIENT_SECRET': '',
+        'AUTH_OPENID_IGNORE_SSL_VERIFICATION': True,
+        'AUTH_OPENID_SHARE_SESSION': True,
 
-            app.config.from_pyfile(os.environ['YOURAPPLICATION_SETTINGS'])
+        'AUTH_RADIUS': False,
+        'RADIUS_SERVER': 'localhost',
+        'RADIUS_PORT': 1812,
+        'RADIUS_SECRET': '',
+        'RADIUS_ENCRYPT_PASSWORD': True,
+        'OTP_IN_RADIUS': False,
 
-        :param variable_name: name of the environment variable
-        :param silent: set to ``True`` if you want silent failure for missing
-                       files.
-        :return: bool. ``True`` if able to load config, ``False`` otherwise.
-        """
-        rv = os.environ.get(variable_name)
-        if not rv:
-            if silent:
+        'OTP_VALID_WINDOW': 2,
+        'OTP_ISSUER_NAME': 'Jumpserver',
+        'EMAIL_SUFFIX': 'jumpserver.org',
+
+        'TERMINAL_PASSWORD_AUTH': True,
+        'TERMINAL_PUBLIC_KEY_AUTH': True,
+        'TERMINAL_HEARTBEAT_INTERVAL': 20,
+        'TERMINAL_ASSET_LIST_SORT_BY': 'hostname',
+        'TERMINAL_ASSET_LIST_PAGE_SIZE': 'auto',
+        'TERMINAL_SESSION_KEEP_DURATION': 9999,
+        'TERMINAL_HOST_KEY': '',
+        'TERMINAL_TELNET_REGEX': '',
+        'TERMINAL_COMMAND_STORAGE': {},
+
+        'SECURITY_MFA_AUTH': False,
+        'SECURITY_SERVICE_ACCOUNT_REGISTRATION': True,
+        'SECURITY_VIEW_AUTH_NEED_MFA': True,
+        'SECURITY_LOGIN_LIMIT_COUNT': 7,
+        'SECURITY_LOGIN_LIMIT_TIME': 30,
+        'SECURITY_MAX_IDLE_TIME': 30,
+        'SECURITY_PASSWORD_EXPIRATION_TIME': 9999,
+        'SECURITY_PASSWORD_MIN_LENGTH': 6,
+        'SECURITY_PASSWORD_UPPER_CASE': False,
+        'SECURITY_PASSWORD_LOWER_CASE': False,
+        'SECURITY_PASSWORD_NUMBER': False,
+        'SECURITY_PASSWORD_SPECIAL_CHAR': False,
+
+        'HTTP_BIND_HOST': '0.0.0.0',
+        'HTTP_LISTEN_PORT': 8080,
+        'WS_LISTEN_PORT': 8070,
+        'LOGIN_LOG_KEEP_DAYS': 90,
+        'ASSETS_PERM_CACHE_TIME': 3600 * 24,
+        'SECURITY_MFA_VERIFY_TTL': 3600,
+        'ASSETS_PERM_CACHE_ENABLE': False,
+        'SYSLOG_ADDR': '',  # '192.168.0.1:514'
+        'SYSLOG_FACILITY': 'user',
+        'PERM_SINGLE_ASSET_TO_UNGROUP_NODE': False,
+        'WINDOWS_SSH_DEFAULT_SHELL': 'cmd',
+        'FLOWER_URL': "127.0.0.1:5555",
+        'DEFAULT_ORG_SHOW_ALL_USERS': True,
+        'PERIOD_TASK_ENABLE': True,
+        'FORCE_SCRIPT_NAME': '',
+        'LOGIN_CONFIRM_ENABLE': False,
+        'WINDOWS_SKIP_ALL_MANUAL_PASSWORD': False,
+    }
+
+    def convert_type(self, k, v):
+        default_value = self.defaults.get(k)
+        if default_value is None:
+            return v
+        tp = type(default_value)
+        # 对bool特殊处理
+        if tp is bool and isinstance(v, str):
+            if v in ("true", "True", "1"):
+                return True
+            else:
                 return False
-            raise RuntimeError('The environment variable %r is not set '
-                               'and as such configuration could not be '
-                               'loaded.  Set this variable and make it '
-                               'point to a configuration file' %
-                               variable_name)
-        return self.from_pyfile(rv, silent=silent)
+        if tp in [list, dict] and isinstance(v, str):
+            try:
+                v = json.loads(v)
+                return v
+            except json.JSONDecodeError:
+                return v
+
+        try:
+            if tp in [list, dict]:
+                v = json.loads(v)
+            else:
+                v = tp(v)
+        except Exception:
+            pass
+        return v
+
+    def __repr__(self):
+        return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self))
+
+    def get_from_config(self, item):
+        try:
+            value = super().__getitem__(item)
+        except KeyError:
+            value = None
+        return value
+
+    def get_from_env(self, item):
+        value = os.environ.get(item, None)
+        if value is not None:
+            value = self.convert_type(item, value)
+        return value
+
+    def get(self, item):
+        # 再从配置文件中获取
+        value = self.get_from_config(item)
+        if value is not None:
+            return value
+        # 其次从环境变量来
+        value = self.get_from_env(item)
+        if value is not None:
+            return value
+        return self.defaults.get(item)
+
+    def __getitem__(self, item):
+        return self.get(item)
+
+    def __getattr__(self, item):
+        return self.get(item)
+
+
+class DynamicConfig:
+    def __init__(self, static_config):
+        self.static_config = static_config
+        self.db_setting = None
+
+    def __getitem__(self, item):
+        return self.dynamic(item)
+
+    def __getattr__(self, item):
+        return self.dynamic(item)
+
+    def dynamic(self, item):
+        return lambda: self.get(item)
+
+    def LOGIN_URL(self):
+        auth_openid = self.get('AUTH_OPENID')
+        if auth_openid:
+            return reverse_lazy("authentication:openid:openid-login")
+        return self.get('LOGIN_URL')
+
+    def AUTHENTICATION_BACKENDS(self):
+        backends = [
+            'authentication.backends.pubkey.PublicKeyAuthBackend',
+            'django.contrib.auth.backends.ModelBackend',
+        ]
+        if self.get('AUTH_LDAP'):
+            backends.insert(0, 'authentication.backends.ldap.LDAPAuthorizationBackend')
+        if self.get('AUTH_OPENID'):
+            backends.insert(0, 'authentication.backends.openid.backends.OpenIDAuthorizationPasswordBackend')
+            backends.insert(0, 'authentication.backends.openid.backends.OpenIDAuthorizationCodeBackend')
+        if self.get('AUTH_RADIUS'):
+            backends.insert(0, 'authentication.backends.radius.RadiusBackend')
+        return backends
+
+    def get_from_db(self, item):
+        if self.db_setting is not None:
+            value = self.db_setting.get(item)
+            if value is not None:
+                return value
+        return None
+
+    def get(self, item):
+        # 先从数据库中获取
+        value = self.get_from_db(item)
+        if value is not None:
+            return value
+        return self.static_config.get(item)
+
+
+class ConfigManager:
+    config_class = Config
+
+    def __init__(self, root_path=None):
+        self.root_path = root_path
+        self.config = self.config_class()
 
     def from_pyfile(self, filename, silent=False):
         """Updates the values in the config from a Python file.  This function
@@ -162,7 +372,7 @@ class Config(dict):
             obj = import_string(obj)
         for key in dir(obj):
             if key.isupper():
-                self[key] = getattr(obj, key)
+                self.config[key] = getattr(obj, key)
 
     def from_json(self, filename, silent=False):
         """Updates the values in the config from a JSON file. This function
@@ -224,218 +434,50 @@ class Config(dict):
         for mapping in mappings:
             for (key, value) in mapping:
                 if key.isupper():
-                    self[key] = value
+                    self.config[key] = value
         return True
 
-    def get_namespace(self, namespace, lowercase=True, trim_namespace=True):
-        """Returns a dictionary containing a subset of configuration options
-        that match the specified namespace/prefix. Example usage::
-
-            app.config['IMAGE_STORE_TYPE'] = 'fs'
-            app.config['IMAGE_STORE_PATH'] = '/var/app/images'
-            app.config['IMAGE_STORE_BASE_URL'] = 'http://img.website.com'
-            image_store_config = app.config.get_namespace('IMAGE_STORE_')
-
-        The resulting dictionary `image_store_config` would look like::
-
-            {
-                'types': 'fs',
-                'path': '/var/app/images',
-                'base_url': 'http://img.website.com'
-            }
-
-        This is often useful when configuration options map directly to
-        keyword arguments in functions or class constructors.
-
-        :param namespace: a configuration namespace
-        :param lowercase: a flag indicating if the keys of the resulting
-                          dictionary should be lowercase
-        :param trim_namespace: a flag indicating if the keys of the resulting
-                          dictionary should not include the namespace
-
-        .. versionadded:: 0.11
-        """
-        rv = {}
-        for k, v in self.items():
-            if not k.startswith(namespace):
-                continue
-            if trim_namespace:
-                key = k[len(namespace):]
-            else:
-                key = k
-            if lowercase:
-                key = key.lower()
-            rv[key] = v
-        return rv
-
-    def convert_type(self, k, v):
-        default_value = self.defaults.get(k)
-        if default_value is None:
-            return v
-        tp = type(default_value)
-        # 对bool特殊处理
-        if tp is bool and isinstance(v, str):
-            if v in ("true", "True", "1"):
-                return True
-            else:
-                return False
-        if tp in [list, dict] and isinstance(v, str):
-            try:
-                v = json.loads(v)
-                return v
-            except json.JSONDecodeError:
-                return v
-
+    def load_from_object(self):
+        sys.path.insert(0, PROJECT_DIR)
         try:
-            if tp in [list, dict]:
-                v = json.loads(v)
-            else:
-                v = tp(v)
-        except Exception:
+            from config import config as c
+            self.from_object(c)
+            return self.config
+        except ImportError:
             pass
-        return v
+        return None
 
-    def __repr__(self):
-        return '<%s %s>' % (self.__class__.__name__, dict.__repr__(self))
+    def load_from_yml(self):
+        for i in ['config.yml', 'config.yaml']:
+            if not os.path.isfile(os.path.join(self.root_path, i)):
+                continue
+            loaded = self.from_yaml(i)
+            if loaded:
+                return self.config
+        return False
 
-    def __getitem__(self, item):
-        # 先从设置的来
-        try:
-            value = super().__getitem__(item)
-        except KeyError:
-            value = None
-        if value is not None:
-            return value
-        # 其次从环境变量来
-        value = os.environ.get(item, None)
-        if value is not None:
-            return self.convert_type(item, value)
-        return self.defaults.get(item)
-
-    def __getattr__(self, item):
-        return self.__getitem__(item)
-
-
-defaults = {
-    'SECRET_KEY': '',
-    'BOOTSTRAP_TOKEN': '',
-    'DEBUG': True,
-    'SITE_URL': 'http://localhost',
-    'LOG_LEVEL': 'DEBUG',
-    'LOG_DIR': os.path.join(PROJECT_DIR, 'logs'),
-    'DB_ENGINE': 'mysql',
-    'DB_NAME': 'jumpserver',
-    'DB_HOST': '127.0.0.1',
-    'DB_PORT': 3306,
-    'DB_USER': 'root',
-    'DB_PASSWORD': '',
-    'REDIS_HOST': '127.0.0.1',
-    'REDIS_PORT': 6379,
-    'REDIS_PASSWORD': '',
-    'REDIS_DB_CELERY': 3,
-    'REDIS_DB_CACHE': 4,
-    'REDIS_DB_SESSION': 5,
-    'REDIS_DB_WS': 6,
-    'CAPTCHA_TEST_MODE': None,
-    'TOKEN_EXPIRATION': 3600 * 24,
-    'DISPLAY_PER_PAGE': 25,
-    'DEFAULT_EXPIRED_YEARS': 70,
-    'SESSION_COOKIE_DOMAIN': None,
-    'CSRF_COOKIE_DOMAIN': None,
-    'SESSION_COOKIE_AGE': 3600 * 24,
-    'SESSION_EXPIRE_AT_BROWSER_CLOSE': False,
-    'AUTH_OPENID': False,
-    'AUTH_OPENID_IGNORE_SSL_VERIFICATION': True,
-    'AUTH_OPENID_SHARE_SESSION': True,
-    'OTP_VALID_WINDOW': 2,
-    'OTP_ISSUER_NAME': 'Jumpserver',
-    'EMAIL_SUFFIX': 'jumpserver.org',
-    'TERMINAL_PASSWORD_AUTH': True,
-    'TERMINAL_PUBLIC_KEY_AUTH': True,
-    'TERMINAL_HEARTBEAT_INTERVAL': 20,
-    'TERMINAL_ASSET_LIST_SORT_BY': 'hostname',
-    'TERMINAL_ASSET_LIST_PAGE_SIZE': 'auto',
-    'TERMINAL_SESSION_KEEP_DURATION': 9999,
-    'TERMINAL_HOST_KEY': '',
-    'TERMINAL_TELNET_REGEX': '',
-    'TERMINAL_COMMAND_STORAGE': {},
-    'SECURITY_MFA_AUTH': False,
-    'SECURITY_SERVICE_ACCOUNT_REGISTRATION': True,
-    'SECURITY_VIEW_AUTH_NEED_MFA': True,
-    'SECURITY_LOGIN_LIMIT_COUNT': 7,
-    'SECURITY_LOGIN_LIMIT_TIME': 30,
-    'SECURITY_MAX_IDLE_TIME': 30,
-    'SECURITY_PASSWORD_EXPIRATION_TIME': 9999,
-    'SECURITY_PASSWORD_MIN_LENGTH': 6,
-    'SECURITY_PASSWORD_UPPER_CASE': False,
-    'SECURITY_PASSWORD_LOWER_CASE': False,
-    'SECURITY_PASSWORD_NUMBER': False,
-    'SECURITY_PASSWORD_SPECIAL_CHAR': False,
-    'AUTH_RADIUS': False,
-    'RADIUS_SERVER': 'localhost',
-    'RADIUS_PORT': 1812,
-    'RADIUS_SECRET': '',
-    'RADIUS_ENCRYPT_PASSWORD': True,
-    'AUTH_LDAP_SEARCH_PAGED_SIZE': 1000,
-    'AUTH_LDAP_SYNC_IS_PERIODIC': False,
-    'AUTH_LDAP_SYNC_INTERVAL': None,
-    'AUTH_LDAP_SYNC_CRONTAB': None,
-    'AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS': False,
-    'AUTH_LDAP_OPTIONS_OPT_REFERRALS': -1,
-    'HTTP_BIND_HOST': '0.0.0.0',
-    'HTTP_LISTEN_PORT': 8080,
-    'WS_LISTEN_PORT': 8070,
-    'LOGIN_LOG_KEEP_DAYS': 90,
-    'ASSETS_PERM_CACHE_TIME': 3600*24,
-    'SECURITY_MFA_VERIFY_TTL': 3600,
-    'ASSETS_PERM_CACHE_ENABLE': False,
-    'SYSLOG_ADDR': '',  # '192.168.0.1:514'
-    'SYSLOG_FACILITY': 'user',
-    'PERM_SINGLE_ASSET_TO_UNGROUP_NODE': False,
-    'WINDOWS_SSH_DEFAULT_SHELL': 'cmd',
-    'FLOWER_URL': "127.0.0.1:5555",
-    'DEFAULT_ORG_SHOW_ALL_USERS': True,
-    'PERIOD_TASK_ENABLE': True,
-    'FORCE_SCRIPT_NAME': '',
-    'LOGIN_CONFIRM_ENABLE': False,
-    'WINDOWS_SKIP_ALL_MANUAL_PASSWORD': False,
-    'OTP_IN_RADIUS': False,
-}
-
-
-def load_from_object(config):
-    try:
-        from config import config as c
-        config.from_object(c)
-        return True
-    except ImportError:
-        pass
-    return False
-
-
-def load_from_yml(config):
-    for i in ['config.yml', 'config.yaml']:
-        if not os.path.isfile(os.path.join(config.root_path, i)):
-            continue
-        loaded = config.from_yaml(i)
-        if loaded:
-            return True
-    return False
-
-
-def load_user_config():
-    sys.path.insert(0, PROJECT_DIR)
-    config = Config(PROJECT_DIR, defaults)
-
-    loaded = load_from_object(config)
-    if not loaded:
-        loaded = load_from_yml(config)
-    if not loaded:
+    @classmethod
+    def load_user_config(cls, root_path=None, config_class=None):
+        config_class = config_class or Config
+        cls.config_class = config_class
+        if not root_path:
+            root_path = PROJECT_DIR
+        manager = cls(root_path=root_path)
+        config = manager.load_from_object()
+        if config:
+            return config
+        config = manager.load_from_yml()
+        if config:
+            return config
         msg = """
-    
+
         Error: No config file found.
-    
+
         You can run `cp config_example.yml config.yml`, and edit it.
         """
         raise ImportError(msg)
-    return config
+
+    @classmethod
+    def get_dynamic_config(cls, config):
+        return DynamicConfig(config)
+

apps/jumpserver/const.py

@@ -1,3 +1,12 @@
 # -*- coding: utf-8 -*-
 #
+import os
+from .conf import ConfigManager
+
+__all__ = ['BASE_DIR', 'PROJECT_DIR', 'VERSION', 'CONFIG', 'DYNAMIC']
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROJECT_DIR = os.path.dirname(BASE_DIR)
 VERSION = '1.5.5'
+CONFIG = ConfigManager.load_user_config()
+DYNAMIC = ConfigManager.get_dynamic_config(CONFIG)

apps/jumpserver/context_processor.py

@@ -19,8 +19,8 @@ def jumpserver_processor(request):
         'SECURITY_COMMAND_EXECUTION': settings.SECURITY_COMMAND_EXECUTION,
         'SECURITY_MFA_VERIFY_TTL': settings.SECURITY_MFA_VERIFY_TTL,
         'FORCE_SCRIPT_NAME': settings.FORCE_SCRIPT_NAME,
-        'SECURITY_VIEW_AUTH_NEED_MFA': settings.CONFIG.SECURITY_VIEW_AUTH_NEED_MFA,
-        'LOGIN_CONFIRM_ENABLE': settings.CONFIG.LOGIN_CONFIRM_ENABLE,
+        'SECURITY_VIEW_AUTH_NEED_MFA': settings.SECURITY_VIEW_AUTH_NEED_MFA,
+        'LOGIN_CONFIRM_ENABLE': settings.LOGIN_CONFIRM_ENABLE,
     }
     return context
 

apps/jumpserver/settings.py

@@ -1,660 +0,0 @@
-"""
-Django settings for jumpserver project.
-
-Generated by 'django-admin startproject' using Django 1.10.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/1.10/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/1.10/ref/settings/
-"""
-
-import os
-import sys
-
-import ldap
-from django.urls import reverse_lazy
-
-from . import const
-from .conf import load_user_config
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-PROJECT_DIR = os.path.dirname(BASE_DIR)
-sys.path.append(PROJECT_DIR)
-
-CONFIG = load_user_config()
-LOG_DIR = os.path.join(PROJECT_DIR, 'logs')
-JUMPSERVER_LOG_FILE = os.path.join(LOG_DIR, 'jumpserver.log')
-ANSIBLE_LOG_FILE = os.path.join(LOG_DIR, 'ansible.log')
-GUNICORN_LOG_FILE = os.path.join(LOG_DIR, 'gunicorn.log')
-VERSION = const.VERSION
-
-if not os.path.isdir(LOG_DIR):
-    os.makedirs(LOG_DIR)
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = CONFIG.SECRET_KEY
-
-# SECURITY WARNING: keep the token secret, remove it if all coco, guacamole ok
-BOOTSTRAP_TOKEN = CONFIG.BOOTSTRAP_TOKEN
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = CONFIG.DEBUG
-
-# Absolute url for some case, for example email link
-SITE_URL = CONFIG.SITE_URL
-
-# LOG LEVEL
-LOG_LEVEL = CONFIG.LOG_LEVEL
-
-ALLOWED_HOSTS = ['*']
-
-# Max post update field num
-DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000
-
-# Application definition
-
-INSTALLED_APPS = [
-    'orgs.apps.OrgsConfig',
-    'users.apps.UsersConfig',
-    'assets.apps.AssetsConfig',
-    'perms.apps.PermsConfig',
-    'ops.apps.OpsConfig',
-    'settings.apps.SettingsConfig',
-    'common.apps.CommonConfig',
-    'terminal.apps.TerminalConfig',
-    'audits.apps.AuditsConfig',
-    'authentication.apps.AuthenticationConfig',  # authentication
-    'applications.apps.ApplicationsConfig',
-    'tickets.apps.TicketsConfig',
-    'rest_framework',
-    'rest_framework_swagger',
-    'drf_yasg',
-    'channels',
-    'django_filters',
-    'bootstrap3',
-    'captcha',
-    'django_celery_beat',
-    'django.contrib.auth',
-    'django.contrib.admin',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-]
-
-
-XPACK_DIR = os.path.join(BASE_DIR, 'xpack')
-XPACK_ENABLED = os.path.isdir(XPACK_DIR)
-XPACK_TEMPLATES_DIR = []
-XPACK_CONTEXT_PROCESSOR = []
-
-if XPACK_ENABLED:
-    from xpack.utils import get_xpack_templates_dir, get_xpack_context_processor
-    INSTALLED_APPS.append('xpack.apps.XpackConfig')
-    XPACK_TEMPLATES_DIR = get_xpack_templates_dir(BASE_DIR)
-    XPACK_CONTEXT_PROCESSOR = get_xpack_context_processor()
-
-MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.locale.LocaleMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    'authentication.backends.openid.middleware.OpenIDAuthenticationMiddleware',
-    'jumpserver.middleware.TimezoneMiddleware',
-    'jumpserver.middleware.DemoMiddleware',
-    'jumpserver.middleware.RequestMiddleware',
-    'orgs.middleware.OrgMiddleware',
-]
-
-
-ROOT_URLCONF = 'jumpserver.urls'
-
-TEMPLATES = [
-    {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [os.path.join(BASE_DIR, 'templates'), *XPACK_TEMPLATES_DIR],
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.i18n',
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
-                'django.template.context_processors.static',
-                'django.template.context_processors.request',
-                'django.template.context_processors.media',
-                'jumpserver.context_processor.jumpserver_processor',
-                'orgs.context_processor.org_processor',
-                *XPACK_CONTEXT_PROCESSOR,
-            ],
-        },
-    },
-]
-
-WSGI_APPLICATION = 'jumpserver.wsgi.application'
-ASGI_APPLICATION = 'jumpserver.routing.application'
-
-LOGIN_REDIRECT_URL = reverse_lazy('index')
-LOGIN_URL = reverse_lazy('authentication:login')
-
-SESSION_COOKIE_DOMAIN = CONFIG.SESSION_COOKIE_DOMAIN
-CSRF_COOKIE_DOMAIN = CONFIG.CSRF_COOKIE_DOMAIN
-SESSION_COOKIE_AGE = CONFIG.SESSION_COOKIE_AGE
-SESSION_EXPIRE_AT_BROWSER_CLOSE = CONFIG.SESSION_EXPIRE_AT_BROWSER_CLOSE
-SESSION_ENGINE = 'redis_sessions.session'
-SESSION_REDIS = {
-    'host': CONFIG.REDIS_HOST,
-    'port': CONFIG.REDIS_PORT,
-    'password': CONFIG.REDIS_PASSWORD,
-    'db': CONFIG.REDIS_DB_SESSION,
-    'prefix': 'auth_session',
-    'socket_timeout': 1,
-    'retry_on_timeout': False
-}
-
-MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage'
-# Database
-# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
-
-DB_OPTIONS = {}
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.{}'.format(CONFIG.DB_ENGINE.lower()),
-        'NAME': CONFIG.DB_NAME,
-        'HOST': CONFIG.DB_HOST,
-        'PORT': CONFIG.DB_PORT,
-        'USER': CONFIG.DB_USER,
-        'PASSWORD': CONFIG.DB_PASSWORD,
-        'ATOMIC_REQUESTS': True,
-        'OPTIONS': DB_OPTIONS
-    }
-}
-DB_CA_PATH = os.path.join(PROJECT_DIR, 'data', 'certs', 'db_ca.pem')
-if CONFIG.DB_ENGINE.lower() == 'mysql':
-    DB_OPTIONS['init_command'] = "SET sql_mode='STRICT_TRANS_TABLES'"
-    if os.path.isfile(DB_CA_PATH):
-        DB_OPTIONS['ssl'] = {'ca': DB_CA_PATH}
-
-
-# Password validation
-# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
-#
-AUTH_PASSWORD_VALIDATORS = [
-    {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
-    },
-]
-
-# Logging setting
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'formatters': {
-        'verbose': {
-            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
-        },
-        'main': {
-            'datefmt': '%Y-%m-%d %H:%M:%S',
-            'format': '%(asctime)s [%(module)s %(levelname)s] %(message)s',
-        },
-        'simple': {
-            'format': '%(levelname)s %(message)s'
-        },
-        'syslog': {
-            'format': 'jumpserver: %(message)s'
-        },
-        'msg': {
-            'format': '%(message)s'
-        }
-    },
-    'handlers': {
-        'null': {
-            'level': 'DEBUG',
-            'class': 'logging.NullHandler',
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'main'
-        },
-        'file': {
-            'encoding': 'utf8',
-            'level': 'DEBUG',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'maxBytes': 1024*1024*100,
-            'backupCount': 7,
-            'formatter': 'main',
-            'filename': JUMPSERVER_LOG_FILE,
-        },
-        'ansible_logs': {
-            'encoding': 'utf8',
-            'level': 'DEBUG',
-            'class': 'logging.handlers.RotatingFileHandler',
-            'formatter': 'main',
-            'maxBytes': 1024*1024*100,
-            'backupCount': 7,
-            'filename': ANSIBLE_LOG_FILE,
-        },
-        'syslog': {
-            'level': 'INFO',
-            'class': 'logging.NullHandler',
-            'formatter': 'syslog'
-        },
-    },
-    'loggers': {
-        'django': {
-            'handlers': ['null'],
-            'propagate': False,
-            'level': LOG_LEVEL,
-        },
-        'django.request': {
-            'handlers': ['console', 'file', 'syslog'],
-            'level': LOG_LEVEL,
-            'propagate': False,
-        },
-        'django.server': {
-            'handlers': ['console', 'file', 'syslog'],
-            'level': LOG_LEVEL,
-            'propagate': False,
-        },
-        'jumpserver': {
-            'handlers': ['console', 'file', 'syslog'],
-            'level': LOG_LEVEL,
-        },
-        'ops.ansible_api': {
-            'handlers': ['console', 'ansible_logs'],
-            'level': LOG_LEVEL,
-        },
-        'django_auth_ldap': {
-            'handlers': ['console', 'file'],
-            'level': "INFO",
-        },
-        'jms.audits': {
-            'handlers': ['syslog'],
-            'level': 'INFO'
-        },
-        # 'django.db': {
-        #     'handlers': ['console', 'file'],
-        #     'level': 'DEBUG'
-        # }
-    }
-}
-
-SYSLOG_ENABLE = False
-
-if CONFIG.SYSLOG_ADDR != '' and len(CONFIG.SYSLOG_ADDR.split(':')) == 2:
-    host, port = CONFIG.SYSLOG_ADDR.split(':')
-    SYSLOG_ENABLE = True
-    LOGGING['handlers']['syslog'].update({
-        'class': 'logging.handlers.SysLogHandler',
-        'facility': CONFIG.SYSLOG_FACILITY,
-        'address': (host, int(port)),
-    })
-
-# Internationalization
-# https://docs.djangoproject.com/en/1.10/topics/i18n/
-# LANGUAGE_CODE = 'en'
-LANGUAGE_CODE = 'zh'
-
-TIME_ZONE = 'Asia/Shanghai'
-
-USE_I18N = True
-
-USE_L10N = True
-
-USE_TZ = True
-
-# I18N translation
-LOCALE_PATHS = [
-    os.path.join(BASE_DIR, 'locale'),
-]
-
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.10/howto/static-files/
-
-STATIC_URL = '{}/static/'.format(CONFIG.FORCE_SCRIPT_NAME)
-STATIC_ROOT = os.path.join(PROJECT_DIR, "data", "static")
-STATIC_DIR = os.path.join(BASE_DIR, "static")
-
-STATICFILES_DIRS = (
-    os.path.join(BASE_DIR, "static"),
-)
-
-# Media files (File, ImageField) will be save these
-
-MEDIA_URL = '/media/'
-
-MEDIA_ROOT = os.path.join(PROJECT_DIR, 'data', 'media').replace('\\', '/') + '/'
-
-# Use django-bootstrap-form to format template, input max width arg
-# BOOTSTRAP_COLUMN_COUNT = 11
-
-# Init data or generate fake data source for development
-FIXTURE_DIRS = [os.path.join(BASE_DIR, 'fixtures'), ]
-
-# Email config
-EMAIL_HOST = 'smtp.jumpserver.org'
-EMAIL_PORT = 25
-EMAIL_HOST_USER = 'noreply@jumpserver.org'
-EMAIL_HOST_PASSWORD = ''
-EMAIL_FROM = ''
-EMAIL_RECIPIENT = ''
-EMAIL_USE_SSL = False
-EMAIL_USE_TLS = False
-EMAIL_SUBJECT_PREFIX = '[JMS] '
-
-# Email custom content
-EMAIL_CUSTOM_USER_CREATED_SUBJECT = ''
-EMAIL_CUSTOM_USER_CREATED_HONORIFIC = ''
-EMAIL_CUSTOM_USER_CREATED_BODY = ''
-EMAIL_CUSTOM_USER_CREATED_SIGNATURE = ''
-
-REST_FRAMEWORK = {
-    # Use Django's standard `django.contrib.auth` permissions,
-    # or allow read-only access for unauthenticated users.
-    'DEFAULT_PERMISSION_CLASSES': (
-        'common.permissions.IsOrgAdmin',
-    ),
-    'DEFAULT_RENDERER_CLASSES': (
-        'rest_framework.renderers.JSONRenderer',
-        'rest_framework.renderers.BrowsableAPIRenderer',
-        'common.renders.JMSCSVRender',
-    ),
-    'DEFAULT_PARSER_CLASSES': (
-        'rest_framework.parsers.JSONParser',
-        'rest_framework.parsers.FormParser',
-        'rest_framework.parsers.MultiPartParser',
-        'common.parsers.JMSCSVParser',
-        'rest_framework.parsers.FileUploadParser',
-    ),
-    'DEFAULT_AUTHENTICATION_CLASSES': (
-        # 'rest_framework.authentication.BasicAuthentication',
-        'authentication.backends.api.AccessKeyAuthentication',
-        'authentication.backends.api.AccessTokenAuthentication',
-        'authentication.backends.api.PrivateTokenAuthentication',
-        'authentication.backends.api.SignatureAuthentication',
-        'authentication.backends.api.SessionAuthentication',
-    ),
-    'DEFAULT_FILTER_BACKENDS': (
-        'django_filters.rest_framework.DjangoFilterBackend',
-        'rest_framework.filters.SearchFilter',
-        'rest_framework.filters.OrderingFilter',
-    ),
-    'DEFAULT_METADATA_CLASS': 'common.drfmetadata.SimpleMetadataWithFilters',
-    'ORDERING_PARAM': "order",
-    'SEARCH_PARAM': "search",
-    'DATETIME_FORMAT': '%Y-%m-%d %H:%M:%S %z',
-    'DATETIME_INPUT_FORMATS': ['iso-8601', '%Y-%m-%d %H:%M:%S %z'],
-    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
-    # 'PAGE_SIZE': 15
-}
-
-AUTHENTICATION_BACKENDS = [
-    'django.contrib.auth.backends.ModelBackend',
-    'authentication.backends.pubkey.PublicKeyAuthBackend',
-]
-
-# Custom User Auth model
-AUTH_USER_MODEL = 'users.User'
-
-# File Upload Permissions
-FILE_UPLOAD_PERMISSIONS = 0o644
-FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o755
-
-# OTP settings
-OTP_ISSUER_NAME = CONFIG.OTP_ISSUER_NAME
-OTP_VALID_WINDOW = CONFIG.OTP_VALID_WINDOW
-
-# Auth LDAP settings
-AUTH_LDAP = False
-AUTH_LDAP_SEARCH_PAGED_SIZE = CONFIG.AUTH_LDAP_SEARCH_PAGED_SIZE
-AUTH_LDAP_SYNC_IS_PERIODIC = CONFIG.AUTH_LDAP_SYNC_IS_PERIODIC
-AUTH_LDAP_SYNC_INTERVAL = CONFIG.AUTH_LDAP_SYNC_INTERVAL
-AUTH_LDAP_SYNC_CRONTAB = CONFIG.AUTH_LDAP_SYNC_CRONTAB
-AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS = CONFIG.AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS
-
-AUTH_LDAP_SERVER_URI = 'ldap://localhost:389'
-AUTH_LDAP_BIND_DN = 'cn=admin,dc=jumpserver,dc=org'
-AUTH_LDAP_BIND_PASSWORD = ''
-AUTH_LDAP_SEARCH_OU = 'ou=tech,dc=jumpserver,dc=org'
-AUTH_LDAP_SEARCH_FILTER = '(cn=%(user)s)'
-AUTH_LDAP_START_TLS = False
-AUTH_LDAP_USER_ATTR_MAP = {"username": "cn", "name": "sn", "email": "mail"}
-AUTH_LDAP_GLOBAL_OPTIONS = {
-    ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
-    ldap.OPT_REFERRALS: CONFIG.AUTH_LDAP_OPTIONS_OPT_REFERRALS
-}
-LDAP_CERT_FILE = os.path.join(PROJECT_DIR, "data", "certs", "ldap_ca.pem")
-if os.path.isfile(LDAP_CERT_FILE):
-    AUTH_LDAP_GLOBAL_OPTIONS[ldap.OPT_X_TLS_CACERTFILE] = LDAP_CERT_FILE
-# AUTH_LDAP_GROUP_SEARCH_OU = CONFIG.AUTH_LDAP_GROUP_SEARCH_OU
-# AUTH_LDAP_GROUP_SEARCH_FILTER = CONFIG.AUTH_LDAP_GROUP_SEARCH_FILTER
-# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
-#    AUTH_LDAP_GROUP_SEARCH_OU, ldap.SCOPE_SUBTREE, AUTH_LDAP_GROUP_SEARCH_FILTER
-# )
-AUTH_LDAP_CONNECTION_OPTIONS = {
-    ldap.OPT_TIMEOUT: 30
-}
-AUTH_LDAP_GROUP_CACHE_TIMEOUT = 1
-AUTH_LDAP_ALWAYS_UPDATE_USER = True
-AUTH_LDAP_BACKEND = 'authentication.backends.ldap.LDAPAuthorizationBackend'
-
-if AUTH_LDAP:
-    AUTHENTICATION_BACKENDS.insert(0, AUTH_LDAP_BACKEND)
-
-# openid
-# Auth OpenID settings
-BASE_SITE_URL = CONFIG.BASE_SITE_URL
-AUTH_OPENID = CONFIG.AUTH_OPENID
-AUTH_OPENID_SERVER_URL = CONFIG.AUTH_OPENID_SERVER_URL
-AUTH_OPENID_REALM_NAME = CONFIG.AUTH_OPENID_REALM_NAME
-AUTH_OPENID_CLIENT_ID = CONFIG.AUTH_OPENID_CLIENT_ID
-AUTH_OPENID_CLIENT_SECRET = CONFIG.AUTH_OPENID_CLIENT_SECRET
-AUTH_OPENID_IGNORE_SSL_VERIFICATION = CONFIG.AUTH_OPENID_IGNORE_SSL_VERIFICATION
-AUTH_OPENID_SHARE_SESSION = CONFIG.AUTH_OPENID_SHARE_SESSION
-AUTH_OPENID_BACKENDS = [
-    'authentication.backends.openid.backends.OpenIDAuthorizationPasswordBackend',
-    'authentication.backends.openid.backends.OpenIDAuthorizationCodeBackend',
-]
-
-if AUTH_OPENID:
-    LOGIN_URL = reverse_lazy("authentication:openid:openid-login")
-    LOGIN_COMPLETE_URL = reverse_lazy("authentication:openid:openid-login-complete")
-    AUTHENTICATION_BACKENDS.insert(0, AUTH_OPENID_BACKENDS[0])
-    AUTHENTICATION_BACKENDS.insert(0, AUTH_OPENID_BACKENDS[1])
-
-# Radius Auth
-AUTH_RADIUS = CONFIG.AUTH_RADIUS
-AUTH_RADIUS_BACKEND = 'authentication.backends.radius.RadiusBackend'
-RADIUS_SERVER = CONFIG.RADIUS_SERVER
-RADIUS_PORT = CONFIG.RADIUS_PORT
-RADIUS_SECRET = CONFIG.RADIUS_SECRET
-
-if AUTH_RADIUS:
-    AUTHENTICATION_BACKENDS.insert(0, AUTH_RADIUS_BACKEND)
-
-# Dump all celery log to here
-CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'celery')
-
-# Celery using redis as broker
-CELERY_BROKER_URL = 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % {
-    'password': CONFIG.REDIS_PASSWORD,
-    'host': CONFIG.REDIS_HOST,
-    'port': CONFIG.REDIS_PORT,
-    'db': CONFIG.REDIS_DB_CELERY,
-}
-CELERY_TASK_SERIALIZER = 'pickle'
-CELERY_RESULT_SERIALIZER = 'pickle'
-CELERY_RESULT_BACKEND = CELERY_BROKER_URL
-CELERY_ACCEPT_CONTENT = ['json', 'pickle']
-CELERY_RESULT_EXPIRES = 3600
-# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
-# CELERY_WORKER_LOG_FORMAT = '%(message)s'
-# CELERY_WORKER_TASK_LOG_FORMAT = '%(task_id)s %(task_name)s %(message)s'
-CELERY_WORKER_TASK_LOG_FORMAT = '%(message)s'
-# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
-CELERY_WORKER_LOG_FORMAT = '%(message)s'
-CELERY_TASK_EAGER_PROPAGATES = True
-CELERY_WORKER_REDIRECT_STDOUTS = True
-CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
-# CELERY_WORKER_HIJACK_ROOT_LOGGER = True
-# CELERY_WORKER_MAX_TASKS_PER_CHILD = 40
-CELERY_TASK_SOFT_TIME_LIMIT = 3600
-
-# Cache use redis
-CACHES = {
-    'default': {
-        'BACKEND': 'redis_cache.RedisCache',
-        'LOCATION': 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % {
-            'password': CONFIG.REDIS_PASSWORD,
-            'host': CONFIG.REDIS_HOST,
-            'port': CONFIG.REDIS_PORT,
-            'db': CONFIG.REDIS_DB_CACHE,
-        }
-    }
-}
-
-# Captcha settings, more see https://django-simple-captcha.readthedocs.io/en/latest/advanced.html
-CAPTCHA_IMAGE_SIZE = (80, 33)
-CAPTCHA_FOREGROUND_COLOR = '#001100'
-CAPTCHA_NOISE_FUNCTIONS = ('captcha.helpers.noise_dots',)
-CAPTCHA_TEST_MODE = CONFIG.CAPTCHA_TEST_MODE
-
-COMMAND_STORAGE = {
-    'ENGINE': 'terminal.backends.command.db',
-}
-
-DEFAULT_TERMINAL_COMMAND_STORAGE = {
-    "default": {
-        "TYPE": "server",
-    },
-}
-
-TERMINAL_COMMAND_STORAGE = CONFIG.TERMINAL_COMMAND_STORAGE
-
-DEFAULT_TERMINAL_REPLAY_STORAGE = {
-    "default": {
-        "TYPE": "server",
-    },
-}
-
-TERMINAL_REPLAY_STORAGE = {
-}
-
-
-SECURITY_MFA_AUTH = False
-SECURITY_COMMAND_EXECUTION = True
-SECURITY_LOGIN_LIMIT_COUNT = 7
-SECURITY_LOGIN_LIMIT_TIME = 30  # Unit: minute
-SECURITY_MAX_IDLE_TIME = 30  # Unit: minute
-SECURITY_PASSWORD_EXPIRATION_TIME = 9999  # Unit: day
-SECURITY_PASSWORD_MIN_LENGTH = 6  # Unit: bit
-SECURITY_PASSWORD_UPPER_CASE = False
-SECURITY_PASSWORD_LOWER_CASE = False
-SECURITY_PASSWORD_NUMBER = False
-SECURITY_PASSWORD_SPECIAL_CHAR = False
-SECURITY_PASSWORD_RULES = [
-    'SECURITY_PASSWORD_MIN_LENGTH',
-    'SECURITY_PASSWORD_UPPER_CASE',
-    'SECURITY_PASSWORD_LOWER_CASE',
-    'SECURITY_PASSWORD_NUMBER',
-    'SECURITY_PASSWORD_SPECIAL_CHAR'
-]
-SECURITY_MFA_VERIFY_TTL = CONFIG.SECURITY_MFA_VERIFY_TTL
-SECURITY_SERVICE_ACCOUNT_REGISTRATION = CONFIG.SECURITY_SERVICE_ACCOUNT_REGISTRATION
-TERMINAL_PASSWORD_AUTH = CONFIG.TERMINAL_PASSWORD_AUTH
-TERMINAL_PUBLIC_KEY_AUTH = CONFIG.TERMINAL_PUBLIC_KEY_AUTH
-TERMINAL_HEARTBEAT_INTERVAL = CONFIG.TERMINAL_HEARTBEAT_INTERVAL
-TERMINAL_ASSET_LIST_SORT_BY = CONFIG.TERMINAL_ASSET_LIST_SORT_BY
-TERMINAL_ASSET_LIST_PAGE_SIZE = CONFIG.TERMINAL_ASSET_LIST_PAGE_SIZE
-TERMINAL_SESSION_KEEP_DURATION = CONFIG.TERMINAL_SESSION_KEEP_DURATION
-TERMINAL_HOST_KEY = CONFIG.TERMINAL_HOST_KEY
-TERMINAL_HEADER_TITLE = CONFIG.TERMINAL_HEADER_TITLE
-TERMINAL_TELNET_REGEX = CONFIG.TERMINAL_TELNET_REGEX
-
-# Django bootstrap3 setting, more see http://django-bootstrap3.readthedocs.io/en/latest/settings.html
-BOOTSTRAP3 = {
-    'horizontal_label_class': 'col-md-2',
-    # Field class to use in horizontal forms
-    'horizontal_field_class': 'col-md-9',
-    # Set placeholder attributes to label if no placeholder is provided
-    'set_placeholder': False,
-    'success_css_class': '',
-    'required_css_class': 'required',
-}
-
-TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION
-DISPLAY_PER_PAGE = CONFIG.DISPLAY_PER_PAGE
-DEFAULT_EXPIRED_YEARS = 70
-USER_GUIDE_URL = ""
-
-
-SWAGGER_SETTINGS = {
-    'DEFAULT_AUTO_SCHEMA_CLASS': 'jumpserver.swagger.CustomSwaggerAutoSchema',
-    'USE_SESSION_AUTH': True,
-    'SECURITY_DEFINITIONS': {
-        'Bearer': {
-            'type': 'apiKey',
-            'name': 'Authorization',
-            'in': 'header'
-        }
-    },
-}
-
-
-# Default email suffix
-EMAIL_SUFFIX = CONFIG.EMAIL_SUFFIX
-LOGIN_LOG_KEEP_DAYS = CONFIG.LOGIN_LOG_KEEP_DAYS
-
-# User or user group permission cache time, default 3600 seconds
-ASSETS_PERM_CACHE_ENABLE = CONFIG.ASSETS_PERM_CACHE_ENABLE
-ASSETS_PERM_CACHE_TIME = CONFIG.ASSETS_PERM_CACHE_TIME
-
-# Asset user auth external backend, default AuthBook backend
-BACKEND_ASSET_USER_AUTH_VAULT = False
-
-DEFAULT_ORG_SHOW_ALL_USERS = CONFIG.DEFAULT_ORG_SHOW_ALL_USERS
-
-PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
-WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
-FLOWER_URL = CONFIG.FLOWER_URL
-
-
-# Django channels support websocket
-CHANNEL_REDIS = "redis://:{}@{}:{}/{}".format(
-    CONFIG.REDIS_PASSWORD, CONFIG.REDIS_HOST, CONFIG.REDIS_PORT,
-    CONFIG.REDIS_DB_WS,
-)
-
-CHANNEL_LAYERS = {
-    'default': {
-        'BACKEND': 'channels_redis.core.RedisChannelLayer',
-        'CONFIG': {
-            "hosts": [CHANNEL_REDIS],
-        },
-    },
-}
-
-# Enable internal period task
-PERIOD_TASK_ENABLED = CONFIG.PERIOD_TASK_ENABLED
-FORCE_SCRIPT_NAME = CONFIG.FORCE_SCRIPT_NAME

apps/jumpserver/settings/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+#
+from .base import *
+from .logging import *
+from .libs import *
+from .auth import *
+from .custom import *
+from ._xpack import *

apps/jumpserver/settings/_xpack.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+#
+
+import os
+from .. import const
+from .base import INSTALLED_APPS, TEMPLATES
+
+XPACK_DIR = os.path.join(const.BASE_DIR, 'xpack')
+XPACK_ENABLED = os.path.isdir(XPACK_DIR)
+XPACK_TEMPLATES_DIR = []
+XPACK_CONTEXT_PROCESSOR = []
+
+if XPACK_ENABLED:
+    from xpack.utils import get_xpack_templates_dir, get_xpack_context_processor
+    INSTALLED_APPS.append('xpack.apps.XpackConfig')
+    XPACK_TEMPLATES_DIR = get_xpack_templates_dir(const.BASE_DIR)
+    XPACK_CONTEXT_PROCESSOR = get_xpack_context_processor()
+    TEMPLATES[0]['DIRS'].extend(XPACK_TEMPLATES_DIR)
+    TEMPLATES[0]['OPTIONS']['context_processors'].extend(XPACK_CONTEXT_PROCESSOR)

apps/jumpserver/settings/auth.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+#
+import os
+import ldap
+from django.urls import reverse_lazy
+
+from ..const import CONFIG, DYNAMIC, PROJECT_DIR
+
+# OTP settings
+OTP_ISSUER_NAME = CONFIG.OTP_ISSUER_NAME
+OTP_VALID_WINDOW = CONFIG.OTP_VALID_WINDOW
+
+# Auth LDAP settings
+AUTH_LDAP = DYNAMIC.AUTH_LDAP
+AUTH_LDAP_SERVER_URI = DYNAMIC.AUTH_LDAP_SERVER_URI
+AUTH_LDAP_BIND_DN = DYNAMIC.AUTH_LDAP_BIND_DN
+AUTH_LDAP_BIND_PASSWORD = DYNAMIC.AUTH_LDAP_BIND_PASSWORD
+AUTH_LDAP_SEARCH_OU = DYNAMIC.AUTH_LDAP_SEARCH_OU
+AUTH_LDAP_SEARCH_FILTER = DYNAMIC.AUTH_LDAP_SEARCH_FILTER
+AUTH_LDAP_START_TLS = DYNAMIC.AUTH_LDAP_START_TLS
+AUTH_LDAP_USER_ATTR_MAP = DYNAMIC.AUTH_LDAP_USER_ATTR_MAP
+AUTH_LDAP_GLOBAL_OPTIONS = {
+    ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER,
+    ldap.OPT_REFERRALS: CONFIG.AUTH_LDAP_OPTIONS_OPT_REFERRALS
+}
+LDAP_CERT_FILE = os.path.join(PROJECT_DIR, "data", "certs", "ldap_ca.pem")
+if os.path.isfile(LDAP_CERT_FILE):
+    AUTH_LDAP_GLOBAL_OPTIONS[ldap.OPT_X_TLS_CACERTFILE] = LDAP_CERT_FILE
+# AUTH_LDAP_GROUP_SEARCH_OU = CONFIG.AUTH_LDAP_GROUP_SEARCH_OU
+# AUTH_LDAP_GROUP_SEARCH_FILTER = CONFIG.AUTH_LDAP_GROUP_SEARCH_FILTER
+# AUTH_LDAP_GROUP_SEARCH = LDAPSearch(
+#    AUTH_LDAP_GROUP_SEARCH_OU, ldap.SCOPE_SUBTREE, AUTH_LDAP_GROUP_SEARCH_FILTER
+# )
+AUTH_LDAP_CONNECTION_OPTIONS = {
+    ldap.OPT_TIMEOUT: CONFIG.AUTH_LDAP_CONNECT_TIMEOUT
+}
+AUTH_LDAP_CACHE_TIMEOUT = 1
+AUTH_LDAP_ALWAYS_UPDATE_USER = True
+
+AUTH_LDAP_SEARCH_PAGED_SIZE = CONFIG.AUTH_LDAP_SEARCH_PAGED_SIZE
+AUTH_LDAP_SYNC_IS_PERIODIC = CONFIG.AUTH_LDAP_SYNC_IS_PERIODIC
+AUTH_LDAP_SYNC_INTERVAL = CONFIG.AUTH_LDAP_SYNC_INTERVAL
+AUTH_LDAP_SYNC_CRONTAB = CONFIG.AUTH_LDAP_SYNC_CRONTAB
+AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS = CONFIG.AUTH_LDAP_USER_LOGIN_ONLY_IN_USERS
+
+# openid
+# Auth OpenID settings
+BASE_SITE_URL = CONFIG.BASE_SITE_URL
+AUTH_OPENID = CONFIG.AUTH_OPENID
+AUTH_OPENID_SERVER_URL = CONFIG.AUTH_OPENID_SERVER_URL
+AUTH_OPENID_REALM_NAME = CONFIG.AUTH_OPENID_REALM_NAME
+AUTH_OPENID_CLIENT_ID = CONFIG.AUTH_OPENID_CLIENT_ID
+AUTH_OPENID_CLIENT_SECRET = CONFIG.AUTH_OPENID_CLIENT_SECRET
+AUTH_OPENID_IGNORE_SSL_VERIFICATION = CONFIG.AUTH_OPENID_IGNORE_SSL_VERIFICATION
+AUTH_OPENID_SHARE_SESSION = CONFIG.AUTH_OPENID_SHARE_SESSION
+AUTH_OPENID_LOGIN_URL = reverse_lazy("authentication:openid:openid-login")
+AUTH_OPENID_LOGIN_COMPLETE_URL = reverse_lazy("authentication:openid:openid-login-complete")
+
+
+# Radius Auth
+AUTH_RADIUS = CONFIG.AUTH_RADIUS
+AUTH_RADIUS_BACKEND = 'authentication.backends.radius.RadiusBackend'
+RADIUS_SERVER = CONFIG.RADIUS_SERVER
+RADIUS_PORT = CONFIG.RADIUS_PORT
+RADIUS_SECRET = CONFIG.RADIUS_SECRET
+
+
+TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION
+LOGIN_CONFIRM_ENABLE = CONFIG.LOGIN_CONFIRM_ENABLE
+OTP_IN_RADIUS = CONFIG.OTP_IN_RADIUS

apps/jumpserver/settings/base.py

@@ -0,0 +1,247 @@
+import os
+
+from django.urls import reverse_lazy
+
+from .. import const
+from ..const import CONFIG, DYNAMIC
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+VERSION = const.VERSION
+BASE_DIR = const.BASE_DIR
+PROJECT_DIR = const.PROJECT_DIR
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = CONFIG.SECRET_KEY
+
+# SECURITY WARNING: keep the token secret, remove it if all coco, guacamole ok
+BOOTSTRAP_TOKEN = CONFIG.BOOTSTRAP_TOKEN
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = CONFIG.DEBUG
+
+# Absolute url for some case, for example email link
+SITE_URL = DYNAMIC.SITE_URL
+
+# LOG LEVEL
+LOG_LEVEL = CONFIG.LOG_LEVEL
+
+ALLOWED_HOSTS = ['*']
+
+# Max post update field num
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000
+
+# Application definition
+
+INSTALLED_APPS = [
+    'orgs.apps.OrgsConfig',
+    'users.apps.UsersConfig',
+    'assets.apps.AssetsConfig',
+    'perms.apps.PermsConfig',
+    'ops.apps.OpsConfig',
+    'settings.apps.SettingsConfig',
+    'common.apps.CommonConfig',
+    'terminal.apps.TerminalConfig',
+    'audits.apps.AuditsConfig',
+    'authentication.apps.AuthenticationConfig',  # authentication
+    'applications.apps.ApplicationsConfig',
+    'tickets.apps.TicketsConfig',
+    'rest_framework',
+    'rest_framework_swagger',
+    'drf_yasg',
+    'channels',
+    'django_filters',
+    'bootstrap3',
+    'captcha',
+    'django_celery_beat',
+    'django.contrib.auth',
+    'django.contrib.admin',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+]
+
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'authentication.backends.openid.middleware.OpenIDAuthenticationMiddleware',
+    'jumpserver.middleware.TimezoneMiddleware',
+    'jumpserver.middleware.DemoMiddleware',
+    'jumpserver.middleware.RequestMiddleware',
+    'orgs.middleware.OrgMiddleware',
+]
+
+
+ROOT_URLCONF = 'jumpserver.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [os.path.join(BASE_DIR, 'templates')],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+                'django.template.context_processors.static',
+                'django.template.context_processors.request',
+                'django.template.context_processors.media',
+                'jumpserver.context_processor.jumpserver_processor',
+                'orgs.context_processor.org_processor',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'jumpserver.wsgi.application'
+
+LOGIN_REDIRECT_URL = reverse_lazy('index')
+LOGIN_URL = reverse_lazy('authentication:login')
+
+SESSION_COOKIE_DOMAIN = CONFIG.SESSION_COOKIE_DOMAIN
+CSRF_COOKIE_DOMAIN = CONFIG.CSRF_COOKIE_DOMAIN
+SESSION_COOKIE_AGE = CONFIG.SESSION_COOKIE_AGE
+SESSION_EXPIRE_AT_BROWSER_CLOSE = CONFIG.SESSION_EXPIRE_AT_BROWSER_CLOSE
+SESSION_ENGINE = 'redis_sessions.session'
+SESSION_REDIS = {
+    'host': CONFIG.REDIS_HOST,
+    'port': CONFIG.REDIS_PORT,
+    'password': CONFIG.REDIS_PASSWORD,
+    'db': CONFIG.REDIS_DB_SESSION,
+    'prefix': 'auth_session',
+    'socket_timeout': 1,
+    'retry_on_timeout': False
+}
+
+MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage'
+# Database
+# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
+
+DB_OPTIONS = {}
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.{}'.format(CONFIG.DB_ENGINE.lower()),
+        'NAME': CONFIG.DB_NAME,
+        'HOST': CONFIG.DB_HOST,
+        'PORT': CONFIG.DB_PORT,
+        'USER': CONFIG.DB_USER,
+        'PASSWORD': CONFIG.DB_PASSWORD,
+        'ATOMIC_REQUESTS': True,
+        'OPTIONS': DB_OPTIONS
+    }
+}
+DB_CA_PATH = os.path.join(PROJECT_DIR, 'data', 'certs', 'db_ca.pem')
+if CONFIG.DB_ENGINE.lower() == 'mysql':
+    DB_OPTIONS['init_command'] = "SET sql_mode='STRICT_TRANS_TABLES'"
+    if os.path.isfile(DB_CA_PATH):
+        DB_OPTIONS['ssl'] = {'ca': DB_CA_PATH}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
+#
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.10/topics/i18n/
+# LANGUAGE_CODE = 'en'
+LANGUAGE_CODE = 'zh'
+
+TIME_ZONE = 'Asia/Shanghai'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+# I18N translation
+LOCALE_PATHS = [
+    os.path.join(BASE_DIR, 'locale'),
+]
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.10/howto/static-files/
+
+STATIC_URL = '{}/static/'.format(CONFIG.FORCE_SCRIPT_NAME)
+STATIC_ROOT = os.path.join(PROJECT_DIR, "data", "static")
+STATIC_DIR = os.path.join(BASE_DIR, "static")
+
+STATICFILES_DIRS = (
+    os.path.join(BASE_DIR, "static"),
+)
+
+# Media files (File, ImageField) will be save these
+
+MEDIA_URL = '/media/'
+
+MEDIA_ROOT = os.path.join(PROJECT_DIR, 'data', 'media').replace('\\', '/') + '/'
+
+# Use django-bootstrap-form to format template, input max width arg
+# BOOTSTRAP_COLUMN_COUNT = 11
+
+# Init data or generate fake data source for development
+FIXTURE_DIRS = [os.path.join(BASE_DIR, 'fixtures'), ]
+
+# Email config
+EMAIL_HOST = DYNAMIC.EMAIL_HOST
+EMAIL_PORT = DYNAMIC.EMAIL_PORT
+EMAIL_HOST_USER = DYNAMIC.EMAIL_HOST_USER
+EMAIL_HOST_PASSWORD = DYNAMIC.EMAIL_HOST_PASSWORD
+EMAIL_FROM = DYNAMIC.EMAIL_FROM
+EMAIL_RECIPIENT = DYNAMIC.EMAIL_RECIPIENT
+EMAIL_USE_SSL = DYNAMIC.EMAIL_USE_SSL
+EMAIL_USE_TLS = DYNAMIC.EMAIL_USE_TLS
+
+
+AUTHENTICATION_BACKENDS = DYNAMIC.AUTHENTICATION_BACKENDS
+
+# Custom User Auth model
+AUTH_USER_MODEL = 'users.User'
+
+# File Upload Permissions
+FILE_UPLOAD_PERMISSIONS = 0o644
+FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o755
+
+# Cache use redis
+CACHES = {
+    'default': {
+        'BACKEND': 'redis_cache.RedisCache',
+        'LOCATION': 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % {
+            'password': CONFIG.REDIS_PASSWORD,
+            'host': CONFIG.REDIS_HOST,
+            'port': CONFIG.REDIS_PORT,
+            'db': CONFIG.REDIS_DB_CACHE,
+        }
+    }
+}
+
+
+FORCE_SCRIPT_NAME = CONFIG.FORCE_SCRIPT_NAME
+

apps/jumpserver/settings/custom.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+#
+from ..const import CONFIG, DYNAMIC
+
+# Storage settings
+COMMAND_STORAGE = {
+    'ENGINE': 'terminal.backends.command.db',
+}
+DEFAULT_TERMINAL_COMMAND_STORAGE = {
+    "default": {
+        "TYPE": "server",
+    },
+}
+TERMINAL_COMMAND_STORAGE = DYNAMIC.TERMINAL_COMMAND_STORAGE or {}
+DEFAULT_TERMINAL_REPLAY_STORAGE = {
+    "default": {
+        "TYPE": "server",
+    },
+}
+TERMINAL_REPLAY_STORAGE = DYNAMIC.TERMINAL_REPLAY_STORAGE
+
+# Security settings
+SECURITY_MFA_AUTH = DYNAMIC.SECURITY_MFA_AUTH
+SECURITY_COMMAND_EXECUTION = DYNAMIC.SECURITY_COMMAND_EXECUTION
+SECURITY_LOGIN_LIMIT_COUNT = DYNAMIC.SECURITY_LOGIN_LIMIT_COUNT
+SECURITY_LOGIN_LIMIT_TIME = DYNAMIC.SECURITY_LOGIN_LIMIT_TIME  # Unit: minute
+SECURITY_MAX_IDLE_TIME = DYNAMIC.SECURITY_MAX_IDLE_TIME  # Unit: minute
+SECURITY_PASSWORD_EXPIRATION_TIME = DYNAMIC.SECURITY_PASSWORD_EXPIRATION_TIME # Unit: day
+SECURITY_PASSWORD_MIN_LENGTH = DYNAMIC.SECURITY_PASSWORD_MIN_LENGTH  # Unit: bit
+SECURITY_PASSWORD_UPPER_CASE = DYNAMIC.SECURITY_PASSWORD_UPPER_CASE
+SECURITY_PASSWORD_LOWER_CASE = DYNAMIC.SECURITY_PASSWORD_LOWER_CASE
+SECURITY_PASSWORD_NUMBER = DYNAMIC.SECURITY_PASSWORD_NUMBER
+SECURITY_PASSWORD_SPECIAL_CHAR = DYNAMIC.SECURITY_PASSWORD_SPECIAL_CHAR
+SECURITY_PASSWORD_RULES = [
+    'SECURITY_PASSWORD_MIN_LENGTH',
+    'SECURITY_PASSWORD_UPPER_CASE',
+    'SECURITY_PASSWORD_LOWER_CASE',
+    'SECURITY_PASSWORD_NUMBER',
+    'SECURITY_PASSWORD_SPECIAL_CHAR'
+]
+SECURITY_MFA_VERIFY_TTL = CONFIG.SECURITY_MFA_VERIFY_TTL
+SECURITY_VIEW_AUTH_NEED_MFA = CONFIG.SECURITY_VIEW_AUTH_NEED_MFA
+SECURITY_SERVICE_ACCOUNT_REGISTRATION = DYNAMIC.SECURITY_SERVICE_ACCOUNT_REGISTRATION
+
+# Terminal other setting
+TERMINAL_PASSWORD_AUTH = DYNAMIC.TERMINAL_PASSWORD_AUTH
+TERMINAL_PUBLIC_KEY_AUTH = DYNAMIC.TERMINAL_PUBLIC_KEY_AUTH
+TERMINAL_HEARTBEAT_INTERVAL = DYNAMIC.TERMINAL_HEARTBEAT_INTERVAL
+TERMINAL_ASSET_LIST_SORT_BY = DYNAMIC.TERMINAL_ASSET_LIST_SORT_BY
+TERMINAL_ASSET_LIST_PAGE_SIZE = DYNAMIC.TERMINAL_ASSET_LIST_PAGE_SIZE
+TERMINAL_SESSION_KEEP_DURATION = DYNAMIC.TERMINAL_SESSION_KEEP_DURATION
+TERMINAL_HOST_KEY = DYNAMIC.TERMINAL_HOST_KEY
+TERMINAL_HEADER_TITLE = DYNAMIC.TERMINAL_HEADER_TITLE
+TERMINAL_TELNET_REGEX = DYNAMIC.TERMINAL_TELNET_REGEX
+
+# User or user group permission cache time, default 3600 seconds
+ASSETS_PERM_CACHE_ENABLE = CONFIG.ASSETS_PERM_CACHE_ENABLE
+ASSETS_PERM_CACHE_TIME = CONFIG.ASSETS_PERM_CACHE_TIME
+
+# Asset user auth external backend, default AuthBook backend
+BACKEND_ASSET_USER_AUTH_VAULT = False
+
+DEFAULT_ORG_SHOW_ALL_USERS = CONFIG.DEFAULT_ORG_SHOW_ALL_USERS
+PERM_SINGLE_ASSET_TO_UNGROUP_NODE = CONFIG.PERM_SINGLE_ASSET_TO_UNGROUP_NODE
+WINDOWS_SSH_DEFAULT_SHELL = CONFIG.WINDOWS_SSH_DEFAULT_SHELL
+FLOWER_URL = CONFIG.FLOWER_URL
+
+# Enable internal period task
+PERIOD_TASK_ENABLED = CONFIG.PERIOD_TASK_ENABLED
+
+# Email custom content
+EMAIL_SUBJECT_PREFIX = DYNAMIC.EMAIL_SUBJECT_PREFIX
+EMAIL_SUFFIX = DYNAMIC.EMAIL_SUFFIX
+EMAIL_CUSTOM_USER_CREATED_SUBJECT = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_SUBJECT
+EMAIL_CUSTOM_USER_CREATED_HONORIFIC = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_HONORIFIC
+EMAIL_CUSTOM_USER_CREATED_BODY = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_BODY
+EMAIL_CUSTOM_USER_CREATED_SIGNATURE = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_SIGNATURE
+
+DISPLAY_PER_PAGE = CONFIG.DISPLAY_PER_PAGE
+DEFAULT_EXPIRED_YEARS = 70
+USER_GUIDE_URL = DYNAMIC.USER_GUIDE_URL
+HTTP_LISTEN_PORT = CONFIG.HTTP_LISTEN_PORT
+WS_LISTEN_PORT = CONFIG.WS_LISTEN_PORT
+LOGIN_LOG_KEEP_DAYS = DYNAMIC.LOGIN_LOG_KEEP_DAYS

apps/jumpserver/settings/libs.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+#
+import os
+from ..const import CONFIG, PROJECT_DIR
+
+REST_FRAMEWORK = {
+    # Use Django's standard `django.contrib.auth` permissions,
+    # or allow read-only access for unauthenticated users.
+    'DEFAULT_PERMISSION_CLASSES': (
+        'common.permissions.IsOrgAdmin',
+    ),
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+        'rest_framework.renderers.BrowsableAPIRenderer',
+        'common.drf.renders.JMSCSVRender',
+    ),
+    'DEFAULT_PARSER_CLASSES': (
+        'rest_framework.parsers.JSONParser',
+        'rest_framework.parsers.FormParser',
+        'rest_framework.parsers.MultiPartParser',
+        'common.drf.parsers.JMSCSVParser',
+        'rest_framework.parsers.FileUploadParser',
+    ),
+    'DEFAULT_AUTHENTICATION_CLASSES': (
+        # 'rest_framework.authentication.BasicAuthentication',
+        'authentication.backends.api.AccessKeyAuthentication',
+        'authentication.backends.api.AccessTokenAuthentication',
+        'authentication.backends.api.PrivateTokenAuthentication',
+        'authentication.backends.api.SignatureAuthentication',
+        'authentication.backends.api.SessionAuthentication',
+    ),
+    'DEFAULT_FILTER_BACKENDS': (
+        'django_filters.rest_framework.DjangoFilterBackend',
+        'rest_framework.filters.SearchFilter',
+        'rest_framework.filters.OrderingFilter',
+    ),
+    'DEFAULT_METADATA_CLASS': 'common.drf.metadata.SimpleMetadataWithFilters',
+    'ORDERING_PARAM': "order",
+    'SEARCH_PARAM': "search",
+    'DATETIME_FORMAT': '%Y-%m-%d %H:%M:%S %z',
+    'DATETIME_INPUT_FORMATS': ['iso-8601', '%Y-%m-%d %H:%M:%S %z'],
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+    # 'PAGE_SIZE': 15
+}
+
+SWAGGER_SETTINGS = {
+    'DEFAULT_AUTO_SCHEMA_CLASS': 'jumpserver.swagger.CustomSwaggerAutoSchema',
+    'USE_SESSION_AUTH': True,
+    'SECURITY_DEFINITIONS': {
+        'Bearer': {
+            'type': 'apiKey',
+            'name': 'Authorization',
+            'in': 'header'
+        }
+    },
+}
+
+
+# Captcha settings, more see https://django-simple-captcha.readthedocs.io/en/latest/advanced.html
+CAPTCHA_IMAGE_SIZE = (80, 33)
+CAPTCHA_FOREGROUND_COLOR = '#001100'
+CAPTCHA_NOISE_FUNCTIONS = ('captcha.helpers.noise_dots',)
+CAPTCHA_TEST_MODE = CONFIG.CAPTCHA_TEST_MODE
+
+# Django bootstrap3 setting, more see http://django-bootstrap3.readthedocs.io/en/latest/settings.html
+BOOTSTRAP3 = {
+    'horizontal_label_class': 'col-md-2',
+    # Field class to use in horizontal forms
+    'horizontal_field_class': 'col-md-9',
+    # Set placeholder attributes to label if no placeholder is provided
+    'set_placeholder': False,
+    'success_css_class': '',
+    'required_css_class': 'required',
+}
+
+
+# Django channels support websocket
+CHANNEL_REDIS = "redis://:{}@{}:{}/{}".format(
+    CONFIG.REDIS_PASSWORD, CONFIG.REDIS_HOST, CONFIG.REDIS_PORT,
+    CONFIG.REDIS_DB_WS,
+)
+
+CHANNEL_LAYERS = {
+    'default': {
+        'BACKEND': 'channels_redis.core.RedisChannelLayer',
+        'CONFIG': {
+            "hosts": [CHANNEL_REDIS],
+        },
+    },
+}
+ASGI_APPLICATION = 'jumpserver.routing.application'
+
+
+# Dump all celery log to here
+CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'celery')
+
+# Celery using redis as broker
+CELERY_BROKER_URL = 'redis://:%(password)s@%(host)s:%(port)s/%(db)s' % {
+    'password': CONFIG.REDIS_PASSWORD,
+    'host': CONFIG.REDIS_HOST,
+    'port': CONFIG.REDIS_PORT,
+    'db': CONFIG.REDIS_DB_CELERY,
+}
+CELERY_TASK_SERIALIZER = 'pickle'
+CELERY_RESULT_SERIALIZER = 'pickle'
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+CELERY_ACCEPT_CONTENT = ['json', 'pickle']
+CELERY_RESULT_EXPIRES = 3600
+# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
+# CELERY_WORKER_LOG_FORMAT = '%(message)s'
+# CELERY_WORKER_TASK_LOG_FORMAT = '%(task_id)s %(task_name)s %(message)s'
+CELERY_WORKER_TASK_LOG_FORMAT = '%(message)s'
+# CELERY_WORKER_LOG_FORMAT = '%(asctime)s [%(module)s %(levelname)s] %(message)s'
+CELERY_WORKER_LOG_FORMAT = '%(message)s'
+CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_WORKER_REDIRECT_STDOUTS = True
+CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
+# CELERY_WORKER_HIJACK_ROOT_LOGGER = True
+# CELERY_WORKER_MAX_TASKS_PER_CHILD = 40
+CELERY_TASK_SOFT_TIME_LIMIT = 3600

apps/jumpserver/settings/logging.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+#
+import os
+from ..const import PROJECT_DIR, CONFIG
+
+LOG_DIR = os.path.join(PROJECT_DIR, 'logs')
+JUMPSERVER_LOG_FILE = os.path.join(LOG_DIR, 'jumpserver.log')
+ANSIBLE_LOG_FILE = os.path.join(LOG_DIR, 'ansible.log')
+GUNICORN_LOG_FILE = os.path.join(LOG_DIR, 'gunicorn.log')
+LOG_LEVEL = CONFIG.LOG_LEVEL
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+        },
+        'main': {
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'format': '%(asctime)s [%(module)s %(levelname)s] %(message)s',
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'syslog': {
+            'format': 'jumpserver: %(message)s'
+        },
+        'msg': {
+            'format': '%(message)s'
+        }
+    },
+    'handlers': {
+        'null': {
+            'level': 'DEBUG',
+            'class': 'logging.NullHandler',
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'main'
+        },
+        'file': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'maxBytes': 1024*1024*100,
+            'backupCount': 7,
+            'formatter': 'main',
+            'filename': JUMPSERVER_LOG_FILE,
+        },
+        'ansible_logs': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'formatter': 'main',
+            'maxBytes': 1024*1024*100,
+            'backupCount': 7,
+            'filename': ANSIBLE_LOG_FILE,
+        },
+        'syslog': {
+            'level': 'INFO',
+            'class': 'logging.NullHandler',
+            'formatter': 'syslog'
+        },
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['null'],
+            'propagate': False,
+            'level': LOG_LEVEL,
+        },
+        'django.request': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'django.server': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'jumpserver': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+        },
+        'ops.ansible_api': {
+            'handlers': ['console', 'ansible_logs'],
+            'level': LOG_LEVEL,
+        },
+        'django_auth_ldap': {
+            'handlers': ['console', 'file'],
+            'level': "INFO",
+        },
+        'jms.audits': {
+            'handlers': ['syslog'],
+            'level': 'INFO'
+        },
+        # 'django.db': {
+        #     'handlers': ['console', 'file'],
+        #     'level': 'DEBUG'
+        # }
+    }
+}
+SYSLOG_ENABLE = CONFIG.SYSLOG_ENABLE
+
+if CONFIG.SYSLOG_ADDR != '' and len(CONFIG.SYSLOG_ADDR.split(':')) == 2:
+    host, port = CONFIG.SYSLOG_ADDR.split(':')
+    LOGGING['handlers']['syslog'].update({
+        'class': 'logging.handlers.SysLogHandler',
+        'facility': CONFIG.SYSLOG_FACILITY,
+        'address': (host, int(port)),
+    })
+
+if not os.path.isdir(LOG_DIR):
+    os.makedirs(LOG_DIR)

apps/jumpserver/urls.py

@@ -7,10 +7,7 @@ from django.conf.urls.static import static
 from django.conf.urls.i18n import i18n_patterns
 from django.views.i18n import JavaScriptCatalog
 
-# from .views import IndexView, LunaView, I18NView, HealthCheckView, redirect_format_api
 from . import views
-from .celery_flower import celery_flower_view
-from .swagger import get_swagger_view
 
 api_v1 = [
     path('users/', include('users.urls.api_urls', namespace='api-users')),
@@ -44,7 +41,7 @@ app_view_patterns = [
     path('auth/', include('authentication.urls.view_urls'), name='auth'),
     path('applications/', include('applications.urls.views_urls', namespace='applications')),
     path('tickets/', include('tickets.urls.views_urls', namespace='tickets')),
-    re_path(r'flower/(?P<path>.*)', celery_flower_view, name='flower-view'),
+    re_path(r'flower/(?P<path>.*)', views.celery_flower_view, name='flower-view'),
 ]
 
 
@@ -82,19 +79,19 @@ urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \
             + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
 urlpatterns += js_i18n_patterns
 
-handler404 = 'jumpserver.error_views.handler404'
-handler500 = 'jumpserver.error_views.handler500'
+handler404 = 'jumpserver.views.handler404'
+handler500 = 'jumpserver.views.handler500'
 
 if settings.DEBUG:
     urlpatterns += [
         re_path('^swagger(?P<format>\.json|\.yaml)$',
-                get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
-        path('docs/', get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"),
-        path('redoc/', get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'),
+                views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
+        path('docs/', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"),
+        path('redoc/', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'),
 
         re_path('^v2/swagger(?P<format>\.json|\.yaml)$',
-                get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
-        path('docs/v2/', get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"),
-        path('redoc/v2/', get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'),
+                views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'),
+        path('docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"),
+        path('redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'),
     ]
 

apps/jumpserver/views/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+#
+from .index import *
+from .other import *
+from .celery_flower import *
+from .swagger import *
+from .error_views import *

apps/jumpserver/views/celery_flower.py

@@ -9,6 +9,8 @@ from proxy.views import proxy_view
 
 flower_url = settings.FLOWER_URL
 
+__all__ = ['celery_flower_view']
+
 
 @csrf_exempt
 def celery_flower_view(request, path):

apps/jumpserver/views/error_views.py

@@ -3,6 +3,8 @@
 from django.shortcuts import render
 from django.http import JsonResponse
 
+__all__ = ['handler404', 'handler500']
+
 
 def handler404(request, *args, **argv):
     if request.content_type.find('application/json') > -1:

apps/jumpserver/views/index.py

@@ -1,17 +1,10 @@
 import datetime
-import re
-import time
 
-from django.http import HttpResponseRedirect, JsonResponse
-from django.conf import settings
-from django.views.generic import TemplateView, View
+from django.views.generic import TemplateView
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 from django.db.models import Count
 from django.shortcuts import redirect
-from rest_framework.views import APIView
-from django.views.decorators.csrf import csrf_exempt
-from django.http import HttpResponse
 
 
 from users.models import User
@@ -19,7 +12,8 @@ from assets.models import Asset
 from terminal.models import Session
 from orgs.utils import current_org
 from common.permissions import PermissionsMixin, IsValidUser
-from common.http import HttpResponseTemporaryRedirect
+
+__all__ = ['IndexView']
 
 
 class IndexView(PermissionsMixin, TemplateView):
@@ -186,58 +180,3 @@ class IndexView(PermissionsMixin, TemplateView):
 
         kwargs.update(context)
         return super(IndexView, self).get_context_data(**kwargs)
-
-
-class LunaView(View):
-    def get(self, request):
-        msg = _("<div>Luna is a separately deployed program, you need to deploy Luna, koko, configure nginx for url distribution,</div> "
-                "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
-        return HttpResponse(msg)
-
-
-class I18NView(View):
-    def get(self, request, lang):
-        referer_url = request.META.get('HTTP_REFERER', '/')
-        response = HttpResponseRedirect(referer_url)
-        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang)
-        return response
-
-
-api_url_pattern = re.compile(r'^/api/(?P<app>\w+)/(?P<version>v\d)/(?P<extra>.*)$')
-
-
-@csrf_exempt
-def redirect_format_api(request, *args, **kwargs):
-    _path, query = request.path, request.GET.urlencode()
-    matched = api_url_pattern.match(_path)
-    if matched:
-        kwargs = matched.groupdict()
-        kwargs["query"] = query
-        _path = '/api/{version}/{app}/{extra}?{query}'.format(**kwargs).rstrip("?")
-        return HttpResponseTemporaryRedirect(_path)
-    else:
-        return JsonResponse({"msg": "Redirect url failed: {}".format(_path)}, status=404)
-
-
-class HealthCheckView(APIView):
-    permission_classes = ()
-
-    def get(self, request):
-        return JsonResponse({"status": 1, "time": int(time.time())})
-
-
-class WsView(APIView):
-    ws_port = settings.CONFIG.HTTP_LISTEN_PORT + 1
-
-    def get(self, request):
-        msg = _("Websocket server run on port: {}, you should proxy it on nginx"
-                .format(self.ws_port))
-        return JsonResponse({"msg": msg})
-
-
-class KokoView(View):
-    def get(self, request):
-        msg = _(
-            "<div>Koko is a separately deployed program, you need to deploy Koko, configure nginx for url distribution,</div> "
-            "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
-        return HttpResponse(msg)

apps/jumpserver/views/other.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+#
+import re
+import time
+
+from django.http import HttpResponseRedirect, JsonResponse
+from django.conf import settings
+from django.views.generic import View
+from django.utils.translation import ugettext_lazy as _
+from rest_framework.views import APIView
+from django.views.decorators.csrf import csrf_exempt
+from django.http import HttpResponse
+
+from common.http import HttpResponseTemporaryRedirect
+
+
+__all__ = [
+    'LunaView', 'I18NView', 'KokoView', 'WsView', 'HealthCheckView',
+    'redirect_format_api'
+]
+
+
+class LunaView(View):
+    def get(self, request):
+        msg = _("<div>Luna is a separately deployed program, you need to deploy Luna, koko, configure nginx for url distribution,</div> "
+                "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
+        return HttpResponse(msg)
+
+
+class I18NView(View):
+    def get(self, request, lang):
+        referer_url = request.META.get('HTTP_REFERER', '/')
+        response = HttpResponseRedirect(referer_url)
+        response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang)
+        return response
+
+
+api_url_pattern = re.compile(r'^/api/(?P<app>\w+)/(?P<version>v\d)/(?P<extra>.*)$')
+
+
+@csrf_exempt
+def redirect_format_api(request, *args, **kwargs):
+    _path, query = request.path, request.GET.urlencode()
+    matched = api_url_pattern.match(_path)
+    if matched:
+        kwargs = matched.groupdict()
+        kwargs["query"] = query
+        _path = '/api/{version}/{app}/{extra}?{query}'.format(**kwargs).rstrip("?")
+        return HttpResponseTemporaryRedirect(_path)
+    else:
+        return JsonResponse({"msg": "Redirect url failed: {}".format(_path)}, status=404)
+
+
+class HealthCheckView(APIView):
+    permission_classes = ()
+
+    def get(self, request):
+        return JsonResponse({"status": 1, "time": int(time.time())})
+
+
+class WsView(APIView):
+    ws_port = settings.HTTP_LISTEN_PORT + 1
+
+    def get(self, request):
+        msg = _("Websocket server run on port: {}, you should proxy it on nginx"
+                .format(self.ws_port))
+        return JsonResponse({"msg": msg})
+
+
+class KokoView(View):
+    def get(self, request):
+        msg = _(
+            "<div>Koko is a separately deployed program, you need to deploy Koko, configure nginx for url distribution,</div> "
+            "</div>If you see this page, prove that you are not accessing the nginx listening port. Good luck.</div>")
+        return HttpResponse(msg)

apps/jumpserver/views/swagger.py

@@ -50,7 +50,7 @@ class CustomSwaggerAutoSchema(SwaggerAutoSchema):
 
 
 def get_swagger_view(version='v1'):
-    from .urls import api_v1, api_v2
+    from ..urls import api_v1, api_v2
     from django.urls import path, include
     api_v1_patterns = [
         path('api/v1/', include(api_v1))

apps/ops/api/celery.py

@@ -5,17 +5,20 @@ import os
 import re
 
 from django.utils.translation import ugettext as _
+from rest_framework import viewsets
 from celery.result import AsyncResult
 from rest_framework import generics
+from django_celery_beat.models import PeriodicTask
 
-from common.permissions import IsValidUser
+from common.permissions import IsValidUser, IsSuperUser
 from common.api import LogTailApi
 from ..models import CeleryTask
-from ..serializers import CeleryResultSerializer
+from ..serializers import CeleryResultSerializer, CeleryPeriodTaskSerializer
 from ..celery.utils import get_celery_task_log_path
+from common.mixins.api import CommonApiMixin
 
 
-__all__ = ['CeleryTaskLogApi', 'CeleryResultApi']
+__all__ = ['CeleryTaskLogApi', 'CeleryResultApi', 'CeleryPeriodTaskViewSet']
 
 
 class CeleryTaskLogApi(LogTailApi):
@@ -62,3 +65,14 @@ class CeleryResultApi(generics.RetrieveAPIView):
         pk = self.kwargs.get('pk')
         return AsyncResult(pk)
 
+
+class CeleryPeriodTaskViewSet(CommonApiMixin, viewsets.ModelViewSet):
+    queryset = PeriodicTask.objects.all()
+    serializer_class = CeleryPeriodTaskSerializer
+    permission_classes = (IsSuperUser,)
+    http_method_names = ('get', 'head', 'options', 'patch')
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        queryset = queryset.exclude(description='')
+        return queryset

apps/ops/celery/decorator.py

@@ -9,51 +9,37 @@ _after_app_shutdown_clean_periodic_tasks = []
 
 def add_register_period_task(task):
     _need_registered_period_tasks.append(task)
-    # key = "__REGISTER_PERIODIC_TASKS"
-    # value = cache.get(key, [])
-    # value.append(name)
-    # cache.set(key, value)
 
 
 def get_register_period_tasks():
-    # key = "__REGISTER_PERIODIC_TASKS"
-    # return cache.get(key, [])
     return _need_registered_period_tasks
 
 
 def add_after_app_shutdown_clean_task(name):
-    # key = "__AFTER_APP_SHUTDOWN_CLEAN_TASKS"
-    # value = cache.get(key, [])
-    # value.append(name)
-    # cache.set(key, value)
     _after_app_shutdown_clean_periodic_tasks.append(name)
 
 
 def get_after_app_shutdown_clean_tasks():
-    # key = "__AFTER_APP_SHUTDOWN_CLEAN_TASKS"
-    # return cache.get(key, [])
     return _after_app_shutdown_clean_periodic_tasks
 
 
 def add_after_app_ready_task(name):
-    # key = "__AFTER_APP_READY_RUN_TASKS"
-    # value = cache.get(key, [])
-    # value.append(name)
-    # cache.set(key, value)
     _after_app_ready_start_tasks.append(name)
 
 
 def get_after_app_ready_tasks():
-    # key = "__AFTER_APP_READY_RUN_TASKS"
-    # return cache.get(key, [])
     return _after_app_ready_start_tasks
 
 
-def register_as_period_task(crontab=None, interval=None):
+def register_as_period_task(
+        crontab=None, interval=None, name=None,
+        description=''):
     """
     Warning: Task must be have not any args and kwargs
     :param crontab:  "* * * * *"
     :param interval:  60*60*60
+    :param description: "
+    :param name: ""
     :return:
     """
     if crontab is None and interval is None:
@@ -65,14 +51,16 @@ def register_as_period_task(crontab=None, interval=None):
 
         # Because when this decorator run, the task was not created,
         # So we can't use func.name
-        name = '{func.__module__}.{func.__name__}'.format(func=func)
+        task = '{func.__module__}.{func.__name__}'.format(func=func)
+        _name = name if name else task
         add_register_period_task({
-           name: {
-               'task': name,
+           _name: {
+               'task': task,
                'interval': interval,
                'crontab': crontab,
                'args': (),
                'enabled': True,
+               'description': description
            }
         })
 

apps/ops/celery/signal_handler.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 import logging
+from django.dispatch import receiver
 
 from django.core.cache import cache
 from celery import subtask
@@ -11,6 +12,7 @@ from kombu.utils.encoding import safe_str
 from django_celery_beat.models import PeriodicTask
 
 from common.utils import get_logger
+from common.signals import django_ready
 from .decorator import get_after_app_ready_tasks, get_after_app_shutdown_clean_tasks
 from .logger import CeleryTaskFileHandler
 

apps/ops/celery/utils.py

@@ -10,6 +10,10 @@ from django_celery_beat.models import (
     PeriodicTask, IntervalSchedule, CrontabSchedule, PeriodicTasks
 )
 
+from common.utils import get_logger
+
+logger = get_logger(__name__)
+
 
 def create_or_update_celery_periodic_tasks(tasks):
     """
@@ -21,6 +25,7 @@ def create_or_update_celery_periodic_tasks(tasks):
             'args': (16, 16),
             'kwargs': {},
             'enabled': False,
+            'description': ''
         },
     }
     :return:
@@ -35,34 +40,30 @@ def create_or_update_celery_periodic_tasks(tasks):
             return None
 
         if isinstance(detail.get("interval"), int):
-            intervals = IntervalSchedule.objects.filter(
-                every=detail["interval"], period=IntervalSchedule.SECONDS
+            kwargs = dict(
+                every=detail['interval'],
+                period=IntervalSchedule.SECONDS,
             )
-            if intervals:
-                interval = intervals[0]
-            else:
-                interval = IntervalSchedule.objects.create(
-                    every=detail['interval'],
-                    period=IntervalSchedule.SECONDS,
-                )
+            # 不能使用 get_or_create，因为可能会有多个
+            interval = IntervalSchedule.objects.filter(**kwargs).first()
+            if interval is None:
+                interval = IntervalSchedule.objects.create(**kwargs)
         elif isinstance(detail.get("crontab"), str):
             try:
                 minute, hour, day, month, week = detail["crontab"].split()
             except ValueError:
-                raise SyntaxError("crontab is not valid")
+                logger.error("crontab is not valid")
+                return
             kwargs = dict(
                 minute=minute, hour=hour, day_of_week=week,
                 day_of_month=day, month_of_year=month, timezone=get_current_timezone()
             )
-            contabs = CrontabSchedule.objects.filter(
-                **kwargs
-            )
-            if contabs:
-                crontab = contabs[0]
-            else:
+            crontab = CrontabSchedule.objects.filter(**kwargs).first()
+            if crontab is None:
                 crontab = CrontabSchedule.objects.create(**kwargs)
         else:
-            raise SyntaxError("Schedule is not valid")
+            logger.error("Schedule is not valid")
+            return
 
         defaults = dict(
             interval=interval,
@@ -71,8 +72,9 @@ def create_or_update_celery_periodic_tasks(tasks):
             task=detail['task'],
             args=json.dumps(detail.get('args', [])),
             kwargs=json.dumps(detail.get('kwargs', {})),
-            enabled=detail.get('enabled', True),
+            description=detail.get('description') or ''
         )
+        print(defaults)
 
         task = PeriodicTask.objects.update_or_create(
             defaults=defaults, name=name,

apps/ops/serializers/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+#
+
+from .celery import *
+from .adhoc import *

apps/ops/serializers/adhoc.py

@@ -3,17 +3,7 @@ from __future__ import unicode_literals
 from rest_framework import serializers
 from django.shortcuts import reverse
 
-from .models import Task, AdHoc, AdHocRunHistory, CommandExecution
-
-
-class CeleryResultSerializer(serializers.Serializer):
-    id = serializers.UUIDField()
-    result = serializers.JSONField()
-    state = serializers.CharField(max_length=16)
-
-
-class CeleryTaskSerializer(serializers.Serializer):
-    pass
+from ..models import Task, AdHoc, AdHocRunHistory, CommandExecution
 
 
 class TaskSerializer(serializers.ModelSerializer):
@@ -87,3 +77,4 @@ class CommandExecutionSerializer(serializers.ModelSerializer):
     @staticmethod
     def get_log_url(obj):
         return reverse('api-ops:celery-task-log', kwargs={'pk': obj.id})
+

apps/ops/serializers/celery.py

@@ -0,0 +1,29 @@
+# ~*~ coding: utf-8 ~*~
+from __future__ import unicode_literals
+from rest_framework import serializers
+
+from django_celery_beat.models import PeriodicTask
+
+__all__ = [
+    'CeleryResultSerializer', 'CeleryTaskSerializer',
+    'CeleryPeriodTaskSerializer'
+]
+
+
+class CeleryResultSerializer(serializers.Serializer):
+    id = serializers.UUIDField()
+    result = serializers.JSONField()
+    state = serializers.CharField(max_length=16)
+
+
+class CeleryTaskSerializer(serializers.Serializer):
+    pass
+
+
+class CeleryPeriodTaskSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = PeriodicTask
+        fields = [
+            'name', 'task', 'enabled', 'description',
+            'last_run_at', 'total_run_count'
+        ]

apps/ops/tasks.py

@@ -1,21 +1,22 @@
 # coding: utf-8
 import os
 import subprocess
-import datetime
 import time
 
 from django.conf import settings
 from celery import shared_task, subtask
 from celery.exceptions import SoftTimeLimitExceeded
 from django.utils import timezone
+from django.utils.translation import ugettext_lazy as _
 
-from common.utils import get_logger, get_object_or_none
+from common.utils import get_logger, get_object_or_none, get_disk_usage
 from .celery.decorator import (
     register_as_period_task, after_app_shutdown_clean_periodic,
     after_app_ready_start
 )
 from .celery.utils import create_or_update_celery_periodic_tasks
 from .models import Task, CommandExecution, CeleryTask
+from .utils import send_server_performance_mail
 
 logger = get_logger(__file__)
 
@@ -59,7 +60,7 @@ def run_command_execution(cid, **kwargs):
 
 @shared_task
 @after_app_shutdown_clean_periodic
-@register_as_period_task(interval=3600*24)
+@register_as_period_task(interval=3600*24, description=_("Clean task history period"))
 def clean_tasks_adhoc_period():
     logger.debug("Start clean task adhoc and run history")
     tasks = Task.objects.all()
@@ -72,7 +73,7 @@ def clean_tasks_adhoc_period():
 
 @shared_task
 @after_app_shutdown_clean_periodic
-@register_as_period_task(interval=3600*24)
+@register_as_period_task(interval=3600*24, description=_("Clean celery log period"))
 def clean_celery_tasks_period():
     expire_days = 30
     logger.debug("Start clean celery task history")
@@ -103,6 +104,19 @@ def create_or_update_registered_periodic_tasks():
         create_or_update_celery_periodic_tasks(task)
 
 
+@shared_task
+@register_as_period_task(interval=3600)
+def check_server_performance_period():
+    usages = get_disk_usage()
+    usages = {path: usage for path, usage in usages.items()
+              if not path.startswith('/etc')}
+
+    for path, usage in usages.items():
+        if usage.percent > 80:
+            send_server_performance_mail(path, usage, usages)
+            return
+
+
 @shared_task(queue="ansible")
 def hello(name, callback=None):
     import time

apps/ops/templates/ops/adhoc_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/adhoc_history.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/adhoc_history_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/command_execution_create.html

@@ -11,8 +11,6 @@
           rel="stylesheet">
     <link href="{% static 'css/plugins/codemirror/ambiance.css' %}"
           rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}"
-          rel="stylesheet">
     <script type="text/javascript"
             src="{% static 'js/plugins/ztree/jquery.ztree.all.min.js' %}"></script>
     <script type="text/javascript"
@@ -22,7 +20,6 @@
     <script src="{% static 'js/plugins/xterm/addons/fit/fit.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/codemirror.js' %}"></script>
     <script src="{% static 'js/plugins/codemirror/mode/shell/shell.js' %}"></script>
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style type="text/css">
         .xterm .xterm-screen canvas {
             position: absolute;

apps/ops/templates/ops/command_execution_list.html

@@ -6,8 +6,6 @@
 {% block custom_head_css_js %}
     <link href="{% static "css/plugins/footable/footable.core.css" %}" rel="stylesheet">
     <link href="{% static 'css/plugins/datepicker/datepicker3.css' %}" rel="stylesheet">
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <style>
         .form-control {
             height: 30px;
@@ -16,6 +14,11 @@
         #search_btn {
             margin-bottom: 0;
         }
+
+        .select2-selection__rendered span.select2-selection, .select2-container .select2-selection--single {
+            height: 30px !important;
+        }
+
     </style>
 {% endblock %}
 

apps/ops/templates/ops/task_adhoc.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/templates/ops/task_detail.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static 'css/plugins/sweetalert/sweetalert.css' %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static 'js/plugins/sweetalert/sweetalert.min.js' %}"></script>
 {% endblock %}
 

apps/ops/templates/ops/task_history.html

@@ -3,9 +3,7 @@
 {% load i18n %}
 
 {% block custom_head_css_js %}
-    <link href="{% static 'css/plugins/select2/select2.min.css' %}" rel="stylesheet">
     <link href="{% static "css/plugins/sweetalert/sweetalert.css" %}" rel="stylesheet">
-    <script src="{% static 'js/plugins/select2/select2.full.min.js' %}"></script>
     <script src="{% static "js/plugins/sweetalert/sweetalert.min.js" %}"></script>
 {% endblock %}
 {% block content %}

apps/ops/urls/api_urls.py

@@ -13,6 +13,7 @@ router.register(r'tasks', api.TaskViewSet, 'task')
 router.register(r'adhoc', api.AdHocViewSet, 'adhoc')
 router.register(r'history', api.AdHocRunHistoryViewSet, 'history')
 router.register(r'command-executions', api.CommandExecutionViewSet, 'command-execution')
+router.register(r'celery/period-tasks', api.CeleryPeriodTaskViewSet, 'celery-period-task')
 
 urlpatterns = [
     path('tasks/<uuid:pk>/run/', api.TaskRun.as_view(), name='task-run'),

apps/ops/utils.py

@@ -1,6 +1,8 @@
 # ~*~ coding: utf-8 ~*~
 from django.utils.translation import ugettext_lazy as _
+
 from common.utils import get_logger, get_object_or_none
+from common.tasks import send_mail_async
 from orgs.utils import set_to_root_org
 from .models import Task, AdHoc
 
@@ -56,4 +58,14 @@ def update_or_create_ansible_task(
     return task, created
 
 
+def send_server_performance_mail(path, usage, usages):
+    from users.models import User
+    subject = _("Disk used more than 80%: {} => {}").format(path, usage.percent)
+    message = subject
+    admins = User.objects.filter(role=User.ROLE_ADMIN)
+    recipient_list = [u.email for u in admins if u.email]
+    logger.info(subject)
+    send_mail_async(subject, message, recipient_list, html_message=message)
+
+
 

apps/ops/views/celery.py

@@ -17,6 +17,6 @@ class CeleryTaskLogView(PermissionsMixin, TemplateView):
         context = super().get_context_data(**kwargs)
         context.update({
             'task_id': self.kwargs.get('pk'),
-            'ws_port': settings.CONFIG.WS_LISTEN_PORT
+            'ws_port': settings.WS_LISTEN_PORT
         })
         return context

apps/ops/views/command.py

@@ -80,7 +80,7 @@ class CommandExecutionStartView(PermissionsMixin, TemplateView):
             'action': _('Command execution'),
             'form': self.get_form(),
             'system_users': system_users,
-            'ws_port': settings.CONFIG.WS_LISTEN_PORT
+            'ws_port': settings.WS_LISTEN_PORT
         }
         kwargs.update(context)
         return super().get_context_data(**kwargs)

apps/orgs/mixins/api.py

@@ -46,7 +46,11 @@ class OrgModelViewSet(CommonApiMixin, OrgQuerySetMixin, ModelViewSet):
 
 class OrgBulkModelViewSet(CommonApiMixin, OrgQuerySetMixin, BulkModelViewSet):
     def allow_bulk_destroy(self, qs, filtered):
-        if qs.count() <= filtered.count():
+        qs_count = qs.count()
+        filtered_count = filtered.count()
+        if filtered_count == 1:
+            return True
+        if qs_count <= filtered_count:
             return False
         if self.request.query_params.get('spm', ''):
             return True