From de41747bb29001f1ac2e3b2304096bc99d6ecc68 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 29 Jun 2022 14:48:54 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E6=B7=BB=E5=8A=A0=20debug=20tool=20bar?= =?UTF-8?q?=20(#8504)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 添加 debug tool bar * perf: 修改 config name Co-authored-by: ibuler --- apps/common/signal_handlers.py | 3 +- apps/jumpserver/conf.py | 1 + apps/jumpserver/settings/base.py | 41 +++++++++-- apps/jumpserver/settings/libs.py | 16 ++-- apps/jumpserver/settings/logging.py | 2 +- apps/jumpserver/urls.py | 5 ++ apps/rbac/tree.py | 8 +- requirements/requirements.txt | 110 +++++++++++++++------------- utils/start_celery_beat.py | 6 +- 9 files changed, 114 insertions(+), 78 deletions(-) diff --git a/apps/common/signal_handlers.py b/apps/common/signal_handlers.py index 0cb3a04ad..e14a9fdf8 100644 --- a/apps/common/signal_handlers.py +++ b/apps/common/signal_handlers.py @@ -14,7 +14,6 @@ from .local import thread_local pattern = re.compile(r'FROM `(\w+)`') logger = logging.getLogger("jumpserver.common") -DEBUG_DB = os.environ.get('DEBUG_DB', '0') == '1' class Counter: @@ -66,7 +65,7 @@ def on_request_finished_release_local(sender, **kwargs): thread_local.__release_local__() -if settings.DEBUG and DEBUG_DB: +if settings.DEBUG_DEV: request_finished.connect(on_request_finished_logging_db_query) else: request_finished.connect(on_request_finished_release_local) diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 8b589ded7..8afa16eb5 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -128,6 +128,7 @@ class Config(dict): 'SECRET_KEY': '', 'BOOTSTRAP_TOKEN': '', 'DEBUG': False, + 'DEBUG_DEV': False, 'LOG_LEVEL': 'DEBUG', 'LOG_DIR': os.path.join(PROJECT_DIR, 'logs'), 'DB_ENGINE': 'mysql', diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index ae3dea267..c66a47d98 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -36,6 +36,8 @@ BOOTSTRAP_TOKEN = CONFIG.BOOTSTRAP_TOKEN # SECURITY WARNING: don't run with debug turned on in production! DEBUG = CONFIG.DEBUG +# SECURITY WARNING: If you run with debug turned on, more debug msg with be log +DEBUG_DEV = CONFIG.DEBUG_DEV # Absolute url for some case, for example email link SITE_URL = CONFIG.SITE_URL @@ -107,6 +109,8 @@ MIDDLEWARE = [ 'simple_history.middleware.HistoryRequestMiddleware', ] + + ROOT_URLCONF = 'jumpserver.urls' TEMPLATES = [ @@ -262,10 +266,10 @@ FILE_UPLOAD_PERMISSIONS = 0o644 FILE_UPLOAD_DIRECTORY_PERMISSIONS = 0o755 # Cache use redis -REDIS_SSL_KEYFILE = exist_or_default(os.path.join(CERTS_DIR, 'redis_client.key'), None) -REDIS_SSL_CERTFILE = exist_or_default(os.path.join(CERTS_DIR, 'redis_client.crt'), None) -REDIS_SSL_CA_CERTS = exist_or_default(os.path.join(CERTS_DIR, 'redis_ca.pem'), None) -REDIS_SSL_CA_CERTS = exist_or_default(os.path.join(CERTS_DIR, 'redis_ca.crt'), REDIS_SSL_CA_CERTS) +REDIS_SSL_KEY = exist_or_default(os.path.join(CERTS_DIR, 'redis_client.key'), None) +REDIS_SSL_CERT = exist_or_default(os.path.join(CERTS_DIR, 'redis_client.crt'), None) +REDIS_SSL_CA = exist_or_default(os.path.join(CERTS_DIR, 'redis_ca.pem'), None) +REDIS_SSL_CA = exist_or_default(os.path.join(CERTS_DIR, 'redis_ca.crt'), REDIS_SSL_CA) REDIS_SSL_REQUIRED = 'none' REDIS_USE_SSL = CONFIG.REDIS_USE_SSL @@ -283,9 +287,9 @@ REDIS_CACHE_DEFAULT = { "REDIS_CLIENT_KWARGS": {"health_check_interval": 30}, "CONNECTION_POOL_KWARGS": { 'ssl_cert_reqs': REDIS_SSL_REQUIRED, - "ssl_keyfile": REDIS_SSL_KEYFILE, - "ssl_certfile": REDIS_SSL_CERTFILE, - "ssl_ca_certs": REDIS_SSL_CA_CERTS + "ssl_keyfile": REDIS_SSL_KEY, + "ssl_certfile": REDIS_SSL_CERT, + "ssl_ca_certs": REDIS_SSL_CA } if REDIS_USE_SSL else {} } } @@ -301,3 +305,26 @@ SESSION_CACHE_ALIAS = "session" FORCE_SCRIPT_NAME = CONFIG.FORCE_SCRIPT_NAME SESSION_COOKIE_SECURE = CONFIG.SESSION_COOKIE_SECURE CSRF_COOKIE_SECURE = CONFIG.CSRF_COOKIE_SECURE + +# For Debug toolbar +INTERNAL_IPS = ["127.0.0.1"] +if DEBUG_DEV: + INSTALLED_APPS = ['debug_toolbar', 'pympler'] + INSTALLED_APPS + MIDDLEWARE.insert(0, 'debug_toolbar.middleware.DebugToolbarMiddleware') + DEBUG_TOOLBAR_PANELS = [ + 'debug_toolbar.panels.history.HistoryPanel', + 'debug_toolbar.panels.versions.VersionsPanel', + 'debug_toolbar.panels.timer.TimerPanel', + 'debug_toolbar.panels.settings.SettingsPanel', + 'debug_toolbar.panels.headers.HeadersPanel', + 'debug_toolbar.panels.request.RequestPanel', + 'debug_toolbar.panels.sql.SQLPanel', + 'debug_toolbar.panels.staticfiles.StaticFilesPanel', + 'debug_toolbar.panels.templates.TemplatesPanel', + 'debug_toolbar.panels.cache.CachePanel', + 'debug_toolbar.panels.signals.SignalsPanel', + 'debug_toolbar.panels.logging.LoggingPanel', + 'debug_toolbar.panels.redirects.RedirectsPanel', + 'debug_toolbar.panels.profiling.ProfilingPanel', + 'pympler.panels.MemoryPanel', + ] diff --git a/apps/jumpserver/settings/libs.py b/apps/jumpserver/settings/libs.py index 8cfc0abb5..03fcadfce 100644 --- a/apps/jumpserver/settings/libs.py +++ b/apps/jumpserver/settings/libs.py @@ -4,7 +4,7 @@ import os import ssl from .base import ( - REDIS_SSL_CA_CERTS, REDIS_SSL_CERTFILE, REDIS_SSL_KEYFILE, + REDIS_SSL_CA, REDIS_SSL_CERT, REDIS_SSL_KEY, REDIS_SSL_REQUIRED, REDIS_USE_SSL ) from ..const import CONFIG, PROJECT_DIR @@ -89,10 +89,10 @@ if not REDIS_USE_SSL: else: redis_ssl = ssl.SSLContext() redis_ssl.check_hostname = bool(CONFIG.REDIS_SSL_REQUIRED) - if REDIS_SSL_CA_CERTS: - redis_ssl.load_verify_locations(REDIS_SSL_CA_CERTS) - if REDIS_SSL_CERTFILE and REDIS_SSL_KEYFILE: - redis_ssl.load_cert_chain(REDIS_SSL_CERTFILE, REDIS_SSL_KEYFILE) + if REDIS_SSL_CA: + redis_ssl.load_verify_locations(REDIS_SSL_CA) + if REDIS_SSL_CERT and REDIS_SSL_KEY: + redis_ssl.load_cert_chain(REDIS_SSL_CERT, REDIS_SSL_KEY) CHANNEL_LAYERS = { 'default': { @@ -136,9 +136,9 @@ CELERY_TASK_SOFT_TIME_LIMIT = 3600 if REDIS_USE_SSL: CELERY_BROKER_USE_SSL = CELERY_REDIS_BACKEND_USE_SSL = { 'ssl_cert_reqs': REDIS_SSL_REQUIRED, - 'ssl_ca_certs': REDIS_SSL_CA_CERTS, - 'ssl_certfile': REDIS_SSL_CERTFILE, - 'ssl_keyfile': REDIS_SSL_KEYFILE + 'ssl_ca_certs': REDIS_SSL_CA, + 'ssl_certfile': REDIS_SSL_CERT, + 'ssl_keyfile': REDIS_SSL_KEY } ANSIBLE_LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'ansible') diff --git a/apps/jumpserver/settings/logging.py b/apps/jumpserver/settings/logging.py index 961cc0c38..81b22b0fc 100644 --- a/apps/jumpserver/settings/logging.py +++ b/apps/jumpserver/settings/logging.py @@ -136,7 +136,7 @@ LOGGING = { } } -if os.environ.get("DEBUG_DB"): +if CONFIG.DEBUG_DEV: LOGGING['loggers']['django.db'] = { 'handlers': ['console', 'file'], 'level': 'DEBUG' diff --git a/apps/jumpserver/urls.py b/apps/jumpserver/urls.py index be8c059ce..6c4f55949 100644 --- a/apps/jumpserver/urls.py +++ b/apps/jumpserver/urls.py @@ -79,6 +79,11 @@ urlpatterns += [ re_path('api/redoc/?', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'), ] +if settings.DEBUG_DEV: + urlpatterns += [ + path('__debug__/', include('debug_toolbar.urls')), + ] + # 兼容之前的 old_app_pattern = '|'.join(apps) diff --git a/apps/rbac/tree.py b/apps/rbac/tree.py index bae0b930e..0b08c565d 100644 --- a/apps/rbac/tree.py +++ b/apps/rbac/tree.py @@ -12,8 +12,6 @@ from django.db.models import F, Count from common.tree import TreeNode from .models import Permission, ContentType -DEBUG_DB = os.environ.get('DEBUG_DB', '0') == '1' - # 根节点 root_node_data = { 'id': '$ROOT$', @@ -349,7 +347,7 @@ class PermissionTreeUtil: # name 要特殊处理，解决 i18n 问题 name, icon = self._get_permission_name_icon(p, content_types_name_mapper) - if DEBUG_DB: + if settings.DEBUG_DEV: name += '[{}]'.format(p.app_label_codename) pid = model_id @@ -394,9 +392,9 @@ class PermissionTreeUtil: } node_data['title'] = node_data['id'] node = TreeNode(**node_data) - if DEBUG_DB: + if settings.DEBUG_DEV: node.name += ('[' + node.id + ']') - if DEBUG_DB: + if settings.DEBUG_DEV: node.name += ('-' + node.id) return node diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f60fbc2f2..7ecf78ef1 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -3,95 +3,48 @@ ansible==2.10.7 asn1crypto==0.24.0 bcrypt==3.1.4 billiard==3.6.4.0 -boto3==1.24.12 -botocore==1.27.12 -celery==5.2.7 certifi==2018.1.18 cffi==1.13.2 chardet==3.0.4 configparser==3.5.0 -coreapi==2.3.3 -coreschema==0.0.4 decorator==4.1.2 -Django==3.1.14 -django-auth-ldap==2.2.0 -django-bootstrap3==14.2.0 -django-celery-beat==2.3.0 -django-filter==2.4.0 -django-formtools==2.2 -django-ranged-response==0.2.0 -django-rest-swagger==2.2.0 -django-simple-captcha==0.5.17 -django-timezone-field==5.0 -djangorestframework==3.13.1 -djangorestframework-bulk==0.2.1 docutils==0.14 ecdsa==0.13.3 enum-compat==0.0.2 ephem==3.7.6.0 -eventlet==0.33.1 future==0.16.0 -ForgeryPy3==0.3.1 -greenlet==1.1.2 -gunicorn==20.1.0 idna==2.6 -itsdangerous==1.1.0 itypes==1.2.0 Jinja2==3.1.2 jmespath==1.0.1 -kombu==5.2.4 -ldap3==2.9.1 MarkupSafe==2.1.1 -mysqlclient==2.1.0 olefile==0.46 -openapi-codec==1.3.2 paramiko==2.11.0 passlib==1.7.4 -Pillow==9.1.1 pyasn1==0.4.8 pycparser==2.21 cryptography==36.0.1 pycryptodome==3.15.0 pycryptodomex==3.15.0 +gmssl==3.2.1 +itsdangerous==1.1.0 pyotp==2.6.0 PyNaCl==1.5.0 python-dateutil==2.8.2 -pytz==2022.1 PyYAML==6.0 -redis==4.3.3 requests==2.28.0 jms-storage==0.0.44 -s3transfer==0.6.0 simplejson==3.17.6 six==1.16.0 sshpubkeys==3.3.1 uritemplate==4.1.1 urllib3==1.26.9 vine==5.0.0 -drf-yasg==1.20.0 Werkzeug==2.1.2 -drf-nested-routers==0.93.4 -rest_condition==1.0.3 -python-ldap==3.4.0 -django-radius==1.5.0 unicodecsv==0.14.1 -python-daemon==2.3.0 httpsig==1.3.0 treelib==1.6.1 -django-proxy==1.2.1 -flower==1.0.0 -channels-redis==3.4.0 -channels==3.0.4 -daphne==3.0.2 psutil==5.9.1 -django-cas-ng==4.0.1 -python-cas==1.5.0 -ipython -django-redis==5.2.0 -python-redis-lock==3.7.0 -jumpserver-django-oidc-rp==0.3.7.8 -django-mysql==3.9.0 -gmssl==3.2.1 msrestazure==0.6.4 adal==1.2.5 openpyxl==3.0.10 @@ -100,18 +53,57 @@ pyexcel-xlsx==0.6.0 data-tree==0.0.1 pyvmomi==7.0.1 termcolor==1.1.0 -django-simple-history==3.1.1 -geoip2==4.5.0 html2text==2020.1.16 pyzipper==0.3.5 python3-saml==1.12.0 -kubernetes==21.7.0 websocket-client==1.2.3 numpy==1.22.0 pandas==1.3.5 pyjwkest==1.4.2 jsonfield2==4.0.0.post0 +geoip2==4.5.0 ipip-ipdb==1.6.1 +# Django environment +Django==3.2.12 +django-bootstrap3==14.2.0 +django-filter==2.4.0 +django-formtools==2.2 +django-ranged-response==0.2.0 +django-rest-swagger==2.2.0 +django-simple-captcha==0.5.17 +django-timezone-field==5.0 +djangorestframework==3.13.1 +djangorestframework-bulk==0.2.1 +django-simple-history==3.1.1 +drf-nested-routers==0.93.4 +rest_condition==1.0.3 +drf-yasg==1.20.0 +coreapi==2.3.3 +coreschema==0.0.4 +openapi-codec==1.3.2 +Pillow==9.1.1 +pytz==2022.1 +# Runtime +django-proxy==1.2.1 +channels-redis==3.4.0 +channels==3.0.4 +daphne==3.0.2 +python-daemon==2.3.0 +eventlet==0.33.1 +greenlet==1.1.2 +gunicorn==20.1.0 +celery==5.2.7 +flower==1.0.0 +django-celery-beat==2.3.0 +kombu==5.2.4 +# Auth +python-ldap==3.4.0 +ldap3==2.9.1 +django-radius==1.5.0 +jumpserver-django-oidc-rp==0.3.7.8 +django-cas-ng==4.0.1 +python-cas==1.5.0 +django-auth-ldap==2.2.0 # Cloud req qingcloud-sdk==1.2.12 azure-mgmt-subscription==1.0.0 @@ -127,8 +119,22 @@ tencentcloud-sdk-python==3.0.662 aliyun-python-sdk-core-v3==2.9.1 aliyun-python-sdk-ecs==4.10.1 huaweicloud-sdk-python==1.0.21 +boto3==1.24.12 +botocore==1.27.12 +s3transfer==0.6.0 +kubernetes==21.7.0 # DB requirements +mysqlclient==2.1.0 PyMySQL==1.0.2 cx-Oracle==8.2.1 psycopg2-binary==2.9.1 pymssql==2.1.5 +django-mysql==3.9.0 +django-redis==5.2.0 +python-redis-lock==3.7.0 +redis==4.3.3 +# Debug +ipython==8.4.0 +ForgeryPy3==0.3.1 +django-debug-toolbar==3.5 +Pympler==1.0.1 diff --git a/utils/start_celery_beat.py b/utils/start_celery_beat.py index f56760cfe..f6259ddcc 100644 --- a/utils/start_celery_beat.py +++ b/utils/start_celery_beat.py @@ -25,9 +25,9 @@ params = { 'password': settings.REDIS_PASSWORD, 'ssl': settings.REDIS_USE_SSL, 'ssl_cert_reqs': settings.REDIS_SSL_REQUIRED, - 'ssl_keyfile': settings.REDIS_SSL_KEYFILE, - 'ssl_certfile': settings.REDIS_SSL_CERTFILE, - 'ssl_ca_certs': settings.REDIS_SSL_CA_CERTS + 'ssl_keyfile': settings.REDIS_SSL_KEY, + 'ssl_certfile': settings.REDIS_SSL_CERT, + 'ssl_ca_certs': settings.REDIS_SSL_CA } print("Pamras: ", params) redis = Redis(**params)