From d04b90b8e86305f1364e858e76d38125b93ded7b Mon Sep 17 00:00:00 2001 From: Bai Date: Thu, 21 Jan 2021 23:43:39 +0800 Subject: [PATCH 01/71] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9copyright=20201?= =?UTF-8?q?4-2021?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/jumpserver/context_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/jumpserver/context_processor.py b/apps/jumpserver/context_processor.py index 7fdc7eab2..48e2a740a 100644 --- a/apps/jumpserver/context_processor.py +++ b/apps/jumpserver/context_processor.py @@ -16,7 +16,7 @@ def jumpserver_processor(request): 'LOGIN_CAS_LOGO_URL': static('img/login_cas_logo.png'), 'JMS_TITLE': 'JumpServer', 'VERSION': settings.VERSION, - 'COPYRIGHT': 'FIT2CLOUD 飞致云' + ' © 2014-2020', + 'COPYRIGHT': 'FIT2CLOUD 飞致云' + ' © 2014-2021', 'SECURITY_COMMAND_EXECUTION': settings.SECURITY_COMMAND_EXECUTION, 'SECURITY_MFA_VERIFY_TTL': settings.SECURITY_MFA_VERIFY_TTL, 'FORCE_SCRIPT_NAME': settings.FORCE_SCRIPT_NAME, From efb9f48c6f09189b93302e609237e213fb72670a Mon Sep 17 00:00:00 2001 From: Bai Date: Fri, 22 Jan 2021 15:57:29 +0800 Subject: [PATCH 02/71] =?UTF-8?q?perf:=20=E5=88=A0=E9=99=A4`pycryptodome`?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=E5=8C=85=E5=AE=89=E8=A3=85(=E5=9B=A0?= =?UTF-8?q?=E4=B8=BA`pycryptodome`=E5=92=8C`pycrypto`=E5=AE=89=E8=A3=85?= =?UTF-8?q?=E5=8C=85=E7=9B=AE=E5=BD=95=E5=86=B2=E7=AA=81);=E5=8F=AA?= =?UTF-8?q?=E5=AE=89=E8=A3=85=20`pycryptodomex`=E4=BE=9D=E8=B5=96=E5=8C=85?= =?UTF-8?q?;=20=E4=BF=AE=E6=94=B9=20`from=20crypto`=20=E4=B8=BA=20`from=20?= =?UTF-8?q?cryptodome`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/utils.py | 6 +++--- apps/common/utils/crypto.py | 6 +++--- requirements/requirements.txt | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/authentication/utils.py b/apps/authentication/utils.py index f6750a73d..f8571d6d7 100644 --- a/apps/authentication/utils.py +++ b/apps/authentication/utils.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # import base64 -from Crypto.PublicKey import RSA -from Crypto.Cipher import PKCS1_v1_5 -from Crypto import Random +from Cryptodome.PublicKey import RSA +from Cryptodome.Cipher import PKCS1_v1_5 +from Cryptodome import Random from common.utils import get_logger diff --git a/apps/common/utils/crypto.py b/apps/common/utils/crypto.py index 6d8d876ef..744e6c367 100644 --- a/apps/common/utils/crypto.py +++ b/apps/common/utils/crypto.py @@ -1,7 +1,7 @@ import base64 -from Crypto.Cipher import AES -from Crypto.Util.Padding import pad -from Crypto.Random import get_random_bytes +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import pad +from Cryptodome.Random import get_random_bytes from gmssl.sm4 import CryptSM4, SM4_ENCRYPT, SM4_DECRYPT from django.conf import settings diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 9a3eeb1b9..7c6771c77 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -53,7 +53,6 @@ Pillow==7.1.0 pyasn1==0.4.8 pycparser==2.19 pycrypto==2.6.1 -pycryptodome==3.9.9 pycryptodomex==3.9.9 pyotp==2.2.6 PyNaCl==1.2.1 From 351d4d81230c5a2663da1bbbdc0697346017f8e7 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Mon, 25 Jan 2021 19:34:41 +0800 Subject: [PATCH 03/71] =?UTF-8?q?refactor(celery):=20=E9=87=8D=E6=9E=84cel?= =?UTF-8?q?ery=EF=BC=8C=E4=BD=BF=E7=94=A8=20threads=20=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=EF=BC=8C=E9=81=BF=E5=85=8D=20=E5=8D=A0=E7=94=A8=E5=A4=AA?= =?UTF-8?q?=E5=A4=9A=E5=86=85=E5=AD=98=20(#5525)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(celery): 重构celery，使用 threads 模型，避免 占用太多内存 * fix: 修复无法关闭fd的bug Co-authored-by: ibuler --- apps/ops/celery/__init__.py | 5 --- apps/ops/celery/logger.py | 65 ++++++++++++++++++++++++++++++- apps/ops/celery/signal_handler.py | 7 +--- jms | 26 ++----------- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/apps/ops/celery/__init__.py b/apps/ops/celery/__init__.py index 0ded6bc52..cb7bdcb88 100644 --- a/apps/ops/celery/__init__.py +++ b/apps/ops/celery/__init__.py @@ -4,7 +4,6 @@ import os from kombu import Exchange, Queue from celery import Celery -from celery.schedules import crontab # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'jumpserver.settings') @@ -20,11 +19,7 @@ configs = {k: v for k, v in settings.__dict__.items() if k.startswith('CELERY')} configs["CELERY_QUEUES"] = [ Queue("celery", Exchange("celery"), routing_key="celery"), Queue("ansible", Exchange("ansible"), routing_key="ansible"), - Queue("celery_node_tree", Exchange("celery_node_tree"), routing_key="celery_node_tree") ] -configs["CELERY_ROUTES"] = { - "ops.tasks.run_ansible_task": {'exchange': 'ansible', 'routing_key': 'ansible'}, -} app.namespace = 'CELERY' app.conf.update(configs) diff --git a/apps/ops/celery/logger.py b/apps/ops/celery/logger.py index 1dd517200..46b7e626c 100644 --- a/apps/ops/celery/logger.py +++ b/apps/ops/celery/logger.py @@ -1,4 +1,5 @@ from logging import StreamHandler +from threading import get_ident from django.conf import settings from celery import current_task @@ -123,6 +124,32 @@ class CeleryTaskLoggerHandler(StreamHandler): pass +class CeleryThreadingLoggerHandler(CeleryTaskLoggerHandler): + @staticmethod + def get_current_thread_id(): + return str(get_ident()) + + def emit(self, record): + thread_id = self.get_current_thread_id() + try: + self.write_thread_task_log(thread_id, record) + self.flush() + except ValueError: + self.handleError(record) + + def write_thread_task_log(self, thread_id, msg): + pass + + def handle_task_start(self, task_id): + pass + + def handle_task_end(self, task_id): + pass + + def handleError(self, record) -> None: + pass + + class CeleryTaskMQLoggerHandler(CeleryTaskLoggerHandler): def __init__(self): self.producer = CeleryLoggerProducer() @@ -137,9 +164,9 @@ class CeleryTaskMQLoggerHandler(CeleryTaskLoggerHandler): class CeleryTaskFileHandler(CeleryTaskLoggerHandler): - def __init__(self): + def __init__(self, *args, **kwargs): self.f = None - super().__init__(stream=None) + super().__init__(*args, **kwargs) def emit(self, record): msg = self.format(record) @@ -158,3 +185,37 @@ class CeleryTaskFileHandler(CeleryTaskLoggerHandler): def handle_task_end(self, task_id): self.f and self.f.close() + + +class CeleryThreadTaskFileHandler(CeleryThreadingLoggerHandler): + def __init__(self, *args, **kwargs): + self.thread_id_fd_mapper = {} + self.task_id_thread_id_mapper = {} + super().__init__(*args, **kwargs) + + def write_thread_task_log(self, thread_id, record): + f = self.thread_id_fd_mapper.get(thread_id, None) + if not f: + raise ValueError('Not found thread task file') + msg = self.format(record) + f.write(msg) + f.write(self.terminator) + f.flush() + + def flush(self): + for f in self.thread_id_fd_mapper.values(): + f.flush() + + def handle_task_start(self, task_id): + log_path = get_celery_task_log_path(task_id) + thread_id = self.get_current_thread_id() + self.task_id_thread_id_mapper[task_id] = thread_id + f = open(log_path, 'a') + self.thread_id_fd_mapper[thread_id] = f + + def handle_task_end(self, task_id): + ident_id = self.task_id_thread_id_mapper.get(task_id, '') + f = self.thread_id_fd_mapper.pop(ident_id, None) + if f and not f.closed: + f.close() + self.task_id_thread_id_mapper.pop(task_id, None) diff --git a/apps/ops/celery/signal_handler.py b/apps/ops/celery/signal_handler.py index 5d5fc4227..5ca29f7ba 100644 --- a/apps/ops/celery/signal_handler.py +++ b/apps/ops/celery/signal_handler.py @@ -1,20 +1,17 @@ # -*- coding: utf-8 -*- # import logging -from django.dispatch import receiver from django.core.cache import cache from celery import subtask from celery.signals import ( worker_ready, worker_shutdown, after_setup_logger ) -from kombu.utils.encoding import safe_str from django_celery_beat.models import PeriodicTask from common.utils import get_logger -from common.signals import django_ready from .decorator import get_after_app_ready_tasks, get_after_app_shutdown_clean_tasks -from .logger import CeleryTaskFileHandler +from .logger import CeleryThreadTaskFileHandler logger = get_logger(__file__) safe_str = lambda x: x @@ -47,7 +44,7 @@ def after_app_shutdown_periodic_tasks(sender=None, **kwargs): def add_celery_logger_handler(sender=None, logger=None, loglevel=None, format=None, **kwargs): if not logger: return - task_handler = CeleryTaskFileHandler() + task_handler = CeleryThreadTaskFileHandler() task_handler.setLevel(loglevel) formatter = logging.Formatter(format) task_handler.setFormatter(formatter) diff --git a/jms b/jms index da671373c..8ba32460f 100755 --- a/jms +++ b/jms @@ -169,8 +169,7 @@ def is_running(s, unlink=True): def parse_service(s): web_services = ['gunicorn', 'flower', 'daphne'] celery_services = [ - "celery_ansible", "celery_default", "celery_node_tree", - "celery_check_asset_perm_expired", "celery_heavy_tasks" + "celery_ansible", "celery_default" ] task_services = celery_services + ['beat'] all_services = web_services + task_services @@ -227,27 +226,12 @@ def get_start_daphne_kwargs(): def get_start_celery_ansible_kwargs(): print("\n- Start Celery as Distributed Task Queue: Ansible") - return get_start_worker_kwargs('ansible', 4) + return get_start_worker_kwargs('ansible', 10) def get_start_celery_default_kwargs(): print("\n- Start Celery as Distributed Task Queue: Celery") - return get_start_worker_kwargs('celery', 4) - - -def get_start_celery_node_tree_kwargs(): - print("\n- Start Celery as Distributed Task Queue: NodeTree") - return get_start_worker_kwargs('node_tree', 2) - - -def get_start_celery_heavy_tasks_kwargs(): - print("\n- Start Celery as Distributed Task Queue: HeavyTasks") - return get_start_worker_kwargs('celery_heavy_tasks', 1) - - -def get_start_celery_check_asset_perm_expired_kwargs(): - print("\n- Start Celery as Distributed Task Queue: CheckAseetPermissionExpired") - return get_start_worker_kwargs('celery_check_asset_perm_expired', 1) + return get_start_worker_kwargs('celery', 10) def get_start_worker_kwargs(queue, num): @@ -263,6 +247,7 @@ def get_start_worker_kwargs(queue, num): cmd = [ 'celery', 'worker', + '-P', 'threads', '-A', 'ops', '-l', 'INFO', '-c', str(num), @@ -385,9 +370,6 @@ def start_service(s): "gunicorn": get_start_gunicorn_kwargs, "celery_ansible": get_start_celery_ansible_kwargs, "celery_default": get_start_celery_default_kwargs, - "celery_node_tree": get_start_celery_node_tree_kwargs, - "celery_heavy_tasks": get_start_celery_heavy_tasks_kwargs, - "celery_check_asset_perm_expired": get_start_celery_check_asset_perm_expired_kwargs, "beat": get_start_beat_kwargs, "flower": get_start_flower_kwargs, "daphne": get_start_daphne_kwargs, From d363118911ba297439809f9bb3bc4d1e05c9bc28 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 26 Jan 2021 17:54:12 +0800 Subject: [PATCH 04/71] =?UTF-8?q?perf(settings):=20=E4=BC=98=E5=8C=96setti?= =?UTF-8?q?ngs=E9=85=8D=E7=BD=AE=20(#5515)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * stash * perf: 优化 动态seting * perf(settings): 优化settings配置 * perf: 完成终端和安全setting * perf: 修改翻译 * perf: 去掉其他位置的DYNAMIC * perf: 还原回来原来的一些代码 * perf: 优化ldap * perf: 移除dynmic config * perf: 去掉debug消息 * perf: 优化 refresh 命名 Co-authored-by: ibuler --- apps/common/local.py | 35 +-- apps/common/signals_handlers.py | 18 -- apps/common/utils/connection.py | 27 ++ apps/jumpserver/conf.py | 96 ------- apps/jumpserver/const.py | 3 +- apps/jumpserver/settings/auth.py | 33 ++- apps/jumpserver/settings/base.py | 20 +- apps/jumpserver/settings/custom.py | 75 +++--- apps/jumpserver/settings/libs.py | 5 + apps/locale/zh/LC_MESSAGES/django.mo | Bin 64146 -> 69926 bytes apps/locale/zh/LC_MESSAGES/django.po | 371 ++++++++++++++++++++++---- apps/settings/api/__init__.py | 2 + apps/settings/api/common.py | 189 +++++++++++++ apps/settings/{api.py => api/ldap.py} | 117 +------- apps/settings/models.py | 71 +++-- apps/settings/serializers/settings.py | 229 ++++++++++------ apps/settings/signals_handler.py | 54 +++- apps/users/models/user.py | 3 +- 18 files changed, 858 insertions(+), 490 deletions(-) create mode 100644 apps/common/utils/connection.py create mode 100644 apps/settings/api/__init__.py create mode 100644 apps/settings/api/common.py rename apps/settings/{api.py => api/ldap.py} (58%) diff --git a/apps/common/local.py b/apps/common/local.py index e075d29ff..d0299b43e 100644 --- a/apps/common/local.py +++ b/apps/common/local.py @@ -1,40 +1,7 @@ -# -*- coding: utf-8 -*- -# -from jumpserver.const import DYNAMIC -from werkzeug.local import Local, LocalProxy +from werkzeug.local import Local thread_local = Local() def _find(attr): return getattr(thread_local, attr, None) - - -class _Settings: - pass - - -def get_dynamic_cfg_from_thread_local(): - KEY = 'dynamic_config' - - try: - cfg = getattr(thread_local, KEY) - except AttributeError: - cfg = _Settings() - setattr(thread_local, KEY, cfg) - - return cfg - - -class DynamicDefaultLocalProxy(LocalProxy): - def __getattr__(self, item): - try: - value = super().__getattr__(item) - except AttributeError: - value = getattr(DYNAMIC, item)() - setattr(self, item, value) - - return value - - -LOCAL_DYNAMIC_SETTINGS = DynamicDefaultLocalProxy(get_dynamic_cfg_from_thread_local) diff --git a/apps/common/signals_handlers.py b/apps/common/signals_handlers.py index d1cf1c7fa..0cb3a04ad 100644 --- a/apps/common/signals_handlers.py +++ b/apps/common/signals_handlers.py @@ -5,16 +5,12 @@ import os import logging from collections import defaultdict from django.conf import settings -from django.dispatch import receiver from django.core.signals import request_finished from django.db import connection -from django.conf import LazySettings -from django.db.utils import ProgrammingError, OperationalError from jumpserver.utils import get_current_request from .local import thread_local -from .signals import django_ready pattern = re.compile(r'FROM `(\w+)`') logger = logging.getLogger("jumpserver.common") @@ -74,17 +70,3 @@ if settings.DEBUG and DEBUG_DB: request_finished.connect(on_request_finished_logging_db_query) else: request_finished.connect(on_request_finished_release_local) - - -@receiver(django_ready) -def monkey_patch_settings(sender, **kwargs): - def monkey_patch_getattr(self, name): - val = getattr(self._wrapped, name) - if callable(val): - val = val() - return val - - try: - LazySettings.__getattr__ = monkey_patch_getattr - except (ProgrammingError, OperationalError): - pass diff --git a/apps/common/utils/connection.py b/apps/common/utils/connection.py new file mode 100644 index 000000000..9bdf39628 --- /dev/null +++ b/apps/common/utils/connection.py @@ -0,0 +1,27 @@ +import redis +from django.conf import settings + + +def get_redis_client(db): + rc = redis.StrictRedis( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, + db=db + ) + return rc + + +class RedisPubSub: + def __init__(self, ch, db=10): + self.ch = ch + self.redis = get_redis_client(db) + + def subscribe(self): + ps = self.redis.pubsub() + ps.subscribe(self.ch) + return ps + + def publish(self, data): + self.redis.publish(self.ch, data) + return True diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 602916cb6..9989103f8 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -426,98 +426,6 @@ class Config(dict): return self.get(item) -class DynamicConfig: - def __init__(self, static_config): - self.static_config = static_config - self.db_setting = None - - def __getitem__(self, item): - return self.dynamic(item) - - def __getattr__(self, item): - return self.dynamic(item) - - def dynamic(self, item): - return lambda: self.get(item) - - def LOGIN_URL(self): - return self.get('LOGIN_URL') - - def AUTHENTICATION_BACKENDS(self): - backends = [ - 'authentication.backends.pubkey.PublicKeyAuthBackend', - 'django.contrib.auth.backends.ModelBackend', - ] - if self.get('AUTH_LDAP'): - backends.insert(0, 'authentication.backends.ldap.LDAPAuthorizationBackend') - if self.static_config.get('AUTH_CAS'): - backends.insert(0, 'authentication.backends.cas.CASBackend') - if self.static_config.get('AUTH_OPENID'): - backends.insert(0, 'jms_oidc_rp.backends.OIDCAuthPasswordBackend') - backends.insert(0, 'jms_oidc_rp.backends.OIDCAuthCodeBackend') - if self.static_config.get('AUTH_RADIUS'): - backends.insert(0, 'authentication.backends.radius.RadiusBackend') - if self.static_config.get('AUTH_SSO'): - backends.insert(0, 'authentication.backends.api.SSOAuthentication') - return backends - - def XPACK_LICENSE_IS_VALID(self): - if not HAS_XPACK: - return False - try: - from xpack.plugins.license.models import License - return License.has_valid_license() - except: - return False - - def XPACK_INTERFACE_LOGIN_TITLE(self): - default_title = _('Welcome to the JumpServer open source fortress') - if not HAS_XPACK: - return default_title - try: - from xpack.plugins.interface.models import Interface - return Interface.get_login_title() - except: - return default_title - - def LOGO_URLS(self): - logo_urls = {'logo_logout': static('img/logo.png'), - 'logo_index': static('img/logo_text.png'), - 'login_image': static('img/login_image.png'), - 'favicon': static('img/facio.ico')} - if not HAS_XPACK: - return logo_urls - try: - from xpack.plugins.interface.models import Interface - obj = Interface.interface() - if obj: - if obj.logo_logout: - logo_urls.update({'logo_logout': obj.logo_logout.url}) - if obj.logo_index: - logo_urls.update({'logo_index': obj.logo_index.url}) - if obj.login_image: - logo_urls.update({'login_image': obj.login_image.url}) - if obj.favicon: - logo_urls.update({'favicon': obj.favicon.url}) - except: - pass - return logo_urls - - def get_from_db(self, item): - if self.db_setting is not None: - value = self.db_setting.get(item) - if value is not None: - return value - return None - - def get(self, item): - # 先从数据库中获取 - value = self.get_from_db(item) - if value is not None: - return value - return self.static_config.get(item) - - class ConfigManager: config_class = Config @@ -694,7 +602,3 @@ class ConfigManager: # 对config进行兼容处理 config.compatible() return config - - @classmethod - def get_dynamic_config(cls, config): - return DynamicConfig(config) diff --git a/apps/jumpserver/const.py b/apps/jumpserver/const.py index c2889870e..c4607808f 100644 --- a/apps/jumpserver/const.py +++ b/apps/jumpserver/const.py @@ -4,12 +4,11 @@ import os from .conf import ConfigManager -__all__ = ['BASE_DIR', 'PROJECT_DIR', 'VERSION', 'CONFIG', 'DYNAMIC'] +__all__ = ['BASE_DIR', 'PROJECT_DIR', 'VERSION', 'CONFIG'] BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PROJECT_DIR = os.path.dirname(BASE_DIR) VERSION = '2.0.0' CONFIG = ConfigManager.load_user_config() -DYNAMIC = ConfigManager.get_dynamic_config(CONFIG) diff --git a/apps/jumpserver/settings/auth.py b/apps/jumpserver/settings/auth.py index 4430aae2f..b0190e8a6 100644 --- a/apps/jumpserver/settings/auth.py +++ b/apps/jumpserver/settings/auth.py @@ -3,21 +3,21 @@ import os import ldap -from ..const import CONFIG, DYNAMIC, PROJECT_DIR +from ..const import CONFIG, PROJECT_DIR # OTP settings OTP_ISSUER_NAME = CONFIG.OTP_ISSUER_NAME OTP_VALID_WINDOW = CONFIG.OTP_VALID_WINDOW # Auth LDAP settings -AUTH_LDAP = DYNAMIC.AUTH_LDAP -AUTH_LDAP_SERVER_URI = DYNAMIC.AUTH_LDAP_SERVER_URI -AUTH_LDAP_BIND_DN = DYNAMIC.AUTH_LDAP_BIND_DN -AUTH_LDAP_BIND_PASSWORD = DYNAMIC.AUTH_LDAP_BIND_PASSWORD -AUTH_LDAP_SEARCH_OU = DYNAMIC.AUTH_LDAP_SEARCH_OU -AUTH_LDAP_SEARCH_FILTER = DYNAMIC.AUTH_LDAP_SEARCH_FILTER -AUTH_LDAP_START_TLS = DYNAMIC.AUTH_LDAP_START_TLS -AUTH_LDAP_USER_ATTR_MAP = DYNAMIC.AUTH_LDAP_USER_ATTR_MAP +AUTH_LDAP = CONFIG.AUTH_LDAP +AUTH_LDAP_SERVER_URI = CONFIG.AUTH_LDAP_SERVER_URI +AUTH_LDAP_BIND_DN = CONFIG.AUTH_LDAP_BIND_DN +AUTH_LDAP_BIND_PASSWORD = CONFIG.AUTH_LDAP_BIND_PASSWORD +AUTH_LDAP_SEARCH_OU = CONFIG.AUTH_LDAP_SEARCH_OU +AUTH_LDAP_SEARCH_FILTER = CONFIG.AUTH_LDAP_SEARCH_FILTER +AUTH_LDAP_START_TLS = CONFIG.AUTH_LDAP_START_TLS +AUTH_LDAP_USER_ATTR_MAP = CONFIG.AUTH_LDAP_USER_ATTR_MAP AUTH_LDAP_USER_QUERY_FIELD = 'username' AUTH_LDAP_GLOBAL_OPTIONS = { ldap.OPT_X_TLS_REQUIRE_CERT: ldap.OPT_X_TLS_NEVER, @@ -105,4 +105,17 @@ TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION LOGIN_CONFIRM_ENABLE = CONFIG.LOGIN_CONFIRM_ENABLE OTP_IN_RADIUS = CONFIG.OTP_IN_RADIUS -AUTHENTICATION_BACKENDS = DYNAMIC.AUTHENTICATION_BACKENDS +AUTHENTICATION_BACKENDS = [ + 'authentication.backends.pubkey.PublicKeyAuthBackend', + 'django.contrib.auth.backends.ModelBackend', +] + +if AUTH_CAS: + AUTHENTICATION_BACKENDS.insert(0, 'authentication.backends.cas.CASBackend') +if AUTH_OPENID: + AUTHENTICATION_BACKENDS.insert(0, 'jms_oidc_rp.backends.OIDCAuthPasswordBackend') + AUTHENTICATION_BACKENDS.insert(0, 'jms_oidc_rp.backends.OIDCAuthCodeBackend') +if AUTH_RADIUS: + AUTHENTICATION_BACKENDS.insert(0, 'authentication.backends.radius.RadiusBackend') +if AUTH_SSO: + AUTHENTICATION_BACKENDS.insert(0, 'authentication.backends.api.SSOAuthentication') diff --git a/apps/jumpserver/settings/base.py b/apps/jumpserver/settings/base.py index 2a6291751..bfb956bdc 100644 --- a/apps/jumpserver/settings/base.py +++ b/apps/jumpserver/settings/base.py @@ -3,7 +3,7 @@ import os from django.urls import reverse_lazy from .. import const -from ..const import CONFIG, DYNAMIC +from ..const import CONFIG # Build paths inside the project like this: os.path.join(BASE_DIR, ...) VERSION = const.VERSION @@ -23,7 +23,7 @@ BOOTSTRAP_TOKEN = CONFIG.BOOTSTRAP_TOKEN DEBUG = CONFIG.DEBUG # Absolute url for some case, for example email link -SITE_URL = DYNAMIC.SITE_URL +SITE_URL = CONFIG.SITE_URL # LOG LEVEL LOG_LEVEL = CONFIG.LOG_LEVEL @@ -216,14 +216,14 @@ MEDIA_ROOT = os.path.join(PROJECT_DIR, 'data', 'media').replace('\\', '/') + '/' FIXTURE_DIRS = [os.path.join(BASE_DIR, 'fixtures'), ] # Email config -EMAIL_HOST = DYNAMIC.EMAIL_HOST -EMAIL_PORT = DYNAMIC.EMAIL_PORT -EMAIL_HOST_USER = DYNAMIC.EMAIL_HOST_USER -EMAIL_HOST_PASSWORD = DYNAMIC.EMAIL_HOST_PASSWORD -EMAIL_FROM = DYNAMIC.EMAIL_FROM -EMAIL_RECIPIENT = DYNAMIC.EMAIL_RECIPIENT -EMAIL_USE_SSL = DYNAMIC.EMAIL_USE_SSL -EMAIL_USE_TLS = DYNAMIC.EMAIL_USE_TLS +EMAIL_HOST = CONFIG.EMAIL_HOST +EMAIL_PORT = CONFIG.EMAIL_PORT +EMAIL_HOST_USER = CONFIG.EMAIL_HOST_USER +EMAIL_HOST_PASSWORD = CONFIG.EMAIL_HOST_PASSWORD +EMAIL_FROM = CONFIG.EMAIL_FROM +EMAIL_RECIPIENT = CONFIG.EMAIL_RECIPIENT +EMAIL_USE_SSL = CONFIG.EMAIL_USE_SSL +EMAIL_USE_TLS = CONFIG.EMAIL_USE_TLS # Custom User Auth model diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index b1f2e1d41..95c1fce16 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -from ..const import CONFIG, DYNAMIC +from ..const import CONFIG # Storage settings COMMAND_STORAGE = { @@ -11,7 +11,7 @@ DEFAULT_TERMINAL_COMMAND_STORAGE = { "TYPE": "server", }, } -TERMINAL_COMMAND_STORAGE = DYNAMIC.TERMINAL_COMMAND_STORAGE or {} +TERMINAL_COMMAND_STORAGE = CONFIG.TERMINAL_COMMAND_STORAGE or {} # Server 类型的录像存储 SERVER_REPLAY_STORAGE = CONFIG.SERVER_REPLAY_STORAGE @@ -28,20 +28,20 @@ DEFAULT_TERMINAL_REPLAY_STORAGE = { "TYPE": "server", }, } -TERMINAL_REPLAY_STORAGE = DYNAMIC.TERMINAL_REPLAY_STORAGE +TERMINAL_REPLAY_STORAGE = CONFIG.TERMINAL_REPLAY_STORAGE # Security settings -SECURITY_MFA_AUTH = DYNAMIC.SECURITY_MFA_AUTH -SECURITY_COMMAND_EXECUTION = DYNAMIC.SECURITY_COMMAND_EXECUTION -SECURITY_LOGIN_LIMIT_COUNT = DYNAMIC.SECURITY_LOGIN_LIMIT_COUNT -SECURITY_LOGIN_LIMIT_TIME = DYNAMIC.SECURITY_LOGIN_LIMIT_TIME # Unit: minute -SECURITY_MAX_IDLE_TIME = DYNAMIC.SECURITY_MAX_IDLE_TIME # Unit: minute -SECURITY_PASSWORD_EXPIRATION_TIME = DYNAMIC.SECURITY_PASSWORD_EXPIRATION_TIME # Unit: day -SECURITY_PASSWORD_MIN_LENGTH = DYNAMIC.SECURITY_PASSWORD_MIN_LENGTH # Unit: bit -SECURITY_PASSWORD_UPPER_CASE = DYNAMIC.SECURITY_PASSWORD_UPPER_CASE -SECURITY_PASSWORD_LOWER_CASE = DYNAMIC.SECURITY_PASSWORD_LOWER_CASE -SECURITY_PASSWORD_NUMBER = DYNAMIC.SECURITY_PASSWORD_NUMBER -SECURITY_PASSWORD_SPECIAL_CHAR = DYNAMIC.SECURITY_PASSWORD_SPECIAL_CHAR +SECURITY_MFA_AUTH = CONFIG.SECURITY_MFA_AUTH +SECURITY_COMMAND_EXECUTION = CONFIG.SECURITY_COMMAND_EXECUTION +SECURITY_LOGIN_LIMIT_COUNT = CONFIG.SECURITY_LOGIN_LIMIT_COUNT +SECURITY_LOGIN_LIMIT_TIME = CONFIG.SECURITY_LOGIN_LIMIT_TIME # Unit: minute +SECURITY_MAX_IDLE_TIME = CONFIG.SECURITY_MAX_IDLE_TIME # Unit: minute +SECURITY_PASSWORD_EXPIRATION_TIME = CONFIG.SECURITY_PASSWORD_EXPIRATION_TIME # Unit: day +SECURITY_PASSWORD_MIN_LENGTH = CONFIG.SECURITY_PASSWORD_MIN_LENGTH # Unit: bit +SECURITY_PASSWORD_UPPER_CASE = CONFIG.SECURITY_PASSWORD_UPPER_CASE +SECURITY_PASSWORD_LOWER_CASE = CONFIG.SECURITY_PASSWORD_LOWER_CASE +SECURITY_PASSWORD_NUMBER = CONFIG.SECURITY_PASSWORD_NUMBER +SECURITY_PASSWORD_SPECIAL_CHAR = CONFIG.SECURITY_PASSWORD_SPECIAL_CHAR SECURITY_PASSWORD_RULES = [ 'SECURITY_PASSWORD_MIN_LENGTH', 'SECURITY_PASSWORD_UPPER_CASE', @@ -51,24 +51,24 @@ SECURITY_PASSWORD_RULES = [ ] SECURITY_MFA_VERIFY_TTL = CONFIG.SECURITY_MFA_VERIFY_TTL SECURITY_VIEW_AUTH_NEED_MFA = CONFIG.SECURITY_VIEW_AUTH_NEED_MFA -SECURITY_SERVICE_ACCOUNT_REGISTRATION = DYNAMIC.SECURITY_SERVICE_ACCOUNT_REGISTRATION +SECURITY_SERVICE_ACCOUNT_REGISTRATION = CONFIG.SECURITY_SERVICE_ACCOUNT_REGISTRATION SECURITY_LOGIN_CAPTCHA_ENABLED = CONFIG.SECURITY_LOGIN_CAPTCHA_ENABLED SECURITY_LOGIN_CHALLENGE_ENABLED = CONFIG.SECURITY_LOGIN_CHALLENGE_ENABLED SECURITY_DATA_CRYPTO_ALGO = CONFIG.SECURITY_DATA_CRYPTO_ALGO -SECURITY_INSECURE_COMMAND = DYNAMIC.SECURITY_INSECURE_COMMAND +SECURITY_INSECURE_COMMAND = CONFIG.SECURITY_INSECURE_COMMAND SECURITY_INSECURE_COMMAND_LEVEL = CONFIG.SECURITY_INSECURE_COMMAND_LEVEL -SECURITY_INSECURE_COMMAND_EMAIL_RECEIVER = DYNAMIC.SECURITY_INSECURE_COMMAND_EMAIL_RECEIVER +SECURITY_INSECURE_COMMAND_EMAIL_RECEIVER = CONFIG.SECURITY_INSECURE_COMMAND_EMAIL_RECEIVER # Terminal other setting -TERMINAL_PASSWORD_AUTH = DYNAMIC.TERMINAL_PASSWORD_AUTH -TERMINAL_PUBLIC_KEY_AUTH = DYNAMIC.TERMINAL_PUBLIC_KEY_AUTH -TERMINAL_HEARTBEAT_INTERVAL = DYNAMIC.TERMINAL_HEARTBEAT_INTERVAL -TERMINAL_ASSET_LIST_SORT_BY = DYNAMIC.TERMINAL_ASSET_LIST_SORT_BY -TERMINAL_ASSET_LIST_PAGE_SIZE = DYNAMIC.TERMINAL_ASSET_LIST_PAGE_SIZE -TERMINAL_SESSION_KEEP_DURATION = DYNAMIC.TERMINAL_SESSION_KEEP_DURATION -TERMINAL_HOST_KEY = DYNAMIC.TERMINAL_HOST_KEY -TERMINAL_HEADER_TITLE = DYNAMIC.TERMINAL_HEADER_TITLE -TERMINAL_TELNET_REGEX = DYNAMIC.TERMINAL_TELNET_REGEX +TERMINAL_PASSWORD_AUTH = CONFIG.TERMINAL_PASSWORD_AUTH +TERMINAL_PUBLIC_KEY_AUTH = CONFIG.TERMINAL_PUBLIC_KEY_AUTH +TERMINAL_HEARTBEAT_INTERVAL = CONFIG.TERMINAL_HEARTBEAT_INTERVAL +TERMINAL_ASSET_LIST_SORT_BY = CONFIG.TERMINAL_ASSET_LIST_SORT_BY +TERMINAL_ASSET_LIST_PAGE_SIZE = CONFIG.TERMINAL_ASSET_LIST_PAGE_SIZE +TERMINAL_SESSION_KEEP_DURATION = CONFIG.TERMINAL_SESSION_KEEP_DURATION +TERMINAL_HOST_KEY = CONFIG.TERMINAL_HOST_KEY +TERMINAL_HEADER_TITLE = CONFIG.TERMINAL_HEADER_TITLE +TERMINAL_TELNET_REGEX = CONFIG.TERMINAL_TELNET_REGEX # User or user group permission cache time, default 3600 seconds ASSETS_PERM_CACHE_ENABLE = CONFIG.ASSETS_PERM_CACHE_ENABLE @@ -90,32 +90,25 @@ PERIOD_TASK_ENABLED = CONFIG.PERIOD_TASK_ENABLED USER_LOGIN_SINGLE_MACHINE_ENABLED = CONFIG.USER_LOGIN_SINGLE_MACHINE_ENABLED # Email custom content -EMAIL_SUBJECT_PREFIX = DYNAMIC.EMAIL_SUBJECT_PREFIX -EMAIL_SUFFIX = DYNAMIC.EMAIL_SUFFIX -EMAIL_CUSTOM_USER_CREATED_SUBJECT = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_SUBJECT -EMAIL_CUSTOM_USER_CREATED_HONORIFIC = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_HONORIFIC -EMAIL_CUSTOM_USER_CREATED_BODY = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_BODY -EMAIL_CUSTOM_USER_CREATED_SIGNATURE = DYNAMIC.EMAIL_CUSTOM_USER_CREATED_SIGNATURE +EMAIL_SUBJECT_PREFIX = CONFIG.EMAIL_SUBJECT_PREFIX +EMAIL_SUFFIX = CONFIG.EMAIL_SUFFIX +EMAIL_CUSTOM_USER_CREATED_SUBJECT = CONFIG.EMAIL_CUSTOM_USER_CREATED_SUBJECT +EMAIL_CUSTOM_USER_CREATED_HONORIFIC = CONFIG.EMAIL_CUSTOM_USER_CREATED_HONORIFIC +EMAIL_CUSTOM_USER_CREATED_BODY = CONFIG.EMAIL_CUSTOM_USER_CREATED_BODY +EMAIL_CUSTOM_USER_CREATED_SIGNATURE = CONFIG.EMAIL_CUSTOM_USER_CREATED_SIGNATURE DISPLAY_PER_PAGE = CONFIG.DISPLAY_PER_PAGE DEFAULT_EXPIRED_YEARS = 70 -USER_GUIDE_URL = DYNAMIC.USER_GUIDE_URL +USER_GUIDE_URL = CONFIG.USER_GUIDE_URL HTTP_LISTEN_PORT = CONFIG.HTTP_LISTEN_PORT WS_LISTEN_PORT = CONFIG.WS_LISTEN_PORT -LOGIN_LOG_KEEP_DAYS = DYNAMIC.LOGIN_LOG_KEEP_DAYS +LOGIN_LOG_KEEP_DAYS = CONFIG.LOGIN_LOG_KEEP_DAYS TASK_LOG_KEEP_DAYS = CONFIG.TASK_LOG_KEEP_DAYS ORG_CHANGE_TO_URL = CONFIG.ORG_CHANGE_TO_URL WINDOWS_SKIP_ALL_MANUAL_PASSWORD = CONFIG.WINDOWS_SKIP_ALL_MANUAL_PASSWORD AUTH_EXPIRED_SECONDS = 60 * 5 -# XPACK -XPACK_LICENSE_IS_VALID = DYNAMIC.XPACK_LICENSE_IS_VALID - -XPACK_INTERFACE_LOGIN_TITLE = DYNAMIC.XPACK_INTERFACE_LOGIN_TITLE - -LOGO_URLS = DYNAMIC.LOGO_URLS - CHANGE_AUTH_PLAN_SECURE_MODE_ENABLED = CONFIG.CHANGE_AUTH_PLAN_SECURE_MODE_ENABLED DATETIME_DISPLAY_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/apps/jumpserver/settings/libs.py b/apps/jumpserver/settings/libs.py index aac0c4d50..49b3d3b2d 100644 --- a/apps/jumpserver/settings/libs.py +++ b/apps/jumpserver/settings/libs.py @@ -127,3 +127,8 @@ CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO" CELERY_TASK_SOFT_TIME_LIMIT = 3600 ANSIBLE_LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'ansible') + +# +REDIS_HOST = CONFIG.REDIS_HOST +REDIS_PORT = CONFIG.REDIS_PORT +REDIS_PASSWORD = CONFIG.REDIS_PASSWORD diff --git a/apps/locale/zh/LC_MESSAGES/django.mo b/apps/locale/zh/LC_MESSAGES/django.mo index e00cb3d92aa8ae1771b51e17d0a4c3a74d6838e3..ea2503cb993c1688fe3ba25e7386495b773ed7b6 100644 GIT binary patch delta 25617 zcma)^2Y6LQ_pf*8y?0O!9V8SB9Sc$vK~VuIiW-stfslk0sd_@MAxLPUcL514^w7nE z73>Wx~xO-nG`OSu?Yy>>c27;m?IPuP@@iP^rjbkLT6G zo>vF(f7WKS&on`gBT@QO+lR}=i8xBJM=y}PoJB>esufVscUq9IMsuc44UfdI&cb1CjPkLTe_}EjPR}4nO zvM>RbhLd46m;&p;jmB4CUGzLy6P9?|^NPUw@K)Fawt$16+RcQ`;8u7u{d?~r+=Af> zYzc1)^Snl|H~bjJ!waz2P|qt5ZwdFjQm`W|1G^g^H9iG3;AmJGj)PU;JXjpAv-+*D z4E=liB*0Tp4c>vuz(rUNUV>`)yV*k`TxLo@4X7Hd4jV(oxeJzryJ^pLJh1AB(C49hoDrngbiSK*ak+y+u<765xxg& z!E(_qKr^V0T0&*0J!}o{H~V;~4yVFWa1PXf7eg)023S((|3w6iXrDP8hB{8KK?V5I zmjCEX!=0Y!;I0;uaS&J=m2L!jrb+l6uu5M(d2GlV=33UoShMLfMDE~{QUxkYAB?jC3 zy>bZZxE54uZh;zc2dGWe&Ge_B1~d{Xa3WOT$xt(%1GO|Oq2g_XmEm5f3BG3bccA+D z0yfq8{~n<^hU!UNBCspe3?@MZoD9|BY~vEBjyFJcupMe(C!hxYI@A(=03U$ALiykO zjEma~YCsRe(mMYGB*1Ve$5_Z$lQ#}3Rp+2y9$&zF;a{fTJ=XK?L4N|O-72WlvKcmm z+o3Y?q1khwI{p@F(_V#s?bagW+)QtR3e?Eh66*N0g9#xNb$g(u*x z@JpzSRvPazbQe@6AB9T&V5lXF8c+UJF%g4CHWg|BbD->LW?uny<*v8-6HrTX#`F)N z+J6DnE(a>3-<$mpsJI~$oV^sh1HIY=@~^e(g+T#FL#2EI)Cgxm&3rl3+HQmDU?0>> zjzaB$H=$DcC6wQfP;vf(@-H>f9lJ`f8hR*H+^&8Et!-bZ5ywGorb$o%W45;>5X8#zf{dv=Kq2hiEHNoGY4;G!|GU9KHpblC=Iot&`^IlM!X$aIv!>v96 z7D7)pPK8SC45$I8LhYGV#uM;P^ovlLs`sp0sy2{cHT+&@gl$wj4sU|Ro^vT{2sMCE z({D4qBW#8J9@q!Q!Pf8uRBC^OnsEu1Rs*X7HNl&p1{ezKz>YBZ{vUv#T^$D1Akp}& zaR$_JN`+e6L$DS+4VCII%$^U`@lQ}4ddY5+m4|BA4Jv*g*c=XorSSEpA!uz=p*GPX zsL$w4ur%BXZ-)DzQv1H~SE%}0lU+(1L(Q-SR3b^Iu-3Wq=qECDLc6zJFP+=NgY zWH4Xs7|thKjp*3hS>6 zWFrPOd;_Y%hft~g4k~58n(j??ex;xWS`}(wP0ij0YLndqtHXX!_d+xb&K%Z5p8^ZR zjZ?|LI@p3iYrGq3gr7kL%7q%>pHLmvpXPoyYy;)@IMl$0L!FX%sM9kAD&>oy4{n1R z=o?TI`xt6KU;7a>!XJ&lK?VK?7J;RwyHr;&R)g9Lb)Zg5Cs+dxGRDIy=yRb4uoi0I z`=OTLjMWFAGVTAv3>9a%hK-?S(g|v$y`TotAF9D%sEmz)>S%)LsZcXq2IapA-Uhe9 zPoU>~wnFa#m62zR zv!F7w9BOlIGrj_AqJIrF)4!nN7n$c+6{=n1dE~z#LR$=4^T&;0P%{}0HIq1~Ogsm* zROzN~hwAuMs5lp)PD6U30xdbzZO^6zE!{Y`%omcl*)Dgz0A1Ray9P=WWuN8xL* zDXg99KCQYz4KxClgJYmNNQT-2bD#$FJX9vr&A!I$o1xW|H(d`Xdj!S)Avv_DUs&>z_Kz_X6}L-&=XL*ItuC(#K3NFCe$f;6Y3tw zgPM@Hi2Q5hr4TgY8c-ehphnmnR)HO%j$a>mD;x>+a#;yAz&%j@r=Vtf4l3TKPy_qg z^vhQNE7THri^+duSZuKyc?+nK-3Aq?Gt{Q+4)1`EKxJx%*;Alqnhv#udtfv85^M{< zgPKsoC9Yj7sDZYHm02#*d&G-EHgvtDrv!HPWF_Yc~d};}ocXo1vCuFI0x!f%5+h zYVTaK`oCah^yBFo(7AlpKpw@0C)QnP~{MSMa zY#UUZgRlua34Ji%SSsC3q&eiK^n2Y9w2Q-_*3b{zzy+{AdsYd97r!Y!~lylD-~2z$cna4CEQ?u0vF$XbrIA7K|lE%-XrOad_2 z(K)?ge)&YZ>c7?f#~)D!d(P$$G+@VFauTr$X(a4N#l) zZK${xO#cD;(7i2gsp|O=G~>2V9X|xs(O_5%MnVOOw)#o1IQlH(0;m}+gJEzz)BuZY zbs4G!6}K*wy*X6-E=GS(b9lrYhCt0M0cs%gp*Bl8jD#=1=CJHG_b%uF)xjvJV>%XU z#xtNMwhZcc?}E3(x1pA*(2K$NtUrPp+zb_{HPpq@87g2Os0=&_)i4H@gnrl@&Nh7? z)E+tx<@YgEoP1aU{slFlqT3yBf^BsE+aPESN5P_S4lDx~z`Nlpr~v1o>T{r$=3A(N zU4=?%;S4u{(oh4e4oAUyupvx=8qgl7&HFM8{{7$U5-@xO?}FJ-9W>yU(7C?_wuBL| z9!!I~;XcT(P~PyJj&q^T|4!Hlo`!0F33h}Hce$l~3TiJ*fWg22TY#VrHbDh=A8Lg8 zuucdc9#9>X-s3XR3TlRJp!Uk$umT(m%fnGn$9f_x2UDTqt%l0r2B;<9x`+JBAq#^7 zybc@E;b-tN^!BgHYq7OOjdE==cnZ>W*FdH6) z!;i2psBeVw2MinwhW5TZLDWh(EI#Q*xCNF*&wyI<6z@(sJ)bE`b6U_qkpjp>!BQXLd|r)>2E@<^?OhqoHt&AI&Qz3Uf~tz zcN0{cCQ$w@P48s9*Vxn5`@M&q;0=LVijhzqJ`0t~nXoC`0|&#;pmufJSDjxREQUS- zHi46kTVXx)b5QkH;Y08bsEep4X%7DW|15&e{~V}YyERzB#(^5h>rgX&-|XL+{SVWN zoN+U)XuR3j9BOm7hdLGaK}~Rg=_6rP?dk~<-~!`js7-go^tX-KPy_h`Dq!W;UB?ZK zt)XUmCschmsEPG6J;C&;up##K(65ePHHY)Y?~H##)t7z4v4ODzRHphE!=X0kSf~L^ zH_n44(bJ#?w951hsHHjZ2B%t^6Svp&4A_L3FA9P{{<7igJp5}%UJ3i*PsSez$V69jh&3$jeVd7G6bnrs@JDzH{0*wZ24`KM7EtwfLd~cf)Jz9M&1|@Ff-wcE{aUDY+hAL` z8!BUY#*lN0uk}Yz;9G(QMg`UIVdDU(fKNhoJi_crrcZ*Uu+KL8BG?#xh1H*e<bQm3+nN0?V=uE0gqqn?Q2yi0KE?FeQ1Q~tzSi^? zVDS5YFG3R>PQv?OK2!#7eb0@!JyhUcP=WeG&1k6U@uvHYsZjoF;1IYK>Xa3K-^HyD zRp0D=@~=R*V^H9(=J2pN41n70VWuZT`Okrx@p7o8%Y+*68K^jCjpvP*jK3O-eBdTl z@dNT-i3SZZs9`&(z}-wAVjO843)Rsy)90GL4C?f(ff`_j*^ffSJ#9P(b@N>?dj1dH zuT1r!0zYgV0M+nG<0!L_gX%!-RG(^G2^*kqg?c-_W%lA9xwut~wV~S8H~L$d&>7xK zMNiY`L1iW#wuakHKX3XEP&2OgvFo4})XdsK4R9dT1vbL!qoLv@Kn-Aqv-`a?Gc0!o z?*&*FhaF~r8)~zB2KCkoz))D~6F0DSur2xs*ch&X)!-{o?ao7Wm}~sW81ktG&ia=N z3Tz~(26aqt3^n2wX1@n2&;zCqFor=b-6&(calA1Zs@+VeQ?>|dB1dG``TxoszK6BZ z|1`b&XD)CNb0&SK+ws3qMA<^PV^Kl$9we+~w9_zP5r z#m>9eZds@XH^V}(sj)dMirx~IgLlAIu&XfvK8n5pDnq|O#VvWk{Yh#CxE}qM3*^5l z!dn>RaM5_l_@nVR*oFFkU@zF^3-`T{0@cAGs2RTr73UMvzchXamHI!dzS5V@zos8S zGi(giu(h!r)Z6ebSOP}DDli_(ZzfcS^NlNvTc9@AKGQ#i8elHe@%#g-U7fF79Dg$e zIkbj4uU%nfcpt0@ABR4e1a+*Iz|t@SD$pUQ_OC$se`5BlX7|2!ewE?P)cc?&d^aTX zey=Bj0z3t^sp5@u&At(;!2#n*s0_RTwP)Ui8pzki%TV$DvikBDU430+DAc!RhoJ7~ zEry_x#X@y7A1dX`f)4!spm7J3UzX{g8gpR<>^~WcX1flnK@Fq<)Ka#E8gLlYM909k z^zY3<&?Y@m$C1Q2tGzGS(4l#)FLURzEkF z^;gAe3>v@z<7rq7{Tx*4K86}t;XJoF%R=e(jLnU0U=!?j8;8Rd=&7(FJPI|Sub~DQ zqJOiYj;iE4)`l8LePb)AtFp7zCqOlvVD%|byM8%*0Pcm##6MPF{*sGV$Jor+7Aj6x zKY|)OW`<#=kB1t_Y~vEBfo_2MMP-}m9~#d?WilJ8-QUJi-?&UwhiYFBYA-Z3-QNvC z4SJbjAk=`KGR8us%nvo<)lh+U!;8GILoH2d~<$n<>-Y-xc7yH)f^&tcHd$%FT zu@|fdpM+&#GE~PYrf-J|bjb8GPys(QUV+L$x$oQno4_*YEsdRw_d~VoAJkd@F$kK` zbLKGJxDe`?tucKc)BsKy--R0R*HC^%zIU6m7Stwe2{nMnp!~w2`k8L_dD3_ed9ML-R70@Sgd^CS7!08V01 zM{mPh;8##7FLTAMT_dOt?t}7w2rA&?W*=i52Q?r+l;3Qq%q=l}A5@%Euo66T&GB<{ z$cJk1i|JLby3g$Tur&78#?Hq3js0LL>Ys+1;V7tqOf*h6rozhD)1e09&oE&x)QGc; zuR)FQ1FO%2%E)D7iJzQ*J*Yq}q3YYf5%3|?4;arv4JhAu71GY{{f(eqTI6RJuq;%^ zwP0n~0_yJW0@cAVsDb*8^Ni`RD)!B$AAuU!8Pm_gqUh(L;#>^Yv;Kbt1zs+{xYU(` zN@XqB0ycu`u&>oW4HX~`YDTl6&UYHT8}5Vh{~2lkg?@GZC5`o=?44j?o&S3g)S!n| z^o7dEVAI2)UNR$1p9ZU<&xJ+cI;f0ohFbeWQ1Q-~{X?h$=bOFIZ*Jh_VetK56G07q z#@0|B-UD@v21CvC8Pk)YI#>zSZmaPStb=|AYCt(q?S3~F{@u;IEL8g{zuWn*XNH!> zPR9F<{h?Ae6sm(Ls0_`3>LAtX(_vHewNMw>8RNIG2738FT>GX_zY%r+gZx)Ocp8Hm zj)8UIa;TXdh8pSXR{s%HVDC@o-vIVO?G1atm9PW62(?5v{pH>Tx5JL;kHhwG39Jp@ z@tg1s)MhLGH-DN5+rYLk1uD>Kr~!Ry`X%T?_x|JB)q~oEePC-i25QFZVM~|=7s5+W zOY`hMj{XG*)i7)@9)|a$pM%|DEiWXP;-{f9F%oJZW1#}gG%h!LhUrIPP3q5@{Ss9B zKa9mgg1X;#J!aq+uC4g3LEh5hFZL8yo!-Z%|vq)VV2H<-R3YJe|6Ey?RpDgF`$ce&LU zF6#VBLA9?4Ro@U!fGw?lE382O-Y#=EX?(*RK7i`zbEuSGF%~N3GEvFc3TjjKh8lRR zaguR1)IgV*z7uN6kHFyj|2+ha@Jn;JWc<}wqOY6- zuqS^lssRjzijxEtXFOCK|5Sut2n(P({2b~6xdZh8|err1u zdpPnuc!V+!`AD!8EBO=7H)CuD@1Wx0w39t5`>P;5kIOgIq{4nsPZTUpot{Lq?T0!{ zn_(BC)PmJ0JMp`VG8(sAwilcj{P~jkNLgM$mfoAvQ-VEDFu>Kwyt%w0_zpDt%l-llC!4$lavyuO zDeqFQpx)yN{c_h3D--Au?eIM%ifYJbaF#q!rQYh(^Z?W}-&%%3Xu!y2t{UGH( z$g8a5La-BhF-pNFk+$8a+g-r;268^-HtP1lakOcVyb1b0<;QW1CoM3dHv+i>4n^TC zxE0$PN;hOZooTaNuxkEBHKxz2I{hpGF02VN~|Q76;qHS+w_4 z9|>cz-GS|GiXPoNUs8hoH^f12_AehDoI!YFwzzKh`#Qh5CONUns# z>r~uFfI7&7DEDLU2LGn$;Uh8l9E00Tmf=I>LikLe@F^U8K8CTBQIuaP^(cDYg!+A` zqv?IL{*l%}(Vz?Iq_VutM5e#dcTOZ4?HFB{35FFN0CQ`~!c2S~n7)tTc$$a=CjKH3R zT=2Ozfh*KcqoM@7_8dTd4y7jT^t7bI!B&6D(eHhW@hatJtLTpNS>*QE^mIc03jPeA zu|PxN^OSsidYWI*nICuH7f0QATpoHQxhk89<84sIpTWo(({~AS48Mxl$1;$;*=CnPG+)w8}1mO^6jaAli z?Y)u&UTwke!uAC9mGSqPtq$@@>~-OJYkL7b9$CK`F2b)jb$=k2#P&E$Q62Vn*zbh? z&HSi>Qt+9K@@)az1>`1_zpxj>zZuMmZH6`Y9a+!Auq^z>^q`R+%g{GcUNQeStmA=3 z`Rzki^`CIPwQr0Z%>S(z9;LQA4&PHgq?E%}0*4PNk0HNcjiyi+jc*6*Y!lpIGF9Gi z^NWFpu#d);N(?oFa02jf6&mhxRX#PK%vmCxho(WgN-_1S_=k1h3*ca2`1mrOV z+Wdq4ss%hjeJ6B1`zej^yW4zcn9t+*jHT!qtNdr0!w?*XQ*mdoioIYBo-+M6xRx>m zpAr^8Rd<`8}=O`Vm-DvFNk^K!3KF6^aYo zA|H7+@}rbTaC#Eker%PAtDPzuxVpap&lHZ?f_M-%6)>AUb_H1+wxc-U7G>i$9G|Hb;^rUe~*qoAuyxm61ww&K9sjAXc45=O)(MMwA|q7s7f5=TTO_=d&C`w}9#X@gziyTL|#HiR9@7_q7MglqkQN-x> zP+Ilj-781OhL7?^#}1E;@kPZjqE-nKF${vL532eMn|T#9raD0$kAa@ z(F~l}F^OK+_{gxtNXuogCLw9)$jI;;Ylg?iCdJ+0J0dnFHa==tRCs3N=+L6w6f!f9Ty*SEmW}lcI%iHv>>c7C z=Ii@lzlVKsvGIvM*5!`2Z9=c*At5qagYEf0T-vm6_g`H8O@wbWS@$K52qVXdp;~qF z5kny5MP!M5{bQmMJNnp0g_9B^Gv7>lKcuoYL*&1U9u^%LpP0FFY?VT#G&%DAOk{l8 zj`0WYj2a!4Ncc!!Ow#C~?7G-tTG6N&-!KwIhy<@UyV4gIHayao5H%svsR^3j(D9iO z6LuB$Cq*Yl#j$R|9q0B{s4pQdIx5k7Fp0KWz%U|)hs8xDhDG0KCT5NpgSF;y|6lc3@VrlVnSCkzU z85!q`NQw{kn^t7r)C$96!V<|6|1U;u&G_idY4aL{l-92F#l=SsiyD`?A?2n*6{2Gz z6D_YC^T=_TM^pQRl>B!aW)^!st%Sd$uW1qs*@_vkjLm#e!-5MFOzVg+4mm3l8R1Ke zRU}Sa@YIL-qP2wBuP>kuP(moj+`42Va+#25ZVj#o&VBIGklVipx25Or7Zqni=;(_G z8{f>wR@Kf34j>{nGQnLOaS=L(s<@W1>rvg|4ZYs%Uzf2_TxXHv;-c7mkZYCkBg6eCQ7p=;mmGSWDE%lkBrml zv#sge`zNrHu`$u(6*7#M1cxlx|Gyi`#y?;LF~W6k1~2>IN^&$c0FC6|n^C7e(T~O5 zxSHnuk?fdjGyH$Vv3oce<%S@3WnW7`LQ;5mWJ1EQq-fT}4Y5Gvrg|U!&mr37e?#cB zt?Azl4z88fIMmmdnA`)Qy4E?rq5s}`*EUWBH!oY3^K|_(y>^O2T_V{sZ0oS0Ttt~^ z%hN+j_wC!8|IcBhuV3%JX-!tfRq{=8SDJ6Mt~+0!{t0PmE87(6(?9Lh%0B+YMD5i$ zwcy2Vnr?vLP8|_DR=X7k_ikZ*%?s=G{S+FUd<-wg(BKcZ7#!Z~F2UH=3m2YmSbXeg z-^6Fr>a5z|A}XG3;O_1ucShMz!?-eh5wS6mBqxHi$Gt?KQHg19tm<2GIIrd4t(8_| z^@z5%$J`-h$MEV8PA5(uD8URTI732YSUevTgyzMVIcIgnV)ch}ZW0sYI<{&R5ji@x z<@Eu#jEx`Ov3=|Ituxa%ZmjA*ayT$+f6mU-?1S3^3wGu%+7j5fJg{PIAY(z^j*WRc zCwqbElk;{Q^m3N%3@q5}l-#UYxi9X_$=ni{G4Jw`d4Z#8IkS^I!UX1Q2`rdT zRrbNd*@tHYcFYb;-x4^yC~xPc!1T=gr5XQ*Yp&q*nltM__R-@A*FJ!JIkR8LTQ@If z_M!Zlse$K@W@l~8y!Y_*qD=}~oX@rIp}_1EFE{J?z>ekm96EQreU04BlXEiG=cZ=` zjxNqwx<7yEUj4UxQD%t~9SgNd-I+J_Xzuc9`N^~M*BxSld>7@+TEY_W^TuzZ0-r|r zobrcMT$vo$_(Ja1!}&}1xVUKtUizkU{*)crS^J3?m@z%DzDCArdkom zI_##Jn>EcUan4!1J#S}5=I+yni*$4?au)3;PWIu$9$k2DSqyHw+R_wUhs>liw->Ii zH1AG!ErQ=c&NK6cH;0G#1DmJhtX`>iEUOnt-N20b8e*05la~h$9?V~voVR(hwbfz! zFaCTQ{da#3rI(YsAa~>8%SV>GhI#v^=N+HPlIKrP$vwIw=k`78*dsYYVwTvX3m!TmNFtl8nHSWZ&f@vjQoLvyY}I#;h5akIc?2`}WHr6;pQSr*Cz8An^R0 zyo}A6XW#icq;&ABFtBo0U{!M3?sJE`<)<%TFT2$aY}}o*xbvcW7d3pP$WgmVa zFl%Yvx-D$={B`@VvaN2cn7=S^I5l(RyVHt{G|uj z0`sN^7QDzl2+T+iWGv0yzWK(AU8(=Qg1&J#-?X*aRWdJsSg}ayz|I-DYdH-Zy0rg% z`dn$Y*sj$4#m57OH)ke(mQ$z<4*3f=*@6zb@O9yq`gF?MyE%}0zS6svL8v#3(C|8!82NL)zoy>F1M!cGRdE^A^Xs*z{)K-85;|p zW=>ndgMNJ<-msrGZe>5Yok^E|E*y)rat)V4j@z6o_U0_v z#+b8s*5xBBe6GS-+57o37P{k$E1?V4lBhO;jaxO)MZ0q|vvRZiYZx4R5i7&Wo4HL_ zX6p0#$&+)l(p|s*eI7n$PK#^TM{tdNO_|pXpF!7;Vlbu~m)j~jzgf%t>?7OO!H;fN z$I0ZzcH^*(^>3u?gUQ*4SJ(xWx*+@5iooI-?#{?NmKK=4NmtpjojLPo<*b^@PQE_; z>@23eC1=$p7d~g-+`OIs#sBlFa~*Z$j9tGcgL9o{sHdiU| z6DfvKqyf-kthM>+8=I2lz|q}-v=r`_ zoRl4b!)YAnz?3a~P9-GdX3Wk$m=Z``>Rgz{%5AxsyMKrXsd{Y}aN(P%53DLzde?Nf zARksftC(aLVA@kxT9gRBv#Cs*bEUmMuzPh*@(Vf3)&_P>^X%9ZFlA?L(AAqfKWFU$ z#;uDx`(VcZ_T#``|4?z&*$3yaWA%L&`~ve8_zq*YnRg9-cV;)Z7WN(scAu7YG0wK5r4=%->AjMjOYw_UjesG5Z!L1Z4Zl}dvTC`ZP;w}YR zv<1rL{lBv(_wmNtV=R7a&ZT?peNIB#dsi%pGyQdd=X%mOGaasR0gjUvoASr*IF7TR zn6i#DwYuYE_jjCW*ob)hSB^6o!@hQ$f%rY9@?&gG$Ei=hBGHZ$LOi*)<4mA^ULD67 zhFR-6&J|pNCutv1&vBCZJ02&xf#aN}A%@w*u}EXb3B($h5bIz9Y>COS3ueSo<_gR} zd<0YBLuCJse-pm)urtQP;nqGLLs;LLsR}MZ4cLG>frFR`kD&%$w)`#B z$vi}D=nba8z~=5e=`n~n7xGk{!WbWmqvnaiP^^X??W_qE-BEXQ5NhHvsD)->He7}} zy5ralUt>{>Y2o%;h+1$BM&bs{j+ZblIxXFPfvA&C){^s2LZt!;?XVtd=gm<0_Nb#A zXz>`-&gY=sg;l7N+lNW>s3U!en)p5HQxw|TeRO$H^OV9gSO?Xw8|p?zd8lZ@nWzcZSiBSU zE}XIW9%dkZZ}~KB+@s8gdfQ8*jyMW+l2uTTt`_P9n`0L2gnHCdQJ)IWd@4Hfqo_MP zhuYaq)JymrHPJuT9vI_J6owjC4E3m@&>!of#x=k=*bFsqYt%_}$ILhu3$wnnhRPQt zZea-wZ_7suqfrxdLM_xCbwUF%KTfdxP7EYIjJmTEs1094J(_!{@lR3n{cY|4VxT_% z!R_1$l3+SIq{J*(1T~-uYJwIRh#gTMw;rgI8jm`GdFEPkk9ivPC~u=)x;Lmt9@L(~ z`c4K_Fo*etS=#&(}}L#Km#lJU_KpExBI z-AP8XAnFlRK)wCVP&@9AK{y39(HzuCEJpR)hWf%fi+am%p&r>AjEjLC-3^7H9#L3F z&R;J_MiQDZF9u@~)WGtVuZ5Y28=>xSFlwPGsH0to>c7d{kGiolsCj=!9sO<8_}7+? z*NOAjM2S1OCy*XBARp>uQxvtq3YL#X^>1i#I}9T3hUzx}GvXN3OT7~H2(F?&|94RR zKVkw5@^p506pq0pvLoLiPC?X>&P3hG2Gm5`PK-5mhp%$KjI)P=V4SP}V${vd^qc(IGHSaUj zyzfy*AE&E(!pV{OJWhHl+F4%I9Y(ngPE|}q+z2DFJ?6q`sNYR?qwe4h>XE%iEf~W1n)onkL&s1jaK^lb>h}m6;tNcO)w{dj z{f#h+xUa?Au^jOQ)VLHq+)qOU=FsOqndNu0h z+Kt)pGG;)hr~4aF1nT5!qfTNdYX0fyNknBHl_a>z8g`;KauBtFW0pT>`75Yz%G=iN z^m30p2-QD1YJ3LNLfKF!nAh?pQ2omH;{4T6jYJ`=kNWsbL_N!OsH5G1+Tan?o&Sz{ zhL2GTyhh!~2MoiI-tNaa0@beoYMxT4{?$;QuG+mhe|;P}lhDuYk*EzVM!hU+Q3JQ2 z7C3}@H_n-tto<5l{2!K&-^U%F5Y;~gYTmS{8;isUEbO78qo|Kspt*JEin`-LsISh+ zs5_6f_NC}gywUWc=GlVU@LtqOoJ2i}TbLdn;%bc7*L{SZ%~W(Gr%?-CviQ2i_c0&& zCs-5H_H%#yc1GRNLe#>mP&cp{wSoPp6FG%C$tRc;-=O*j_4ny#A7X4Q3te&Qk!j!Ahw0I%8J!47Shz zauV9nCe&NL3-$J&KppLMjKEi@jfM|#?<^B)L%C5KENMoeHc%Dy@vDbA;l^eQ)T3@Y zg!4~FWi*MDxYXQ*$%xORHt-m=b7!c#!7$X(W=1WL2la>wqE4bJYJ6>r+o3kz9o2sb z7Q_)AD!Hlrgaz<6rp07zOur@~bJWqcLG82$YT*H>g-4)1R#Q5VN8tY>2h7BSzwG%z*b%8w?!l zK8i5Z#F;PWG@R4{9TxNmTUgr=#A=*{BItqKdtmy z5S}tGV?5$Js15vq@$oI{MEobXHHhdNJcHhE^_zv~Z@S}W8`>A_H3h=p-IX2QFe7~@TIzra#pM&fd)`CH*P zIKg9;huDEc<>~G>-A2@p&(qixxhW~(-^GIxo1yLt3!`kPf zHn<#h$2%>5(DJ8H8^4EI$XV#VTM4iLVNvv*7$zd_vyk)G9gHHOozB8^xE^)XCr}ez zL-l)zn&2Jk3n_4sJu+1P9H<*8fw{1v#oeuaAnId25w+foMV!AXYe^KwLzo`lpI)^)_xxnUikKPSn5mbz??4&MO1=(i`~WP0Ueu%VJfos# z8no0MkO8$oA&Xn1HrOB2`tf$7?tI=d_s$lhCR&Yp1cy;4aRK!)ypMX;?=TJqEqBK! zLLRBdNk&B-vY{p@j9FNyGX9C3R=8hGO;)Nm|DXZeV2)LMo@t+g5&n+z!)i{D_{duB zlJ@iKcpO-BJzqX}7WdLVbpxBhXpZei%~;**J_{n>4#QB-E+y(&XF+!A=-;|k9Csq|BbVqHdsKaQigiG-VK0@~9Y~JGj&enFT z`%O5+oQOKHxfZWRy+hl~!{%AkJ9NY1`{qmZXn?cL9heZ+F*Sx`MvIH0-r_Q-{#DF+ zn3A}a#e-1&#-Zk!Zuz+uuQ1n{KW^jvHQ`R{cm(yiK98E{A?he!U?is7&fnp&GU{#q z9@Xy(YQcM$9iN!VcDVUcsC6o!KHkx&m$Tar&R-29Nob;}m=foq@?OjDv-p^K(Y$Lu zN4>=VLB0J6ce*!{4i)D|ooG3;zS-GBMK9L~OUy7=qc*Y+HNj=nh8~%3PeMs1+FSsQhNjZhnI zZE-KmNIV2}1K*(@Cts`*_qlFw64G z&9$Z%HQx@4f589`iC?Ltz_X|Y9;5CcXrJ9V>Wr=&D$!DzF&+0`6^dQ_9l zd6r*^nr|z5RM~Hd%mY8@|Xp54}d zazE#yV#3fMAzP#BOHE}Q0d?PKMi|W71+=#wAN3FO2 z0Ozj`zgprv1{2>#<)30Ud}Zxv4!R3PqBdF#HBoi5p&5f3-_zm&sD;N^eyZhXd#tj| z8aAOiZnOA+#Ya&KowNK+iyxS;P~Uuhhxm;dbD%cR1GUlqsCg%%=9!5)G0zff*o@zh z*oFF-q(1CUP#86^6ly?a)CBddy|uM>M7_MdEsjO?Ux<2C>rjv8XVk{7BJ+5hJ8s4K z%lu#l|LiW50(A#j%%YaBhMKs6#of&z<^&9+eYVAmEM9~9RBXn?`uy*+hSR7!yJ+6U zjKt5)uwVEcfVeOQVr#P_YFrOc!rLHBmU~Wz2+n6eTVHwb=wUu05v1Zm3V$WDk`hR90dL-oRjdV)5UomnhCjcY5M+j8c{--R>evW#;d0a+ zok4Bzy2ba*7pR}6AFVybX}5oR)D7f9jVtVv=U<9SP7)O`K6b@q*w5F2AD^fNW6j0p zI@CL}!{QsLJAZ=uczv{d#xw4Mc~R|!QJ=ah7{>Ze4JtaK78rp8Q6HE2sD(G92JS`m zJ8StTmVaw;$XWL*I2Gzni=mD>3X@_(%Xc#SphrjHp^_NKpcb5EF2$t88&T~CEq~6u ziIL==TAcixyMfH84Md?%v<7P4dS)xs&x&s6?DIdx8s=jt4J*yi4e zh5zQh{aI0;qB5vE?t}W+PQ?89gTO{~2=>PEVmBRp1_X|6=w*-rDg<*%TQ@R|7@wQ&54 z?ujHuEm#7>u%g8c%~ob-)E8b~)3b<59uhyJ-sV@RXOrrZyVGK*g{qsiQ5$Jww#6dE zy{vsT>JB$q`ytdzdj_lFAE+BFa@jYJ$EoaAoH}L;vkPjXfv5qKEnaBxM$|_3nI};j z{T=l^am(U_zq|Qln38-N)VM-EdH&_8=;*#iP0$eajGJ3L2sL1Y#owYfG|OCpINcx}#&PFY?%;JNn{--Uz ziD`(Rnn73X(}CJh8Pq(L%-ZPt`+svP8razy#-Rqxw0MEJ3iT=2V(|&o1}>YAP#b=W z>X+u4`!W_ly>yjO8|Z@SH}D#te=W4Y8kVBsji{sBYVJ3Wo99s{bOSZvKd5nWue;wL zA*drxhngops$U7z7t)uOkG{_Nrzg?aIt;*u#A7fw-oQ*4a>JbGeZFb}G|C~Cnn zs7G5Jm2ZcduPbV!o>5fv`JI6pa2&PJCDd2#AE={Fe#?Dk`A`$pLiKNmnlQ%lgUk`A z4UI?ji$$H>VvF}8^LU&SRJ7nZYq)Q|L=A9myAx!<7)zk;pb~1KdS-L81L~Ww z7iz;}%_*o2&GhNvw-PGazy|AZ0CmSl&D*E}@2x%l9k)F(wkIEH@icQaYC{K63!g;2 zbibkIxrSQrG5UV~f1skT-XwS34n@ogW;E)Ezp=O%YT{uQk460oHUqWsdDiYlozxE0 zSNvhr24130&bi0=Yd|O!?KlT!#4oTc)p%z?%>c7f7VEHQ;K>oJH_fdEJ)Z#ao ziui-YDek*Bl>R=?KMoCrNa#q4qjpvUwQy6*cR(#X*zz+`8(xBexE3|eYaT=`cn*Ev zh^YQS58OBmHGjSbJbw)=MZ&i*rX_BQ+Ry;hz)9vT3?yERn&1cXM{~dVt9iwIh#L1A zbu#~==8fxl=r$zAND|306P7hwq3(P%YGaEqHSR?HrgQ-#@d0MQ&`0jZ@}u&lQS-I6 zcs$l5-i(#eKz+ z9EL&oGlt-4)T6zIn&+wM{HZ+8Ka7eRGNWFaBEAL|MorMd>|zcy$D*F;4Ag@2QIBZ3 z+v%R$Pwy8F3t!;#1TY(zL(b?}fFf1&*M;fUaOp zjPsBCSFe1i?~&F>Kab;~5>8?|YQRd=M$Vu%a2=ET@xuzWG6nIS8vKrtNYHHIG)A(u{CjSG^-UMp>mh6A!0;hykJ0 z^*i8Y>h-B-U=zCZHT#uqV5zy5KKUr^XiG*JLDAKe_E_?LFpc%q*ZeoWxmdpV?;Wl9z>zS4~5pBhZCtEH9aiGn8m-Z&a^=zE) z{#TLs4~ZXbKn)zpfLr80T?L4DeHO2$e@{w2${F&3;P5MNbfRU{zKVBt}p&Uz6X5%gwDFg&{mE53yUwBv#}ffdeP^yuP0wx#9gRYV9b}a2UE(Ct6}qdv6+oOtMC6M zl);oKG(^+UYmJ&uKYBl1_o;NJPfhv+U|)(KeY2vj_*30lwT~Vfn6A&EB|M?J;a6RA4A#jS^hY2O3H0=59xE7GQ`>sk{?9L zL&-rt4S9Zo`>tQzuMLZ!2c5Rj;k6BJ%q9lfMuKTyOM4XM5akgiGyTR>I#d62g%aPk zU@PkHm#ru#=yRX8oLGl4fLujCo_~KTx;BtJjQZ`R4U3i`p2&cc#3v}z;Oa;DjW%8Fm}E8cZK9q5bRd_C*u>%<@5k^jf~ zEXTUUg{;pz+T&3l{n?nT)Em%m3vD6j`~FKvr45Nl1{bx?56I~{M(Ibr4sD;VPQ+(F z1?FRN=O{7OFFEr}CJv=N3FQU(Y?SGg^5pIE`2Lf41(_+=XdFekPQ8?MN@W|WXsWNS z2E?7*s&kPsTPUr`1(JKm*sq8?5)VLKiOmJHO(Xu6xU7EucO|$+@;xOE$%2&sP<|oS z^@MVYxF&H^EW)JY$yKMsw?0FuN7FV8Yta7b%0YcK!7S@{8ROG0pY`$6^VgM&L|)4G zbf|=>twVC+o7QIp{@`}^{kf7q6H@f|$33)F#bE3E5A~lIdkS@BX1*(K)!9dW0rmFu z^*#UeH2y|X*Hg^uR-NqRZ&|)N)*`M;38y}tK405nIk6$}1@bqQw=pNlH=;h&`ss(A zu4g#at@H1HNt~e+r_=Y?AE%(MK zua@<3TmSdZbmXhkuB$C&BqbwpT`cK4e=21u)kyZVj*FSJI0JW+&tnVrq`re(Y5HWN z-k16)>h-L>F!kP)O2l<&Pe9R?hFp63mPcLrT%0KMWTWBJ6+vY%r4eNeogdr8gicf9 zKe00X%V8ocM(z;x3G^F>#cZxx#LbA;Q*w}-ig9hO>BJF?>q0zQ|NlQ_sT`%lT*@~z z=D|sn`_v=pc!c)Klv$LY$(5iyrTqsiN={b;N*rQcRT#4uN7B{;7g@eAaeLZ+z;L`k zo4x;hk5j!vLncf?V+i%J4Ez#zQ*`}Kc}}hxaT`i$;>5&_=&vg-r6%PHIbFfzw_~8i z%B5r6Ow`9dDMkOYPG=h(3K6tN{gaTX)OA&(Tq5_CFXC4*Cd@!Roc4uS1#f&d`PbGj zKK%+)5BjVv$l4a!+zNl9?&-)Mh3WXwI*+BHwAEA4wu*Ypr#qxx|Fa3M5=YYiD7jr2 zM!!4QnWF1Y@?FU#qP(PU4T`Q8l$W#>)6f6c1Uc!Ts|U$*_&XhUQEm_?r`#nMK%Zjx z3uPR+lQvly8%x@GK>3mUSLB9bed3=e{?_*b<66-+k6b3|v-SJmaVp7dk`Wq2eH><_ zB&1DO9&-JOw@~Jj>r1>4XAqBZyZrah?`=#;`W*OgmVdxaJuSK3j0vHhiTd(S-~SHD z5+s`2;HGrQOQ}M>4tZT|sOQ8Kl;Pyg6BnWXSnBU_tvYe}6ThLYJN0-NMO=z7(2&5Bumu?tpqicPnwKy3Y0Rb#_e?~jeYrbcY*HLZeU zdUom1t5=6^U3+=ouSpW<&A)j`c-+y$AFki#9eiYT+~BB~*4?_qIQPa+j*a}an0LUh z#r?gHP7m>m^_;zv_};kr_jc}hFk|KYaa$kk9OYefZbzKp2dj2C_oi+j>z#h-ivaJn zD{cL}Z>}Bk^PaqM(=T@Wt?k~ew@1eDZheq6PV9w0276\n" "Language-Team: JumpServer team\n" @@ -36,10 +36,10 @@ msgstr "自定义" #: assets/models/base.py:234 assets/models/cluster.py:18 #: assets/models/cmd_filter.py:21 assets/models/domain.py:21 #: assets/models/group.py:20 assets/models/label.py:18 ops/mixin.py:24 -#: orgs/models.py:23 perms/models/base.py:48 settings/models.py:27 +#: orgs/models.py:23 perms/models/base.py:48 settings/models.py:29 #: terminal/models/storage.py:15 terminal/models/storage.py:55 #: terminal/models/task.py:16 terminal/models/terminal.py:131 -#: users/forms/profile.py:20 users/models/group.py:15 users/models/user.py:518 +#: users/forms/profile.py:20 users/models/group.py:15 users/models/user.py:517 #: users/templates/users/_select_user_modal.html:13 #: users/templates/users/user_asset_permission.html:37 #: users/templates/users/user_asset_permission.html:154 @@ -95,10 +95,10 @@ msgstr "" #: assets/models/cmd_filter.py:57 assets/models/domain.py:22 #: assets/models/domain.py:56 assets/models/group.py:23 #: assets/models/label.py:23 ops/models/adhoc.py:37 orgs/models.py:26 -#: perms/models/base.py:56 settings/models.py:32 terminal/models/storage.py:21 +#: perms/models/base.py:56 settings/models.py:34 terminal/models/storage.py:21 #: terminal/models/storage.py:61 terminal/models/terminal.py:145 #: tickets/models/ticket.py:73 users/models/group.py:16 -#: users/models/user.py:551 users/templates/users/user_detail.html:115 +#: users/models/user.py:550 users/templates/users/user_detail.html:115 #: users/templates/users/user_granted_database_app.html:38 #: users/templates/users/user_granted_remote_app.html:37 #: users/templates/users/user_group_detail.html:62 @@ -164,7 +164,7 @@ msgstr "目标URL" #: assets/models/base.py:235 assets/models/gathered_user.py:15 #: audits/models.py:99 authentication/forms.py:11 #: authentication/templates/authentication/login.html:101 -#: ops/models/adhoc.py:148 users/forms/profile.py:19 users/models/user.py:516 +#: ops/models/adhoc.py:148 users/forms/profile.py:19 users/models/user.py:515 #: users/templates/users/_select_user_modal.html:14 #: users/templates/users/user_detail.html:53 #: users/templates/users/user_list.html:15 @@ -181,7 +181,8 @@ msgstr "用户名" #: assets/models/base.py:236 assets/serializers/asset_user.py:71 #: audits/signals_handler.py:42 authentication/forms.py:13 #: authentication/templates/authentication/login.html:109 -#: users/forms/user.py:22 users/forms/user.py:193 +#: settings/serializers/settings.py:84 users/forms/user.py:22 +#: users/forms/user.py:193 #: users/templates/users/user_otp_check_password.html:13 #: users/templates/users/user_password_update.html:43 #: users/templates/users/user_password_verify.html:18 @@ -204,7 +205,7 @@ msgstr "目标URL" #: applications/serializers/attrs/application_type/mysql_workbench.py:18 #: assets/models/asset.py:190 assets/models/domain.py:52 -#: assets/serializers/asset_user.py:46 settings/serializers/settings.py:52 +#: assets/serializers/asset_user.py:46 settings/serializers/settings.py:103 #: users/templates/users/_granted_assets.html:26 #: users/templates/users/user_asset_permission.html:156 msgid "IP" @@ -260,7 +261,7 @@ msgid "Platform" msgstr "系统平台" #: assets/models/asset.py:191 assets/serializers/asset_user.py:45 -#: assets/serializers/gathered_user.py:20 settings/serializers/settings.py:51 +#: assets/serializers/gathered_user.py:20 settings/serializers/settings.py:102 #: users/templates/users/_granted_assets.html:25 #: users/templates/users/user_asset_permission.html:157 msgid "Hostname" @@ -368,7 +369,7 @@ msgstr "标签管理" #: assets/models/cluster.py:28 assets/models/cmd_filter.py:26 #: assets/models/cmd_filter.py:60 assets/models/group.py:21 #: common/db/models.py:67 common/mixins/models.py:49 orgs/models.py:24 -#: orgs/models.py:427 perms/models/base.py:54 users/models/user.py:559 +#: orgs/models.py:427 perms/models/base.py:54 users/models/user.py:558 #: users/serializers/group.py:35 users/templates/users/user_detail.html:97 #: xpack/plugins/change_auth_plan/models.py:81 xpack/plugins/cloud/models.py:58 #: xpack/plugins/cloud/models.py:156 xpack/plugins/gathered_user/models.py:30 @@ -430,7 +431,7 @@ msgstr "带宽" msgid "Contact" msgstr "联系人" -#: assets/models/cluster.py:22 users/models/user.py:537 +#: assets/models/cluster.py:22 users/models/user.py:536 #: users/templates/users/user_detail.html:62 msgid "Phone" msgstr "手机" @@ -456,7 +457,7 @@ msgid "Default" msgstr "默认" #: assets/models/cluster.py:36 assets/models/label.py:14 -#: users/models/user.py:678 +#: users/models/user.py:677 msgid "System" msgstr "系统" @@ -560,7 +561,7 @@ msgstr "默认资产组" #: templates/index.html:78 terminal/backends/command/models.py:18 #: terminal/backends/command/serializers.py:12 terminal/models/session.py:37 #: tickets/models/comment.py:17 users/forms/group.py:15 -#: users/models/user.py:159 users/models/user.py:666 +#: users/models/user.py:158 users/models/user.py:665 #: users/serializers/group.py:20 #: users/templates/users/user_asset_permission.html:38 #: users/templates/users/user_asset_permission.html:64 @@ -574,7 +575,7 @@ msgstr "默认资产组" msgid "User" msgstr "用户" -#: assets/models/label.py:19 assets/models/node.py:413 settings/models.py:28 +#: assets/models/label.py:19 assets/models/node.py:413 settings/models.py:30 msgid "Value" msgstr "值" @@ -741,14 +742,14 @@ msgid "Backend" msgstr "后端" #: assets/serializers/asset_user.py:75 users/forms/profile.py:148 -#: users/models/user.py:548 users/templates/users/user_password_update.html:48 +#: users/models/user.py:547 users/templates/users/user_password_update.html:48 #: users/templates/users/user_profile.html:69 #: users/templates/users/user_profile_update.html:46 #: users/templates/users/user_pubkey_update.html:46 msgid "Public key" msgstr "SSH公钥" -#: assets/serializers/asset_user.py:79 users/models/user.py:545 +#: assets/serializers/asset_user.py:79 users/models/user.py:544 msgid "Private key" msgstr "ssh私钥" @@ -1045,7 +1046,7 @@ msgstr "修改者" msgid "Disabled" msgstr "禁用" -#: audits/models.py:90 settings/models.py:31 +#: audits/models.py:90 settings/models.py:33 #: users/templates/users/user_detail.html:82 msgid "Enabled" msgstr "启用" @@ -1079,7 +1080,7 @@ msgstr "用户代理" #: audits/models.py:104 #: authentication/templates/authentication/_mfa_confirm_modal.html:14 #: authentication/templates/authentication/login_otp.html:6 -#: users/forms/profile.py:52 users/models/user.py:540 +#: users/forms/profile.py:52 users/models/user.py:539 #: users/serializers/user.py:232 users/templates/users/user_detail.html:77 #: users/templates/users/user_profile.html:87 msgid "MFA" @@ -1354,7 +1355,7 @@ msgid "Show" msgstr "显示" #: authentication/templates/authentication/_access_key_modal.html:66 -#: users/models/user.py:444 users/serializers/user.py:229 +#: users/models/user.py:443 users/serializers/user.py:229 #: users/templates/users/user_profile.html:94 #: users/templates/users/user_profile.html:163 #: users/templates/users/user_profile.html:166 @@ -1363,7 +1364,7 @@ msgid "Disable" msgstr "禁用" #: authentication/templates/authentication/_access_key_modal.html:67 -#: users/models/user.py:445 users/serializers/user.py:230 +#: users/models/user.py:444 users/serializers/user.py:230 #: users/templates/users/user_profile.html:92 #: users/templates/users/user_profile.html:170 msgid "Enable" @@ -1600,11 +1601,6 @@ msgstr "字段必须唯一" msgid "Should not contains special characters" msgstr "不能包含特殊字符" -#: jumpserver/conf.py:474 xpack/plugins/interface/api.py:18 -#: xpack/plugins/interface/models.py:36 -msgid "Welcome to the JumpServer open source fortress" -msgstr "欢迎使用JumpServer开源堡垒机" - #: jumpserver/views/celery_flower.py:23 msgid "

Flow service unavailable, check it

" msgstr "" @@ -1813,7 +1809,7 @@ msgstr "组织管理员" msgid "Organization auditor" msgstr "组织审计员" -#: orgs/models.py:424 users/forms/user.py:27 users/models/user.py:528 +#: orgs/models.py:424 users/forms/user.py:27 users/models/user.py:527 #: users/templates/users/_select_user_modal.html:15 #: users/templates/users/user_detail.html:73 #: users/templates/users/user_list.html:16 @@ -1837,7 +1833,7 @@ msgstr "管理员正在修改授权，请稍等" msgid "The authorization cannot be revoked for the time being" msgstr "该授权暂时不能撤销" -#: perms/models/application_permission.py:27 users/models/user.py:160 +#: perms/models/application_permission.py:27 users/models/user.py:159 msgid "Application" msgstr "应用程序" @@ -1845,7 +1841,7 @@ msgstr "应用程序" msgid "Application permission" msgstr "应用管理" -#: perms/models/asset_permission.py:34 settings/serializers/settings.py:56 +#: perms/models/asset_permission.py:34 settings/serializers/settings.py:107 msgid "All" msgstr "全部" @@ -1887,7 +1883,7 @@ msgid "Asset permission" msgstr "资产授权" #: perms/models/base.py:50 templates/_nav.html:21 users/forms/user.py:168 -#: users/models/group.py:31 users/models/user.py:524 +#: users/models/group.py:31 users/models/user.py:523 #: users/templates/users/_select_user_modal.html:16 #: users/templates/users/user_asset_permission.html:39 #: users/templates/users/user_asset_permission.html:67 @@ -1905,7 +1901,7 @@ msgstr "用户组" #: tickets/serializers/ticket/meta/ticket_type/apply_application.py:77 #: tickets/serializers/ticket/meta/ticket_type/apply_asset.py:43 #: tickets/serializers/ticket/meta/ticket_type/apply_asset.py:81 -#: users/models/user.py:556 users/templates/users/user_detail.html:93 +#: users/models/user.py:555 users/templates/users/user_detail.html:93 #: users/templates/users/user_profile.html:120 msgid "Date expired" msgstr "失效日期" @@ -1944,27 +1940,310 @@ msgstr "收藏夹" msgid "Please wait while your data is being initialized" msgstr "数据正在初始化，请稍等" -#: settings/api.py:34 +#: settings/api/common.py:24 msgid "Test mail sent to {}, please check" msgstr "邮件已经发送{}, 请检查" -#: settings/api.py:244 +#: settings/api/common.py:110 xpack/plugins/interface/api.py:18 +#: xpack/plugins/interface/models.py:36 +msgid "Welcome to the JumpServer open source fortress" +msgstr "欢迎使用JumpServer开源堡垒机" + +#: settings/api/ldap.py:189 msgid "Get ldap users is None" msgstr "获取 LDAP 用户为 None" -#: settings/api.py:251 +#: settings/api/ldap.py:196 msgid "Imported {} users successfully" msgstr "导入 {} 个用户成功" -#: settings/models.py:96 users/templates/users/reset_password.html:29 +#: settings/models.py:123 users/templates/users/reset_password.html:29 #: users/templates/users/user_profile.html:20 msgid "Setting" msgstr "设置" -#: settings/serializers/settings.py:57 +#: settings/serializers/settings.py:15 +msgid "Site url" +msgstr "当前站点URL" + +#: settings/serializers/settings.py:16 +msgid "eg: http://demo.jumpserver.org:8080" +msgstr "如: http://demo.jumpserver.org:8080" + +#: settings/serializers/settings.py:19 +msgid "User guide url" +msgstr "用户向导URL" + +#: settings/serializers/settings.py:20 +msgid "User first login update profile done redirect to it" +msgstr "用户第一次登录，修改profile后重定向到地址, 可以是 wiki 或 其他说明文档" + +#: settings/serializers/settings.py:27 +msgid "SMTP host" +msgstr "SMTP 主机" + +#: settings/serializers/settings.py:28 +msgid "SMTP port" +msgstr "SMTP 端口" + +#: settings/serializers/settings.py:29 +msgid "SMTP account" +msgstr "SMTP 账号" + +#: settings/serializers/settings.py:31 +msgid "SMTP password" +msgstr "SMTP 密码" + +#: settings/serializers/settings.py:32 +msgid "Tips: Some provider use token except password" +msgstr "提示：一些邮件提供商需要输入的是授权码" + +#: settings/serializers/settings.py:35 +msgid "Send user" +msgstr "发件人" + +#: settings/serializers/settings.py:36 +msgid "Tips: Send mail account, default SMTP account as the send account" +msgstr "提示：发送邮件账号，默认使用 SMTP 账号作为发送账号" + +#: settings/serializers/settings.py:39 +msgid "Test recipient" +msgstr "测试收件人" + +#: settings/serializers/settings.py:40 +msgid "Tips: Used only as a test mail recipient" +msgstr "提示：仅用来作为测试邮件收件人" + +#: settings/serializers/settings.py:43 +msgid "Use SSL" +msgstr "使用 SSL" + +#: settings/serializers/settings.py:44 +msgid "If SMTP port is 465, may be select" +msgstr "如果SMTP端口是465，通常需要启用 SSL" + +#: settings/serializers/settings.py:47 +msgid "Use TLS" +msgstr "使用 TLS" + +#: settings/serializers/settings.py:48 +msgid "If SMTP port is 587, may be select" +msgstr "如果SMTP端口是587，通常需要启用 TLS" + +#: settings/serializers/settings.py:51 +msgid "Subject prefix" +msgstr "主题前缀" + +#: settings/serializers/settings.py:58 +msgid "Create user email subject" +msgstr "邮件主题" + +#: settings/serializers/settings.py:59 +msgid "" +"Tips: When creating a user, send the subject of the email (eg:Create account " +"successfully)" +msgstr "提示: 创建用户时，发送设置密码邮件的主题 (例如: 创建用户成功)" + +#: settings/serializers/settings.py:63 +msgid "Create user honorific" +msgstr "邮件的敬语" + +#: settings/serializers/settings.py:64 +msgid "Tips: When creating a user, send the honorific of the email (eg:Hello)" +msgstr "提示: 创建用户时，发送设置密码邮件的敬语 (例如: 您好)" + +#: settings/serializers/settings.py:68 +msgid "Create user email content" +msgstr "邮件的内容" + +#: settings/serializers/settings.py:69 +msgid "Tips:When creating a user, send the content of the email" +msgstr "提示: 创建用户时，发送设置密码邮件的内容" + +#: settings/serializers/settings.py:72 +msgid "Signature" +msgstr "署名" + +#: settings/serializers/settings.py:73 +msgid "Tips: Email signature (eg:jumpserver)" +msgstr "邮件署名 (如:jumpserver)" + +#: settings/serializers/settings.py:81 +msgid "LDAP server" +msgstr "LDAP 地址" + +#: settings/serializers/settings.py:81 +msgid "eg: ldap://localhost:389" +msgstr "" + +#: settings/serializers/settings.py:83 +msgid "Bind DN" +msgstr "绑定 DN" + +#: settings/serializers/settings.py:86 +msgid "User OU" +msgstr "用户 OU" + +#: settings/serializers/settings.py:87 +msgid "Use | split multi OUs" +msgstr "多个 OU 使用 | 分割" + +#: settings/serializers/settings.py:90 +msgid "User search filter" +msgstr "用户过滤器" + +#: settings/serializers/settings.py:91 +#, python-format +msgid "Choice may be (cn|uid|sAMAccountName)=%(user)s)" +msgstr "可能的选项是(cn或uid或sAMAccountName=%(user)s)" + +#: settings/serializers/settings.py:94 +msgid "User attr map" +msgstr "用户属性映射" + +#: settings/serializers/settings.py:95 +msgid "" +"User attr map present how to map LDAP user attr to jumpserver, username,name," +"email is jumpserver attr" +msgstr "" +"用户属性映射代表怎样将LDAP中用户属性映射到jumpserver用户上，username, name," +"email 是jumpserver的用户需要属性" + +#: settings/serializers/settings.py:97 +msgid "Enable LDAP auth" +msgstr "启用 LDAP 认证" + +#: settings/serializers/settings.py:108 msgid "Auto" msgstr "自动" +#: settings/serializers/settings.py:114 +msgid "Password auth" +msgstr "密码认证" + +#: settings/serializers/settings.py:115 +msgid "Public key auth" +msgstr "密钥认证" + +#: settings/serializers/settings.py:116 +msgid "List sort by" +msgstr "资产列表排序" + +#: settings/serializers/settings.py:117 +msgid "List page size" +msgstr "资产列表每页数量" + +#: settings/serializers/settings.py:119 +msgid "Session keep duration" +msgstr "会话日志保存时间" + +#: settings/serializers/settings.py:120 +msgid "" +"Units: days, Session, record, command will be delete if more than duration, " +"only in database" +msgstr "" +"单位：天。 会话、录像、命令记录超过该时长将会被删除(仅影响数据库存储, oss等不" +"受影响)" + +#: settings/serializers/settings.py:122 +msgid "Telnet login regex" +msgstr "Telnet 成功正则表达式" + +#: settings/serializers/settings.py:127 +msgid "Global MFA auth" +msgstr "全局启用 MFA 认证" + +#: settings/serializers/settings.py:128 +msgid "All user enable MFA" +msgstr "强制每个启用多因子认证" + +#: settings/serializers/settings.py:131 +msgid "Batch command execution" +msgstr "批量命令执行" + +#: settings/serializers/settings.py:132 +msgid "Allow user run batch command or not using ansible" +msgstr "是否允许用户使用 ansible 执行批量命令" + +#: settings/serializers/settings.py:135 +msgid "Enable terminal register" +msgstr "终端注册" + +#: settings/serializers/settings.py:136 +msgid "" +"Allow terminal register, after all terminal setup, you should disable this " +"for security" +msgstr "是否允许终端注册，当所有终端启动后，为了安全应该关闭" + +#: settings/serializers/settings.py:140 +msgid "Limit the number of login failures" +msgstr "限制登录失败次数" + +#: settings/serializers/settings.py:144 +msgid "Block logon interval" +msgstr "禁止登录时间间隔" + +#: settings/serializers/settings.py:145 +msgid "" +"Tip: (unit/minute) if the user has failed to log in for a limited number of " +"times, no login is allowed during this time interval." +msgstr "" +"提示：（单位：分）当用户登录失败次数达到限制后，那么在此时间间隔内禁止登录" + +#: settings/serializers/settings.py:149 +msgid "Connection max idle time" +msgstr "连接最大空闲时间" + +#: settings/serializers/settings.py:150 +msgid "If idle time more than it, disconnect connection Unit: minute" +msgstr "提示：如果超过该配置没有操作，连接会被断开 （单位：分）" + +#: settings/serializers/settings.py:154 +msgid "User password expiration" +msgstr "用户密码过期时间" + +#: settings/serializers/settings.py:155 +msgid "" +"Tip: (unit: day) If the user does not update the password during the time, " +"the user password will expire failure;The password expiration reminder mail " +"will be automatic sent to the user by system within 5 days (daily) before " +"the password expires" +msgstr "" +"提示：（单位：天）如果用户在此期间没有更新密码，用户密码将过期失效； 密码过期" +"提醒邮件将在密码过期前5天内由系统（每天）自动发送给用户" + +#: settings/serializers/settings.py:159 +msgid "Password minimum length" +msgstr "密码最小长度" + +#: settings/serializers/settings.py:162 +msgid "Must contain capital" +msgstr "必须包含大写字符" + +#: settings/serializers/settings.py:164 +msgid "Must contain lowercase" +msgstr "必须包含小写字符" + +#: settings/serializers/settings.py:165 +msgid "Must contain numeric" +msgstr "必须包含数字" + +#: settings/serializers/settings.py:166 +msgid "Must contain special" +msgstr "必须包含特殊字符" + +#: settings/serializers/settings.py:167 +msgid "Insecure command alert" +msgstr "危险命令告警" + +#: settings/serializers/settings.py:169 +msgid "Email recipient" +msgstr "邮件收件人" + +#: settings/serializers/settings.py:170 +msgid "Multiple user using , split" +msgstr "多个用户，使用 , 分割" + #: settings/utils/ldap.py:411 msgid "Host or port is disconnected: {}" msgstr "主机或端口不可连接: {}" @@ -3215,7 +3494,7 @@ msgstr "确认密码" msgid "Password does not match" msgstr "密码不一致" -#: users/forms/profile.py:89 users/models/user.py:520 +#: users/forms/profile.py:89 users/models/user.py:519 #: users/templates/users/user_detail.html:57 #: users/templates/users/user_profile.html:59 msgid "Email" @@ -3256,7 +3535,7 @@ msgstr "不能和原来的密钥相同" msgid "Not a valid ssh public key" msgstr "SSH密钥不合法" -#: users/forms/user.py:31 users/models/user.py:563 +#: users/forms/user.py:31 users/models/user.py:562 #: users/templates/users/user_detail.html:89 #: users/templates/users/user_list.html:18 #: users/templates/users/user_profile.html:102 @@ -3290,39 +3569,39 @@ msgstr "设置密码" msgid "Password strategy" msgstr "密码策略" -#: users/models/user.py:157 +#: users/models/user.py:156 msgid "System administrator" msgstr "系统管理员" -#: users/models/user.py:158 +#: users/models/user.py:157 msgid "System auditor" msgstr "系统审计员" -#: users/models/user.py:446 users/templates/users/user_profile.html:90 +#: users/models/user.py:445 users/templates/users/user_profile.html:90 msgid "Force enable" msgstr "强制启用" -#: users/models/user.py:508 +#: users/models/user.py:507 msgid "Local" msgstr "数据库" -#: users/models/user.py:531 +#: users/models/user.py:530 msgid "Avatar" msgstr "头像" -#: users/models/user.py:534 users/templates/users/user_detail.html:68 +#: users/models/user.py:533 users/templates/users/user_detail.html:68 msgid "Wechat" msgstr "微信" -#: users/models/user.py:567 +#: users/models/user.py:566 msgid "Date password last updated" msgstr "最后更新密码日期" -#: users/models/user.py:674 +#: users/models/user.py:673 msgid "Administrator" msgstr "管理员" -#: users/models/user.py:677 +#: users/models/user.py:676 msgid "Administrator is the super user of system" msgstr "Administrator是初始的超级管理员" diff --git a/apps/settings/api/__init__.py b/apps/settings/api/__init__.py new file mode 100644 index 000000000..151617be5 --- /dev/null +++ b/apps/settings/api/__init__.py @@ -0,0 +1,2 @@ +from .common import * +from .ldap import * diff --git a/apps/settings/api/common.py b/apps/settings/api/common.py new file mode 100644 index 000000000..ba7945ec7 --- /dev/null +++ b/apps/settings/api/common.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# + +from smtplib import SMTPSenderRefused +from rest_framework import generics +from rest_framework.views import Response, APIView +from rest_framework.permissions import AllowAny +from django.conf import settings +from django.core.mail import send_mail, get_connection +from django.utils.translation import ugettext_lazy as _ +from django.templatetags.static import static + +from common.permissions import IsSuperUser +from common.utils import get_logger +from .. import serializers +from ..models import Setting + +logger = get_logger(__file__) + + +class MailTestingAPI(APIView): + permission_classes = (IsSuperUser,) + serializer_class = serializers.MailTestSerializer + success_message = _("Test mail sent to {}, please check") + + def post(self, request): + serializer = self.serializer_class(data=request.data) + serializer.is_valid(raise_exception=True) + + email_host = serializer.validated_data['EMAIL_HOST'] + email_port = serializer.validated_data['EMAIL_PORT'] + email_host_user = serializer.validated_data["EMAIL_HOST_USER"] + email_host_password = serializer.validated_data['EMAIL_HOST_PASSWORD'] + email_from = serializer.validated_data["EMAIL_FROM"] + email_recipient = serializer.validated_data["EMAIL_RECIPIENT"] + email_use_ssl = serializer.validated_data['EMAIL_USE_SSL'] + email_use_tls = serializer.validated_data['EMAIL_USE_TLS'] + + # 设置 settings 的值，会导致动态配置在当前进程失效 + # for k, v in serializer.validated_data.items(): + # if k.startswith('EMAIL'): + # setattr(settings, k, v) + try: + subject = "Test" + message = "Test smtp setting" + email_from = email_from or email_host_user + email_recipient = email_recipient or email_from + connection = get_connection( + host=email_host, port=email_port, + username=email_host_user, password=email_host_password, + use_tls=email_use_tls, use_ssl=email_use_ssl, + ) + send_mail( + subject, message, email_from, [email_recipient], + connection=connection + ) + except SMTPSenderRefused as e: + error = e.smtp_error + if isinstance(error, bytes): + for coding in ('gbk', 'utf8'): + try: + error = error.decode(coding) + except UnicodeDecodeError: + continue + else: + break + return Response({"error": str(error)}, status=400) + except Exception as e: + logger.error(e) + return Response({"error": str(e)}, status=400) + return Response({"msg": self.success_message.format(email_recipient)}) + + +class PublicSettingApi(generics.RetrieveAPIView): + permission_classes = (AllowAny,) + serializer_class = serializers.PublicSettingSerializer + + @staticmethod + def get_logo_urls(): + logo_urls = { + 'logo_logout': static('img/logo.png'), + 'logo_index': static('img/logo_text.png'), + 'login_image': static('img/login_image.png'), + 'favicon': static('img/facio.ico') + } + if not settings.XPACK_ENABLED: + return logo_urls + from xpack.plugins.interface.models import Interface + obj = Interface.interface() + if not obj: + return logo_urls + for attr in ['logo_logout', 'logo_index', 'login_image', 'favicon']: + if getattr(obj, attr, '') and getattr(obj, attr).url: + logo_urls.update({attr: getattr(obj, attr).url}) + return logo_urls + + @staticmethod + def get_xpack_license_is_valid(): + if not settings.XPACK_ENABLED: + return False + try: + from xpack.plugins.license.models import License + return License.has_valid_license() + except Exception as e: + logger.error(e) + return False + + @staticmethod + def get_login_title(): + default_title = _('Welcome to the JumpServer open source fortress') + if not settings.XPACK_ENABLED: + return default_title + from xpack.plugins.interface.models import Interface + return Interface.get_login_title() + + def get_object(self): + instance = { + "data": { + "WINDOWS_SKIP_ALL_MANUAL_PASSWORD": settings.WINDOWS_SKIP_ALL_MANUAL_PASSWORD, + "SECURITY_MAX_IDLE_TIME": settings.SECURITY_MAX_IDLE_TIME, + "XPACK_ENABLED": settings.XPACK_ENABLED, + "LOGIN_CONFIRM_ENABLE": settings.LOGIN_CONFIRM_ENABLE, + "SECURITY_VIEW_AUTH_NEED_MFA": settings.SECURITY_VIEW_AUTH_NEED_MFA, + "SECURITY_MFA_VERIFY_TTL": settings.SECURITY_MFA_VERIFY_TTL, + "SECURITY_COMMAND_EXECUTION": settings.SECURITY_COMMAND_EXECUTION, + "SECURITY_PASSWORD_EXPIRATION_TIME": settings.SECURITY_PASSWORD_EXPIRATION_TIME, + "XPACK_LICENSE_IS_VALID": self.get_xpack_license_is_valid(), + "LOGIN_TITLE": self.get_login_title(), + "LOGO_URLS": self.get_logo_urls(), + "TICKETS_ENABLED": settings.TICKETS_ENABLED, + "PASSWORD_RULE": { + 'SECURITY_PASSWORD_MIN_LENGTH': settings.SECURITY_PASSWORD_MIN_LENGTH, + 'SECURITY_PASSWORD_UPPER_CASE': settings.SECURITY_PASSWORD_UPPER_CASE, + 'SECURITY_PASSWORD_LOWER_CASE': settings.SECURITY_PASSWORD_LOWER_CASE, + 'SECURITY_PASSWORD_NUMBER': settings.SECURITY_PASSWORD_NUMBER, + 'SECURITY_PASSWORD_SPECIAL_CHAR': settings.SECURITY_PASSWORD_SPECIAL_CHAR, + } + } + } + return instance + + +class SettingsApi(generics.RetrieveUpdateAPIView): + permission_classes = (IsSuperUser,) + serializer_class_mapper = { + 'all': serializers.SettingsSerializer, + 'basic': serializers.BasicSettingSerializer, + 'terminal': serializers.TerminalSettingSerializer, + 'security': serializers.SecuritySettingSerializer, + 'ldap': serializers.LDAPSettingSerializer, + 'email': serializers.EmailSettingSerializer, + 'email_content': serializers.EmailContentSettingSerializer, + } + + def get_serializer_class(self): + category = self.request.query_params.get('category', serializers.BasicSettingSerializer) + return self.serializer_class_mapper.get(category, serializers.BasicSettingSerializer) + + def get_fields(self): + serializer = self.get_serializer_class()() + fields = serializer.get_fields() + return fields + + def get_object(self): + items = self.get_fields().keys() + return {item: getattr(settings, item) for item in items} + + def parse_serializer_data(self, serializer): + data = [] + fields = self.get_fields() + encrypted_items = [name for name, field in fields.items() if field.write_only] + category = self.request.query_params.get('category', '') + for name, value in serializer.validated_data.items(): + encrypted = name in encrypted_items + data.append({ + 'name': name, 'value': value, + 'encrypted': encrypted, 'category': category + }) + return data + + def perform_update(self, serializer): + settings_items = self.parse_serializer_data(serializer) + serializer_data = getattr(serializer, 'data', {}) + for item in settings_items: + changed, setting = Setting.update_or_create(**item) + if not changed: + continue + serializer_data[setting.name] = setting.cleaned_value + setattr(serializer, '_data', serializer_data) diff --git a/apps/settings/api.py b/apps/settings/api/ldap.py similarity index 58% rename from apps/settings/api.py rename to apps/settings/api/ldap.py index 1fba03e6a..0c982d1b1 100644 --- a/apps/settings/api.py +++ b/apps/settings/api/ldap.py @@ -10,16 +10,15 @@ from rest_framework.views import Response, APIView from django.conf import settings from django.core.mail import send_mail, get_connection from django.utils.translation import ugettext_lazy as _ -from rest_framework import serializers -from .utils import ( +from ..utils import ( LDAPServerUtil, LDAPCacheUtil, LDAPImportUtil, LDAPSyncUtil, LDAP_USE_CACHE_FLAGS, LDAPTestUtil, ObjectDict ) -from .tasks import sync_ldap_user +from ..tasks import sync_ldap_user from common.permissions import IsOrgAdmin, IsSuperUser from common.utils import get_logger -from .serializers import ( +from ..serializers import ( MailTestSerializer, LDAPTestConfigSerializer, LDAPUserSerializer, PublicSettingSerializer, LDAPTestLoginSerializer, SettingsSerializer ) @@ -28,60 +27,6 @@ from users.models import User logger = get_logger(__file__) -class MailTestingAPI(APIView): - permission_classes = (IsSuperUser,) - serializer_class = MailTestSerializer - success_message = _("Test mail sent to {}, please check") - - def post(self, request): - serializer = self.serializer_class(data=request.data) - if serializer.is_valid(): - email_host = serializer.validated_data['EMAIL_HOST'] - email_port = serializer.validated_data['EMAIL_PORT'] - email_host_user = serializer.validated_data["EMAIL_HOST_USER"] - email_host_password = serializer.validated_data['EMAIL_HOST_PASSWORD'] - email_from = serializer.validated_data["EMAIL_FROM"] - email_recipient = serializer.validated_data["EMAIL_RECIPIENT"] - email_use_ssl = serializer.validated_data['EMAIL_USE_SSL'] - email_use_tls = serializer.validated_data['EMAIL_USE_TLS'] - - # 设置 settings 的值，会导致动态配置在当前进程失效 - # for k, v in serializer.validated_data.items(): - # if k.startswith('EMAIL'): - # setattr(settings, k, v) - try: - subject = "Test" - message = "Test smtp setting" - email_from = email_from or email_host_user - email_recipient = email_recipient or email_from - connection = get_connection( - host=email_host, port=email_port, - username=email_host_user, password=email_host_password, - use_tls=email_use_tls, use_ssl=email_use_ssl, - ) - send_mail( - subject, message, email_from, [email_recipient], - connection=connection - ) - except SMTPSenderRefused as e: - resp = e.smtp_error - if isinstance(resp, bytes): - for coding in ('gbk', 'utf8'): - try: - resp = resp.decode(coding) - except UnicodeDecodeError: - continue - else: - break - return Response({"error": str(resp)}, status=400) - except Exception as e: - print(e) - return Response({"error": str(e)}, status=400) - return Response({"msg": self.success_message.format(email_recipient)}) - else: - return Response({"error": str(serializer.errors)}, status=400) - - class LDAPTestingConfigAPI(APIView): permission_classes = (IsSuperUser,) serializer_class = LDAPTestConfigSerializer @@ -262,59 +207,3 @@ class LDAPCacheRefreshAPI(generics.RetrieveAPIView): return Response(data={'msg': str(e)}, status=400) return Response(data={'msg': 'success'}) - -class PublicSettingApi(generics.RetrieveAPIView): - permission_classes = () - serializer_class = PublicSettingSerializer - - def get_object(self): - instance = { - "data": { - "WINDOWS_SKIP_ALL_MANUAL_PASSWORD": settings.WINDOWS_SKIP_ALL_MANUAL_PASSWORD, - "SECURITY_MAX_IDLE_TIME": settings.SECURITY_MAX_IDLE_TIME, - "XPACK_ENABLED": settings.XPACK_ENABLED, - "XPACK_LICENSE_IS_VALID": settings.XPACK_LICENSE_IS_VALID, - "LOGIN_CONFIRM_ENABLE": settings.LOGIN_CONFIRM_ENABLE, - "SECURITY_VIEW_AUTH_NEED_MFA": settings.SECURITY_VIEW_AUTH_NEED_MFA, - "SECURITY_MFA_VERIFY_TTL": settings.SECURITY_MFA_VERIFY_TTL, - "SECURITY_COMMAND_EXECUTION": settings.SECURITY_COMMAND_EXECUTION, - "LOGIN_TITLE": settings.XPACK_INTERFACE_LOGIN_TITLE, - "SECURITY_PASSWORD_EXPIRATION_TIME": settings.SECURITY_PASSWORD_EXPIRATION_TIME, - "LOGO_URLS": settings.LOGO_URLS, - "TICKETS_ENABLED": settings.TICKETS_ENABLED, - "PASSWORD_RULE": { - 'SECURITY_PASSWORD_MIN_LENGTH': settings.SECURITY_PASSWORD_MIN_LENGTH, - 'SECURITY_PASSWORD_UPPER_CASE': settings.SECURITY_PASSWORD_UPPER_CASE, - 'SECURITY_PASSWORD_LOWER_CASE': settings.SECURITY_PASSWORD_LOWER_CASE, - 'SECURITY_PASSWORD_NUMBER': settings.SECURITY_PASSWORD_NUMBER, - 'SECURITY_PASSWORD_SPECIAL_CHAR': settings.SECURITY_PASSWORD_SPECIAL_CHAR, - } - } - } - return instance - - -class SettingsApi(generics.RetrieveUpdateAPIView): - permission_classes = (IsSuperUser,) - serializer_class = SettingsSerializer - - def get_object(self): - instance = {category: self._get_setting_fields_obj(list(category_serializer.get_fields())) - for category, category_serializer in self.serializer_class().get_fields().items() - if isinstance(category_serializer, serializers.Serializer)} - return ObjectDict(instance) - - def perform_update(self, serializer): - serializer.save() - - def _get_setting_fields_obj(self, category_fields): - if isinstance(category_fields, Iterable): - fields_data = {field_name: getattr(settings, field_name) - for field_name in category_fields} - return ObjectDict(fields_data) - - if isinstance(category_fields, str): - fields_data = {category_fields: getattr(settings, category_fields)} - return ObjectDict(fields_data) - - return ObjectDict() diff --git a/apps/settings/models.py b/apps/settings/models.py index 30732a00c..5617b010d 100644 --- a/apps/settings/models.py +++ b/apps/settings/models.py @@ -3,9 +3,11 @@ import json from django.db import models from django.db.utils import ProgrammingError, OperationalError from django.utils.translation import ugettext_lazy as _ -from django.core.cache import cache +from django.conf import settings -from common.utils import signer +from common.utils import signer, get_logger + +logger = get_logger(__name__) class SettingQuerySet(models.QuerySet): @@ -37,24 +39,6 @@ class Setting(models.Model): def __str__(self): return self.name - @classmethod - def get(cls, item): - cached = cls.get_from_cache(item) - if cached is not None: - return cached - instances = cls.objects.filter(name=item) - if len(instances) == 1: - s = instances[0] - s.refresh_setting() - return s.cleaned_value - return None - - @classmethod - def get_from_cache(cls, item): - key = cls.cache_key_prefix + item - cached = cache.get(key) - return cached - @property def cleaned_value(self): try: @@ -87,9 +71,52 @@ class Setting(models.Model): except (ProgrammingError, OperationalError): pass + @classmethod + def refresh_item(cls, name): + item = cls.objects.filter(name=name).first() + if not item: + return + item.refresh_setting() + def refresh_setting(self): - key = self.cache_key_prefix + self.name - cache.set(key, self.cleaned_value, None) + logger.debug(f"Refresh setting: {self.name}") + if hasattr(self.__class__, f'refresh_{self.name}'): + getattr(self.__class__, f'refresh_{self.name}')() + else: + setattr(settings, self.name, self.cleaned_value) + + @classmethod + def refresh_AUTH_LDAP(cls): + setting = cls.objects.filter(name='AUTH_LDAP').first() + if not setting: + return + ldap_backend = 'authentication.backends.ldap.LDAPAuthorizationBackend' + backends = settings.AUTHENTICATION_BACKENDS + has = ldap_backend in backends + if setting.cleaned_value and not has: + settings.AUTHENTICATION_BACKENDS.insert(0, ldap_backend) + + if not setting.cleaned_value and has: + index = backends.index(ldap_backend) + backends.pop(index) + settings.AUTH_LDAP = setting.cleaned_value + + @classmethod + def update_or_create(cls, name='', value='', encrypted=False, category=''): + """ + 不能使用 Model 提供的，update_or_create 因为这里有 encrypted 和 cleaned_value + :return: (changed, instance) + """ + setting = cls.objects.filter(name=name).first() + changed = False + if not setting: + setting = Setting(name=name, encrypted=encrypted, category=category) + if setting.cleaned_value != value: + setting.encrypted = encrypted + setting.cleaned_value = value + setting.save() + changed = True + return changed, setting class Meta: db_table = "settings_setting" diff --git a/apps/settings/serializers/settings.py b/apps/settings/serializers/settings.py index 208c86078..982d78ff4 100644 --- a/apps/settings/serializers/settings.py +++ b/apps/settings/serializers/settings.py @@ -1,49 +1,100 @@ # coding: utf-8 -from django.db import transaction from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from ..models import Setting -__all__ = ['SettingsSerializer'] +__all__ = [ + 'BasicSettingSerializer', 'EmailSettingSerializer', 'EmailContentSettingSerializer', + 'LDAPSettingSerializer', 'TerminalSettingSerializer', 'SecuritySettingSerializer', + 'SettingsSerializer' +] class BasicSettingSerializer(serializers.Serializer): - SITE_URL = serializers.URLField(required=True) - USER_GUIDE_URL = serializers.URLField(required=False, allow_blank=True, ) - EMAIL_SUBJECT_PREFIX = serializers.CharField(max_length=1024, required=True) + SITE_URL = serializers.URLField( + required=True, label=_("Site url"), + help_text=_('eg: http://demo.jumpserver.org:8080') + ) + USER_GUIDE_URL = serializers.URLField( + required=False, allow_blank=True, label=_("User guide url"), + help_text=_('User first login update profile done redirect to it') + ) class EmailSettingSerializer(serializers.Serializer): - encrypt_fields = ["EMAIL_HOST_PASSWORD", ] + # encrypt_fields 现在使用 write_only 来判断了 - EMAIL_HOST = serializers.CharField(max_length=1024, required=True) - EMAIL_PORT = serializers.CharField(max_length=5, required=True) - EMAIL_HOST_USER = serializers.CharField(max_length=128, required=True) - EMAIL_HOST_PASSWORD = serializers.CharField(max_length=1024, write_only=True, required=False, ) - EMAIL_FROM = serializers.CharField(max_length=128, allow_blank=True, required=False) - EMAIL_RECIPIENT = serializers.CharField(max_length=128, allow_blank=True, required=False) - EMAIL_USE_SSL = serializers.BooleanField(required=False) - EMAIL_USE_TLS = serializers.BooleanField(required=False) + EMAIL_HOST = serializers.CharField(max_length=1024, required=True, label=_("SMTP host")) + EMAIL_PORT = serializers.CharField(max_length=5, required=True, label=_("SMTP port")) + EMAIL_HOST_USER = serializers.CharField(max_length=128, required=True, label=_("SMTP account")) + EMAIL_HOST_PASSWORD = serializers.CharField( + max_length=1024, write_only=True, required=False, label=_("SMTP password"), + help_text=_("Tips: Some provider use token except password") + ) + EMAIL_FROM = serializers.CharField( + max_length=128, allow_blank=True, required=False, label=_('Send user'), + help_text=_('Tips: Send mail account, default SMTP account as the send account') + ) + EMAIL_RECIPIENT = serializers.CharField( + max_length=128, allow_blank=True, required=False, label=_('Test recipient'), + help_text=_('Tips: Used only as a test mail recipient') + ) + EMAIL_USE_SSL = serializers.BooleanField( + required=False, label=_('Use SSL'), + help_text=_('If SMTP port is 465, may be select') + ) + EMAIL_USE_TLS = serializers.BooleanField( + required=False, label=_("Use TLS"), + help_text=_('If SMTP port is 587, may be select') + ) + EMAIL_SUBJECT_PREFIX = serializers.CharField( + max_length=1024, required=True, label=_('Subject prefix') + ) class EmailContentSettingSerializer(serializers.Serializer): - EMAIL_CUSTOM_USER_CREATED_SUBJECT = serializers.CharField(max_length=1024, allow_blank=True, required=False, ) - EMAIL_CUSTOM_USER_CREATED_HONORIFIC = serializers.CharField(max_length=1024, allow_blank=True, required=False, ) - EMAIL_CUSTOM_USER_CREATED_BODY = serializers.CharField(max_length=4096, allow_blank=True, required=False) - EMAIL_CUSTOM_USER_CREATED_SIGNATURE = serializers.CharField(max_length=512, allow_blank=True, required=False) + EMAIL_CUSTOM_USER_CREATED_SUBJECT = serializers.CharField( + max_length=1024, allow_blank=True, required=False, + label=_('Create user email subject'), + help_text=_('Tips: When creating a user, send the subject of the email (eg:Create account successfully)') + ) + EMAIL_CUSTOM_USER_CREATED_HONORIFIC = serializers.CharField( + max_length=1024, allow_blank=True, required=False, + label=_('Create user honorific'), + help_text=_('Tips: When creating a user, send the honorific of the email (eg:Hello)') + ) + EMAIL_CUSTOM_USER_CREATED_BODY = serializers.CharField( + max_length=4096, allow_blank=True, required=False, + label=_('Create user email content'), + help_text=_('Tips:When creating a user, send the content of the email') + ) + EMAIL_CUSTOM_USER_CREATED_SIGNATURE = serializers.CharField( + max_length=512, allow_blank=True, required=False, label=_('Signature'), + help_text=_('Tips: Email signature (eg:jumpserver)') + ) -class LdapSettingSerializer(serializers.Serializer): - encrypt_fields = ["AUTH_LDAP_BIND_PASSWORD", ] +class LDAPSettingSerializer(serializers.Serializer): + # encrypt_fields 现在使用 write_only 来判断了 - AUTH_LDAP_SERVER_URI = serializers.CharField(required=True) - AUTH_LDAP_BIND_DN = serializers.CharField(required=False) - AUTH_LDAP_BIND_PASSWORD = serializers.CharField(max_length=1024, write_only=True, required=False) - AUTH_LDAP_SEARCH_OU = serializers.CharField(max_length=1024, allow_blank=True, required=False) - AUTH_LDAP_SEARCH_FILTER = serializers.CharField(max_length=1024, required=True) - AUTH_LDAP_USER_ATTR_MAP = serializers.DictField(required=True) - AUTH_LDAP = serializers.BooleanField(required=False) + AUTH_LDAP_SERVER_URI = serializers.CharField( + required=True, max_length=1024, label=_('LDAP server'), help_text=_('eg: ldap://localhost:389') + ) + AUTH_LDAP_BIND_DN = serializers.CharField(required=False, max_length=1024, label=_('Bind DN')) + AUTH_LDAP_BIND_PASSWORD = serializers.CharField(max_length=1024, write_only=True, required=False, label=_('Password')) + AUTH_LDAP_SEARCH_OU = serializers.CharField( + max_length=1024, allow_blank=True, required=False, label=_('User OU'), + help_text=_('Use | split multi OUs') + ) + AUTH_LDAP_SEARCH_FILTER = serializers.CharField( + max_length=1024, required=True, label=_('User search filter'), + help_text=_('Choice may be (cn|uid|sAMAccountName)=%(user)s)') + ) + AUTH_LDAP_USER_ATTR_MAP = serializers.DictField( + required=True, label=_('User attr map'), + help_text=_('User attr map present how to map LDAP user attr to jumpserver, username,name,email is jumpserver attr') + ) + AUTH_LDAP = serializers.BooleanField(required=False, label=_('Enable LDAP auth')) class TerminalSettingSerializer(serializers.Serializer): @@ -60,67 +111,75 @@ class TerminalSettingSerializer(serializers.Serializer): ('25', '25'), ('50', '50'), ) - TERMINAL_PASSWORD_AUTH = serializers.BooleanField(required=False) - TERMINAL_PUBLIC_KEY_AUTH = serializers.BooleanField(required=False) - TERMINAL_HEARTBEAT_INTERVAL = serializers.IntegerField(min_value=5, max_value=99999, required=False) - TERMINAL_ASSET_LIST_SORT_BY = serializers.ChoiceField(SORT_BY_CHOICES, required=False) - TERMINAL_ASSET_LIST_PAGE_SIZE = serializers.ChoiceField(PAGE_SIZE_CHOICES, required=False) - TERMINAL_SESSION_KEEP_DURATION = serializers.IntegerField(min_value=1, max_value=99999, required=True) - TERMINAL_TELNET_REGEX = serializers.CharField(allow_blank=True, required=False) + TERMINAL_PASSWORD_AUTH = serializers.BooleanField(required=False, label=_('Password auth')) + TERMINAL_PUBLIC_KEY_AUTH = serializers.BooleanField(required=False, label=_('Public key auth')) + TERMINAL_ASSET_LIST_SORT_BY = serializers.ChoiceField(SORT_BY_CHOICES, required=False, label=_('List sort by')) + TERMINAL_ASSET_LIST_PAGE_SIZE = serializers.ChoiceField(PAGE_SIZE_CHOICES, required=False, label=_('List page size')) + TERMINAL_SESSION_KEEP_DURATION = serializers.IntegerField( + min_value=1, max_value=99999, required=True, label=_('Session keep duration'), + help_text=_('Units: days, Session, record, command will be delete if more than duration, only in database') + ) + TERMINAL_TELNET_REGEX = serializers.CharField(allow_blank=True, max_length=1024, required=False, label=_('Telnet login regex')) class SecuritySettingSerializer(serializers.Serializer): - SECURITY_MFA_AUTH = serializers.BooleanField(required=False) - SECURITY_COMMAND_EXECUTION = serializers.BooleanField(required=False) - SECURITY_SERVICE_ACCOUNT_REGISTRATION = serializers.BooleanField(required=True) - SECURITY_LOGIN_LIMIT_COUNT = serializers.IntegerField(min_value=3, max_value=99999, required=True) - SECURITY_LOGIN_LIMIT_TIME = serializers.IntegerField(min_value=5, max_value=99999, required=True) - SECURITY_MAX_IDLE_TIME = serializers.IntegerField(min_value=1, max_value=99999, required=False) - SECURITY_PASSWORD_EXPIRATION_TIME = serializers.IntegerField(min_value=1, max_value=99999, required=True) - SECURITY_PASSWORD_MIN_LENGTH = serializers.IntegerField(min_value=6, max_value=30, required=True) - SECURITY_PASSWORD_UPPER_CASE = serializers.BooleanField(required=False) - SECURITY_PASSWORD_LOWER_CASE = serializers.BooleanField(required=False) - SECURITY_PASSWORD_NUMBER = serializers.BooleanField(required=False) - SECURITY_PASSWORD_SPECIAL_CHAR = serializers.BooleanField(required=False) - SECURITY_INSECURE_COMMAND = serializers.BooleanField(required=False) - SECURITY_INSECURE_COMMAND_EMAIL_RECEIVER = serializers.CharField(max_length=8192, required=False, allow_blank=True) + SECURITY_MFA_AUTH = serializers.BooleanField( + required=False, label=_("Global MFA auth"), + help_text=_('All user enable MFA') + ) + SECURITY_COMMAND_EXECUTION = serializers.BooleanField( + required=False, label=_('Batch command execution'), + help_text=_('Allow user run batch command or not using ansible') + ) + SECURITY_SERVICE_ACCOUNT_REGISTRATION = serializers.BooleanField( + required=True, label=_('Enable terminal register'), + help_text=_("Allow terminal register, after all terminal setup, you should disable this for security") + ) + SECURITY_LOGIN_LIMIT_COUNT = serializers.IntegerField( + min_value=3, max_value=99999, + label=_('Limit the number of login failures') + ) + SECURITY_LOGIN_LIMIT_TIME = serializers.IntegerField( + min_value=5, max_value=99999, required=True, + label=_('Block logon interval'), + help_text=_('Tip: (unit/minute) if the user has failed to log in for a limited number of times, no login is allowed during this time interval.') + ) + SECURITY_MAX_IDLE_TIME = serializers.IntegerField( + min_value=1, max_value=99999, required=False, + label=_('Connection max idle time'), + help_text=_('If idle time more than it, disconnect connection Unit: minute') + ) + SECURITY_PASSWORD_EXPIRATION_TIME = serializers.IntegerField( + min_value=1, max_value=99999, required=True, + label=_('User password expiration'), + help_text=_('Tip: (unit: day) If the user does not update the password during the time, the user password will expire failure;The password expiration reminder mail will be automatic sent to the user by system within 5 days (daily) before the password expires') + ) + SECURITY_PASSWORD_MIN_LENGTH = serializers.IntegerField( + min_value=6, max_value=30, required=True, + label=_('Password minimum length') + ) + SECURITY_PASSWORD_UPPER_CASE = serializers.BooleanField( + required=False, label=_('Must contain capital') + ) + SECURITY_PASSWORD_LOWER_CASE = serializers.BooleanField(required=False, label=_('Must contain lowercase')) + SECURITY_PASSWORD_NUMBER = serializers.BooleanField(required=False, label=_('Must contain numeric')) + SECURITY_PASSWORD_SPECIAL_CHAR = serializers.BooleanField(required=False, label=_('Must contain special')) + SECURITY_INSECURE_COMMAND = serializers.BooleanField(required=False, label=_('Insecure command alert')) + SECURITY_INSECURE_COMMAND_EMAIL_RECEIVER = serializers.CharField( + max_length=8192, required=False, allow_blank=True, label=_('Email recipient'), + help_text=_('Multiple user using , split') + ) -class SettingsSerializer(serializers.Serializer): - basic = BasicSettingSerializer(required=False) - email = EmailSettingSerializer(required=False) - email_content = EmailContentSettingSerializer(required=False) - ldap = LdapSettingSerializer(required=False) - terminal = TerminalSettingSerializer(required=False) - security = SecuritySettingSerializer(required=False) +class SettingsSerializer( + BasicSettingSerializer, + EmailSettingSerializer, + EmailContentSettingSerializer, + LDAPSettingSerializer, + TerminalSettingSerializer, + SecuritySettingSerializer +): - encrypt_fields = ["EMAIL_HOST_PASSWORD", "AUTH_LDAP_BIND_PASSWORD"] + # encrypt_fields 现在使用 write_only 来判断了 + pass - def create(self, validated_data): - pass - - def update(self, instance, validated_data): - for category, category_data in validated_data.items(): - if not category_data: - continue - self.update_validated_settings(category_data) - for field_name, field_value in category_data.items(): - setattr(getattr(instance, category), field_name, field_value) - - return instance - - def update_validated_settings(self, validated_data, category='default'): - if not validated_data: - return - with transaction.atomic(): - for field_name, field_value in validated_data.items(): - try: - setting = Setting.objects.get(name=field_name) - except Setting.DoesNotExist: - setting = Setting() - encrypted = True if field_name in self.encrypt_fields else False - setting.name = field_name - setting.category = category - setting.encrypted = encrypted - setting.cleaned_value = field_value - setting.save() diff --git a/apps/settings/signals_handler.py b/apps/settings/signals_handler.py index 94c4569bd..9625df3f6 100644 --- a/apps/settings/signals_handler.py +++ b/apps/settings/signals_handler.py @@ -1,28 +1,44 @@ # -*- coding: utf-8 -*- # import json +import threading from django.dispatch import receiver from django.db.models.signals import post_save, pre_save +from django.utils.functional import LazyObject from jumpserver.utils import current_request +from common.decorator import on_transaction_commit from common.utils import get_logger, ssh_key_gen +from common.utils.connection import RedisPubSub from common.signals import django_ready from .models import Setting logger = get_logger(__file__) -@receiver(post_save, sender=Setting, dispatch_uid="my_unique_identifier") +def get_settings_pub_sub(): + return RedisPubSub('settings') + + +class SettingSubPub(LazyObject): + def _setup(self): + self._wrapped = get_settings_pub_sub() + + +setting_pub_sub = SettingSubPub() + + +@receiver(post_save, sender=Setting) +@on_transaction_commit def refresh_settings_on_changed(sender, instance=None, **kwargs): if instance: - instance.refresh_setting() + setting_pub_sub.publish(instance.name) @receiver(django_ready) def on_django_ready_add_db_config(sender, **kwargs): - from django.conf import settings - settings.DYNAMIC.db_setting = Setting + Setting.refresh_all_settings() @receiver(django_ready) @@ -41,9 +57,27 @@ def auto_generate_terminal_host_key(sender, **kwargs): def on_create_set_created_by(sender, instance=None, **kwargs): if getattr(instance, '_ignore_auto_created_by', False) is True: return - if hasattr(instance, 'created_by') and not instance.created_by: - if current_request and current_request.user.is_authenticated: - user_name = current_request.user.name - if isinstance(user_name, str): - user_name = user_name[:30] - instance.created_by = user_name + if not hasattr(instance, 'created_by') or instance.created_by: + return + if current_request and current_request.user.is_authenticated: + user_name = current_request.user.name + if isinstance(user_name, str): + user_name = user_name[:30] + instance.created_by = user_name + + +@receiver(django_ready) +def subscribe_settings_change(sender, **kwargs): + logger.debug("Start subscribe setting change") + + def keep_subscribe(): + sub = setting_pub_sub.subscribe() + for msg in sub.listen(): + if msg["type"] != "message": + continue + item = msg['data'].decode() + logger.debug("Found setting change: {}".format(str(item))) + Setting.refresh_item(item) + t = threading.Thread(target=keep_subscribe) + t.daemon = True + t.start() diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 36e5eb330..1d8590ed0 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -16,7 +16,6 @@ from django.utils.translation import ugettext_lazy as _ from django.utils import timezone from django.shortcuts import reverse -from common.local import LOCAL_DYNAMIC_SETTINGS from orgs.utils import current_org from orgs.models import OrganizationMember, Organization from common.utils import date_expired_default, get_logger, lazyproperty @@ -452,7 +451,7 @@ class MFAMixin: @property def mfa_force_enabled(self): - if LOCAL_DYNAMIC_SETTINGS.SECURITY_MFA_AUTH: + if settings.SECURITY_MFA_AUTH: return True return self.mfa_level == 2 From dd5b2b910172c0df27f675a82b7af14a670bdcb0 Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 20 Jan 2021 18:33:38 +0800 Subject: [PATCH 05/71] =?UTF-8?q?perf:=20=E5=8E=BB=E6=8E=89=20v2=20?= =?UTF-8?q?=E7=9A=84api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/jumpserver/urls.py | 13 - apps/jumpserver/views/swagger.py | 12 +- apps/terminal/api/terminal.py | 22 +- apps/terminal/api_v2/__init__.py | 3 - apps/terminal/api_v2/terminal.py | 44 ---- apps/terminal/serializers/terminal.py | 38 ++- apps/terminal/serializers_v2/__init__.py | 4 - apps/terminal/serializers_v2/terminal.py | 60 ----- apps/terminal/urls/api_urls.py | 1 + apps/terminal/urls/api_urls_v2.py | 26 -- apps/users/api/__init__.py | 1 + .../user.py => api/service_account.py} | 2 +- apps/users/api_v2/__init__.py | 4 - apps/users/serializers/__init__.py | 1 + apps/users/serializers/profile.py | 177 ++++++++++++++ apps/users/serializers/user.py | 222 ++++-------------- apps/users/serializers_v2/__init__.py | 3 - apps/users/serializers_v2/user.py | 48 ---- apps/users/urls/api_urls.py | 1 + apps/users/urls/api_urls_v2.py | 23 -- 20 files changed, 285 insertions(+), 420 deletions(-) delete mode 100644 apps/terminal/api_v2/__init__.py delete mode 100644 apps/terminal/api_v2/terminal.py delete mode 100644 apps/terminal/serializers_v2/__init__.py delete mode 100644 apps/terminal/serializers_v2/terminal.py delete mode 100644 apps/terminal/urls/api_urls_v2.py rename apps/users/{api_v2/user.py => api/service_account.py} (87%) delete mode 100644 apps/users/api_v2/__init__.py create mode 100644 apps/users/serializers/profile.py delete mode 100644 apps/users/serializers_v2/__init__.py delete mode 100644 apps/users/serializers_v2/user.py delete mode 100644 apps/users/urls/api_urls_v2.py diff --git a/apps/jumpserver/urls.py b/apps/jumpserver/urls.py index 713db5791..306b1d9ea 100644 --- a/apps/jumpserver/urls.py +++ b/apps/jumpserver/urls.py @@ -4,8 +4,6 @@ from __future__ import unicode_literals from django.urls import path, include, re_path from django.conf import settings from django.conf.urls.static import static -from django.conf.urls.i18n import i18n_patterns -from django.http import HttpResponse from django.views.i18n import JavaScriptCatalog from . import views, api @@ -27,11 +25,6 @@ api_v1 = [ path('prometheus/metrics/', api.PrometheusMetricsApi.as_view()) ] -api_v2 = [ - path('terminal/', include('terminal.urls.api_urls_v2', namespace='api-terminal-v2')), - path('users/', include('users.urls.api_urls_v2', namespace='api-users-v2')), -] - app_view_patterns = [ path('auth/', include('authentication.urls.view_urls'), name='auth'), path('ops/', include('ops.urls.view_urls'), name='ops'), @@ -53,7 +46,6 @@ apps = [ urlpatterns = [ path('', views.IndexView.as_view(), name='index'), path('api/v1/', include(api_v1)), - path('api/v2/', include(api_v2)), re_path('api/(?P\w+)/(?Pv\d)/.*', views.redirect_format_api), path('api/health/', views.HealthCheckView.as_view(), name="health"), # External apps url @@ -77,11 +69,6 @@ urlpatterns += [ views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'), re_path('api/docs/?', views.get_swagger_view().with_ui('swagger', cache_timeout=1), name="docs"), re_path('api/redoc/?', views.get_swagger_view().with_ui('redoc', cache_timeout=1), name='redoc'), - - re_path('^api/v2/swagger(?P\.json|\.yaml)$', - views.get_swagger_view().without_ui(cache_timeout=1), name='schema-json'), - path('api/docs/v2/', views.get_swagger_view("v2").with_ui('swagger', cache_timeout=1), name="docs"), - path('api/redoc/v2/', views.get_swagger_view("v2").with_ui('redoc', cache_timeout=1), name='redoc'), ] diff --git a/apps/jumpserver/views/swagger.py b/apps/jumpserver/views/swagger.py index 34a5777c6..b42d283b6 100644 --- a/apps/jumpserver/views/swagger.py +++ b/apps/jumpserver/views/swagger.py @@ -60,20 +60,12 @@ api_info = openapi.Info( def get_swagger_view(version='v1'): - from ..urls import api_v1, api_v2 + from ..urls import api_v1 from django.urls import path, include api_v1_patterns = [ path('api/v1/', include(api_v1)) ] - - api_v2_patterns = [ - path('api/v2/', include(api_v2)) - ] - - if version == "v2": - patterns = api_v2_patterns - else: - patterns = api_v1_patterns + patterns = api_v1_patterns schema_view = get_schema_view( api_info, patterns=patterns, diff --git a/apps/terminal/api/terminal.py b/apps/terminal/api/terminal.py index 26b030ff8..444061035 100644 --- a/apps/terminal/api/terminal.py +++ b/apps/terminal/api/terminal.py @@ -5,18 +5,22 @@ import uuid from django.core.cache import cache from django.shortcuts import get_object_or_404 -from rest_framework import viewsets +from rest_framework import viewsets, generics from rest_framework.views import APIView, Response +from rest_framework import status +from django.conf import settings + from common.drf.api import JMSBulkModelViewSet from common.utils import get_object_or_none -from common.permissions import IsAppUser, IsOrgAdminOrAppUser, IsSuperUser +from common.permissions import IsAppUser, IsOrgAdminOrAppUser, IsSuperUser, WithBootstrapToken from ..models import Terminal, Status, Session from .. import serializers from .. import exceptions __all__ = [ 'TerminalViewSet', 'StatusViewSet', 'TerminalConfig', + 'TerminalRegistrationApi', ] logger = logging.getLogger(__file__) @@ -112,4 +116,16 @@ class TerminalConfig(APIView): def get(self, request): config = request.user.terminal.config - return Response(config, status=200) \ No newline at end of file + return Response(config, status=200) + + +class TerminalRegistrationApi(generics.CreateAPIView): + serializer_class = serializers.TerminalRegistrationSerializer + permission_classes = [WithBootstrapToken] + http_method_names = ['post'] + + def create(self, request, *args, **kwargs): + if not settings.SECURITY_SERVICE_ACCOUNT_REGISTRATION: + data = {"error": "service account registration disabled"} + return Response(data=data, status=status.HTTP_400_BAD_REQUEST) + return super().create(request, *args, **kwargs) diff --git a/apps/terminal/api_v2/__init__.py b/apps/terminal/api_v2/__init__.py deleted file mode 100644 index 0dfa92ec1..000000000 --- a/apps/terminal/api_v2/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- coding: utf-8 -*- -# -from .terminal import * diff --git a/apps/terminal/api_v2/terminal.py b/apps/terminal/api_v2/terminal.py deleted file mode 100644 index 57b8ab3e3..000000000 --- a/apps/terminal/api_v2/terminal.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# -from rest_framework import viewsets, generics -from rest_framework import status -from rest_framework.response import Response -from django.conf import settings - -from common.permissions import IsSuperUser, WithBootstrapToken - - -from ..models import Terminal -from .. import serializers_v2 as serializers - -__all__ = ['TerminalViewSet', 'TerminalRegistrationApi'] - - -class TerminalViewSet(viewsets.ModelViewSet): - queryset = Terminal.objects.filter(is_deleted=False) - serializer_class = serializers.TerminalSerializer - permission_classes = [IsSuperUser] - http_method_names = [ - 'get', 'put', 'patch', 'delete', 'head', 'options', 'trace' - ] - - -class TerminalRegistrationApi(generics.CreateAPIView): - serializer_class = serializers.TerminalRegistrationSerializer - permission_classes = [WithBootstrapToken] - http_method_names = ['post'] - - def create(self, request, *args, **kwargs): - data = {k: v for k, v in request.data.items()} - serializer = serializers.TerminalSerializer( - data=data, context={'request': request} - ) - if not settings.SECURITY_SERVICE_ACCOUNT_REGISTRATION: - data = {"error": "service account registration disabled"} - return Response(data=data, status=status.HTTP_400_BAD_REQUEST) - serializer.is_valid(raise_exception=True) - terminal = serializer.save() - sa_serializer = serializer.sa_serializer_class(instance=terminal.user) - data['service_account'] = sa_serializer.data - return Response(data, status=status.HTTP_201_CREATED) - diff --git a/apps/terminal/serializers/terminal.py b/apps/terminal/serializers/terminal.py index 33d46ada1..caffba522 100644 --- a/apps/terminal/serializers/terminal.py +++ b/apps/terminal/serializers/terminal.py @@ -3,6 +3,9 @@ from django.utils.translation import ugettext_lazy as _ from common.drf.serializers import BulkModelSerializer, AdaptedBulkListSerializer from common.utils import is_uuid +from users.serializers import ServiceAccountSerializer +from common.utils import get_request_ip + from ..models import ( Terminal, Status, Session, Task, CommandStorage, ReplayStorage ) @@ -63,9 +66,42 @@ class StatusSerializer(serializers.ModelSerializer): class TaskSerializer(BulkModelSerializer): - class Meta: fields = '__all__' model = Task list_serializer_class = AdaptedBulkListSerializer ref_name = 'TerminalTaskSerializer' + + +class TerminalRegistrationSerializer(serializers.ModelSerializer): + service_account = ServiceAccountSerializer(read_only=True) + + class Meta: + model = Terminal + fields = ['name', 'type', 'comment', 'service_account', 'remote_addr'] + extra_fields = { + 'remote_addr': {'readonly': True} + } + + def is_valid(self, raise_exception=False): + valid = super().is_valid(raise_exception=raise_exception) + if not valid: + return valid + data = {'name': self.validated_data.get('name')} + kwargs = {'data': data} + if self.instance and self.instance.user: + kwargs['instance'] = self.instance.user + self.service_account = ServiceAccountSerializer(**kwargs) + valid = self.service_account.is_valid(raise_exception=True) + return valid + + def save(self, **kwargs): + instance = super().save(**kwargs) + request = self.context.get('request') + instance.is_accepted = True + if request: + instance.remote_addr = get_request_ip(request) + sa = self.service_account.save() + instance.user = sa + instance.save() + return instance diff --git a/apps/terminal/serializers_v2/__init__.py b/apps/terminal/serializers_v2/__init__.py deleted file mode 100644 index 9161be085..000000000 --- a/apps/terminal/serializers_v2/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from .terminal import * diff --git a/apps/terminal/serializers_v2/terminal.py b/apps/terminal/serializers_v2/terminal.py deleted file mode 100644 index 8121410e6..000000000 --- a/apps/terminal/serializers_v2/terminal.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# -from django.conf import settings -from rest_framework import serializers - -from common.utils import get_request_ip -from users.serializers_v2 import ServiceAccountSerializer -from ..models import Terminal - - -__all__ = ['TerminalSerializer', 'TerminalRegistrationSerializer'] - - -class TerminalSerializer(serializers.ModelSerializer): - sa_serializer_class = ServiceAccountSerializer - sa_serializer = None - - class Meta: - model = Terminal - fields = [ - 'id', 'name', 'type', 'remote_addr', 'command_storage', - 'replay_storage', 'user', 'is_accepted', 'is_deleted', - 'date_created', 'comment' - ] - read_only_fields = ['remote_addr', 'user', 'date_created'] - - def is_valid(self, raise_exception=False): - valid = super().is_valid(raise_exception=raise_exception) - if not valid: - return valid - data = {'name': self.validated_data.get('name')} - kwargs = {'data': data} - if self.instance and self.instance.user: - kwargs['instance'] = self.instance.user - self.sa_serializer = ServiceAccountSerializer(**kwargs) - valid = self.sa_serializer.is_valid(raise_exception=True) - return valid - - def save(self, **kwargs): - instance = super().save(**kwargs) - sa = self.sa_serializer.save() - instance.user = sa - instance.save() - return instance - - def create(self, validated_data): - request = self.context.get('request') - instance = super().create(validated_data) - instance.is_accepted = True - if request: - instance.remote_addr = get_request_ip(request) - instance.save() - return instance - - -class TerminalRegistrationSerializer(serializers.Serializer): - name = serializers.CharField(max_length=128) - comment = serializers.CharField(max_length=128) - type = serializers.CharField(max_length=64) - service_account = ServiceAccountSerializer(read_only=True) diff --git a/apps/terminal/urls/api_urls.py b/apps/terminal/urls/api_urls.py index 077494d01..38b6df976 100644 --- a/apps/terminal/urls/api_urls.py +++ b/apps/terminal/urls/api_urls.py @@ -22,6 +22,7 @@ router.register(r'replay-storages', api.ReplayStorageViewSet, 'replay-storage') router.register(r'command-storages', api.CommandStorageViewSet, 'command-storage') urlpatterns = [ + path('terminal-registrations/', api.TerminalRegistrationApi.as_view(), name='terminal-registration'), path('sessions/join/validate/', api.SessionJoinValidateAPI.as_view(), name='join-session-validate'), path('sessions//replay/', api.SessionReplayViewSet.as_view({'get': 'retrieve', 'post': 'create'}), diff --git a/apps/terminal/urls/api_urls_v2.py b/apps/terminal/urls/api_urls_v2.py deleted file mode 100644 index 42c9ffdb5..000000000 --- a/apps/terminal/urls/api_urls_v2.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# - -from django.urls import path, re_path -from rest_framework_bulk.routes import BulkRouter - -from common import api as capi -from .. import api_v2 as api - -app_name = 'terminal' - -router = BulkRouter() -router.register(r'terminals', api.TerminalViewSet, 'terminal') - - -urlpatterns = [ - path('terminal-registrations/', api.TerminalRegistrationApi.as_view(), - name='terminal-registration') -] - -old_version_urlpatterns = [ - re_path('(?Pterminal)/.*', capi.redirect_plural_name_api) -] - -urlpatterns += router.urls diff --git a/apps/users/api/__init__.py b/apps/users/api/__init__.py index df750c107..d5db61888 100644 --- a/apps/users/api/__init__.py +++ b/apps/users/api/__init__.py @@ -4,4 +4,5 @@ from .user import * from .group import * from .profile import * +from .service_account import * from .relation import * diff --git a/apps/users/api_v2/user.py b/apps/users/api/service_account.py similarity index 87% rename from apps/users/api_v2/user.py rename to apps/users/api/service_account.py index fe097fa3f..783ba9162 100644 --- a/apps/users/api_v2/user.py +++ b/apps/users/api/service_account.py @@ -3,7 +3,7 @@ from rest_framework import viewsets from common.permissions import WithBootstrapToken -from .. import serializers_v2 as serializers +from .. import serializers class ServiceAccountRegistrationViewSet(viewsets.ModelViewSet): diff --git a/apps/users/api_v2/__init__.py b/apps/users/api_v2/__init__.py deleted file mode 100644 index 3c486a98c..000000000 --- a/apps/users/api_v2/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from .user import * diff --git a/apps/users/serializers/__init__.py b/apps/users/serializers/__init__.py index 41812a5c8..3dc95be36 100644 --- a/apps/users/serializers/__init__.py +++ b/apps/users/serializers/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # from .user import * +from .profile import * from .group import * from .realtion import * diff --git a/apps/users/serializers/profile.py b/apps/users/serializers/profile.py new file mode 100644 index 000000000..b1611307a --- /dev/null +++ b/apps/users/serializers/profile.py @@ -0,0 +1,177 @@ +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from common.utils import validate_ssh_public_key +from ..models import User + +from .user import UserSerializer + + +class UserOrgSerializer(serializers.Serializer): + id = serializers.CharField() + name = serializers.CharField() + + +class UserOrgLabelSerializer(serializers.Serializer): + value = serializers.CharField(source='id') + label = serializers.CharField(source='name') + + +class UserUpdatePasswordSerializer(serializers.ModelSerializer): + old_password = serializers.CharField(required=True, max_length=128, write_only=True) + new_password = serializers.CharField(required=True, max_length=128, write_only=True) + new_password_again = serializers.CharField(required=True, max_length=128, write_only=True) + + class Meta: + model = User + fields = ['old_password', 'new_password', 'new_password_again'] + + def validate_old_password(self, value): + if not self.instance.check_password(value): + msg = _('The old password is incorrect') + raise serializers.ValidationError(msg) + return value + + @staticmethod + def validate_new_password(value): + from ..utils import check_password_rules + if not check_password_rules(value): + msg = _('Password does not match security rules') + raise serializers.ValidationError(msg) + return value + + def validate_new_password_again(self, value): + if value != self.initial_data.get('new_password', ''): + msg = _('The newly set password is inconsistent') + raise serializers.ValidationError(msg) + return value + + def update(self, instance, validated_data): + new_password = self.validated_data.get('new_password') + instance.reset_password(new_password) + return instance + + +class UserUpdatePublicKeySerializer(serializers.ModelSerializer): + public_key_comment = serializers.CharField( + source='get_public_key_comment', required=False, read_only=True, max_length=128 + ) + public_key_hash_md5 = serializers.CharField( + source='get_public_key_hash_md5', required=False, read_only=True, max_length=128 + ) + + class Meta: + model = User + fields = ['public_key_comment', 'public_key_hash_md5', 'public_key'] + extra_kwargs = { + 'public_key': {'required': True, 'write_only': True, 'max_length': 2048} + } + + @staticmethod + def validate_public_key(value): + if not validate_ssh_public_key(value): + raise serializers.ValidationError(_('Not a valid ssh public key')) + return value + + def update(self, instance, validated_data): + new_public_key = self.validated_data.get('public_key') + instance.set_public_key(new_public_key) + return instance + + +class UserRoleSerializer(serializers.Serializer): + name = serializers.CharField(max_length=24) + display = serializers.CharField(max_length=64) + + +class UserProfileSerializer(UserSerializer): + admin_or_audit_orgs = UserOrgSerializer(many=True, read_only=True) + user_all_orgs = UserOrgLabelSerializer(many=True, read_only=True) + current_org_roles = serializers.ListField(read_only=True) + public_key_comment = serializers.CharField( + source='get_public_key_comment', required=False, read_only=True, max_length=128 + ) + public_key_hash_md5 = serializers.CharField( + source='get_public_key_hash_md5', required=False, read_only=True, max_length=128 + ) + MFA_LEVEL_CHOICES = ( + (0, _('Disable')), + (1, _('Enable')), + ) + mfa_level = serializers.ChoiceField(choices=MFA_LEVEL_CHOICES, label=_('MFA'), required=False) + guide_url = serializers.SerializerMethodField() + + class Meta(UserSerializer.Meta): + fields = UserSerializer.Meta.fields + [ + 'public_key_comment', 'public_key_hash_md5', 'admin_or_audit_orgs', 'current_org_roles', + 'guide_url', 'user_all_orgs' + ] + read_only_fields = [ + 'date_joined', 'last_login', 'created_by', 'source' + ] + extra_kwargs = dict(UserSerializer.Meta.extra_kwargs) + extra_kwargs.update({ + 'name': {'read_only': True, 'max_length': 128}, + 'username': {'read_only': True, 'max_length': 128}, + 'email': {'read_only': True}, + 'is_first_login': {'label': _('Is first login'), 'read_only': False}, + 'source': {'read_only': True}, + 'is_valid': {'read_only': True}, + 'is_active': {'read_only': True}, + 'groups': {'read_only': True}, + 'roles': {'read_only': True}, + 'password_strategy': {'read_only': True}, + 'date_expired': {'read_only': True}, + 'date_joined': {'read_only': True}, + 'last_login': {'read_only': True}, + 'role': {'read_only': True}, + }) + + if 'password' in fields: + fields.remove('password') + extra_kwargs.pop('password', None) + + @staticmethod + def get_guide_url(obj): + return settings.USER_GUIDE_URL + + def validate_mfa_level(self, mfa_level): + if self.instance and self.instance.mfa_force_enabled: + return 2 + return mfa_level + + def validate_public_key(self, public_key): + if self.instance and self.instance.can_update_ssh_key(): + if not validate_ssh_public_key(public_key): + raise serializers.ValidationError(_('Not a valid ssh public key')) + return public_key + return None + + +class UserPKUpdateSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ['id', 'public_key'] + + @staticmethod + def validate_public_key(value): + if not validate_ssh_public_key(value): + raise serializers.ValidationError(_('Not a valid ssh public key')) + return value + + +class ChangeUserPasswordSerializer(serializers.ModelSerializer): + + class Meta: + model = User + fields = ['password'] + +class ResetOTPSerializer(serializers.Serializer): + msg = serializers.CharField(read_only=True) + + def create(self, validated_data): + pass + + def update(self, instance, validated_data): + pass diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index b44ac050f..f1af1bfe9 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # from django.core.cache import cache -from django.conf import settings from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from common.utils import validate_ssh_public_key from common.mixins import CommonBulkSerializerMixin from common.permissions import CanUpdateDeleteUser from orgs.models import ROLE as ORG_ROLE @@ -13,24 +11,11 @@ from ..models import User __all__ = [ - 'UserSerializer', 'UserPKUpdateSerializer', - 'ChangeUserPasswordSerializer', 'ResetOTPSerializer', - 'UserProfileSerializer', 'UserOrgSerializer', - 'UserUpdatePasswordSerializer', 'UserUpdatePublicKeySerializer', - 'UserRetrieveSerializer', 'MiniUserSerializer', 'InviteSerializer' + 'UserSerializer', 'UserRetrieveSerializer', 'MiniUserSerializer', + 'InviteSerializer', 'ServiceAccountSerializer', ] -class UserOrgSerializer(serializers.Serializer): - id = serializers.CharField() - name = serializers.CharField() - - -class UserOrgLabelSerializer(serializers.Serializer): - value = serializers.CharField(source='id') - label = serializers.CharField(source='name') - - class UserSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer): EMAIL_SET_PASSWORD = _('Reset link will be generated and sent to the user') CUSTOM_PASSWORD = _('Set password') @@ -181,166 +166,6 @@ class UserRetrieveSerializer(UserSerializer): fields = UserSerializer.Meta.fields + ['login_confirm_settings'] -class UserPKUpdateSerializer(serializers.ModelSerializer): - class Meta: - model = User - fields = ['id', 'public_key'] - - @staticmethod - def validate_public_key(value): - if not validate_ssh_public_key(value): - raise serializers.ValidationError(_('Not a valid ssh public key')) - return value - - -class ChangeUserPasswordSerializer(serializers.ModelSerializer): - - class Meta: - model = User - fields = ['password'] - - -class ResetOTPSerializer(serializers.Serializer): - msg = serializers.CharField(read_only=True) - - def create(self, validated_data): - pass - - def update(self, instance, validated_data): - pass - - -class UserRoleSerializer(serializers.Serializer): - name = serializers.CharField(max_length=24) - display = serializers.CharField(max_length=64) - - -class UserProfileSerializer(UserSerializer): - admin_or_audit_orgs = UserOrgSerializer(many=True, read_only=True) - user_all_orgs = UserOrgLabelSerializer(many=True, read_only=True) - current_org_roles = serializers.ListField(read_only=True) - public_key_comment = serializers.CharField( - source='get_public_key_comment', required=False, read_only=True, max_length=128 - ) - public_key_hash_md5 = serializers.CharField( - source='get_public_key_hash_md5', required=False, read_only=True, max_length=128 - ) - MFA_LEVEL_CHOICES = ( - (0, _('Disable')), - (1, _('Enable')), - ) - mfa_level = serializers.ChoiceField(choices=MFA_LEVEL_CHOICES, label=_('MFA'), required=False) - guide_url = serializers.SerializerMethodField() - - class Meta(UserSerializer.Meta): - fields = UserSerializer.Meta.fields + [ - 'public_key_comment', 'public_key_hash_md5', 'admin_or_audit_orgs', 'current_org_roles', - 'guide_url', 'user_all_orgs' - ] - read_only_fields = [ - 'date_joined', 'last_login', 'created_by', 'source' - ] - extra_kwargs = dict(UserSerializer.Meta.extra_kwargs) - extra_kwargs.update({ - 'name': {'read_only': True, 'max_length': 128}, - 'username': {'read_only': True, 'max_length': 128}, - 'email': {'read_only': True}, - 'is_first_login': {'label': _('Is first login'), 'read_only': False}, - 'source': {'read_only': True}, - 'is_valid': {'read_only': True}, - 'is_active': {'read_only': True}, - 'groups': {'read_only': True}, - 'roles': {'read_only': True}, - 'password_strategy': {'read_only': True}, - 'date_expired': {'read_only': True}, - 'date_joined': {'read_only': True}, - 'last_login': {'read_only': True}, - 'role': {'read_only': True}, - }) - - if 'password' in fields: - fields.remove('password') - extra_kwargs.pop('password', None) - - @staticmethod - def get_guide_url(obj): - return settings.USER_GUIDE_URL - - def validate_mfa_level(self, mfa_level): - if self.instance and self.instance.mfa_force_enabled: - return 2 - return mfa_level - - def validate_public_key(self, public_key): - if self.instance and self.instance.can_update_ssh_key(): - if not validate_ssh_public_key(public_key): - raise serializers.ValidationError(_('Not a valid ssh public key')) - return public_key - return None - - -class UserUpdatePasswordSerializer(serializers.ModelSerializer): - old_password = serializers.CharField(required=True, max_length=128, write_only=True) - new_password = serializers.CharField(required=True, max_length=128, write_only=True) - new_password_again = serializers.CharField(required=True, max_length=128, write_only=True) - - class Meta: - model = User - fields = ['old_password', 'new_password', 'new_password_again'] - - def validate_old_password(self, value): - if not self.instance.check_password(value): - msg = _('The old password is incorrect') - raise serializers.ValidationError(msg) - return value - - @staticmethod - def validate_new_password(value): - from ..utils import check_password_rules - if not check_password_rules(value): - msg = _('Password does not match security rules') - raise serializers.ValidationError(msg) - return value - - def validate_new_password_again(self, value): - if value != self.initial_data.get('new_password', ''): - msg = _('The newly set password is inconsistent') - raise serializers.ValidationError(msg) - return value - - def update(self, instance, validated_data): - new_password = self.validated_data.get('new_password') - instance.reset_password(new_password) - return instance - - -class UserUpdatePublicKeySerializer(serializers.ModelSerializer): - public_key_comment = serializers.CharField( - source='get_public_key_comment', required=False, read_only=True, max_length=128 - ) - public_key_hash_md5 = serializers.CharField( - source='get_public_key_hash_md5', required=False, read_only=True, max_length=128 - ) - - class Meta: - model = User - fields = ['public_key_comment', 'public_key_hash_md5', 'public_key'] - extra_kwargs = { - 'public_key': {'required': True, 'write_only': True, 'max_length': 2048} - } - - @staticmethod - def validate_public_key(value): - if not validate_ssh_public_key(value): - raise serializers.ValidationError(_('Not a valid ssh public key')) - return value - - def update(self, instance, validated_data): - new_public_key = self.validated_data.get('public_key') - instance.set_public_key(new_public_key) - return instance - - class MiniUserSerializer(serializers.ModelSerializer): class Meta: model = User @@ -352,3 +177,46 @@ class InviteSerializer(serializers.Serializer): queryset=User.objects.exclude(role=User.ROLE.APP) ) role = serializers.ChoiceField(choices=ORG_ROLE.choices) + + +class ServiceAccountSerializer(serializers.ModelSerializer): + + class Meta: + model = User + fields = ['id', 'name', 'access_key'] + read_only_fields = ['access_key'] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + from authentication.serializers import AccessKeySerializer + self.fields['access_key'] = AccessKeySerializer(read_only=True) + + def get_username(self): + return self.initial_data.get('name') + + def get_email(self): + name = self.initial_data.get('name') + return '{}@serviceaccount.local'.format(name) + + def validate_name(self, name): + email = self.get_email() + username = self.get_username() + if self.instance: + users = User.objects.exclude(id=self.instance.id) + else: + users = User.objects.all() + if users.filter(email=email) or \ + users.filter(username=username): + raise serializers.ValidationError(_('name not unique'), code='unique') + return name + + def save(self, **kwargs): + self.validated_data['email'] = self.get_email() + self.validated_data['username'] = self.get_username() + self.validated_data['role'] = User.ROLE.APP + return super().save(**kwargs) + + def create(self, validated_data): + instance = super().create(validated_data) + instance.create_access_key() + return instance diff --git a/apps/users/serializers_v2/__init__.py b/apps/users/serializers_v2/__init__.py deleted file mode 100644 index c2dce9535..000000000 --- a/apps/users/serializers_v2/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- coding: utf-8 -*- -# -from .user import * diff --git a/apps/users/serializers_v2/user.py b/apps/users/serializers_v2/user.py deleted file mode 100644 index d2b9cfa1c..000000000 --- a/apps/users/serializers_v2/user.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -# -from django.utils.translation import ugettext as _ -from rest_framework import serializers -from ..models import User - -from authentication.serializers import AccessKeySerializer - -__all__ = ['ServiceAccountSerializer'] - - -class ServiceAccountSerializer(serializers.ModelSerializer): - access_key = AccessKeySerializer(read_only=True) - - class Meta: - model = User - fields = ['id', 'name', 'access_key'] - read_only_fields = ['access_key'] - - def get_username(self): - return self.initial_data.get('name') - - def get_email(self): - name = self.initial_data.get('name') - return '{}@serviceaccount.local'.format(name) - - def validate_name(self, name): - email = self.get_email() - username = self.get_username() - if self.instance: - users = User.objects.exclude(id=self.instance.id) - else: - users = User.objects.all() - if users.filter(email=email) or \ - users.filter(username=username): - raise serializers.ValidationError(_('name not unique'), code='unique') - return name - - def save(self, **kwargs): - self.validated_data['email'] = self.get_email() - self.validated_data['username'] = self.get_username() - self.validated_data['role'] = User.ROLE.APP - return super().save(**kwargs) - - def create(self, validated_data): - instance = super().create(validated_data) - instance.create_access_key() - return instance diff --git a/apps/users/urls/api_urls.py b/apps/users/urls/api_urls.py index ea1101eae..e6b6974c2 100644 --- a/apps/users/urls/api_urls.py +++ b/apps/users/urls/api_urls.py @@ -15,6 +15,7 @@ router = BulkRouter() router.register(r'users', api.UserViewSet, 'user') router.register(r'groups', api.UserGroupViewSet, 'user-group') router.register(r'users-groups-relations', api.UserUserGroupRelationViewSet, 'users-groups-relation') +router.register(r'service-account-registrations', api.ServiceAccountRegistrationViewSet, 'service-account-registration') urlpatterns = [ diff --git a/apps/users/urls/api_urls_v2.py b/apps/users/urls/api_urls_v2.py deleted file mode 100644 index dc3c6e249..000000000 --- a/apps/users/urls/api_urls_v2.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# ~*~ coding: utf-8 ~*~ -# -from __future__ import absolute_import - -from django.urls import path, include -from rest_framework_bulk.routes import BulkRouter -from .. import api_v2 as api - -app_name = 'users' - -router = BulkRouter() -router.register(r'service-account-registrations', - api.ServiceAccountRegistrationViewSet, - 'service-account-registration') - - -urlpatterns = [ - # path('token/', api.UserToken.as_view(), name='user-token'), -] -urlpatterns += router.urls - - From 23afe81ff5703ead90c523a98c0e6098c71ec05a Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 21 Jan 2021 13:50:29 +0800 Subject: [PATCH 06/71] =?UTF-8?q?perf(authentication):=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96connection=20token=E7=9A=84=E4=BD=BF=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/api/auth.py | 57 ++++++++++++++++-------------- apps/authentication/serializers.py | 31 ++++++++++++++++ apps/jumpserver/conf.py | 3 +- apps/jumpserver/settings/custom.py | 2 ++ 4 files changed, 66 insertions(+), 27 deletions(-) diff --git a/apps/authentication/api/auth.py b/apps/authentication/api/auth.py index 6886fa2f0..0b24950e2 100644 --- a/apps/authentication/api/auth.py +++ b/apps/authentication/api/auth.py @@ -2,54 +2,59 @@ # import uuid +from django.conf import settings from django.core.cache import cache -from django.shortcuts import get_object_or_404 from rest_framework.response import Response -from rest_framework.views import APIView +from rest_framework.generics import CreateAPIView from common.utils import get_logger -from common.permissions import IsOrgAdminOrAppUser +from common.permissions import IsSuperUserOrAppUser from orgs.mixins.api import RootOrgViewMixin -from users.models import User -from assets.models import Asset, SystemUser + +from ..serializers import ConnectionTokenSerializer logger = get_logger(__name__) -__all__ = [ - 'UserConnectionTokenApi', -] +__all__ = ['UserConnectionTokenApi'] -class UserConnectionTokenApi(RootOrgViewMixin, APIView): - permission_classes = (IsOrgAdminOrAppUser,) +class UserConnectionTokenApi(RootOrgViewMixin, CreateAPIView): + permission_classes = (IsSuperUserOrAppUser,) + serializer_class = ConnectionTokenSerializer - def post(self, request): - user_id = request.data.get('user', '') - asset_id = request.data.get('asset', '') - system_user_id = request.data.get('system_user', '') + def perform_create(self, serializer): + user = serializer.validated_data['user'] + asset = serializer.validated_data['asset'] + system_user = serializer.validated_data['system_user'] token = str(uuid.uuid4()) - user = get_object_or_404(User, id=user_id) - asset = get_object_or_404(Asset, id=asset_id) - system_user = get_object_or_404(SystemUser, id=system_user_id) value = { - 'user': user_id, + 'user': str(user.id), 'username': user.username, - 'asset': asset_id, + 'asset': str(asset.id), 'hostname': asset.hostname, - 'system_user': system_user_id, + 'system_user': str(system_user.id), 'system_user_name': system_user.name } cache.set(token, value, timeout=20) + return token + + def create(self, request, *args, **kwargs): + if not settings.CONNECTION_TOKEN_ENABLED: + data = {'error': 'Connection token disabled'} + return Response(data, status=400) + + if not request.user.is_superuser: + data = {'error': 'Only super user can create token'} + return Response(data, status=403) + + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + token = self.perform_create(serializer) return Response({"token": token}, status=201) def get(self, request): token = request.query_params.get('token') - user_only = request.query_params.get('user-only', None) value = cache.get(token, None) if not value: return Response('', status=404) - - if not user_only: - return Response(value) - else: - return Response({'user': value['user']}) + return Response(value) diff --git a/apps/authentication/serializers.py b/apps/authentication/serializers.py index 7d666db4c..d50173ff9 100644 --- a/apps/authentication/serializers.py +++ b/apps/authentication/serializers.py @@ -11,6 +11,7 @@ from .models import AccessKey, LoginConfirmSetting, SSOToken __all__ = [ 'AccessKeySerializer', 'OtpVerifySerializer', 'BearerTokenSerializer', 'MFAChallengeSerializer', 'LoginConfirmSettingSerializer', 'SSOTokenSerializer', + 'ConnectionTokenSerializer', ] @@ -82,3 +83,33 @@ class SSOTokenSerializer(serializers.Serializer): username = serializers.CharField(write_only=True) login_url = serializers.CharField(read_only=True) next = serializers.CharField(write_only=True, allow_blank=True, required=False, allow_null=True) + + +class ConnectionTokenSerializer(serializers.Serializer): + user = serializers.CharField(max_length=128, required=True) + system_user = serializers.CharField(max_length=128, required=True) + asset = serializers.CharField(max_length=128, required=True) + + @staticmethod + def validate_user(user_id): + from users.models import User + user = User.objects.filter(id=user_id).first() + if user is None: + raise serializers.ValidationError('user id not exist') + return user + + @staticmethod + def validate_system_user(system_user_id): + from assets.models import SystemUser + system_user = SystemUser.objects.filter(id=system_user_id).first() + if system_user is None: + raise serializers.ValidationError('system_user id not exist') + return system_user + + @staticmethod + def validate_asset(asset_id): + from assets.models import Asset + asset = Asset.objects.filter(id=asset_id).first() + if asset is None: + raise serializers.ValidationError('asset id not exist') + return asset diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 9989103f8..edfbae2c1 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -280,7 +280,8 @@ class Config(dict): 'SESSION_COOKIE_SECURE': False, 'CSRF_COOKIE_SECURE': False, 'REFERER_CHECK_ENABLED': False, - 'SERVER_REPLAY_STORAGE': {} + 'SERVER_REPLAY_STORAGE': {}, + 'CONNECTION_TOKEN_ENABLED': False, } def compatible_auth_openid_of_key(self): diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index 95c1fce16..9b4417ced 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -115,3 +115,5 @@ DATETIME_DISPLAY_FORMAT = '%Y-%m-%d %H:%M:%S' TICKETS_ENABLED = CONFIG.TICKETS_ENABLED REFERER_CHECK_ENABLED = CONFIG.REFERER_CHECK_ENABLED + +CONNECTION_TOKEN_ENABLED = CONFIG.CONNECTION_TOKEN_ENABLED \ No newline at end of file From 087a3f2914755760ac236a0a9e20277f041f7c34 Mon Sep 17 00:00:00 2001 From: Bai Date: Fri, 29 Jan 2021 23:52:16 +0800 Subject: [PATCH 07/71] =?UTF-8?q?fix:=20=E6=B3=A8=E9=87=8Afake=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=94=9F=E6=88=90=E6=A8=A1=E5=9D=97=E7=9A=84=E5=AF=BC?= =?UTF-8?q?=E5=85=A5(system)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/generate_fake_data/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/generate_fake_data/generate.py b/utils/generate_fake_data/generate.py index c87a8dbe4..839295049 100644 --- a/utils/generate_fake_data/generate.py +++ b/utils/generate_fake_data/generate.py @@ -15,7 +15,7 @@ django.setup() from resources.assets import AssetsGenerator, NodesGenerator, SystemUsersGenerator, AdminUsersGenerator from resources.users import UserGroupGenerator, UserGenerator from resources.perms import AssetPermissionGenerator -from resources.system import StatGenerator +# from resources.system import StatGenerator resource_generator_mapper = { @@ -26,7 +26,7 @@ resource_generator_mapper = { 'user': UserGenerator, 'user_group': UserGroupGenerator, 'asset_permission': AssetPermissionGenerator, - 'stat': StatGenerator + # 'stat': StatGenerator } From d852d2f670fb814c91a5e09c5c5287eb4d520a56 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 2 Feb 2021 16:28:27 +0800 Subject: [PATCH 08/71] =?UTF-8?q?perf:=20=E8=BF=98=E5=8E=9F=E5=9B=9E?= =?UTF-8?q?=E5=8E=9F=E6=9D=A5=E7=9A=84=E7=94=A8=E6=88=B7=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/users/serializers/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index f1af1bfe9..9eb10bc5a 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -54,7 +54,7 @@ class UserSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer): ] read_only_fields = [ - 'date_joined', 'last_login', 'created_by', 'is_first_login', 'source' + 'date_joined', 'last_login', 'created_by', 'is_first_login', ] extra_kwargs = { From 609d2710facd2045a87323f963e1886cee0401d6 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 3 Feb 2021 11:51:02 +0800 Subject: [PATCH 09/71] =?UTF-8?q?perf:=20=E4=BC=9A=E8=AF=9D=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E6=B7=BB=E5=8A=A0search=5Ffields=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/api/session.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/terminal/api/session.py b/apps/terminal/api/session.py index e8688819e..805b9b4a1 100644 --- a/apps/terminal/api/session.py +++ b/apps/terminal/api/session.py @@ -42,10 +42,10 @@ class SessionViewSet(OrgBulkModelViewSet): 'display': serializers.SessionDisplaySerializer, } permission_classes = (IsOrgAdminOrAppUser, ) - filterset_fields = [ - "user", "asset", "system_user", "remote_addr", - "protocol", "terminal", "is_finished", 'login_from', + search_fields = [ + "user", "asset", "system_user", "remote_addr", "protocol", "is_finished", 'login_from', ] + filterset_fields = search_fields + ['terminal'] date_range_filter_fields = [ ('date_start', ('date_from', 'date_to')) ] From 542eb25e7be1b783f28e1cf1dec7d926caced417 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 3 Feb 2021 12:01:18 +0800 Subject: [PATCH 10/71] =?UTF-8?q?fix(perms):=20=E4=BF=AE=E5=A4=8D=E6=9D=83?= =?UTF-8?q?=E9=99=90=E6=A0=A1=E9=AA=8C=E6=97=B6=E7=9A=84=E7=BB=84=E7=BB=87?= =?UTF-8?q?=E5=88=87=E6=8D=A2=E9=97=AE=E9=A2=98=20(#5546)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(perms): 修复权限校验时的组织切换问题 * fix(perms): 修复获取actions的切换组织问题 * perf: 继续添加 application 的验证组织 Co-authored-by: ibuler --- apps/perms/api/application/user_permission/common.py | 3 +++ apps/perms/api/asset/user_permission/common.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/apps/perms/api/application/user_permission/common.py b/apps/perms/api/application/user_permission/common.py index a147a45ce..428f6bdc9 100644 --- a/apps/perms/api/application/user_permission/common.py +++ b/apps/perms/api/application/user_permission/common.py @@ -2,11 +2,13 @@ # import uuid from django.shortcuts import get_object_or_404 +from django.utils.decorators import method_decorator from rest_framework.views import APIView, Response from rest_framework.generics import ( ListAPIView, get_object_or_404 ) +from orgs.utils import tmp_to_root_org from applications.models import Application from perms.utils.application.permission import ( get_application_system_users_id @@ -49,6 +51,7 @@ class MyGrantedApplicationSystemUsersApi(ForUserMixin, GrantedApplicationSystemU pass +@method_decorator(tmp_to_root_org(), name='get') class ValidateUserApplicationPermissionApi(APIView): permission_classes = (IsOrgAdminOrAppUser,) diff --git a/apps/perms/api/asset/user_permission/common.py b/apps/perms/api/asset/user_permission/common.py index 89df323cc..8746beb2a 100644 --- a/apps/perms/api/asset/user_permission/common.py +++ b/apps/perms/api/asset/user_permission/common.py @@ -30,6 +30,7 @@ __all__ = [ ] +@method_decorator(tmp_to_root_org(), name='get') class GetUserAssetPermissionActionsApi(RetrieveAPIView): permission_classes = (IsOrgAdminOrAppUser,) serializer_class = serializers.ActionsSerializer @@ -57,6 +58,7 @@ class GetUserAssetPermissionActionsApi(RetrieveAPIView): return {"actions": actions} +@method_decorator(tmp_to_root_org(), name='get') class ValidateUserAssetPermissionApi(APIView): permission_classes = (IsOrgAdminOrAppUser,) From 93474766f68b8ce6fe522e1fc0e8ba57b401076e Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 3 Feb 2021 10:52:51 +0800 Subject: [PATCH 11/71] =?UTF-8?q?perf(permission):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E6=9D=83=E9=99=90=E6=8E=A7=E5=88=B6=EF=BC=8C=E6=98=BE=E5=BC=8F?= =?UTF-8?q?=E7=9A=84=E5=A3=B0=E6=98=8E=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/mixin.py | 4 +--- apps/authentication/api/__init__.py | 2 +- apps/authentication/api/{auth.py => connection_token.py} | 0 apps/authentication/api/login_confirm.py | 6 +++--- apps/authentication/api/sso.py | 4 +++- apps/common/permissions.py | 2 +- apps/jumpserver/api.py | 3 ++- apps/jumpserver/settings/libs.py | 2 +- apps/jumpserver/views/other.py | 3 ++- 9 files changed, 14 insertions(+), 12 deletions(-) rename apps/authentication/api/{auth.py => connection_token.py} (100%) diff --git a/apps/assets/api/mixin.py b/apps/assets/api/mixin.py index 54eb91b41..386a1f507 100644 --- a/apps/assets/api/mixin.py +++ b/apps/assets/api/mixin.py @@ -2,13 +2,11 @@ from typing import List from assets.models import Node, Asset from assets.pagination import AssetLimitOffsetPagination -from common.utils import lazyproperty, dict_get_any, is_uuid, get_object_or_none +from common.utils import lazyproperty from assets.utils import get_node, is_query_node_all_assets class SerializeToTreeNodeMixin: - permission_classes = () - def serialize_nodes(self, nodes: List[Node], with_asset_amount=False): if with_asset_amount: def _name(node: Node): diff --git a/apps/authentication/api/__init__.py b/apps/authentication/api/__init__.py index af5d8d1b4..12b83421f 100644 --- a/apps/authentication/api/__init__.py +++ b/apps/authentication/api/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # -from .auth import * +from .connection_token import * from .token import * from .mfa import * from .access_key import * diff --git a/apps/authentication/api/auth.py b/apps/authentication/api/connection_token.py similarity index 100% rename from apps/authentication/api/auth.py rename to apps/authentication/api/connection_token.py diff --git a/apps/authentication/api/login_confirm.py b/apps/authentication/api/login_confirm.py index 6561962a9..527e473d8 100644 --- a/apps/authentication/api/login_confirm.py +++ b/apps/authentication/api/login_confirm.py @@ -3,10 +3,10 @@ from rest_framework.generics import UpdateAPIView from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework.permissions import AllowAny from django.shortcuts import get_object_or_404 -from django.utils.translation import ugettext as _ -from common.utils import get_logger, get_object_or_none +from common.utils import get_logger from common.permissions import IsOrgAdmin from ..models import LoginConfirmSetting from ..serializers import LoginConfirmSettingSerializer @@ -32,7 +32,7 @@ class LoginConfirmSettingUpdateApi(UpdateAPIView): class TicketStatusApi(mixins.AuthMixin, APIView): - permission_classes = () + permission_classes = (AllowAny,) def get(self, request, *args, **kwargs): try: diff --git a/apps/authentication/api/sso.py b/apps/authentication/api/sso.py index a2d87e6db..04149e9a5 100644 --- a/apps/authentication/api/sso.py +++ b/apps/authentication/api/sso.py @@ -7,6 +7,7 @@ from django.http.response import HttpResponseRedirect from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.request import Request +from rest_framework.permissions import AllowAny from common.utils.timezone import utcnow from common.const.http import POST, GET @@ -31,6 +32,7 @@ class SSOViewSet(AuthMixin, JmsGenericViewSet): 'login_url': SSOTokenSerializer, 'login': EmptySerializer } + permission_classes = (IsSuperUser,) @action(methods=[POST], detail=False, permission_classes=[IsSuperUser], url_path='login-url') def login_url(self, request, *args, **kwargs): @@ -54,7 +56,7 @@ class SSOViewSet(AuthMixin, JmsGenericViewSet): login_url = '%s?%s' % (reverse('api-auth:sso-login', external=True), urlencode(query)) return Response(data={'login_url': login_url}) - @action(methods=[GET], detail=False, filter_backends=[AuthKeyQueryDeclaration], permission_classes=[]) + @action(methods=[GET], detail=False, filter_backends=[AuthKeyQueryDeclaration], permission_classes=[AllowAny]) def login(self, request: Request, *args, **kwargs): """ 此接口违反了 `Restful` 的规范 diff --git a/apps/common/permissions.py b/apps/common/permissions.py index 40f665af1..43779204c 100644 --- a/apps/common/permissions.py +++ b/apps/common/permissions.py @@ -97,7 +97,7 @@ class WithBootstrapToken(permissions.BasePermission): class PermissionsMixin(UserPassesTestMixin): - permission_classes = [] + permission_classes = [permissions.IsAuthenticated] def get_permissions(self): return self.permission_classes diff --git a/apps/jumpserver/api.py b/apps/jumpserver/api.py index b74099fa3..e31a1d843 100644 --- a/apps/jumpserver/api.py +++ b/apps/jumpserver/api.py @@ -4,6 +4,7 @@ from django.utils.timesince import timesince from django.db.models import Count, Max from django.http.response import JsonResponse, HttpResponse from rest_framework.views import APIView +from rest_framework.permissions import AllowAny from collections import Counter from users.models import User @@ -307,7 +308,7 @@ class IndexApi(TotalCountMixin, DatesLoginMetricMixin, APIView): class PrometheusMetricsApi(APIView): - permission_classes = () + permission_classes = (AllowAny,) def get(self, request, *args, **kwargs): util = ComponentsPrometheusMetricsUtil() diff --git a/apps/jumpserver/settings/libs.py b/apps/jumpserver/settings/libs.py index 49b3d3b2d..639d542e7 100644 --- a/apps/jumpserver/settings/libs.py +++ b/apps/jumpserver/settings/libs.py @@ -7,7 +7,7 @@ REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. 'DEFAULT_PERMISSION_CLASSES': ( - 'common.permissions.IsOrgAdmin', + 'common.permissions.IsSuperUser', ), 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', diff --git a/apps/jumpserver/views/other.py b/apps/jumpserver/views/other.py index 134d599a6..da8046bfc 100644 --- a/apps/jumpserver/views/other.py +++ b/apps/jumpserver/views/other.py @@ -9,6 +9,7 @@ from django.views.generic import View from django.shortcuts import redirect from django.utils.translation import ugettext_lazy as _ from rest_framework.views import APIView +from rest_framework.permissions import AllowAny from django.views.decorators.csrf import csrf_exempt from django.http import HttpResponse @@ -64,7 +65,7 @@ def redirect_old_apps_view(request, *args, **kwargs): class HealthCheckView(APIView): - permission_classes = () + permission_classes = (AllowAny,) def get(self, request): return JsonResponse({"status": 1, "time": int(time.time())}) From 709e7af953efa068c035cd2098d5238afbfbfd40 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 3 Feb 2021 14:35:41 +0800 Subject: [PATCH 12/71] =?UTF-8?q?perf:=20=E4=BF=AE=E6=94=B9=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E7=89=88=E6=9C=ACjumpserver-django-oidc-rp=3D0.3.7.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 7c6771c77..33306c712 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -97,7 +97,7 @@ ipython huaweicloud-sdk-python==1.0.21 django-redis==4.11.0 python-redis-lock==3.5.0 -jumpserver-django-oidc-rp==0.3.7.5 +jumpserver-django-oidc-rp==0.3.7.6 django-mysql==3.9.0 gmssl==3.2.1 azure-mgmt-compute==4.6.2 From 7cf6e54f01e9691769ca4be2fe0b74f9d3a91b27 Mon Sep 17 00:00:00 2001 From: "Jiangjie.Bai" <32935519+BaiJiangJie@users.noreply.github.com> Date: Fri, 5 Feb 2021 13:29:29 +0800 Subject: [PATCH 13/71] =?UTF-8?q?refactor=20tree=20(=E9=87=8D=E6=9E=84&?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B5=84=E4=BA=A7=E6=A0=91/=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E6=8E=88=E6=9D=83=E6=A0=91=E5=8A=A0=E8=BD=BD=E9=80=9F?= =?UTF-8?q?=E5=BA=A6)=20(#5548)=20(#5549)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) (#5548) * tree: v0.1 * tree: v0.2 * tree: v0.3 * tree: v0.4 * tree: 添加并发锁未请求到时的debug日志 * 以空间换时间的方式优化资产树 * Reactor tree togther v2 (#5576) * Bai reactor tree ( 重构获取完整资产树中节点下资产总数的逻辑) (#5548) * tree: v0.1 * tree: v0.2 * tree: v0.3 * tree: v0.4 * tree: 添加并发锁未请求到时的debug日志 * 以空间换时间的方式优化资产树 * 修改授权适配新方案 * 添加树处理工具 * 完成新的用户授权树计算以及修改一些信号 * 重构了获取资产的一些 api * 重构了一些节点的api * 整理了一些代码 * 完成了api 的重构 * 重构检查节点数量功能 * 完成重构授权树工具类 * api 添加强制刷新参数 * 整理一些信号 * 处理一些信号的问题 * 完成了信号的处理 * 重构了资产树相关的锁机制 * RebuildUserTreeTask 还得添加回来 * 优化下不能在root组织的检查函数 * 优化资产树变化时锁的使用 * 修改一些算法的小工具 * 资产树锁不再校验是否在具体组织里 * 整理了一些信号的位置 * 修复资产与节点关系维护的bug * 去掉一些调试代码 * 修复资产授权过期检查刷新授权树的 bug * 添加了可重入锁 * 添加一些计时，优化一些sql * 增加 union 查询的支持 * 尝试用 sql 解决节点资产数量问题 * 开始优化计算授权树节点资产数量不用冗余表 * 新代码能跑起来了，修复一下bug * 去掉 UserGrantedMappingNode 换成 UserAssetGrantedTreeNodeRelation * 修了些bug，做了些优化 * 优化QuerySetStage 执行逻辑 * 与小白的内存结合了 * 删掉老的表，迁移新的 assets_amount 字段 * 优化用户授权页面资产列表 count 慢 * 修复批量命令数量不对 * 修改获取非直接授权节点的 children 的逻辑 * 获取整棵树的节点 * 回退锁 * 整理迁移脚本 * 改变更新树策略 * perf: 修改一波缩进 * fix: 修改handler名称 * 修复授权树获取资产sql 泛滥 * 修复授权规则有效bug * 修复一些bug * 修复一些bug * 又修了一些小bug * 去掉了老的 get_nodes_all_assets * 修改一些写法 * Reactor tree togther b2 (#5570) * fix: 修改handler名称 * perf: 优化生成树 * perf: 去掉注释 * 优化了一些 * 重新生成迁移脚本 * 去掉周期检查节点资产数量的任务 * Pr@reactor tree togther guang@perf mapping (#5573) * fix: 修改handler名称 * perf: mapping 拆分出来 * 修改名称 * perf: 修改锁名 * perf: 去掉检查节点任务 * perf: 修改一下名称 * perf: 优化一波 Co-authored-by: Jiangjie.Bai <32935519+BaiJiangJie@users.noreply.github.com> Co-authored-by: Bai Co-authored-by: xinwen Co-authored-by: xinwen Co-authored-by: 老广 --- apps/assets/api/asset.py | 4 +- apps/assets/api/mixin.py | 4 + apps/assets/api/node.py | 21 +- apps/assets/locks.py | 21 + .../0066_remove_node_assets_amount.py | 17 + apps/assets/models/asset.py | 10 +- apps/assets/models/base.py | 1 + apps/assets/models/favorite_asset.py | 12 - apps/assets/models/node.py | 185 ++- apps/assets/models/user.py | 2 +- apps/assets/serializers/asset.py | 9 +- apps/assets/signals_handler/__init__.py | 2 + .../common.py} | 139 +- .../signals_handler/maintain_nodes_tree.py | 88 ++ apps/assets/tasks/common.py | 3 +- apps/assets/tasks/gather_asset_users.py | 3 +- apps/assets/tasks/nodes_amount.py | 27 - apps/assets/tests/tree.py | 33 + apps/assets/urls/api_urls.py | 7 +- apps/assets/utils.py | 113 +- apps/common/const/distributed_lock_key.py | 2 - apps/common/db/models.py | 4 + apps/common/utils/common.py | 19 + apps/common/utils/lock.py | 53 +- apps/orgs/lock.py | 131 -- apps/orgs/utils.py | 5 + .../api/application/user_permission/common.py | 6 +- .../user_permission_applications.py | 6 +- apps/perms/api/asset/user_permission/mixin.py | 36 +- .../user_permission/user_permission_assets.py | 156 --- .../user_permission_assets/__init__.py | 1 + .../user_permission_assets/mixin.py | 127 ++ .../user_permission_assets/views.py | 99 ++ .../user_permission/user_permission_nodes.py | 55 +- .../user_permission_nodes_with_assets.py | 184 +-- apps/perms/api/system_user_permission.py | 6 +- apps/perms/async_tasks/__init__.py | 0 apps/perms/async_tasks/mapping_node_task.py | 47 - apps/perms/locks.py | 11 + .../migrations/0014_build_users_perm_tree.py | 14 - .../migrations/0018_auto_20210204_1749.py | 65 + apps/perms/models/asset_permission.py | 123 +- apps/perms/pagination.py | 38 +- apps/perms/signals_handler/__init__.py | 2 + .../common.py} | 88 +- apps/perms/signals_handler/refresh_perms.py | 115 ++ apps/perms/tasks.py | 90 +- apps/perms/utils/asset/user_permission.py | 1122 ++++++++++------- utils/generate_fake_data/resources/assets.py | 3 +- 49 files changed, 1829 insertions(+), 1480 deletions(-) create mode 100644 apps/assets/locks.py create mode 100644 apps/assets/migrations/0066_remove_node_assets_amount.py create mode 100644 apps/assets/signals_handler/__init__.py rename apps/assets/{signals_handler.py => signals_handler/common.py} (60%) create mode 100644 apps/assets/signals_handler/maintain_nodes_tree.py create mode 100644 apps/assets/tests/tree.py delete mode 100644 apps/common/const/distributed_lock_key.py delete mode 100644 apps/orgs/lock.py delete mode 100644 apps/perms/api/asset/user_permission/user_permission_assets.py create mode 100644 apps/perms/api/asset/user_permission/user_permission_assets/__init__.py create mode 100644 apps/perms/api/asset/user_permission/user_permission_assets/mixin.py create mode 100644 apps/perms/api/asset/user_permission/user_permission_assets/views.py delete mode 100644 apps/perms/async_tasks/__init__.py delete mode 100644 apps/perms/async_tasks/mapping_node_task.py create mode 100644 apps/perms/locks.py create mode 100644 apps/perms/migrations/0018_auto_20210204_1749.py create mode 100644 apps/perms/signals_handler/__init__.py rename apps/perms/{signals_handler.py => signals_handler/common.py} (71%) create mode 100644 apps/perms/signals_handler/refresh_perms.py diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index 4de9fb899..9d8a6bf89 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -3,10 +3,10 @@ from assets.api import FilterAssetByNodeMixin from rest_framework.viewsets import ModelViewSet from rest_framework.generics import RetrieveAPIView -from rest_framework.response import Response -from rest_framework import status from django.shortcuts import get_object_or_404 +from django.utils.decorators import method_decorator +from assets.locks import NodeTreeUpdateLock from common.utils import get_logger, get_object_or_none from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser, IsSuperUser from orgs.mixins.api import OrgBulkModelViewSet diff --git a/apps/assets/api/mixin.py b/apps/assets/api/mixin.py index 386a1f507..763f025f4 100644 --- a/apps/assets/api/mixin.py +++ b/apps/assets/api/mixin.py @@ -1,5 +1,6 @@ from typing import List +from common.utils.common import timeit from assets.models import Node, Asset from assets.pagination import AssetLimitOffsetPagination from common.utils import lazyproperty @@ -7,6 +8,8 @@ from assets.utils import get_node, is_query_node_all_assets class SerializeToTreeNodeMixin: + + @timeit def serialize_nodes(self, nodes: List[Node], with_asset_amount=False): if with_asset_amount: def _name(node: Node): @@ -43,6 +46,7 @@ class SerializeToTreeNodeMixin: return platform return default + @timeit def serialize_assets(self, assets, node_key=None): if node_key is None: get_pid = lambda asset: getattr(asset, 'parent_key', '') diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index a64326042..c793ad384 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -17,12 +17,9 @@ from common.const.signals import PRE_REMOVE, POST_REMOVE from assets.models import Asset from common.utils import get_logger, get_object_or_none from common.tree import TreeNodeSerializer -from common.const.distributed_lock_key import UPDATE_NODE_TREE_LOCK_KEY from orgs.mixins.api import OrgModelViewSet from orgs.mixins import generics -from orgs.lock import org_level_transaction_lock from orgs.utils import current_org -from assets.tasks import check_node_assets_amount_task from ..hands import IsOrgAdmin from ..models import Node from ..tasks import ( @@ -31,6 +28,7 @@ from ..tasks import ( ) from .. import serializers from .mixin import SerializeToTreeNodeMixin +from assets.locks import NodeTreeUpdateLock logger = get_logger(__file__) @@ -50,11 +48,6 @@ class NodeViewSet(OrgModelViewSet): permission_classes = (IsOrgAdmin,) serializer_class = serializers.NodeSerializer - @action(methods=[POST], detail=False, url_name='launch-check-assets-amount-task') - def launch_check_assets_amount_task(self, request): - task = check_node_assets_amount_task.delay(current_org.id) - return Response(data={'task': task.id}) - # 仅支持根节点指直接创建，子节点下的节点需要通过children接口创建 def perform_create(self, serializer): child_key = Node.org_root().get_next_child_key() @@ -184,9 +177,9 @@ class NodeChildrenAsTreeApi(SerializeToTreeNodeMixin, NodeChildrenApi): if not include_assets: return [] assets = self.instance.get_assets().only( - "id", "hostname", "ip", "os", - "org_id", "protocols", "is_active" - ) + "id", "hostname", "ip", "os", "platform_id", + "org_id", "protocols", "is_active", + ).prefetch_related('platform') return self.serialize_assets(assets, self.instance.key) @@ -219,8 +212,6 @@ class NodeAddChildrenApi(generics.UpdateAPIView): return Response("OK") -@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='patch') -@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='put') class NodeAddAssetsApi(generics.UpdateAPIView): model = Node serializer_class = serializers.NodeAssetsSerializer @@ -233,8 +224,6 @@ class NodeAddAssetsApi(generics.UpdateAPIView): instance.assets.add(*tuple(assets)) -@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='patch') -@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='put') class NodeRemoveAssetsApi(generics.UpdateAPIView): model = Node serializer_class = serializers.NodeAssetsSerializer @@ -251,8 +240,6 @@ class NodeRemoveAssetsApi(generics.UpdateAPIView): Node.org_root().assets.add(*orphan_assets) -@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='patch') -@method_decorator(org_level_transaction_lock(UPDATE_NODE_TREE_LOCK_KEY), name='put') class MoveAssetsToNodeApi(generics.UpdateAPIView): model = Node serializer_class = serializers.NodeAssetsSerializer diff --git a/apps/assets/locks.py b/apps/assets/locks.py new file mode 100644 index 000000000..b80db3ff8 --- /dev/null +++ b/apps/assets/locks.py @@ -0,0 +1,21 @@ +from orgs.utils import current_org +from common.utils.lock import DistributedLock + + +class NodeTreeUpdateLock(DistributedLock): + name_template = 'assets.node.tree.update.' + + def get_name(self): + if current_org: + org_id = current_org.id + else: + org_id = 'current_org_is_null' + name = self.name_template.format( + org_id=org_id + ) + return name + + def __init__(self, blocking=True): + name = self.get_name() + super().__init__(name=name, blocking=blocking, + release_lock_on_transaction_commit=True) diff --git a/apps/assets/migrations/0066_remove_node_assets_amount.py b/apps/assets/migrations/0066_remove_node_assets_amount.py new file mode 100644 index 000000000..5d7044179 --- /dev/null +++ b/apps/assets/migrations/0066_remove_node_assets_amount.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2021-02-04 09:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0065_auto_20210121_1549'), + ] + + operations = [ + migrations.RemoveField( + model_name='node', + name='assets_amount', + ), + ] diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index d4c787bbc..5d133f40d 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -17,7 +17,7 @@ from orgs.mixins.models import OrgModelMixin, OrgManager from .base import ConnectivityMixin from .utils import Connectivity -__all__ = ['Asset', 'ProtocolsMixin', 'Platform'] +__all__ = ['Asset', 'ProtocolsMixin', 'Platform', 'AssetQuerySet'] logger = logging.getLogger(__name__) @@ -41,13 +41,6 @@ def default_node(): class AssetManager(OrgManager): - def get_queryset(self): - return super().get_queryset().annotate( - platform_base=models.F('platform__base') - ) - - -class AssetOrgManager(OrgManager): pass @@ -230,7 +223,6 @@ class Asset(ProtocolsMixin, NodesRelationMixin, OrgModelMixin): comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) objects = AssetManager.from_queryset(AssetQuerySet)() - org_objects = AssetOrgManager.from_queryset(AssetQuerySet)() _connectivity = None def __str__(self): diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index b7239da75..094f029bc 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -11,6 +11,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from django.conf import settings +from common.utils.common import timeit from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, lazyproperty ) diff --git a/apps/assets/models/favorite_asset.py b/apps/assets/models/favorite_asset.py index b176c8105..3abc69c8c 100644 --- a/apps/assets/models/favorite_asset.py +++ b/apps/assets/models/favorite_asset.py @@ -18,15 +18,3 @@ class FavoriteAsset(CommonModelMixin): @classmethod def get_user_favorite_assets_id(cls, user): return cls.objects.filter(user=user).values_list('asset', flat=True) - - @classmethod - def get_user_favorite_assets(cls, user, asset_perms_id=None): - from assets.models import Asset - from perms.utils.asset.user_permission import get_user_granted_all_assets - asset_ids = get_user_granted_all_assets( - user, - via_mapping_node=False, - asset_perms_id=asset_perms_id - ).values_list('id', flat=True) - query_name = cls.asset.field.related_query_name() - return Asset.org_objects.filter(**{f'{query_name}__user_id': user.id}, id__in=asset_ids).distinct() diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 58e267f70..e5b53eb45 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -1,23 +1,32 @@ # -*- coding: utf-8 -*- # -import uuid import re +import time +import uuid +import threading +import os +import time +import uuid +from collections import defaultdict from django.db import models, transaction -from django.db.models import Q +from django.db.models import Q, Manager from django.db.utils import IntegrityError from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext from django.db.transaction import atomic +from django.core.cache import cache +from common.utils.lock import DistributedLock +from common.utils.common import timeit +from common.db.models import output_as_string from common.utils import get_logger -from common.utils.common import lazyproperty from orgs.mixins.models import OrgModelMixin, OrgManager from orgs.utils import get_current_org, tmp_to_org from orgs.models import Organization -__all__ = ['Node', 'FamilyMixin', 'compute_parent_key'] +__all__ = ['Node', 'FamilyMixin', 'compute_parent_key', 'NodeQuerySet'] logger = get_logger(__name__) @@ -247,9 +256,125 @@ class FamilyMixin: return [*tuple(ancestors), self, *tuple(children)] -class NodeAssetsMixin: +class NodeAllAssetsMappingMixin: + # Use a new plan + + # { org_id: { node_key: [ asset1_id, asset2_id ] } } + orgid_nodekey_assetsid_mapping = defaultdict(dict) + + @classmethod + def get_node_all_assets_id_mapping(cls, org_id): + _mapping = cls.get_node_all_assets_id_mapping_from_memory(org_id) + if _mapping: + return _mapping + + _mapping = cls.get_node_all_assets_id_mapping_from_cache_or_generate_to_cache(org_id) + cls.set_node_all_assets_id_mapping_to_memory(org_id, mapping=_mapping) + return _mapping + + # from memory + @classmethod + def get_node_all_assets_id_mapping_from_memory(cls, org_id): + mapping = cls.orgid_nodekey_assetsid_mapping.get(org_id, {}) + return mapping + + @classmethod + def set_node_all_assets_id_mapping_to_memory(cls, org_id, mapping): + cls.orgid_nodekey_assetsid_mapping[org_id] = mapping + + @classmethod + def expire_node_all_assets_id_mapping_from_memory(cls, org_id): + org_id = str(org_id) + cls.orgid_nodekey_assetsid_mapping.pop(org_id, None) + + # get order: from memory -> (from cache -> to generate) + @classmethod + def get_node_all_assets_id_mapping_from_cache_or_generate_to_cache(cls, org_id): + mapping = cls.get_node_all_assets_id_mapping_from_cache(org_id) + if mapping: + return mapping + + lock_key = f'KEY_LOCK_GENERATE_ORG_{org_id}_NODE_ALL_ASSETS_ID_MAPPING' + logger.info(f'Thread[{threading.get_ident()}] acquiring lock[{lock_key}] ...') + with DistributedLock(lock_key): + logger.info(f'Thread[{threading.get_ident()}] acquire lock[{lock_key}] ok') + # 这里使用无限期锁，原因是如果这里卡住了，就卡在数据库了，说明 + # 数据库繁忙，所以不应该再有线程执行这个操作，使数据库忙上加忙 + + # 这里最好先判断内存中有没有，防止同一进程的多个线程重复从 cache 中获取数据， + # 但逻辑过于繁琐，直接判断 cache 吧 + _mapping = cls.get_node_all_assets_id_mapping_from_cache(org_id) + if _mapping: + return _mapping + + _mapping = cls.generate_node_all_assets_id_mapping(org_id) + cls.set_node_all_assets_id_mapping_to_cache(org_id=org_id, mapping=_mapping) + return _mapping + + @classmethod + def get_node_all_assets_id_mapping_from_cache(cls, org_id): + cache_key = cls._get_cache_key_for_node_all_assets_id_mapping(org_id) + mapping = cache.get(cache_key) + return mapping + + @classmethod + def set_node_all_assets_id_mapping_to_cache(cls, org_id, mapping): + cache_key = cls._get_cache_key_for_node_all_assets_id_mapping(org_id) + cache.set(cache_key, mapping, timeout=None) + + @classmethod + def expire_node_all_assets_id_mapping_from_cache(cls, org_id): + cache_key = cls._get_cache_key_for_node_all_assets_id_mapping(org_id) + cache.delete(cache_key) + + @staticmethod + def _get_cache_key_for_node_all_assets_id_mapping(org_id): + return 'ASSETS_ORG_NODE_ALL_ASSETS_ID_MAPPING_{}'.format(org_id) + + @classmethod + def generate_node_all_assets_id_mapping(cls, org_id): + from .asset import Asset + + t1 = time.time() + with tmp_to_org(org_id): + nodes_id_key = Node.objects.filter(org_id=org_id) \ + .annotate(char_id=output_as_string('id')) \ + .values_list('char_id', 'key') + + # * 直接取出全部. filter(node__org_id=org_id)(大规模下会更慢) + nodes_assets_id = Asset.nodes.through.objects.all() \ + .annotate(char_node_id=output_as_string('node_id')) \ + .annotate(char_asset_id=output_as_string('asset_id')) \ + .values_list('char_node_id', 'char_asset_id') + + node_id_ancestor_keys_mapping = { + node_id: cls.get_node_ancestor_keys(node_key, with_self=True) + for node_id, node_key in nodes_id_key + } + + nodeid_assetsid_mapping = defaultdict(set) + for node_id, asset_id in nodes_assets_id: + nodeid_assetsid_mapping[node_id].add(asset_id) + + t2 = time.time() + + mapping = defaultdict(set) + for node_id, node_key in nodes_id_key: + assets_id = nodeid_assetsid_mapping[node_id] + node_ancestor_keys = node_id_ancestor_keys_mapping[node_id] + for ancestor_key in node_ancestor_keys: + mapping[ancestor_key].update(assets_id) + + t3 = time.time() + logger.debug('t1-t2(DB Query): {} s, t3-t2(Generate mapping): {} s'.format(t2-t1, t3-t2)) + return mapping + + +class NodeAssetsMixin(NodeAllAssetsMappingMixin): + org_id: str key = '' id = None + objects: Manager def get_all_assets(self): from .asset import Asset @@ -263,8 +388,7 @@ class NodeAssetsMixin: # 可是 startswith 会导致表关联时 Asset 索引失效 from .asset import Asset node_ids = cls.objects.filter( - Q(key__startswith=f'{key}:') | - Q(key=key) + Q(key__startswith=f'{key}:') | Q(key=key) ).values_list('id', flat=True).distinct() assets = Asset.objects.filter( nodes__id__in=list(node_ids) @@ -283,29 +407,39 @@ class NodeAssetsMixin: return self.get_all_assets().valid() @classmethod - def get_nodes_all_assets_ids(cls, nodes_keys): - assets_ids = cls.get_nodes_all_assets(nodes_keys).values_list('id', flat=True) + def get_nodes_all_assets_ids_by_keys(cls, nodes_keys): + nodes = Node.objects.filter(key__in=nodes_keys) + assets_ids = cls.get_nodes_all_assets(*nodes).values_list('id', flat=True) return assets_ids @classmethod - def get_nodes_all_assets(cls, nodes_keys, extra_assets_ids=None): + def get_nodes_all_assets(cls, *nodes): from .asset import Asset - nodes_keys = cls.clean_children_keys(nodes_keys) - q = Q() - node_ids = () - for key in nodes_keys: - q |= Q(key__startswith=f'{key}:') - q |= Q(key=key) - if q: - node_ids = Node.objects.filter(q).distinct().values_list('id', flat=True) + node_ids = set() + descendant_node_query = Q() + for n in nodes: + node_ids.add(n.id) + descendant_node_query |= Q(key__istartswith=f'{n.key}:') + if descendant_node_query: + _ids = Node.objects.order_by().filter(descendant_node_query).values_list('id', flat=True) + node_ids.update(_ids) + return Asset.objects.order_by().filter(nodes__id__in=node_ids).distinct() - q = Q(nodes__id__in=list(node_ids)) - if extra_assets_ids: - q |= Q(id__in=extra_assets_ids) - if q: - return Asset.org_objects.filter(q).distinct() - else: - return Asset.objects.none() + @property + def assets_amount(self): + assets_id = self.get_all_assets_id() + return len(assets_id) + + def get_all_assets_id(self): + assets_id = self.get_all_assets_id_by_node_key(org_id=self.org_id, node_key=self.key) + return set(assets_id) + + @classmethod + def get_all_assets_id_by_node_key(cls, org_id, node_key): + org_id = str(org_id) + nodekey_assetsid_mapping = cls.get_node_all_assets_id_mapping(org_id) + assets_id = nodekey_assetsid_mapping.get(node_key, []) + return set(assets_id) class SomeNodesMixin: @@ -416,7 +550,6 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): date_create = models.DateTimeField(auto_now_add=True) parent_key = models.CharField(max_length=64, verbose_name=_("Parent key"), db_index=True, default='') - assets_amount = models.IntegerField(default=0) objects = OrgManager.from_queryset(NodeQuerySet)() is_node = True diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 5b800311b..885543796 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -199,7 +199,7 @@ class SystemUser(BaseUser): from assets.models import Node nodes_keys = self.nodes.all().values_list('key', flat=True) assets_ids = set(self.assets.all().values_list('id', flat=True)) - nodes_assets_ids = Node.get_nodes_all_assets_ids(nodes_keys) + nodes_assets_ids = Node.get_nodes_all_assets_ids_by_keys(nodes_keys) assets_ids.update(nodes_assets_ids) assets = Asset.objects.filter(id__in=assets_ids) return assets diff --git a/apps/assets/serializers/asset.py b/apps/assets/serializers/asset.py index a8ce0f3ee..7efe30186 100644 --- a/apps/assets/serializers/asset.py +++ b/apps/assets/serializers/asset.py @@ -111,7 +111,7 @@ class AssetSerializer(BulkOrgResourceModelSerializer): @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.select_related('admin_user', 'domain', 'platform') + queryset = queryset.prefetch_related('admin_user', 'domain', 'platform') queryset = queryset.prefetch_related('nodes', 'labels') return queryset @@ -166,13 +166,6 @@ class AssetDisplaySerializer(AssetSerializer): 'connectivity', ] - @classmethod - def setup_eager_loading(cls, queryset): - queryset = super().setup_eager_loading(queryset) - queryset = queryset\ - .annotate(admin_user_username=F('admin_user__username')) - return queryset - class PlatformSerializer(serializers.ModelSerializer): meta = serializers.DictField(required=False, allow_null=True) diff --git a/apps/assets/signals_handler/__init__.py b/apps/assets/signals_handler/__init__.py new file mode 100644 index 000000000..0c3980565 --- /dev/null +++ b/apps/assets/signals_handler/__init__.py @@ -0,0 +1,2 @@ +from .common import * +from .maintain_nodes_tree import * diff --git a/apps/assets/signals_handler.py b/apps/assets/signals_handler/common.py similarity index 60% rename from apps/assets/signals_handler.py rename to apps/assets/signals_handler/common.py index 061d7d84c..6625e493e 100644 --- a/apps/assets/signals_handler.py +++ b/apps/assets/signals_handler/common.py @@ -1,21 +1,17 @@ # -*- coding: utf-8 -*- # -from operator import add, sub - -from assets.utils import is_asset_exists_in_node from django.db.models.signals import ( post_save, m2m_changed, pre_delete, post_delete, pre_save ) -from django.db.models import Q, F from django.dispatch import receiver from common.exceptions import M2MReverseNotAllowed -from common.const.signals import PRE_ADD, POST_ADD, POST_REMOVE, PRE_CLEAR, PRE_REMOVE +from common.const.signals import POST_ADD, POST_REMOVE, PRE_REMOVE from common.utils import get_logger from common.decorator import on_transaction_commit -from .models import Asset, SystemUser, Node, compute_parent_key +from assets.models import Asset, SystemUser, Node from users.models import User -from .tasks import ( +from assets.tasks import ( update_assets_hardware_info_util, test_asset_connectivity_util, push_system_user_to_assets_manual, @@ -23,7 +19,6 @@ from .tasks import ( add_nodes_assets_to_system_users ) - logger = get_logger(__file__) @@ -202,134 +197,6 @@ def on_asset_nodes_add(instance, action, reverse, pk_set, **kwargs): m2m_model.objects.bulk_create(to_create) -def _update_node_assets_amount(node: Node, asset_pk_set: set, operator=add): - """ - 一个节点与多个资产关系变化时，更新计数 - - :param node: 节点实例 - :param asset_pk_set: 资产的`id`集合, 内部不会修改该值 - :param operator: 操作 - * -> Node - # -> Asset - - * [3] - / \ - * * [2] - / \ - * * [1] - / / \ - * [a] # # [b] - - """ - # 获取节点[1]祖先节点的 `key` 含自己，也就是[1, 2, 3]节点的`key` - ancestor_keys = node.get_ancestor_keys(with_self=True) - ancestors = Node.objects.filter(key__in=ancestor_keys).order_by('-key') - to_update = [] - for ancestor in ancestors: - # 迭代祖先节点的`key`，顺序是 [1] -> [2] -> [3] - # 查询该节点及其后代节点是否包含要操作的资产，将包含的从要操作的 - # 资产集合中去掉，他们是重复节点，无论增加或删除都不会影响节点的资产数量 - - asset_pk_set -= set(Asset.objects.filter( - id__in=asset_pk_set - ).filter( - Q(nodes__key__istartswith=f'{ancestor.key}:') | - Q(nodes__key=ancestor.key) - ).distinct().values_list('id', flat=True)) - if not asset_pk_set: - # 要操作的资产集合为空，说明都是重复资产，不用改变节点资产数量 - # 而且既然它包含了，它的祖先节点肯定也包含了，所以祖先节点都不用 - # 处理了 - break - ancestor.assets_amount = operator(F('assets_amount'), len(asset_pk_set)) - to_update.append(ancestor) - Node.objects.bulk_update(to_update, fields=('assets_amount', 'parent_key')) - - -def _remove_ancestor_keys(ancestor_key, tree_set): - # 这里判断 `ancestor_key` 不能是空，防止数据错误导致的死循环 - # 判断是否在集合里，来区分是否已被处理过 - while ancestor_key and ancestor_key in tree_set: - tree_set.remove(ancestor_key) - ancestor_key = compute_parent_key(ancestor_key) - - -def _update_nodes_asset_amount(node_keys, asset_pk, operator): - """ - 一个资产与多个节点关系变化时，更新计数 - - :param node_keys: 节点 id 的集合 - :param asset_pk: 资产 id - :param operator: 操作 - """ - - # 所有相关节点的祖先节点，组成一棵局部树 - ancestor_keys = set() - for key in node_keys: - ancestor_keys.update(Node.get_node_ancestor_keys(key)) - - # 相关节点可能是其他相关节点的祖先节点，如果是从相关节点里干掉 - node_keys -= ancestor_keys - - to_update_keys = [] - for key in node_keys: - # 遍历相关节点，处理它及其祖先节点 - # 查询该节点是否包含待处理资产 - exists = is_asset_exists_in_node(asset_pk, key) - parent_key = compute_parent_key(key) - - if exists: - # 如果资产在该节点，那么他及其祖先节点都不用处理 - _remove_ancestor_keys(parent_key, ancestor_keys) - continue - else: - # 不存在，要更新本节点 - to_update_keys.append(key) - # 这里判断 `parent_key` 不能是空，防止数据错误导致的死循环 - # 判断是否在集合里，来区分是否已被处理过 - while parent_key and parent_key in ancestor_keys: - exists = is_asset_exists_in_node(asset_pk, parent_key) - if exists: - _remove_ancestor_keys(parent_key, ancestor_keys) - break - else: - to_update_keys.append(parent_key) - ancestor_keys.remove(parent_key) - parent_key = compute_parent_key(parent_key) - - Node.objects.filter(key__in=to_update_keys).update( - assets_amount=operator(F('assets_amount'), 1) - ) - - -@receiver(m2m_changed, sender=Asset.nodes.through) -def update_nodes_assets_amount(action, instance, reverse, pk_set, **kwargs): - # 不允许 `pre_clear` ，因为该信号没有 `pk_set` - # [官网](https://docs.djangoproject.com/en/3.1/ref/signals/#m2m-changed) - refused = (PRE_CLEAR,) - if action in refused: - raise ValueError - - mapper = { - PRE_ADD: add, - POST_REMOVE: sub - } - if action not in mapper: - return - - operator = mapper[action] - - if reverse: - node: Node = instance - asset_pk_set = set(pk_set) - _update_node_assets_amount(node, asset_pk_set, operator) - else: - asset_pk = instance.id - # 与资产直接关联的节点 - node_keys = set(Node.objects.filter(id__in=pk_set).values_list('key', flat=True)) - _update_nodes_asset_amount(node_keys, asset_pk, operator) - - RELATED_NODE_IDS = '_related_node_ids' diff --git a/apps/assets/signals_handler/maintain_nodes_tree.py b/apps/assets/signals_handler/maintain_nodes_tree.py new file mode 100644 index 000000000..d6615c885 --- /dev/null +++ b/apps/assets/signals_handler/maintain_nodes_tree.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# +import os +import threading + +from django.db.models.signals import ( + m2m_changed, post_save, post_delete +) +from django.dispatch import receiver +from django.utils.functional import LazyObject + +from common.signals import django_ready +from common.utils.connection import RedisPubSub +from common.utils import get_logger +from assets.models import Asset, Node + + +logger = get_logger(__file__) + +# clear node assets mapping for memory +# ------------------------------------ + + +def get_node_assets_mapping_for_memory_pub_sub(): + return RedisPubSub('fm.node_all_assets_id_memory_mapping') + + +class NodeAssetsMappingForMemoryPubSub(LazyObject): + def _setup(self): + self._wrapped = get_node_assets_mapping_for_memory_pub_sub() + + +node_assets_mapping_for_memory_pub_sub = NodeAssetsMappingForMemoryPubSub() + + +def expire_node_assets_mapping_for_memory(org_id): + # 所有进程清除(自己的 memory 数据) + org_id = str(org_id) + node_assets_mapping_for_memory_pub_sub.publish(org_id) + # 当前进程清除(cache 数据) + logger.debug( + "Expire node assets id mapping from cache of org={}, pid={}" + "".format(org_id, os.getpid()) + ) + Node.expire_node_all_assets_id_mapping_from_cache(org_id) + + +@receiver(post_save, sender=Node) +def on_node_post_create(sender, instance, created, update_fields, **kwargs): + if created: + need_expire = True + elif update_fields and 'key' in update_fields: + need_expire = True + else: + need_expire = False + + if need_expire: + expire_node_assets_mapping_for_memory(instance.org_id) + + +@receiver(post_delete, sender=Node) +def on_node_post_delete(sender, instance, **kwargs): + expire_node_assets_mapping_for_memory(instance.org_id) + + +@receiver(m2m_changed, sender=Asset.nodes.through) +def on_node_asset_change(sender, instance, **kwargs): + expire_node_assets_mapping_for_memory(instance.org_id) + + +@receiver(django_ready) +def subscribe_node_assets_mapping_expire(sender, **kwargs): + logger.debug("Start subscribe for expire node assets id mapping from memory") + + def keep_subscribe(): + subscribe = node_assets_mapping_for_memory_pub_sub.subscribe() + for message in subscribe.listen(): + if message["type"] != "message": + continue + org_id = message['data'].decode() + Node.expire_node_all_assets_id_mapping_from_memory(org_id) + logger.debug( + "Expire node assets id mapping from memory of org={}, pid={}" + "".format(str(org_id), os.getpid()) + ) + t = threading.Thread(target=keep_subscribe) + t.daemon = True + t.start() diff --git a/apps/assets/tasks/common.py b/apps/assets/tasks/common.py index b743300e1..5a92ec039 100644 --- a/apps/assets/tasks/common.py +++ b/apps/assets/tasks/common.py @@ -12,6 +12,7 @@ __all__ = ['add_nodes_assets_to_system_users'] @tmp_to_root_org() def add_nodes_assets_to_system_users(nodes_keys, system_users): from ..models import Node - assets = Node.get_nodes_all_assets(nodes_keys).values_list('id', flat=True) + nodes = Node.objects.filter(key__in=nodes_keys) + assets = Node.get_nodes_all_assets(*nodes) for system_user in system_users: system_user.assets.add(*tuple(assets)) diff --git a/apps/assets/tasks/gather_asset_users.py b/apps/assets/tasks/gather_asset_users.py index 5d8372451..0187a29aa 100644 --- a/apps/assets/tasks/gather_asset_users.py +++ b/apps/assets/tasks/gather_asset_users.py @@ -141,7 +141,8 @@ def gather_asset_users(assets, task_name=None): @shared_task(queue="ansible") def gather_nodes_asset_users(nodes_key): - assets = Node.get_nodes_all_assets(nodes_key) + nodes = Node.objects.filter(key__in=nodes_key) + assets = Node.get_nodes_all_assets(*nodes) assets_groups_by_100 = [assets[i:i+100] for i in range(0, len(assets), 100)] for _assets in assets_groups_by_100: gather_asset_users(_assets) diff --git a/apps/assets/tasks/nodes_amount.py b/apps/assets/tasks/nodes_amount.py index e1e437797..e69de29bb 100644 --- a/apps/assets/tasks/nodes_amount.py +++ b/apps/assets/tasks/nodes_amount.py @@ -1,27 +0,0 @@ -from celery import shared_task -from django.utils.translation import gettext_lazy as _ - -from orgs.models import Organization -from orgs.utils import tmp_to_org -from ops.celery.decorator import register_as_period_task -from assets.utils import check_node_assets_amount - -from common.utils.lock import AcquireFailed -from common.utils import get_logger - -logger = get_logger(__file__) - - -@shared_task(queue='celery_heavy_tasks') -def check_node_assets_amount_task(org_id=Organization.ROOT_ID): - try: - with tmp_to_org(Organization.get_instance(org_id)): - check_node_assets_amount() - except AcquireFailed: - logger.error(_('The task of self-checking is already running and cannot be started repeatedly')) - - -@register_as_period_task(crontab='0 2 * * *') -@shared_task(queue='celery_heavy_tasks') -def check_node_assets_amount_period_task(): - check_node_assets_amount_task() diff --git a/apps/assets/tests/tree.py b/apps/assets/tests/tree.py new file mode 100644 index 000000000..99bfd2275 --- /dev/null +++ b/apps/assets/tests/tree.py @@ -0,0 +1,33 @@ +from assets.tree import Tree + + +def test(): + from orgs.models import Organization + from assets.models import Node, Asset + import time + Organization.objects.get(id='1863cf22-f666-474e-94aa-935fe175203c').change_to() + + t1 = time.time() + nodes = list(Node.objects.exclude(key__startswith='-').only('id', 'key', 'parent_key')) + node_asset_id_pairs = Asset.nodes.through.objects.all().values_list('node_id', 'asset_id') + t2 = time.time() + node_asset_id_pairs = list(node_asset_id_pairs) + tree = Tree(nodes, node_asset_id_pairs) + tree.build_tree() + tree.nodes = None + tree.node_asset_id_pairs = None + import pickle + d = pickle.dumps(tree) + print('------------', len(d)) + return tree + tree.compute_tree_node_assets_amount() + + print(f'校对算法准确性 ......') + for node in nodes: + tree_node = tree.key_tree_node_mapper[node.key] + if tree_node.assets_amount != node.assets_amount: + print(f'ERROR: {tree_node.assets_amount} {node.assets_amount}') + # print(f'OK {tree_node.asset_amount} {node.assets_amount}') + + print(f'数据库时间: {t2 - t1}') + return tree \ No newline at end of file diff --git a/apps/assets/urls/api_urls.py b/apps/assets/urls/api_urls.py index 707a8e73d..5a5e6d803 100644 --- a/apps/assets/urls/api_urls.py +++ b/apps/assets/urls/api_urls.py @@ -2,7 +2,6 @@ from django.urls import path, re_path from rest_framework_nested import routers from rest_framework_bulk.routes import BulkRouter -from django.db.transaction import non_atomic_requests from common import api as capi @@ -57,9 +56,9 @@ urlpatterns = [ path('nodes/children/', api.NodeChildrenApi.as_view(), name='node-children-2'), path('nodes//children/add/', api.NodeAddChildrenApi.as_view(), name='node-add-children'), path('nodes//assets/', api.NodeAssetsApi.as_view(), name='node-assets'), - path('nodes//assets/add/', non_atomic_requests(api.NodeAddAssetsApi.as_view()), name='node-add-assets'), - path('nodes//assets/replace/', non_atomic_requests(api.MoveAssetsToNodeApi.as_view()), name='node-replace-assets'), - path('nodes//assets/remove/', non_atomic_requests(api.NodeRemoveAssetsApi.as_view()), name='node-remove-assets'), + path('nodes//assets/add/', api.NodeAddAssetsApi.as_view(), name='node-add-assets'), + path('nodes//assets/replace/', api.MoveAssetsToNodeApi.as_view(), name='node-replace-assets'), + path('nodes//assets/remove/', api.NodeRemoveAssetsApi.as_view(), name='node-remove-assets'), path('nodes//tasks/', api.NodeTaskCreateApi.as_view(), name='node-task-create'), path('gateways//test-connective/', api.GatewayTestConnectionApi.as_view(), name='test-gateway-connective'), diff --git a/apps/assets/utils.py b/apps/assets/utils.py index 2805ac034..343fa704b 100644 --- a/apps/assets/utils.py +++ b/apps/assets/utils.py @@ -1,43 +1,16 @@ # ~*~ coding: utf-8 ~*~ # -import time - -from django.db.models import Q - -from common.utils import get_logger, dict_get_any, is_uuid, get_object_or_none -from common.utils.lock import DistributedLock +from collections import defaultdict +from common.utils import get_logger, dict_get_any, is_uuid, get_object_or_none, timeit from common.http import is_true -from .models import Asset, Node +from common.struct import Stack +from common.db.models import output_as_string +from .models import Node logger = get_logger(__file__) -@DistributedLock(name="assets.node.check_node_assets_amount", blocking=False) -def check_node_assets_amount(): - for node in Node.objects.all(): - logger.info(f'Check node assets amount: {node}') - assets_amount = Asset.objects.filter( - Q(nodes__key__istartswith=f'{node.key}:') | Q(nodes=node) - ).distinct().count() - - if node.assets_amount != assets_amount: - logger.warn(f'Node wrong assets amount ' - f'{node.assets_amount} right is {assets_amount}') - node.assets_amount = assets_amount - node.save() - # 防止自检程序给数据库的压力太大 - time.sleep(0.1) - - -def is_asset_exists_in_node(asset_pk, node_key): - return Asset.objects.filter( - id=asset_pk - ).filter( - Q(nodes__key__istartswith=f'{node_key}:') | Q(nodes__key=node_key) - ).exists() - - def is_query_node_all_assets(request): request = request query_all_arg = request.query_params.get('all', 'true') @@ -57,3 +30,79 @@ def get_node(request): else: node = get_object_or_none(Node, key=node_id) return node + + +class NodeAssetsInfo: + __slots__ = ('key', 'assets_amount', 'assets') + + def __init__(self, key, assets_amount, assets): + self.key = key + self.assets_amount = assets_amount + self.assets = assets + + def __str__(self): + return self.key + + +class NodeAssetsUtil: + def __init__(self, nodes, nodekey_assetsid_mapper): + """ + :param nodes: 节点 + :param nodekey_assetsid_mapper: 节点直接资产id的映射 {"key1": set(), "key2": set()} + """ + self.nodes = nodes + # node_id --> set(asset_id1, asset_id2) + self.nodekey_assetsid_mapper = nodekey_assetsid_mapper + self.nodekey_assetsinfo_mapper = {} + + @timeit + def generate(self): + # 准备排序好的资产信息数据 + infos = [] + for node in self.nodes: + assets = self.nodekey_assetsid_mapper.get(node.key, set()) + info = NodeAssetsInfo(key=node.key, assets_amount=0, assets=assets) + infos.append(info) + infos = sorted(infos, key=lambda i: [int(i) for i in i.key.split(':')]) + # 这个守卫需要添加一下，避免最后一个无法出栈 + guarder = NodeAssetsInfo(key='', assets_amount=0, assets=set()) + infos.append(guarder) + + stack = Stack() + for info in infos: + # 如果栈顶的不是这个节点的父祖节点，那么可以出栈了，可以计算资产数量了 + while stack.top and not info.key.startswith(f'{stack.top.key}:'): + pop_info = stack.pop() + pop_info.assets_amount = len(pop_info.assets) + self.nodekey_assetsinfo_mapper[pop_info.key] = pop_info + if not stack.top: + continue + stack.top.assets.update(pop_info.assets) + stack.push(info) + + def get_assets_by_key(self, key): + info = self.nodekey_assetsinfo_mapper[key] + return info['assets'] + + def get_assets_amount(self, key): + info = self.nodekey_assetsinfo_mapper[key] + return info.assets_amount + + @classmethod + def test_it(cls): + from assets.models import Node, Asset + + nodes = list(Node.objects.all()) + nodes_assets = Asset.nodes.through.objects.all()\ + .annotate(aid=output_as_string('asset_id'))\ + .values_list('node__key', 'aid') + + mapping = defaultdict(set) + for key, asset_id in nodes_assets: + mapping[key].add(asset_id) + + util = cls(nodes, mapping) + util.generate() + return util + + diff --git a/apps/common/const/distributed_lock_key.py b/apps/common/const/distributed_lock_key.py deleted file mode 100644 index 735781841..000000000 --- a/apps/common/const/distributed_lock_key.py +++ /dev/null @@ -1,2 +0,0 @@ -UPDATE_NODE_TREE_LOCK_KEY = 'org_level_transaction_lock_{org_id}_assets_update_node_tree' -UPDATE_MAPPING_NODE_TASK_LOCK_KEY = 'org_level_transaction_lock_{user_id}_update_mapping_node_task' diff --git a/apps/common/db/models.py b/apps/common/db/models.py index 502df31e9..df5d6a46d 100644 --- a/apps/common/db/models.py +++ b/apps/common/db/models.py @@ -82,3 +82,7 @@ class JMSModel(JMSBaseModel): def concated_display(name1, name2): return Concat(F(name1), Value('('), F(name2), Value(')')) + + +def output_as_string(field_name): + return ExpressionWrapper(F(field_name), output_field=CharField()) diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index 8bc7377e5..f6808b0ac 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -254,3 +254,22 @@ def get_disk_usage(): mount_points = [p.mountpoint for p in partitions] usages = {p: psutil.disk_usage(p) for p in mount_points} return usages + + +class Time: + def __init__(self): + self._timestamps = [] + self._msgs = [] + + def begin(self): + self._timestamps.append(time.time()) + + def time(self, msg): + self._timestamps.append(time.time()) + self._msgs.append(msg) + + def print(self): + last, *timestamps = self._timestamps + for timestamp, msg in zip(timestamps, self._msgs): + logger.debug(f'TIME_IT: {msg} {timestamp-last}') + last = timestamp diff --git a/apps/common/utils/lock.py b/apps/common/utils/lock.py index 04ee1520f..d7d7acbed 100644 --- a/apps/common/utils/lock.py +++ b/apps/common/utils/lock.py @@ -1,8 +1,9 @@ from functools import wraps import threading -from redis_lock import Lock as RedisLock +from redis_lock import Lock as RedisLock, NotAcquired from redis import Redis +from django.db import transaction from common.utils import get_logger from common.utils.inspect import copy_function_args @@ -16,7 +17,8 @@ class AcquireFailed(RuntimeError): class DistributedLock(RedisLock): - def __init__(self, name, blocking=True, expire=60*2, auto_renewal=True): + def __init__(self, name, blocking=True, expire=None, release_lock_on_transaction_commit=False, + release_raise_exc=False, auto_renewal_seconds=60*2): """ 使用 redis 构造的分布式锁 @@ -25,31 +27,46 @@ class DistributedLock(RedisLock): :param blocking: 该参数只在锁作为装饰器或者 `with` 时有效。 :param expire: - 锁的过期时间，注意不一定是锁到这个时间就释放了，分两种情况 - 当 `auto_renewal=False` 时，锁会释放 - 当 `auto_renewal=True` 时，如果过期之前程序还没释放锁，我们会延长锁的存活时间。 - 这里的作用是防止程序意外终止没有释放锁，导致死锁。 + 锁的过期时间 + :param release_lock_on_transaction_commit: + 是否在当前事务结束后再释放锁 + :param release_raise_exc: + 释放锁时，如果没有持有锁是否抛异常或静默 + :param auto_renewal_seconds: + 当持有一个无限期锁的时候，刷新锁的时间，具体参考 `redis_lock.Lock#auto_renewal` """ self.kwargs_copy = copy_function_args(self.__init__, locals()) redis = Redis(host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, password=CONFIG.REDIS_PASSWORD) + + if expire is None: + expire = auto_renewal_seconds + auto_renewal = True + else: + auto_renewal = False + super().__init__(redis_client=redis, name=name, expire=expire, auto_renewal=auto_renewal) self._blocking = blocking + self._release_lock_on_transaction_commit = release_lock_on_transaction_commit + self._release_raise_exc = release_raise_exc def __enter__(self): thread_id = threading.current_thread().ident - logger.debug(f'DISTRIBUTED_LOCK: attempt to acquire ...') + logger.debug(f'Attempt to acquire global lock: thread {thread_id} lock {self._name}') acquired = self.acquire(blocking=self._blocking) if self._blocking and not acquired: - logger.debug(f'DISTRIBUTED_LOCK: was not acquired , but blocking=True') + logger.debug(f'Not acquired lock, but blocking=True, thread {thread_id} lock {self._name}') raise EnvironmentError("Lock wasn't acquired, but blocking=True") if not acquired: - logger.debug(f'DISTRIBUTED_LOCK: acquire failed') + logger.debug(f'Not acquired the lock, thread {thread_id} lock {self._name}') raise AcquireFailed - logger.debug(f'DISTRIBUTED_LOCK: acquire ok') + logger.debug(f'Acquire lock success, thread {thread_id} lock {self._name}') return self def __exit__(self, exc_type=None, exc_value=None, traceback=None): - self.release() + if self._release_lock_on_transaction_commit: + transaction.on_commit(self.release) + else: + self.release() def __call__(self, func): @wraps(func) @@ -57,5 +74,17 @@ class DistributedLock(RedisLock): # 要创建一个新的锁对象 with self.__class__(**self.kwargs_copy): return func(*args, **kwds) - return inner + + def locked_by_me(self): + if self.locked(): + if self.get_owner_id() == self.id: + return True + return False + + def release(self): + try: + super().release() + except AcquireFailed as e: + if self._release_raise_exc: + raise e diff --git a/apps/orgs/lock.py b/apps/orgs/lock.py deleted file mode 100644 index c129b8bcd..000000000 --- a/apps/orgs/lock.py +++ /dev/null @@ -1,131 +0,0 @@ -from uuid import uuid4 -from functools import wraps - -from django.core.cache import cache -from django.db.transaction import atomic -from rest_framework.request import Request -from rest_framework.exceptions import NotAuthenticated - -from orgs.utils import current_org -from common.exceptions import SomeoneIsDoingThis, Timeout -from common.utils.timezone import dt_formater, now - -# Redis 中锁值得模板，该模板提供了很强的可读性，方便调试与排错 -VALUE_TEMPLATE = '{stage}:{username}:{user_id}:{now}:{rand_str}' - -# 锁的状态 -DOING = 'doing' # 处理中，此状态的锁可以被干掉 -COMMITING = 'commiting' # 提交事务中，此状态很重要，要确保事务在锁消失之前返回了，不要轻易删除该锁 - -client = cache.client.get_client(write=True) - - -""" -将锁的状态从 `doing` 切换到 `commiting` -KEYS[1]： key -ARGV[1]: doingvalue -ARGV[2]: commitingvalue -ARGV[3]: timeout -""" -change_lock_state_to_commiting_lua = ''' -if (redis.call("get", KEYS[1]) == ARGV[1]) -then - return redis.call("set", KEYS[1], ARGV[2], "EX", ARGV[3], "XX") -else - return 0 -end -''' -change_lock_state_to_commiting_lua_obj = client.register_script(change_lock_state_to_commiting_lua) - - -""" -释放锁，两种`value`都要检查`doing`和`commiting` -KEYS[1]： key -ARGV[1]: 两个 `value` 中的其中一个 -ARGV[2]: 两个 `value` 中的其中一个 -""" -release_lua = ''' -if (redis.call("get",KEYS[1]) == ARGV[1] or redis.call("get",KEYS[1]) == ARGV[2]) -then - return redis.call("del",KEYS[1]) -else - return 0 -end -''' -release_lua_obj = client.register_script(release_lua) - - -def acquire(key, value, timeout): - return client.set(key, value, ex=timeout, nx=True) - - -def get(key): - return client.get(key) - - -def change_lock_state_to_commiting(key, doingvalue, commitingvalue, timeout=600): - # 将锁的状态从 `doing` 切换到 `commiting` - return bool(change_lock_state_to_commiting_lua_obj(keys=(key,), args=(doingvalue, commitingvalue, timeout))) - - -def release(key, value1, value2): - # 释放锁，两种`value` `doing`和`commiting` 都要检查 - return release_lua_obj(keys=(key,), args=(value1, value2)) - - -def _generate_value(request: Request, stage=DOING): - # 不支持匿名用户 - user = request.user - if user.is_anonymous: - raise NotAuthenticated - - return VALUE_TEMPLATE.format( - stage=stage, username=user.username, user_id=user.id, - now=dt_formater(now()), rand_str=uuid4() - ) - - -default_wait_msg = SomeoneIsDoingThis.default_detail - - -def org_level_transaction_lock(key, timeout=300, wait_msg=default_wait_msg): - """ - 被装饰的 `View` 必须取消自身的 `ATOMIC_REQUESTS`，因为该装饰器要有事务的完全控制权 - [官网](https://docs.djangoproject.com/en/3.1/topics/db/transactions/#tying-transactions-to-http-requests) - - 1. 获取锁：只有当锁对应的 `key` 不存在时成功获取，`value` 设置为 `doing` - 2. 开启事务：本次请求的事务必须确保在这里开启 - 3. 执行 `View` 体 - 4. `View` 体执行结束未异常，此时事务还未提交 - 5. 检查锁是否过时，过时事务回滚，不过时，重新设置`key`延长`key`有效期，已确保足够时间提交事务，同时把`key`的状态改为`commiting` - 6. 提交事务 - 7. 释放锁，释放的时候会检查`doing`与`commiting`的值，因为删除或者更改锁必须提供与当前锁的`value`相同的值，确保不误删 - [锁参考文章](http://doc.redisfans.com/string/set.html#id2) - """ - - def decorator(fun): - @wraps(fun) - def wrapper(request, *args, **kwargs): - # `key`可能是组织相关的，如果是把组织`id`加上 - _key = key.format(org_id=current_org.id) - doing_value = _generate_value(request) - commiting_value = _generate_value(request, stage=COMMITING) - try: - lock = acquire(_key, doing_value, timeout) - if not lock: - raise SomeoneIsDoingThis(detail=wait_msg) - with atomic(savepoint=False): - ret = fun(request, *args, **kwargs) - # 提交事务前，检查一下锁是否还在 - # 锁在的话，更新锁的状态为 `commiting`，延长锁时间，确保事务提交 - # 锁不在的话回滚 - ok = change_lock_state_to_commiting(_key, doing_value, commiting_value) - if not ok: - # 超时或者被中断了 - raise Timeout - return ret - finally: - # 释放锁，锁的两个值都要尝试，不确定异常是从什么位置抛出的 - release(_key, commiting_value, doing_value) - return wrapper - return decorator diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index c10a5dacc..d01ae3f77 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -184,3 +184,8 @@ def org_aware_func(org_arg_name): current_org = LocalProxy(get_current_org) + + +def ensure_in_real_or_default_org(): + if not current_org or current_org.is_root(): + raise ValueError('You must in a real or default org!') diff --git a/apps/perms/api/application/user_permission/common.py b/apps/perms/api/application/user_permission/common.py index 428f6bdc9..272f84378 100644 --- a/apps/perms/api/application/user_permission/common.py +++ b/apps/perms/api/application/user_permission/common.py @@ -13,7 +13,7 @@ from applications.models import Application from perms.utils.application.permission import ( get_application_system_users_id ) -from perms.api.asset.user_permission.mixin import ForAdminMixin, ForUserMixin +from perms.api.asset.user_permission.mixin import RoleAdminMixin, RoleUserMixin from common.permissions import IsOrgAdminOrAppUser from perms.hands import User, SystemUser from perms import serializers @@ -43,11 +43,11 @@ class GrantedApplicationSystemUsersMixin(ListAPIView): return system_users -class UserGrantedApplicationSystemUsersApi(ForAdminMixin, GrantedApplicationSystemUsersMixin): +class UserGrantedApplicationSystemUsersApi(RoleAdminMixin, GrantedApplicationSystemUsersMixin): pass -class MyGrantedApplicationSystemUsersApi(ForUserMixin, GrantedApplicationSystemUsersMixin): +class MyGrantedApplicationSystemUsersApi(RoleUserMixin, GrantedApplicationSystemUsersMixin): pass diff --git a/apps/perms/api/application/user_permission/user_permission_applications.py b/apps/perms/api/application/user_permission/user_permission_applications.py index 2b8b71847..6916f6f29 100644 --- a/apps/perms/api/application/user_permission/user_permission_applications.py +++ b/apps/perms/api/application/user_permission/user_permission_applications.py @@ -8,7 +8,7 @@ from applications.api.mixin import ( SerializeApplicationToTreeNodeMixin ) from perms import serializers -from perms.api.asset.user_permission.mixin import ForAdminMixin, ForUserMixin +from perms.api.asset.user_permission.mixin import RoleAdminMixin, RoleUserMixin from perms.utils.application.user_permission import ( get_user_granted_all_applications ) @@ -34,11 +34,11 @@ class AllGrantedApplicationsMixin(CommonApiMixin, ListAPIView): return queryset.only(*self.only_fields) -class UserAllGrantedApplicationsApi(ForAdminMixin, AllGrantedApplicationsMixin): +class UserAllGrantedApplicationsApi(RoleAdminMixin, AllGrantedApplicationsMixin): pass -class MyAllGrantedApplicationsApi(ForUserMixin, AllGrantedApplicationsMixin): +class MyAllGrantedApplicationsApi(RoleUserMixin, AllGrantedApplicationsMixin): pass diff --git a/apps/perms/api/asset/user_permission/mixin.py b/apps/perms/api/asset/user_permission/mixin.py index 9961a0cd9..5c5db7729 100644 --- a/apps/perms/api/asset/user_permission/mixin.py +++ b/apps/perms/api/asset/user_permission/mixin.py @@ -4,37 +4,23 @@ from rest_framework.request import Request from common.permissions import IsOrgAdminOrAppUser, IsValidUser from common.utils import lazyproperty +from common.http import is_true from orgs.utils import tmp_to_root_org from users.models import User -from perms.models import UserGrantedMappingNode +from perms.utils.asset.user_permission import UserGrantedTreeRefreshController -class UserNodeGrantStatusDispatchMixin: +class PermBaseMixin: + user: User - @staticmethod - def get_mapping_node_by_key(key, user): - return UserGrantedMappingNode.objects.get(key=key, user=user) - - def dispatch_get_data(self, key, user): - status = UserGrantedMappingNode.get_node_granted_status(key, user) - if status == UserGrantedMappingNode.GRANTED_DIRECT: - return self.get_data_on_node_direct_granted(key) - elif status == UserGrantedMappingNode.GRANTED_INDIRECT: - return self.get_data_on_node_indirect_granted(key) - else: - return self.get_data_on_node_not_granted(key) - - def get_data_on_node_direct_granted(self, key): - raise NotImplementedError - - def get_data_on_node_indirect_granted(self, key): - raise NotImplementedError - - def get_data_on_node_not_granted(self, key): - raise NotImplementedError + def get(self, request, *args, **kwargs): + force = is_true(request.query_params.get('rebuild_tree')) + controller = UserGrantedTreeRefreshController(self.user) + controller.refresh_if_need(force) + return super().get(request, *args, **kwargs) -class ForAdminMixin: +class RoleAdminMixin(PermBaseMixin): permission_classes = (IsOrgAdminOrAppUser,) kwargs: dict @@ -44,7 +30,7 @@ class ForAdminMixin: return User.objects.get(id=user_id) -class ForUserMixin: +class RoleUserMixin(PermBaseMixin): permission_classes = (IsValidUser,) request: Request diff --git a/apps/perms/api/asset/user_permission/user_permission_assets.py b/apps/perms/api/asset/user_permission/user_permission_assets.py deleted file mode 100644 index 209b59625..000000000 --- a/apps/perms/api/asset/user_permission/user_permission_assets.py +++ /dev/null @@ -1,156 +0,0 @@ -# -*- coding: utf-8 -*- -# -from perms.api.asset.user_permission.mixin import UserNodeGrantStatusDispatchMixin -from rest_framework.generics import ListAPIView -from rest_framework.response import Response -from rest_framework.request import Request -from django.conf import settings - -from assets.api.mixin import SerializeToTreeNodeMixin -from common.utils import get_logger -from perms.pagination import GrantedAssetLimitOffsetPagination -from assets.models import Asset, Node, FavoriteAsset -from perms import serializers -from perms.utils.asset.user_permission import ( - get_node_all_granted_assets, get_user_direct_granted_assets, - get_user_granted_all_assets -) -from .mixin import ForAdminMixin, ForUserMixin - - -logger = get_logger(__name__) - - -class UserDirectGrantedAssetsApi(ListAPIView): - """ - 用户直接授权的资产的列表，也就是授权规则上直接授权的资产，并非是来自节点的 - """ - serializer_class = serializers.AssetGrantedSerializer - only_fields = serializers.AssetGrantedSerializer.Meta.only_fields - filterset_fields = ['hostname', 'ip', 'id', 'comment'] - search_fields = ['hostname', 'ip', 'comment'] - - def get_queryset(self): - if getattr(self, 'swagger_fake_view', False): - return Asset.objects.none() - user = self.user - assets = get_user_direct_granted_assets(user)\ - .prefetch_related('platform')\ - .only(*self.only_fields) - return assets - - -class UserFavoriteGrantedAssetsApi(ListAPIView): - serializer_class = serializers.AssetGrantedSerializer - only_fields = serializers.AssetGrantedSerializer.Meta.only_fields - filterset_fields = ['hostname', 'ip', 'id', 'comment'] - search_fields = ['hostname', 'ip', 'comment'] - - def get_queryset(self): - if getattr(self, 'swagger_fake_view', False): - return Asset.objects.none() - user = self.user - assets = FavoriteAsset.get_user_favorite_assets(user)\ - .prefetch_related('platform')\ - .only(*self.only_fields) - return assets - - -class AssetsAsTreeMixin(SerializeToTreeNodeMixin): - """ - 将 资产 序列化成树的结构返回 - """ - def list(self, request: Request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - if request.query_params.get('search'): - # 如果用户搜索的条件不精准，会导致返回大量的无意义数据。 - # 这里限制一下返回数据的最大条数 - queryset = queryset[:999] - data = self.serialize_assets(queryset, None) - return Response(data=data) - - -class UserDirectGrantedAssetsForAdminApi(ForAdminMixin, UserDirectGrantedAssetsApi): - pass - - -class MyDirectGrantedAssetsApi(ForUserMixin, UserDirectGrantedAssetsApi): - pass - - -class UserFavoriteGrantedAssetsForAdminApi(ForAdminMixin, UserFavoriteGrantedAssetsApi): - pass - - -class MyFavoriteGrantedAssetsApi(ForUserMixin, UserFavoriteGrantedAssetsApi): - pass - - -class UserDirectGrantedAssetsAsTreeForAdminApi(ForAdminMixin, AssetsAsTreeMixin, UserDirectGrantedAssetsApi): - pass - - -class MyUngroupAssetsAsTreeApi(ForUserMixin, AssetsAsTreeMixin, UserDirectGrantedAssetsApi): - def get_queryset(self): - queryset = super().get_queryset() - if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - queryset = queryset.none() - return queryset - - -class UserAllGrantedAssetsApi(ForAdminMixin, ListAPIView): - only_fields = serializers.AssetGrantedSerializer.Meta.only_fields - serializer_class = serializers.AssetGrantedSerializer - filterset_fields = ['hostname', 'ip', 'id', 'comment'] - search_fields = ['hostname', 'ip', 'comment'] - - def get_queryset(self): - if getattr(self, 'swagger_fake_view', False): - return Asset.objects.none() - queryset = get_user_granted_all_assets(self.user) - queryset = queryset.prefetch_related('platform') - return queryset.only(*self.only_fields) - - -class MyAllGrantedAssetsApi(ForUserMixin, UserAllGrantedAssetsApi): - pass - - -class MyAllAssetsAsTreeApi(ForUserMixin, AssetsAsTreeMixin, UserAllGrantedAssetsApi): - search_fields = ['hostname', 'ip'] - - -class UserGrantedNodeAssetsApi(UserNodeGrantStatusDispatchMixin, ListAPIView): - serializer_class = serializers.AssetGrantedSerializer - only_fields = serializers.AssetGrantedSerializer.Meta.only_fields - filterset_fields = ['hostname', 'ip', 'id', 'comment'] - search_fields = ['hostname', 'ip', 'comment'] - pagination_class = GrantedAssetLimitOffsetPagination - pagination_node: Node - - def get_queryset(self): - if getattr(self, 'swagger_fake_view', False): - return Asset.objects.none() - node_id = self.kwargs.get("node_id") - node = Node.objects.get(id=node_id) - self.pagination_node = node - return self.dispatch_get_data(node.key, self.user) - - def get_data_on_node_direct_granted(self, key): - # 如果这个节点是直接授权的(或者说祖先节点直接授权的), 获取下面的所有资产 - return Node.get_node_all_assets_by_key_v2(key) - - def get_data_on_node_indirect_granted(self, key): - self.pagination_node = self.get_mapping_node_by_key(key, self.user) - return get_node_all_granted_assets(self.user, key) - - def get_data_on_node_not_granted(self, key): - return Asset.objects.none() - - -class UserGrantedNodeAssetsForAdminApi(ForAdminMixin, UserGrantedNodeAssetsApi): - pass - - -class MyGrantedNodeAssetsApi(ForUserMixin, UserGrantedNodeAssetsApi): - pass diff --git a/apps/perms/api/asset/user_permission/user_permission_assets/__init__.py b/apps/perms/api/asset/user_permission/user_permission_assets/__init__.py new file mode 100644 index 000000000..6b274abdd --- /dev/null +++ b/apps/perms/api/asset/user_permission/user_permission_assets/__init__.py @@ -0,0 +1 @@ +from .views import * diff --git a/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py b/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py new file mode 100644 index 000000000..0b92da278 --- /dev/null +++ b/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py @@ -0,0 +1,127 @@ +from rest_framework.response import Response +from rest_framework.request import Request + +from users.models import User +from assets.api.mixin import SerializeToTreeNodeMixin +from common.utils import get_logger +from perms.pagination import NodeGrantedAssetPagination, AllGrantedAssetPagination +from assets.models import Asset, Node +from perms import serializers +from perms.utils.asset.user_permission import UserGrantedAssetsQueryUtils, QuerySetStage + +logger = get_logger(__name__) + + +# 获取数据的 ------------------------------------------------------------ + +class UserDirectGrantedAssetsQuerysetMixin: + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + user: User + + def get_queryset(self): + if getattr(self, 'swagger_fake_view', False): + return Asset.objects.none() + user = self.user + assets = UserGrantedAssetsQueryUtils(user) \ + .get_direct_granted_assets() \ + .prefetch_related('platform') \ + .only(*self.only_fields) + return assets + + +class UserAllGrantedAssetsQuerysetMixin: + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + pagination_class = AllGrantedAssetPagination + user: User + + def get_union_queryset(self, qs_stage: QuerySetStage): + if getattr(self, 'swagger_fake_view', False): + return Asset.objects.none() + qs_stage.prefetch_related('platform').only(*self.only_fields) + queryset = UserGrantedAssetsQueryUtils(self.user) \ + .get_all_granted_assets(qs_stage) + return queryset + + +class UserFavoriteGrantedAssetsMixin: + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + user: User + + def get_union_queryset(self, qs_stage: QuerySetStage): + if getattr(self, 'swagger_fake_view', False): + return Asset.objects.none() + user = self.user + qs_stage.prefetch_related('platform').only(*self.only_fields) + utils = UserGrantedAssetsQueryUtils(user) + assets = utils.get_favorite_assets(qs_stage=qs_stage) + return assets + + +class UserGrantedNodeAssetsMixin: + only_fields = serializers.AssetGrantedSerializer.Meta.only_fields + pagination_class = NodeGrantedAssetPagination + pagination_node: Node + user: User + + def get_union_queryset(self, qs_stage: QuerySetStage): + if getattr(self, 'swagger_fake_view', False): + return Asset.objects.none() + node_id = self.kwargs.get("node_id") + qs_stage.prefetch_related('platform').only(*self.only_fields) + node, assets = UserGrantedAssetsQueryUtils(self.user).get_node_all_assets( + node_id, qs_stage=qs_stage + ) + self.pagination_node = node + return assets + + +# 控制格式的 ---------------------------------------------------- + +class AssetsUnionQuerysetMixin: + def get_queryset_union_prefer(self): + if hasattr(self, 'get_union_queryset'): + # 为了支持 union 查询 + queryset = Asset.objects.all().distinct() + queryset = self.filter_queryset(queryset) + qs_stage = QuerySetStage() + qs_stage.and_with_queryset(queryset) + queryset = self.get_union_queryset(qs_stage) + else: + queryset = self.filter_queryset(self.get_queryset()) + return queryset + + +class AssetsSerializerFormatMixin(AssetsUnionQuerysetMixin): + serializer_class = serializers.AssetGrantedSerializer + filterset_fields = ['hostname', 'ip', 'id', 'comment'] + search_fields = ['hostname', 'ip', 'comment'] + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset_union_prefer() + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + +class AssetsTreeFormatMixin(AssetsUnionQuerysetMixin, SerializeToTreeNodeMixin): + """ + 将 资产 序列化成树的结构返回 + """ + + def list(self, request: Request, *args, **kwargs): + queryset = self.get_queryset_union_prefer() + + if request.query_params.get('search'): + # 如果用户搜索的条件不精准，会导致返回大量的无意义数据。 + # 这里限制一下返回数据的最大条数 + queryset = queryset[:999] + data = self.serialize_assets(queryset, None) + return Response(data=data) + + # def get_serializer_class(self): + # return EmptySerializer diff --git a/apps/perms/api/asset/user_permission/user_permission_assets/views.py b/apps/perms/api/asset/user_permission/user_permission_assets/views.py new file mode 100644 index 000000000..05b09442a --- /dev/null +++ b/apps/perms/api/asset/user_permission/user_permission_assets/views.py @@ -0,0 +1,99 @@ +from rest_framework.generics import ListAPIView +from django.conf import settings + +from common.utils import get_logger +from ..mixin import RoleAdminMixin, RoleUserMixin +from .mixin import ( + UserAllGrantedAssetsQuerysetMixin, UserDirectGrantedAssetsQuerysetMixin, UserFavoriteGrantedAssetsMixin, + UserGrantedNodeAssetsMixin, AssetsSerializerFormatMixin, AssetsTreeFormatMixin, +) + +__all__ = [ + 'UserDirectGrantedAssetsForAdminApi', 'MyDirectGrantedAssetsApi', 'UserFavoriteGrantedAssetsForAdminApi', + 'MyFavoriteGrantedAssetsApi', 'UserDirectGrantedAssetsAsTreeForAdminApi', 'MyUngroupAssetsAsTreeApi', + 'UserAllGrantedAssetsApi', 'MyAllGrantedAssetsApi', 'MyAllAssetsAsTreeApi', 'UserGrantedNodeAssetsForAdminApi', + 'MyGrantedNodeAssetsApi', +] + +logger = get_logger(__name__) + + +class UserDirectGrantedAssetsForAdminApi(UserDirectGrantedAssetsQuerysetMixin, + RoleAdminMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class MyDirectGrantedAssetsApi(UserDirectGrantedAssetsQuerysetMixin, + RoleUserMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class UserFavoriteGrantedAssetsForAdminApi(UserFavoriteGrantedAssetsMixin, + RoleAdminMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class MyFavoriteGrantedAssetsApi(UserFavoriteGrantedAssetsMixin, + RoleUserMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class UserDirectGrantedAssetsAsTreeForAdminApi(UserDirectGrantedAssetsQuerysetMixin, + RoleAdminMixin, + AssetsTreeFormatMixin, + ListAPIView): + pass + + +class MyUngroupAssetsAsTreeApi(UserDirectGrantedAssetsQuerysetMixin, + RoleUserMixin, + AssetsTreeFormatMixin, + ListAPIView): + def get_queryset(self): + queryset = super().get_queryset() + if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + queryset = queryset.none() + return queryset + + +class UserAllGrantedAssetsApi(UserAllGrantedAssetsQuerysetMixin, + RoleAdminMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class MyAllGrantedAssetsApi(UserAllGrantedAssetsQuerysetMixin, + RoleUserMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class MyAllAssetsAsTreeApi(UserAllGrantedAssetsQuerysetMixin, + RoleUserMixin, + AssetsTreeFormatMixin, + ListAPIView): + search_fields = ['hostname', 'ip'] + + +class UserGrantedNodeAssetsForAdminApi(UserGrantedNodeAssetsMixin, + RoleAdminMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass + + +class MyGrantedNodeAssetsApi(UserGrantedNodeAssetsMixin, + RoleUserMixin, + AssetsSerializerFormatMixin, + ListAPIView): + pass diff --git a/apps/perms/api/asset/user_permission/user_permission_nodes.py b/apps/perms/api/asset/user_permission/user_permission_nodes.py index 58af37090..2a5cfacf2 100644 --- a/apps/perms/api/asset/user_permission/user_permission_nodes.py +++ b/apps/perms/api/asset/user_permission/user_permission_nodes.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # import abc -from django.conf import settings from rest_framework.generics import ( ListAPIView ) @@ -10,16 +9,11 @@ from rest_framework.request import Request from assets.api.mixin import SerializeToTreeNodeMixin from common.utils import get_logger -from .mixin import ForAdminMixin, ForUserMixin, UserNodeGrantStatusDispatchMixin -from perms.hands import Node, User +from .mixin import RoleAdminMixin, RoleUserMixin +from perms.hands import User from perms import serializers -from perms.utils.asset.user_permission import ( - get_indirect_granted_node_children, - get_user_granted_nodes_list_via_mapping_node, - get_top_level_granted_nodes, - rebuild_user_tree_if_need, get_favorite_node, - get_ungrouped_node -) + +from perms.utils.asset.user_permission import UserGrantedNodesQueryUtils logger = get_logger(__name__) @@ -61,7 +55,6 @@ class BaseGrantedNodeApi(_GrantedNodeStructApi, metaclass=abc.ABCMeta): serializer_class = serializers.NodeGrantedSerializer def list(self, request, *args, **kwargs): - rebuild_user_tree_if_need(request, self.user) nodes = self.get_nodes() serializer = self.get_serializer(nodes, many=True) return Response(serializer.data) @@ -73,7 +66,6 @@ class BaseNodeChildrenApi(NodeChildrenMixin, BaseGrantedNodeApi, metaclass=abc.A class BaseGrantedNodeAsTreeApi(SerializeToTreeNodeMixin, _GrantedNodeStructApi, metaclass=abc.ABCMeta): def list(self, request: Request, *args, **kwargs): - rebuild_user_tree_if_need(request, self.user) nodes = self.get_nodes() nodes = self.serialize_nodes(nodes, with_asset_amount=True) return Response(data=nodes) @@ -83,30 +75,16 @@ class BaseNodeChildrenAsTreeApi(NodeChildrenMixin, BaseGrantedNodeAsTreeApi, met pass -class UserGrantedNodeChildrenMixin(UserNodeGrantStatusDispatchMixin): +class UserGrantedNodeChildrenMixin: user: User request: Request def get_children(self): user = self.user key = self.request.query_params.get('key') - - if not key: - nodes = list(get_top_level_granted_nodes(user)) - else: - nodes = self.dispatch_get_data(key, user) + nodes = UserGrantedNodesQueryUtils(user).get_node_children(key) return nodes - def get_data_on_node_direct_granted(self, key): - return Node.objects.filter(parent_key=key) - - def get_data_on_node_indirect_granted(self, key): - nodes = get_indirect_granted_node_children(self.user, key) - return nodes - - def get_data_on_node_not_granted(self, key): - return Node.objects.none() - class UserGrantedNodesMixin: """ @@ -115,41 +93,38 @@ class UserGrantedNodesMixin: user: User def get_nodes(self): - nodes = [] - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - nodes.append(get_ungrouped_node(self.user)) - nodes.append(get_favorite_node(self.user)) - nodes.extend(get_user_granted_nodes_list_via_mapping_node(self.user)) + utils = UserGrantedNodesQueryUtils(self.user) + nodes = utils.get_whole_tree_nodes() return nodes # ------------------------------------------ # 最终的 api -class UserGrantedNodeChildrenForAdminApi(ForAdminMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenApi): +class UserGrantedNodeChildrenForAdminApi(RoleAdminMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenApi): pass -class MyGrantedNodeChildrenApi(ForUserMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenApi): +class MyGrantedNodeChildrenApi(RoleUserMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenApi): pass -class UserGrantedNodeChildrenAsTreeForAdminApi(ForAdminMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenAsTreeApi): +class UserGrantedNodeChildrenAsTreeForAdminApi(RoleAdminMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenAsTreeApi): pass -class MyGrantedNodeChildrenAsTreeApi(ForUserMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenAsTreeApi): +class MyGrantedNodeChildrenAsTreeApi(RoleUserMixin, UserGrantedNodeChildrenMixin, BaseNodeChildrenAsTreeApi): pass -class UserGrantedNodesForAdminApi(ForAdminMixin, UserGrantedNodesMixin, BaseGrantedNodeApi): +class UserGrantedNodesForAdminApi(RoleAdminMixin, UserGrantedNodesMixin, BaseGrantedNodeApi): pass -class MyGrantedNodesApi(ForUserMixin, UserGrantedNodesMixin, BaseGrantedNodeApi): +class MyGrantedNodesApi(RoleUserMixin, UserGrantedNodesMixin, BaseGrantedNodeApi): pass -class MyGrantedNodesAsTreeApi(ForUserMixin, UserGrantedNodesMixin, BaseGrantedNodeAsTreeApi): +class MyGrantedNodesAsTreeApi(RoleUserMixin, UserGrantedNodesMixin, BaseGrantedNodeAsTreeApi): pass # ------------------------------------------ diff --git a/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py b/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py index 44d8f22b3..253a925ca 100644 --- a/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py +++ b/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py @@ -1,29 +1,23 @@ # -*- coding: utf-8 -*- # -from itertools import chain - from rest_framework.generics import ListAPIView from rest_framework.request import Request from rest_framework.response import Response -from django.db.models import F, Value, CharField, Q +from django.db.models import F, Value, CharField from django.conf import settings +from common.utils.common import timeit from orgs.utils import tmp_to_root_org from common.permissions import IsValidUser from common.utils import get_logger, get_object_or_none -from .mixin import UserNodeGrantStatusDispatchMixin, ForUserMixin, ForAdminMixin +from .mixin import RoleUserMixin, RoleAdminMixin from perms.utils.asset.user_permission import ( - get_indirect_granted_node_children, UNGROUPED_NODE_KEY, FAVORITE_NODE_KEY, - get_user_direct_granted_assets, get_top_level_granted_nodes, - get_user_granted_nodes_list_via_mapping_node, - get_user_granted_all_assets, rebuild_user_tree_if_need, - get_user_all_assetpermissions_id, get_favorite_node, - get_ungrouped_node, compute_tmp_mapping_node_from_perm, - TMP_GRANTED_FIELD, count_direct_granted_node_assets, - count_node_all_granted_assets + UserGrantedTreeBuildUtils, get_user_all_asset_perm_ids, + UserGrantedNodesQueryUtils, UserGrantedAssetsQueryUtils, + QuerySetStage, ) -from perms.models import AssetPermission -from assets.models import Asset, FavoriteAsset +from perms.models import AssetPermission, PermNode +from assets.models import Asset from assets.api import SerializeToTreeNodeMixin from perms.hands import Node @@ -33,76 +27,45 @@ logger = get_logger(__name__) class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): permission_classes = (IsValidUser,) - def add_ungrouped_resource(self, data: list, user, asset_perms_id): + @timeit + def add_ungrouped_resource(self, data: list, nodes_query_utils, assets_query_utils): if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: return + ungrouped_node = nodes_query_utils.get_ungrouped_node() - ungrouped_node = get_ungrouped_node(user, asset_perms_id=asset_perms_id) - direct_granted_assets = get_user_direct_granted_assets( - user, asset_perms_id=asset_perms_id - ).annotate( + direct_granted_assets = assets_query_utils.get_direct_granted_assets().annotate( parent_key=Value(ungrouped_node.key, output_field=CharField()) ).prefetch_related('platform') data.extend(self.serialize_nodes([ungrouped_node], with_asset_amount=True)) data.extend(self.serialize_assets(direct_granted_assets)) - def add_favorite_resource(self, data: list, user, asset_perms_id): - favorite_node = get_favorite_node(user, asset_perms_id) - favorite_assets = FavoriteAsset.get_user_favorite_assets( - user, asset_perms_id=asset_perms_id - ).annotate( + @timeit + def add_favorite_resource(self, data: list, nodes_query_utils, assets_query_utils): + favorite_node = nodes_query_utils.get_favorite_node() + + qs_state = QuerySetStage().annotate( parent_key=Value(favorite_node.key, output_field=CharField()) ).prefetch_related('platform') + favorite_assets = assets_query_utils.get_favorite_assets(qs_stage=qs_state, only=()) data.extend(self.serialize_nodes([favorite_node], with_asset_amount=True)) data.extend(self.serialize_assets(favorite_assets)) + @timeit def add_node_filtered_by_system_user(self, data: list, user, asset_perms_id): - tmp_nodes = compute_tmp_mapping_node_from_perm(user, asset_perms_id=asset_perms_id) - granted_nodes_key = [] - for _node in tmp_nodes: - _granted = getattr(_node, TMP_GRANTED_FIELD, False) - if not _granted: - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - assets_amount = count_direct_granted_node_assets(user, _node.key, asset_perms_id) - else: - assets_amount = count_node_all_granted_assets(user, _node.key, asset_perms_id) - _node.assets_amount = assets_amount - else: - granted_nodes_key.append(_node.key) + utils = UserGrantedTreeBuildUtils(user, asset_perms_id) + nodes = utils.get_whole_tree_nodes() + data.extend(self.serialize_nodes(nodes, with_asset_amount=True)) - # 查询他们的子节点 - q = Q() - for _key in granted_nodes_key: - q |= Q(key__startswith=f'{_key}:') + def add_assets(self, data: list, assets_query_utils: UserGrantedAssetsQueryUtils): + qs_stage = QuerySetStage().annotate(parent_key=F('nodes__key')).prefetch_related('platform') - if q: - descendant_nodes = Node.objects.filter(q).distinct() - else: - descendant_nodes = Node.objects.none() - - data.extend(self.serialize_nodes(chain(tmp_nodes, descendant_nodes), with_asset_amount=True)) - - def add_assets(self, data: list, user, asset_perms_id): if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - all_assets = get_user_granted_all_assets( - user, - via_mapping_node=False, - include_direct_granted_assets=False, - asset_perms_id=asset_perms_id - ) + all_assets = assets_query_utils.get_direct_granted_nodes_assets(qs_stage=qs_stage) else: - all_assets = get_user_granted_all_assets( - user, - via_mapping_node=False, - include_direct_granted_assets=True, - asset_perms_id=asset_perms_id - ) + all_assets = assets_query_utils.get_all_granted_assets(qs_stage=qs_stage) - all_assets = all_assets.annotate( - parent_key=F('nodes__key') - ).prefetch_related('platform') data.extend(self.serialize_assets(all_assets)) @tmp_to_root_org() @@ -117,7 +80,7 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): user = request.user data = [] - asset_perms_id = get_user_all_assetpermissions_id(user) + asset_perms_id = get_user_all_asset_perm_ids(user) system_user_id = request.query_params.get('system_user') if system_user_id: @@ -125,89 +88,72 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): id__in=asset_perms_id, system_users__id=system_user_id, actions__gt=0 ).values_list('id', flat=True).distinct()) - self.add_ungrouped_resource(data, user, asset_perms_id) - self.add_favorite_resource(data, user, asset_perms_id) + nodes_query_utils = UserGrantedNodesQueryUtils(user, asset_perms_id) + assets_query_utils = UserGrantedAssetsQueryUtils(user, asset_perms_id) + + self.add_ungrouped_resource(data, nodes_query_utils, assets_query_utils) + self.add_favorite_resource(data, nodes_query_utils, assets_query_utils) if system_user_id: + # 有系统用户筛选的需要重新计算树结构 self.add_node_filtered_by_system_user(data, user, asset_perms_id) else: - rebuild_user_tree_if_need(request, user) - all_nodes = get_user_granted_nodes_list_via_mapping_node(user) + all_nodes = nodes_query_utils.get_whole_tree_nodes(with_special=False) data.extend(self.serialize_nodes(all_nodes, with_asset_amount=True)) - self.add_assets(data, user, asset_perms_id) + self.add_assets(data, assets_query_utils) return Response(data=data) -class GrantedNodeChildrenWithAssetsAsTreeApiMixin(UserNodeGrantStatusDispatchMixin, - SerializeToTreeNodeMixin, +class GrantedNodeChildrenWithAssetsAsTreeApiMixin(SerializeToTreeNodeMixin, ListAPIView): """ 带资产的授权树 """ user: None - def get_data_on_node_direct_granted(self, key): - nodes = Node.objects.filter(parent_key=key) - assets = Asset.org_objects.filter(nodes__key=key).distinct() - assets = assets.prefetch_related('platform') - return nodes, assets + def ensure_key(self): + key = self.request.query_params.get('key', None) + id = self.request.query_params.get('id', None) - def get_data_on_node_indirect_granted(self, key): - user = self.user - asset_perms_id = get_user_all_assetpermissions_id(user) + if key is not None: + return key - nodes = get_indirect_granted_node_children(user, key) - - assets = Asset.org_objects.filter( - nodes__key=key, - ).filter( - granted_by_permissions__id__in=asset_perms_id - ).distinct() - assets = assets.prefetch_related('platform') - return nodes, assets - - def get_data_on_node_not_granted(self, key): - return Node.objects.none(), Asset.objects.none() - - def get_data(self, key, user): - assets, nodes = [], [] - if not key: - root_nodes = get_top_level_granted_nodes(user) - nodes.extend(root_nodes) - elif key == UNGROUPED_NODE_KEY: - assets = get_user_direct_granted_assets(user) - assets = assets.prefetch_related('platform') - elif key == FAVORITE_NODE_KEY: - assets = FavoriteAsset.get_user_favorite_assets(user) - else: - nodes, assets = self.dispatch_get_data(key, user) - return nodes, assets - - def id2key_if_have(self): - id = self.request.query_params.get('id') - if id is not None: - node = get_object_or_none(Node, id=id) - if node: - return node.key + node = get_object_or_none(Node, id=id) + if node: + return node.key def list(self, request: Request, *args, **kwargs): - key = self.request.query_params.get('key') - if key is None: - key = self.id2key_if_have() + user = self.user + key = self.ensure_key() + + nodes_query_utils = UserGrantedNodesQueryUtils(user) + assets_query_utils = UserGrantedAssetsQueryUtils(user) + + nodes = PermNode.objects.none() + assets = Asset.objects.none() + + if not key: + nodes = nodes_query_utils.get_top_level_nodes() + elif key == PermNode.UNGROUPED_NODE_KEY: + assets = assets_query_utils.get_ungroup_assets() + elif key == PermNode.FAVORITE_NODE_KEY: + assets = assets_query_utils.get_favorite_assets() + else: + nodes = nodes_query_utils.get_node_children(key) + assets = assets_query_utils.get_node_assets(key) + assets = assets.prefetch_related('platform') user = self.user - rebuild_user_tree_if_need(request, user) - nodes, assets = self.get_data(key, user) tree_nodes = self.serialize_nodes(nodes, with_asset_amount=True) tree_assets = self.serialize_assets(assets, key) return Response(data=[*tree_nodes, *tree_assets]) -class UserGrantedNodeChildrenWithAssetsAsTreeApi(ForAdminMixin, GrantedNodeChildrenWithAssetsAsTreeApiMixin): +class UserGrantedNodeChildrenWithAssetsAsTreeApi(RoleAdminMixin, GrantedNodeChildrenWithAssetsAsTreeApiMixin): pass -class MyGrantedNodeChildrenWithAssetsAsTreeApi(ForUserMixin, GrantedNodeChildrenWithAssetsAsTreeApiMixin): +class MyGrantedNodeChildrenWithAssetsAsTreeApi(RoleUserMixin, GrantedNodeChildrenWithAssetsAsTreeApiMixin): pass diff --git a/apps/perms/api/system_user_permission.py b/apps/perms/api/system_user_permission.py index 0c026b54d..17ddfc786 100644 --- a/apps/perms/api/system_user_permission.py +++ b/apps/perms/api/system_user_permission.py @@ -1,10 +1,10 @@ from rest_framework import generics -from django.db.models import Q from django.utils.decorators import method_decorator from assets.models import SystemUser from common.permissions import IsValidUser from orgs.utils import tmp_to_root_org +from perms.utils.asset.user_permission import get_user_all_asset_perm_ids from .. import serializers @@ -16,9 +16,9 @@ class SystemUserPermission(generics.ListAPIView): def get_queryset(self): user = self.request.user + asset_perms_id = get_user_all_asset_perm_ids(user) queryset = SystemUser.objects.filter( - Q(granted_by_permissions__users=user) | - Q(granted_by_permissions__user_groups__users=user) + granted_by_permissions__id__in=asset_perms_id ).distinct() return queryset diff --git a/apps/perms/async_tasks/__init__.py b/apps/perms/async_tasks/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/perms/async_tasks/mapping_node_task.py b/apps/perms/async_tasks/mapping_node_task.py deleted file mode 100644 index c45527ab7..000000000 --- a/apps/perms/async_tasks/mapping_node_task.py +++ /dev/null @@ -1,47 +0,0 @@ -from django.utils.crypto import get_random_string -from perms.utils import rebuild_user_mapping_nodes_if_need_with_lock - -from common.thread_pools import SingletonThreadPoolExecutor -from common.utils import get_logger -from perms.models import RebuildUserTreeTask - -logger = get_logger(__name__) - - -class Executor(SingletonThreadPoolExecutor): - pass - - -executor = Executor() - - -def run_mapping_node_tasks(): - failed_user_ids = [] - - ident = get_random_string() - logger.debug(f'[{ident}]mapping_node_tasks running') - - while True: - task = RebuildUserTreeTask.objects.exclude( - user_id__in=failed_user_ids - ).first() - - if task is None: - break - - user = task.user - try: - rebuild_user_mapping_nodes_if_need_with_lock(user) - except: - logger.exception(f'[{ident}]mapping_node_tasks_exception') - failed_user_ids.append(user.id) - - logger.debug(f'[{ident}]mapping_node_tasks finished') - - -def submit_update_mapping_node_task(): - executor.submit(run_mapping_node_tasks) - - -def submit_update_mapping_node_task_for_user(user): - executor.submit(rebuild_user_mapping_nodes_if_need_with_lock, user) diff --git a/apps/perms/locks.py b/apps/perms/locks.py new file mode 100644 index 000000000..e1bd67f09 --- /dev/null +++ b/apps/perms/locks.py @@ -0,0 +1,11 @@ +from common.utils.lock import DistributedLock + + +class UserGrantedTreeRebuildLock(DistributedLock): + name_template = 'perms.user.asset.node.tree.rebuid..' + + def __init__(self, org_id, user_id): + name = self.name_template.format( + org_id=org_id, user_id=user_id + ) + super().__init__(name=name) diff --git a/apps/perms/migrations/0014_build_users_perm_tree.py b/apps/perms/migrations/0014_build_users_perm_tree.py index 85df89b35..a4fd97b96 100644 --- a/apps/perms/migrations/0014_build_users_perm_tree.py +++ b/apps/perms/migrations/0014_build_users_perm_tree.py @@ -1,19 +1,6 @@ # Generated by Django 2.2.13 on 2020-08-21 08:20 from django.db import migrations -from perms.tasks import dispatch_mapping_node_tasks - - -def start_build_users_perm_tree_task(apps, schema_editor): - User = apps.get_model('users', 'User') - RebuildUserTreeTask = apps.get_model('perms', 'RebuildUserTreeTask') - - user_ids = User.objects.all().values_list('id', flat=True).distinct() - RebuildUserTreeTask.objects.bulk_create( - [RebuildUserTreeTask(user_id=i) for i in user_ids] - ) - - dispatch_mapping_node_tasks.delay() class Migration(migrations.Migration): @@ -23,5 +10,4 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(start_build_users_perm_tree_task) ] diff --git a/apps/perms/migrations/0018_auto_20210204_1749.py b/apps/perms/migrations/0018_auto_20210204_1749.py new file mode 100644 index 000000000..00d567c2f --- /dev/null +++ b/apps/perms/migrations/0018_auto_20210204_1749.py @@ -0,0 +1,65 @@ +# Generated by Django 3.1 on 2021-02-04 09:49 + +import assets.models.node +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('assets', '0066_remove_node_assets_amount'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('perms', '0017_auto_20210104_0435'), + ] + + operations = [ + migrations.CreateModel( + name='UserAssetGrantedTreeNodeRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Created by')), + ('updated_by', models.CharField(blank=True, max_length=32, null=True, verbose_name='Updated by')), + ('date_created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Date created')), + ('date_updated', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('org_id', models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization')), + ('node_key', models.CharField(db_index=True, max_length=64, verbose_name='Key')), + ('node_parent_key', models.CharField(db_index=True, default='', max_length=64, verbose_name='Parent key')), + ('node_from', models.CharField(choices=[('granted', 'Direct node granted'), ('child', 'Have children node'), ('asset', 'Direct asset granted')], db_index=True, max_length=16)), + ('node_assets_amount', models.IntegerField(default=0)), + ('node', models.ForeignKey(db_constraint=False, default=None, on_delete=django.db.models.deletion.CASCADE, related_name='granted_node_rels', to='assets.node')), + ('user', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + bases=(assets.models.node.FamilyMixin, models.Model), + ), + migrations.RemoveField( + model_name='usergrantedmappingnode', + name='node', + ), + migrations.RemoveField( + model_name='usergrantedmappingnode', + name='user', + ), + migrations.CreateModel( + name='PermNode', + fields=[ + ], + options={ + 'ordering': [], + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('assets.node',), + ), + migrations.DeleteModel( + name='RebuildUserTreeTask', + ), + migrations.DeleteModel( + name='UserGrantedMappingNode', + ), + ] diff --git a/apps/perms/models/asset_permission.py b/apps/perms/models/asset_permission.py index 00c0d5a97..e5a03c879 100644 --- a/apps/perms/models/asset_permission.py +++ b/apps/perms/models/asset_permission.py @@ -2,7 +2,10 @@ import logging from functools import reduce from django.utils.translation import ugettext_lazy as _ +from django.db.models import F +from common.db.models import ChoiceSet +from orgs.mixins.models import OrgModelMixin from common.db import models from common.utils import lazyproperty from assets.models import Asset, SystemUser, Node, FamilyMixin @@ -11,7 +14,7 @@ from .base import BasePermission __all__ = [ - 'AssetPermission', 'Action', 'UserGrantedMappingNode', 'RebuildUserTreeTask', + 'AssetPermission', 'Action', 'PermNode', 'UserAssetGrantedTreeNodeRelation', ] # 使用场景 @@ -135,39 +138,109 @@ class AssetPermission(BasePermission): from assets.models import Node nodes_keys = self.nodes.all().values_list('key', flat=True) assets_ids = set(self.assets.all().values_list('id', flat=True)) - nodes_assets_ids = Node.get_nodes_all_assets_ids(nodes_keys) + nodes_assets_ids = Node.get_nodes_all_assets_ids_by_keys(nodes_keys) assets_ids.update(nodes_assets_ids) assets = Asset.objects.filter(id__in=assets_ids) return assets +class UserAssetGrantedTreeNodeRelation(OrgModelMixin, FamilyMixin, models.JMSBaseModel): + class NodeFrom(ChoiceSet): + granted = 'granted', 'Direct node granted' + child = 'child', 'Have children node' + asset = 'asset', 'Direct asset granted' -class UserGrantedMappingNode(FamilyMixin, models.JMSBaseModel): - node = models.ForeignKey('assets.Node', default=None, on_delete=models.CASCADE, - db_constraint=False, null=True, related_name='mapping_nodes') - key = models.CharField(max_length=64, verbose_name=_("Key"), db_index=True) # '1:1:1:1' user = models.ForeignKey('users.User', db_constraint=False, on_delete=models.CASCADE) - granted = models.BooleanField(default=False, db_index=True) - asset_granted = models.BooleanField(default=False, db_index=True) - parent_key = models.CharField(max_length=64, default='', verbose_name=_('Parent key'), db_index=True) # '1:1:1:1' - assets_amount = models.IntegerField(default=0) + node = models.ForeignKey('assets.Node', default=None, on_delete=models.CASCADE, + db_constraint=False, null=False, related_name='granted_node_rels') + node_key = models.CharField(max_length=64, verbose_name=_("Key"), db_index=True) + node_parent_key = models.CharField(max_length=64, default='', verbose_name=_('Parent key'), db_index=True) + node_from = models.CharField(choices=NodeFrom.choices, max_length=16, db_index=True) + node_assets_amount = models.IntegerField(default=0) - GRANTED_DIRECT = 1 - GRANTED_INDIRECT = 2 - GRANTED_NONE = 0 + @property + def key(self): + return self.node_key + + @property + def parent_key(self): + return self.node_parent_key @classmethod - def get_node_granted_status(cls, key, user): - ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True) - has_granted = UserGrantedMappingNode.objects.filter( - key__in=ancestor_keys, user=user - ).values_list('granted', flat=True) - if not has_granted: - return cls.GRANTED_NONE - if any(list(has_granted)): - return cls.GRANTED_DIRECT - return cls.GRANTED_INDIRECT + def get_node_granted_status(cls, user, key): + ancestor_keys = set(cls.get_node_ancestor_keys(key, with_self=True)) + ancestor_rel_nodes = cls.objects.filter(user=user, node_key__in=ancestor_keys) + + for rel_node in ancestor_rel_nodes: + if rel_node.key == key: + return rel_node.node_from, rel_node + if rel_node.node_from == cls.NodeFrom.granted: + return cls.NodeFrom.granted, None + return '', None -class RebuildUserTreeTask(models.JMSBaseModel): - user = models.ForeignKey('users.User', on_delete=models.CASCADE, verbose_name=_('User')) +class PermNode(Node): + class Meta: + proxy = True + ordering = [] + + # 特殊节点 + UNGROUPED_NODE_KEY = 'ungrouped' + UNGROUPED_NODE_VALUE = _('Ungrouped') + FAVORITE_NODE_KEY = 'favorite' + FAVORITE_NODE_VALUE = _('Favorite') + + node_from = '' + granted_assets_amount = 0 + + # 提供可以设置 资产数量的字段 + _assets_amount = None + + annotate_granted_node_rel_fields = { + 'granted_assets_amount': F('granted_node_rels__node_assets_amount'), + 'node_from': F('granted_node_rels__node_from') + } + + @property + def assets_amount(self): + _assets_amount = getattr(self, '_assets_amount') + if isinstance(_assets_amount, int): + return _assets_amount + return super().assets_amount + + @assets_amount.setter + def assets_amount(self, value): + self._assets_amount = value + + def use_granted_assets_amount(self): + self.assets_amount = self.granted_assets_amount + + @classmethod + def get_ungrouped_node(cls, assets_amount): + return cls( + id=cls.UNGROUPED_NODE_KEY, + key=cls.UNGROUPED_NODE_KEY, + value=cls.UNGROUPED_NODE_VALUE, + assets_amount=assets_amount + ) + + @classmethod + def get_favorite_node(cls, assets_amount): + node = cls( + id=cls.FAVORITE_NODE_KEY, + key=cls.FAVORITE_NODE_KEY, + value=cls.FAVORITE_NODE_VALUE, + ) + node.assets_amount = assets_amount + return node + + def get_granted_status(self, user): + status, rel_node = UserAssetGrantedTreeNodeRelation.get_node_granted_status(user, self.key) + self.node_from = status + if rel_node: + self.granted_assets_amount = rel_node.node_assets_amount + return status + + def save(self): + # 这是个只读 Model + raise NotImplementedError diff --git a/apps/perms/pagination.py b/apps/perms/pagination.py index 75cf6c493..fc5e43de7 100644 --- a/apps/perms/pagination.py +++ b/apps/perms/pagination.py @@ -1,30 +1,54 @@ from rest_framework.pagination import LimitOffsetPagination from rest_framework.request import Request +from django.db.models import Sum +from perms.models import UserAssetGrantedTreeNodeRelation from common.utils import get_logger logger = get_logger(__name__) -class GrantedAssetLimitOffsetPagination(LimitOffsetPagination): +class GrantedAssetPaginationBase(LimitOffsetPagination): + + def paginate_queryset(self, queryset, request: Request, view=None): + self._request = request + self._view = view + self._user = request.user + return super().paginate_queryset(queryset, request, view=None) + def get_count(self, queryset): exclude_query_params = { self.limit_query_param, self.offset_query_param, 'key', 'all', 'show_current_asset', - 'cache_policy', 'display', 'draw' + 'cache_policy', 'display', 'draw', + 'order', } for k, v in self._request.query_params.items(): if k not in exclude_query_params and v is not None: + logger.warn(f'Not hit node.assets_amount because find a unknow query_param `{k}` -> {self._request.get_full_path()}') return super().get_count(queryset) + return self.get_count_from_nodes(queryset) + + def get_count_from_nodes(self, queryset): + raise NotImplementedError + + +class NodeGrantedAssetPagination(GrantedAssetPaginationBase): + def get_count_from_nodes(self, queryset): node = getattr(self._view, 'pagination_node', None) if node: - logger.debug(f'{self._request.get_full_path()} hit node.assets_amount[{node.assets_amount}]') + logger.debug(f'Hit node.assets_amount[{node.assets_amount}] -> {self._request.get_full_path()}') return node.assets_amount else: + logger.warn(f'Not hit node.assets_amount[{node}] because {self._view} not has `pagination_node` -> {self._request.get_full_path()}') return super().get_count(queryset) - def paginate_queryset(self, queryset, request: Request, view=None): - self._request = request - self._view = view - return super().paginate_queryset(queryset, request, view=None) + +class AllGrantedAssetPagination(GrantedAssetPaginationBase): + def get_count_from_nodes(self, queryset): + assets_amount = sum(UserAssetGrantedTreeNodeRelation.objects.filter( + user=self._user, node_parent_key='' + ).values_list('node_assets_amount', flat=True)) + logger.debug(f'Hit all assets amount {assets_amount} -> {self._request.get_full_path()}') + return assets_amount diff --git a/apps/perms/signals_handler/__init__.py b/apps/perms/signals_handler/__init__.py new file mode 100644 index 000000000..e0b84afea --- /dev/null +++ b/apps/perms/signals_handler/__init__.py @@ -0,0 +1,2 @@ +from . import common +from . import refresh_perms diff --git a/apps/perms/signals_handler.py b/apps/perms/signals_handler/common.py similarity index 71% rename from apps/perms/signals_handler.py rename to apps/perms/signals_handler/common.py index 9e0bfbaeb..c714de834 100644 --- a/apps/perms/signals_handler.py +++ b/apps/perms/signals_handler/common.py @@ -1,31 +1,22 @@ # -*- coding: utf-8 -*- # -from django.db.models.signals import m2m_changed, pre_delete, pre_save +from django.db.models.signals import m2m_changed from django.dispatch import receiver -from perms.tasks import create_rebuild_user_tree_task, \ - create_rebuild_user_tree_task_by_related_nodes_or_assets from users.models import User, UserGroup -from assets.models import Asset, SystemUser +from assets.models import SystemUser from applications.models import Application from common.utils import get_logger from common.exceptions import M2MReverseNotAllowed -from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR -from .models import AssetPermission, ApplicationPermission +from common.const.signals import POST_ADD +from perms.models import AssetPermission, ApplicationPermission logger = get_logger(__file__) -def handle_rebuild_user_tree(instance, action, reverse, pk_set, **kwargs): - if action.startswith('post'): - if reverse: - create_rebuild_user_tree_task(pk_set) - else: - create_rebuild_user_tree_task([instance.id]) - - -def handle_bind_groups_systemuser(instance, action, reverse, pk_set, **kwargs): +@receiver(m2m_changed, sender=User.groups.through) +def on_user_groups_change(sender, instance, action, reverse, pk_set, **kwargs): """ UserGroup 增加 User 时，增加的 User 需要与 UserGroup 关联的动态系统用户相关联 """ @@ -47,53 +38,11 @@ def handle_bind_groups_systemuser(instance, action, reverse, pk_set, **kwargs): system_user.users.add(*users_id) -@receiver(m2m_changed, sender=User.groups.through) -def on_user_groups_change(**kwargs): - handle_rebuild_user_tree(**kwargs) - handle_bind_groups_systemuser(**kwargs) - - -@receiver([pre_save], sender=AssetPermission) -def on_asset_perm_deactive(instance: AssetPermission, **kwargs): - try: - old = AssetPermission.objects.only('is_active').get(id=instance.id) - if instance.is_active != old.is_active: - create_rebuild_user_tree_task_by_asset_perm(instance) - except AssetPermission.DoesNotExist: - pass - - -@receiver([pre_delete], sender=AssetPermission) -def on_asset_permission_delete(instance, **kwargs): - # 授权删除之前，查出所有相关用户 - create_rebuild_user_tree_task_by_asset_perm(instance) - - -def create_rebuild_user_tree_task_by_asset_perm(asset_perm: AssetPermission): - user_ids = set() - user_ids.update( - UserGroup.objects.filter( - assetpermissions=asset_perm, users__id__isnull=False - ).distinct().values_list('users__id', flat=True) - ) - user_ids.update( - User.objects.filter(assetpermissions=asset_perm).distinct().values_list('id', flat=True) - ) - create_rebuild_user_tree_task(user_ids) - - -def need_rebuild_mapping_node(action): - return action in (POST_REMOVE, POST_ADD, POST_CLEAR) - - @receiver(m2m_changed, sender=AssetPermission.nodes.through) def on_permission_nodes_changed(instance, action, reverse, pk_set, model, **kwargs): if reverse: raise M2MReverseNotAllowed - if need_rebuild_mapping_node(action): - create_rebuild_user_tree_task_by_asset_perm(instance) - if action != POST_ADD: return logger.debug("Asset permission nodes change signal received") @@ -110,9 +59,6 @@ def on_permission_assets_changed(instance, action, reverse, pk_set, model, **kwa if reverse: raise M2MReverseNotAllowed - if need_rebuild_mapping_node(action): - create_rebuild_user_tree_task_by_asset_perm(instance) - if action != POST_ADD: return logger.debug("Asset permission assets change signal received") @@ -150,9 +96,6 @@ def on_asset_permission_users_changed(instance, action, reverse, pk_set, model, if reverse: raise M2MReverseNotAllowed - if need_rebuild_mapping_node(action): - create_rebuild_user_tree_task(pk_set) - if action != POST_ADD: return logger.debug("Asset permission users change signal received") @@ -171,10 +114,6 @@ def on_asset_permission_user_groups_changed(instance, action, pk_set, model, if reverse: raise M2MReverseNotAllowed - if need_rebuild_mapping_node(action): - user_ids = User.objects.filter(groups__id__in=pk_set).distinct().values_list('id', flat=True) - create_rebuild_user_tree_task(user_ids) - if action != POST_ADD: return logger.debug("Asset permission user groups change signal received") @@ -187,21 +126,6 @@ def on_asset_permission_user_groups_changed(instance, action, pk_set, model, system_user.groups.add(*tuple(groups)) -@receiver(m2m_changed, sender=Asset.nodes.through) -def on_node_asset_change(action, instance, reverse, pk_set, **kwargs): - if not need_rebuild_mapping_node(action): - return - - if reverse: - asset_pk_set = pk_set - node_pk_set = [instance.id] - else: - asset_pk_set = [instance.id] - node_pk_set = pk_set - - create_rebuild_user_tree_task_by_related_nodes_or_assets.delay(node_pk_set, asset_pk_set) - - @receiver(m2m_changed, sender=ApplicationPermission.system_users.through) def on_application_permission_system_users_changed(sender, instance: ApplicationPermission, action, reverse, pk_set, **kwargs): if not instance.category_remote_app: diff --git a/apps/perms/signals_handler/refresh_perms.py b/apps/perms/signals_handler/refresh_perms.py new file mode 100644 index 000000000..d91594444 --- /dev/null +++ b/apps/perms/signals_handler/refresh_perms.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# +from django.db.models.signals import m2m_changed, pre_delete, pre_save, post_save +from django.dispatch import receiver + +from users.models import User +from assets.models import Asset +from orgs.utils import current_org +from common.utils import get_logger +from common.exceptions import M2MReverseNotAllowed +from common.const.signals import POST_ADD, POST_REMOVE, POST_CLEAR +from perms.models import AssetPermission +from perms.utils.asset.user_permission import UserGrantedTreeRefreshController + + +logger = get_logger(__file__) + + +@receiver(m2m_changed, sender=User.groups.through) +def on_user_groups_change(sender, instance, action, reverse, pk_set, **kwargs): + if action.startswith('post'): + if reverse: + group_ids = [instance.id] + user_ids = pk_set + else: + group_ids = pk_set + user_ids = [instance.id] + + exists = AssetPermission.user_groups.through.objects.filter(usergroup_id__in=group_ids).exists() + if exists: + org_ids = [current_org.id] + UserGrantedTreeRefreshController.add_need_refresh_orgs_for_users(org_ids, user_ids) + + +@receiver([pre_delete], sender=AssetPermission) +def on_asset_perm_pre_delete(sender, instance, **kwargs): + # 授权删除之前，查出所有相关用户 + UserGrantedTreeRefreshController.add_need_refresh_by_asset_perm_ids([instance.id]) + + +@receiver([pre_save], sender=AssetPermission) +def on_asset_perm_pre_save(sender, instance, **kwargs): + try: + old = AssetPermission.objects.get(id=instance.id) + + if old.is_valid != instance.is_valid: + UserGrantedTreeRefreshController.add_need_refresh_by_asset_perm_ids([instance.id]) + except AssetPermission.DoesNotExist: + pass + + +@receiver([post_save], sender=AssetPermission) +def on_asset_perm_post_save(sender, instance, created, **kwargs): + if created: + UserGrantedTreeRefreshController.add_need_refresh_by_asset_perm_ids([instance.id]) + + +def need_rebuild_mapping_node(action): + return action in (POST_REMOVE, POST_ADD, POST_CLEAR) + + +@receiver(m2m_changed, sender=AssetPermission.nodes.through) +def on_permission_nodes_changed(sender, instance, action, reverse, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + UserGrantedTreeRefreshController.add_need_refresh_by_asset_perm_ids([instance.id]) + + +@receiver(m2m_changed, sender=AssetPermission.assets.through) +def on_permission_assets_changed(sender, instance, action, reverse, pk_set, model, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + UserGrantedTreeRefreshController.add_need_refresh_by_asset_perm_ids([instance.id]) + + +@receiver(m2m_changed, sender=AssetPermission.users.through) +def on_asset_permission_users_changed(sender, action, reverse, pk_set, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + UserGrantedTreeRefreshController.add_need_refresh_orgs_for_users( + [current_org.id], pk_set + ) + + +@receiver(m2m_changed, sender=AssetPermission.user_groups.through) +def on_asset_permission_user_groups_changed(sender, action, pk_set, reverse, **kwargs): + if reverse: + raise M2MReverseNotAllowed + + if need_rebuild_mapping_node(action): + user_ids = User.groups.through.objects.filter(usergroup_id__in=pk_set).distinct().values_list('user_id', flat=True) + UserGrantedTreeRefreshController.add_need_refresh_orgs_for_users( + [current_org.id], user_ids + ) + + +@receiver(m2m_changed, sender=Asset.nodes.through) +def on_node_asset_change(action, instance, reverse, pk_set, **kwargs): + if not need_rebuild_mapping_node(action): + return + + if reverse: + asset_pk_set = pk_set + node_pk_set = [instance.id] + else: + asset_pk_set = [instance.id] + node_pk_set = pk_set + + UserGrantedTreeRefreshController.add_need_refresh_on_nodes_assets_relate_change(node_pk_set, asset_pk_set) diff --git a/apps/perms/tasks.py b/apps/perms/tasks.py index fbf2ce8be..8c796ad27 100644 --- a/apps/perms/tasks.py +++ b/apps/perms/tasks.py @@ -2,39 +2,18 @@ from __future__ import absolute_import, unicode_literals from datetime import timedelta -from django.db import transaction -from django.db.models import Q from django.db.transaction import atomic from django.conf import settings from celery import shared_task from common.utils import get_logger from common.utils.timezone import now, dt_formater, dt_parser -from users.models import User from ops.celery.decorator import register_as_period_task -from assets.models import Node -from perms.models import RebuildUserTreeTask, AssetPermission -from perms.utils.asset.user_permission import rebuild_user_mapping_nodes_if_need_with_lock, lock +from perms.models import AssetPermission +from perms.utils.asset.user_permission import UserGrantedTreeRefreshController logger = get_logger(__file__) -@shared_task(queue='node_tree') -def rebuild_user_mapping_nodes_celery_task(user_id): - user = User.objects.get(id=user_id) - try: - rebuild_user_mapping_nodes_if_need_with_lock(user) - except lock.SomeoneIsDoingThis: - pass - - -@shared_task(queue='node_tree') -def dispatch_mapping_node_tasks(): - user_ids = RebuildUserTreeTask.objects.all().values_list('user_id', flat=True).distinct() - logger.info(f'>>> dispatch_mapping_node_tasks for users {list(user_ids)}') - for id in user_ids: - rebuild_user_mapping_nodes_celery_task.delay(id) - - @register_as_period_task(interval=settings.PERM_EXPIRED_CHECK_PERIODIC) @shared_task(queue='celery_check_asset_perm_expired') @atomic() @@ -60,66 +39,9 @@ def check_asset_permission_expired(): setting.value = dt_formater(end) setting.save() - ids = AssetPermission.objects.filter( + asset_perm_ids = AssetPermission.objects.filter( date_expired__gte=start, date_expired__lte=end ).distinct().values_list('id', flat=True) - logger.info(f'>>> checking {start} to {end} have {ids} expired') - dispatch_process_expired_asset_permission.delay(list(ids)) - - -@shared_task(queue='node_tree') -def dispatch_process_expired_asset_permission(asset_perms_id): - user_ids = User.objects.filter( - Q(assetpermissions__id__in=asset_perms_id) | - Q(groups__assetpermissions__id__in=asset_perms_id) - ).distinct().values_list('id', flat=True) - RebuildUserTreeTask.objects.bulk_create( - [RebuildUserTreeTask(user_id=user_id) for user_id in user_ids] - ) - - dispatch_mapping_node_tasks.delay() - - -def create_rebuild_user_tree_task(user_ids): - RebuildUserTreeTask.objects.bulk_create( - [RebuildUserTreeTask(user_id=i) for i in user_ids] - ) - transaction.on_commit(dispatch_mapping_node_tasks.delay) - - -@shared_task(queue='node_tree') -def create_rebuild_user_tree_task_by_related_nodes_or_assets(node_ids, asset_ids): - node_ids = set(node_ids) - node_keys = set() - nodes = Node.objects.filter(id__in=node_ids) - for _node in nodes: - node_keys.update(_node.get_ancestor_keys()) - node_ids.update( - Node.objects.filter(key__in=node_keys).values_list('id', flat=True) - ) - - asset_perms_id = set() - asset_perms_id.update( - AssetPermission.objects.filter( - assets__id__in=asset_ids - ).values_list('id', flat=True).distinct() - ) - asset_perms_id.update( - AssetPermission.objects.filter( - nodes__id__in=node_ids - ).values_list('id', flat=True).distinct() - ) - - user_ids = set() - user_ids.update( - User.objects.filter( - assetpermissions__id__in=asset_perms_id - ).distinct().values_list('id', flat=True) - ) - user_ids.update( - User.objects.filter( - groups__assetpermissions__id__in=asset_perms_id - ).distinct().values_list('id', flat=True) - ) - - create_rebuild_user_tree_task(user_ids) + asset_perm_ids = list(asset_perm_ids) + logger.info(f'>>> checking {start} to {end} have {asset_perm_ids} expired') + UserGrantedTreeRefreshController.add_need_refresh_by_asset_perm_ids_cross_orgs(asset_perm_ids) diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 5b9836400..6319f7f4c 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -1,524 +1,748 @@ -from functools import reduce, wraps -from operator import or_, and_ -from uuid import uuid4 -import threading -import inspect +from collections import defaultdict +from typing import List, Tuple +from django.core.cache import cache from django.conf import settings -from django.db.models import F, Q, Value, BooleanField -from django.utils.translation import ugettext_lazy as _ +from django.db.models import Q, QuerySet -from common.http import is_true +from common.db.models import output_as_string +from common.utils.common import lazyproperty, timeit, Time +from assets.utils import NodeAssetsUtil from common.utils import get_logger -from common.const.distributed_lock_key import UPDATE_MAPPING_NODE_TASK_LOCK_KEY -from orgs.utils import tmp_to_root_org -from common.utils.timezone import dt_formater, now -from assets.models import Node, Asset, FavoriteAsset -from django.db.transaction import atomic -from orgs import lock -from perms.models import UserGrantedMappingNode, RebuildUserTreeTask, AssetPermission +from common.decorator import on_transaction_commit +from orgs.utils import tmp_to_org, current_org, ensure_in_real_or_default_org +from assets.models import ( + Asset, FavoriteAsset, AssetQuerySet, NodeQuerySet +) +from orgs.models import Organization +from perms.models import ( + AssetPermission, PermNode, UserAssetGrantedTreeNodeRelation, +) from users.models import User +from perms.locks import UserGrantedTreeRebuildLock + +NodeFrom = UserAssetGrantedTreeNodeRelation.NodeFrom +NODE_ONLY_FIELDS = ('id', 'key', 'parent_key', 'org_id') logger = get_logger(__name__) -ADD = 'add' -REMOVE = 'remove' -UNGROUPED_NODE_KEY = 'ungrouped' -UNGROUPED_NODE_VALUE = _('Ungrouped') -FAVORITE_NODE_KEY = 'favorite' -FAVORITE_NODE_VALUE = _('Favorite') +def get_user_all_asset_perm_ids(user) -> set: + asset_perm_ids = set() + user_perm_id = AssetPermission.users.through.objects\ + .filter(user_id=user.id) \ + .values_list('assetpermission_id', flat=True) \ + .distinct() + asset_perm_ids.update(user_perm_id) -TMP_GRANTED_FIELD = '_granted' -TMP_ASSET_GRANTED_FIELD = '_asset_granted' -TMP_GRANTED_ASSETS_AMOUNT_FIELD = '_granted_assets_amount' + group_ids = user.groups.through.objects \ + .filter(user_id=user.id) \ + .values_list('usergroup_id', flat=True) \ + .distinct() + group_ids = list(group_ids) + groups_perm_id = AssetPermission.user_groups.through.objects\ + .filter(usergroup_id__in=group_ids)\ + .values_list('assetpermission_id', flat=True) \ + .distinct() + asset_perm_ids.update(groups_perm_id) + + asset_perm_ids = AssetPermission.objects.filter( + id__in=asset_perm_ids).valid().values_list('id', flat=True) + return asset_perm_ids -# 使用场景 -# `Node.objects.annotate(**node_annotate_mapping_node)` -node_annotate_mapping_node = { - TMP_GRANTED_FIELD: F('mapping_nodes__granted'), - TMP_ASSET_GRANTED_FIELD: F('mapping_nodes__asset_granted'), - TMP_GRANTED_ASSETS_AMOUNT_FIELD: F('mapping_nodes__assets_amount') -} +class QuerySetStage: + def __init__(self): + self._prefetch_related = set() + self._only = () + self._filters = [] + self._querysets_and = [] + self._querysets_or = [] + self._order_by = None + self._annotate = [] + self._before_union_merge_funs = set() + self._after_union_merge_funs = set() + + def annotate(self, *args, **kwargs): + self._annotate.append((args, kwargs)) + self._before_union_merge_funs.add(self._merge_annotate) + return self + + def prefetch_related(self, *lookups): + self._prefetch_related.update(lookups) + self._before_union_merge_funs.add(self._merge_prefetch_related) + return self + + def only(self, *fields): + self._only = fields + self._before_union_merge_funs.add(self._merge_only) + return self + + def order_by(self, *field_names): + self._order_by = field_names + self._after_union_merge_funs.add(self._merge_order_by) + return self + + def filter(self, *args, **kwargs): + self._filters.append((args, kwargs)) + self._before_union_merge_funs.add(self._merge_filters) + return self + + def and_with_queryset(self, qs: QuerySet): + assert isinstance(qs, QuerySet), f'Must be `QuerySet`' + self._order_by = qs.query.order_by + self._after_union_merge_funs.add(self._merge_order_by) + self._querysets_and.append(qs.order_by()) + self._before_union_merge_funs.add(self._merge_querysets_and) + return self + + def or_with_queryset(self, qs: QuerySet): + assert isinstance(qs, QuerySet), f'Must be `QuerySet`' + self._order_by = qs.query.order_by + self._after_union_merge_funs.add(self._merge_order_by) + self._querysets_or.append(qs.order_by()) + self._before_union_merge_funs.add(self._merge_querysets_or) + return self + + def merge_multi_before_union(self, *querysets): + ret = [] + for qs in querysets: + qs = self.merge_before_union(qs) + ret.append(qs) + return ret + + def _merge_only(self, qs: QuerySet): + if self._only: + qs = qs.only(*self._only) + return qs + + def _merge_filters(self, qs: QuerySet): + if self._filters: + for args, kwargs in self._filters: + qs = qs.filter(*args, **kwargs) + return qs + + def _merge_querysets_and(self, qs: QuerySet): + if self._querysets_and: + for qs_and in self._querysets_and: + qs &= qs_and + return qs + + def _merge_annotate(self, qs: QuerySet): + if self._annotate: + for args, kwargs in self._annotate: + qs = qs.annotate(*args, **kwargs) + return qs + + def _merge_querysets_or(self, qs: QuerySet): + if self._querysets_or: + for qs_or in self._querysets_or: + qs |= qs_or + return qs + + def _merge_prefetch_related(self, qs: QuerySet): + if self._prefetch_related: + qs = qs.prefetch_related(*self._prefetch_related) + return qs + + def _merge_order_by(self, qs: QuerySet): + if self._order_by is not None: + qs = qs.order_by(*self._order_by) + return qs + + def merge_before_union(self, qs: QuerySet) -> QuerySet: + assert isinstance(qs, QuerySet), f'Must be `QuerySet`' + for fun in self._before_union_merge_funs: + qs = fun(qs) + return qs + + def merge_after_union(self, qs: QuerySet) -> QuerySet: + for fun in self._after_union_merge_funs: + qs = fun(qs) + return qs + + def merge(self, qs: QuerySet) -> QuerySet: + qs = self.merge_before_union(qs) + qs = self.merge_after_union(qs) + return qs -# 使用场景 -# `Node.objects.annotate(**node_annotate_set_granted)` -node_annotate_set_granted = { - TMP_GRANTED_FIELD: Value(True, output_field=BooleanField()), -} +class UserGrantedTreeRefreshController: + key_template = 'perms.user.node_tree.builded_orgs.user_id:{user_id}' + def __init__(self, user): + self.user = user + self.key = self.key_template.format(user_id=user.id) + self.client = self.get_redis_client() -def is_direct_granted_by_annotate(node): - return getattr(node, TMP_GRANTED_FIELD, False) + @classmethod + def get_redis_client(cls): + return cache.client.get_client(write=True) + def get_need_refresh_org_ids(self): + org_ids = self.client.smembers(self.key) + return {org_id.decode() for org_id in org_ids} -def is_asset_granted(node): - return getattr(node, TMP_ASSET_GRANTED_FIELD, False) + def set_all_orgs_as_builed(self): + orgs_id = [str(org_id) for org_id in self.orgs_id] + self.client.sadd(self.key, *orgs_id) + def get_need_refresh_orgs_and_fill_up(self): + orgs_id = set(str(org_id) for org_id in self.orgs_id) -def get_granted_assets_amount(node): - return getattr(node, TMP_GRANTED_ASSETS_AMOUNT_FIELD, 0) + with self.client.pipeline() as p: + p.smembers(self.key) + p.sadd(self.key, *orgs_id) + ret = p.execute() + builded_orgs_id = {org_id.decode() for org_id in ret[0]} + ids = orgs_id - builded_orgs_id + orgs = [] + if Organization.DEFAULT_ID in ids: + ids.remove(Organization.DEFAULT_ID) + orgs.append(Organization.default()) + orgs.extend(Organization.objects.filter(id__in=ids)) + logger.info(f'Need rebuild orgs are {orgs}, builed orgs are {ret[0]}, all orgs are {orgs_id}') + return orgs + @classmethod + @on_transaction_commit + def remove_builed_orgs_from_users(cls, orgs_id, users_id): + client = cls.get_redis_client() + org_ids = [str(org_id) for org_id in orgs_id] -def set_granted(obj): - setattr(obj, TMP_GRANTED_FIELD, True) + with client.pipeline() as p: + for user_id in users_id: + key = cls.key_template.format(user_id=user_id) + p.srem(key, *org_ids) + p.execute() + logger.info(f'Remove orgs from users builded tree, users:{users_id} orgs:{orgs_id}') + @classmethod + def add_need_refresh_orgs_for_users(cls, orgs_id, users_id): + cls.remove_builed_orgs_from_users(orgs_id, users_id) -def set_asset_granted(obj): - setattr(obj, TMP_ASSET_GRANTED_FIELD, True) - - -VALUE_TEMPLATE = '{stage}:{rand_str}:thread:{thread_name}:{thread_id}:{now}' - - -def _generate_value(stage=lock.DOING): - cur_thread = threading.current_thread() - - return VALUE_TEMPLATE.format( - stage=stage, - thread_name=cur_thread.name, - thread_id=cur_thread.ident, - now=dt_formater(now()), - rand_str=uuid4() - ) - - -def build_user_mapping_node_lock(func): - @wraps(func) - def wrapper(*args, **kwargs): - call_args = inspect.getcallargs(func, *args, **kwargs) - user = call_args.get('user') - if user is None or not isinstance(user, User): - raise ValueError('You function must have `user` argument') - - key = UPDATE_MAPPING_NODE_TASK_LOCK_KEY.format(user_id=user.id) - doing_value = _generate_value() - commiting_value = _generate_value(stage=lock.COMMITING) - - try: - locked = lock.acquire(key, doing_value, timeout=600) - if not locked: - logger.error(f'update_mapping_node_task_locked_failed for user: {user.id}') - raise lock.SomeoneIsDoingThis - - with atomic(savepoint=False): - func(*args, **kwargs) - ok = lock.change_lock_state_to_commiting(key, doing_value, commiting_value) - if not ok: - logger.error(f'update_mapping_node_task_timeout for user: {user.id}') - raise lock.Timeout - finally: - lock.release(key, commiting_value, doing_value) - return wrapper - - -@build_user_mapping_node_lock -def rebuild_user_mapping_nodes_if_need_with_lock(user: User): - tasks = RebuildUserTreeTask.objects.filter(user=user) - if tasks: - tasks.delete() - rebuild_user_mapping_nodes(user) - - -@build_user_mapping_node_lock -def rebuild_user_mapping_nodes_with_lock(user: User): - rebuild_user_mapping_nodes(user) - - -def compute_tmp_mapping_node_from_perm(user: User, asset_perms_id=None): - node_only_fields = ('id', 'key', 'parent_key', 'assets_amount') - - if asset_perms_id is None: - asset_perms_id = get_user_all_assetpermissions_id(user) - - # 查询直接授权节点 - nodes = Node.objects.filter( - granted_by_permissions__id__in=asset_perms_id - ).distinct().only(*node_only_fields) - granted_key_set = {_node.key for _node in nodes} - - def _has_ancestor_granted(node): + @classmethod + def add_need_refresh_on_nodes_assets_relate_change(cls, node_ids, asset_ids): """ - 判断一个节点是否有授权过的祖先节点 + 1，计算与这些资产有关的授权 + 2，计算与这些节点以及祖先节点有关的授权 """ - ancestor_keys = set(node.get_ancestor_keys()) - return ancestor_keys & granted_key_set + ensure_in_real_or_default_org() - key2leaf_nodes_mapper = {} + node_ids = set(node_ids) + ancestor_node_keys = set() + asset_perm_ids = set() - # 给授权节点设置 _granted 标识，同时去重 - for _node in nodes: - if _has_ancestor_granted(_node): - continue + nodes = PermNode.objects.filter(id__in=node_ids).only('id', 'key') + for node in nodes: + ancestor_node_keys.update(node.get_ancestor_keys()) - if _node.key not in key2leaf_nodes_mapper: - set_granted(_node) - key2leaf_nodes_mapper[_node.key] = _node + ancestor_id = PermNode.objects.filter(key__in=ancestor_node_keys).values_list('id', flat=True) + node_ids.update(ancestor_id) - # 查询授权资产关联的节点设置 - def process_direct_granted_assets(): - # 查询直接授权资产 - asset_ids = Asset.objects.filter( - granted_by_permissions__id__in=asset_perms_id - ).distinct().values_list('id', flat=True) - # 查询授权资产关联的节点设置 - granted_asset_nodes = Node.objects.filter( - assets__id__in=asset_ids - ).distinct().only(*node_only_fields) + assets_related_perms_id = AssetPermission.nodes.through.objects.filter( + node_id__in=node_ids + ).values_list('assetpermission_id', flat=True) + asset_perm_ids.update(assets_related_perms_id) - # 给资产授权关联的节点设置 _asset_granted 标识，同时去重 - for _node in granted_asset_nodes: - if _has_ancestor_granted(_node): - continue + nodes_related_perms_id = AssetPermission.assets.through.objects.filter( + asset_id__in=asset_ids + ).values_list('assetpermission_id', flat=True) + asset_perm_ids.update(nodes_related_perms_id) - if _node.key not in key2leaf_nodes_mapper: - key2leaf_nodes_mapper[_node.key] = _node - set_asset_granted(key2leaf_nodes_mapper[_node.key]) + cls.add_need_refresh_by_asset_perm_ids(asset_perm_ids) - if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - process_direct_granted_assets() + @classmethod + def add_need_refresh_by_asset_perm_ids_cross_orgs(cls, asset_perm_ids): + org_id_perm_ids_mapper = defaultdict(set) + pairs = AssetPermission.objects.filter(id__in=asset_perm_ids).values_list('org_id', 'id') + for org_id, perm_id in pairs: + org_id_perm_ids_mapper[org_id].add(perm_id) + for org_id, perm_ids in org_id_perm_ids_mapper.items(): + with tmp_to_org(org_id): + cls.add_need_refresh_by_asset_perm_ids(perm_ids) - leaf_nodes = key2leaf_nodes_mapper.values() + @classmethod + def add_need_refresh_by_asset_perm_ids(cls, asset_perm_ids): + ensure_in_real_or_default_org() - # 计算所有祖先节点 - ancestor_keys = set() - for _node in leaf_nodes: - ancestor_keys.update(_node.get_ancestor_keys()) + group_ids = AssetPermission.user_groups.through.objects.filter( + assetpermission_id__in=asset_perm_ids + ).values_list('usergroup_id', flat=True) - # 从祖先节点 key 中去掉同时也是叶子节点的 key - ancestor_keys -= key2leaf_nodes_mapper.keys() - # 查出祖先节点 - ancestors = Node.objects.filter(key__in=ancestor_keys).only(*node_only_fields) - return [*leaf_nodes, *ancestors] + user_ids = set() + direct_user_id = AssetPermission.users.through.objects.filter( + assetpermission_id__in=asset_perm_ids + ).values_list('user_id', flat=True) + user_ids.update(direct_user_id) + group_user_ids = User.groups.through.objects.filter( + usergroup_id__in=group_ids + ).values_list('user_id', flat=True) + user_ids.update(group_user_ids) -def create_mapping_nodes(user, nodes): - to_create = [] - for node in nodes: - _granted = getattr(node, TMP_GRANTED_FIELD, False) - _asset_granted = getattr(node, TMP_ASSET_GRANTED_FIELD, False) - _granted_assets_amount = getattr(node, TMP_GRANTED_ASSETS_AMOUNT_FIELD, 0) - to_create.append(UserGrantedMappingNode( - user=user, - node=node, - key=node.key, - parent_key=node.parent_key, - granted=_granted, - asset_granted=_asset_granted, - assets_amount=_granted_assets_amount, - )) - - UserGrantedMappingNode.objects.bulk_create(to_create) - - -def set_node_granted_assets_amount(user, node, asset_perms_id=None): - """ - 不依赖`UserGrantedMappingNode`直接查询授权计算资产数量 - """ - _granted = getattr(node, TMP_GRANTED_FIELD, False) - if _granted: - assets_amount = node.assets_amount - else: - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - assets_amount = count_direct_granted_node_assets(user, node.key, asset_perms_id) - else: - assets_amount = count_node_all_granted_assets(user, node.key, asset_perms_id) - setattr(node, TMP_GRANTED_ASSETS_AMOUNT_FIELD, assets_amount) - - -@tmp_to_root_org() -def rebuild_user_mapping_nodes(user): - logger.info(f'>>> {dt_formater(now())} start rebuild {user} mapping nodes') - - # 先删除旧的授权树🌲 - UserGrantedMappingNode.objects.filter(user=user).delete() - asset_perms_id = get_user_all_assetpermissions_id(user) - if not asset_perms_id: - # 没有授权直接返回 - return - tmp_nodes = compute_tmp_mapping_node_from_perm(user, asset_perms_id=asset_perms_id) - for _node in tmp_nodes: - set_node_granted_assets_amount(user, _node, asset_perms_id) - create_mapping_nodes(user, tmp_nodes) - logger.info(f'>>> {dt_formater(now())} end rebuild {user} mapping nodes') - - -def rebuild_all_user_mapping_nodes(): - from users.models import User - users = User.objects.all() - for user in users: - rebuild_user_mapping_nodes(user) - - -def get_user_granted_nodes_list_via_mapping_node(user): - """ - 这里的 granted nodes, 是整棵树需要的node，推算出来的也算 - :param user: - :return: - """ - # 获取 `UserGrantedMappingNode` 中对应的 `Node` - nodes = Node.objects.filter( - mapping_nodes__user=user, - ).annotate( - **node_annotate_mapping_node - ).distinct() - - key_to_node_mapper = {} - nodes_descendant_q = Q() - - for node in nodes: - if not is_direct_granted_by_annotate(node): - # 未授权的节点资产数量设置为 `UserGrantedMappingNode` 中的数量 - node.assets_amount = get_granted_assets_amount(node) - else: - # 直接授权的节点 - # 增加查询后代节点的过滤条件 - nodes_descendant_q |= Q(key__startswith=f'{node.key}:') - key_to_node_mapper[node.key] = node - - if nodes_descendant_q: - descendant_nodes = Node.objects.filter( - nodes_descendant_q - ).annotate( - **node_annotate_set_granted + cls.remove_builed_orgs_from_users( + [current_org.id], user_ids ) - for node in descendant_nodes: - key_to_node_mapper[node.key] = node - all_nodes = key_to_node_mapper.values() - return all_nodes + @lazyproperty + def orgs_id(self): + ret = [org.id for org in self.orgs] + return ret + + @lazyproperty + def orgs(self): + orgs = [*self.user.orgs.all(), Organization.default()] + return orgs + + @timeit + def refresh_if_need(self, force=False): + user = self.user + exists = UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exists() + + if force or not exists: + orgs = self.orgs + self.set_all_orgs_as_builed() + else: + orgs = self.get_need_refresh_orgs_and_fill_up() + + for org in orgs: + with tmp_to_org(org): + utils = UserGrantedTreeBuildUtils(user) + utils.rebuild_user_granted_tree() -def get_user_granted_all_assets( - user, via_mapping_node=True, - include_direct_granted_assets=True, asset_perms_id=None): - if asset_perms_id is None: - asset_perms_id = get_user_all_assetpermissions_id(user) +class UserGrantedUtilsBase: + user: User - if via_mapping_node: - granted_node_keys = UserGrantedMappingNode.objects.filter( - user=user, granted=True, - ).values_list('key', flat=True).distinct() - else: - granted_node_keys = Node.objects.filter( - granted_by_permissions__id__in=asset_perms_id - ).distinct().values_list('key', flat=True) - granted_node_keys = Node.clean_children_keys(granted_node_keys) + def __init__(self, user, asset_perm_ids=None): + self.user = user + self._asset_perm_ids = asset_perm_ids - granted_node_q = Q() - for _key in granted_node_keys: - granted_node_q |= Q(nodes__key__startswith=f'{_key}:') - granted_node_q |= Q(nodes__key=_key) + @lazyproperty + def asset_perm_ids(self) -> set: + if self._asset_perm_ids: + return self._asset_perm_ids - if include_direct_granted_assets: - assets__id = get_user_direct_granted_assets(user, asset_perms_id).values_list('id', flat=True) - q = granted_node_q | Q(id__in=list(assets__id)) - else: - q = granted_node_q - - if q: - return Asset.org_objects.filter(q).distinct() - else: - return Asset.org_objects.none() + asset_perm_ids = get_user_all_asset_perm_ids(self.user) + return asset_perm_ids -def get_node_all_granted_assets(user: User, key): - """ - 此算法依据 `UserGrantedMappingNode` 的数据查询 - 1. 查询该节点下的直接授权节点 - 2. 查询该节点下授权资产关联的节点 - """ +class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): - assets = Asset.objects.none() + def get_direct_granted_nodes(self) -> NodeQuerySet: + # 查询直接授权节点 + nodes = PermNode.objects.filter( + granted_by_permissions__id__in=self.asset_perm_ids + ).distinct() + return nodes - # 查询该节点下的授权节点 - granted_mapping_nodes = UserGrantedMappingNode.objects.filter( - user=user, granted=True, - ).filter( - Q(key__startswith=f'{key}:') | Q(key=key) - ) + @lazyproperty + def direct_granted_asset_ids(self) -> list: + # 3.15 + asset_ids = AssetPermission.assets.through.objects.filter( + assetpermission_id__in=self.asset_perm_ids + ).annotate( + asset_id_str=output_as_string('asset_id') + ).values_list( + 'asset_id_str', flat=True + ).distinct() - # 根据授权节点构建资产查询条件 - granted_nodes_qs = [] - for _node in granted_mapping_nodes: - granted_nodes_qs.append(Q(nodes__key__startswith=f'{_node.key}:')) - granted_nodes_qs.append(Q(nodes__key=_node.key)) + asset_ids = list(asset_ids) + return asset_ids - # 查询该节点下的资产授权节点 - only_asset_granted_mapping_nodes = UserGrantedMappingNode.objects.filter( - user=user, - asset_granted=True, - granted=False, - ).filter(Q(key__startswith=f'{key}:') | Q(key=key)) + @timeit + def rebuild_user_granted_tree(self): + ensure_in_real_or_default_org() + logger.info(f'Rebuild user:{self.user} tree in org:{current_org}') - # 根据资产授权节点构建查询 - only_asset_granted_nodes_qs = [] - for _node in only_asset_granted_mapping_nodes: - only_asset_granted_nodes_qs.append(Q(nodes__id=_node.node_id)) + user = self.user + org_id = current_org.id - q = [] - if granted_nodes_qs: - q.append(reduce(or_, granted_nodes_qs)) + with UserGrantedTreeRebuildLock(org_id, user.id): + # 先删除旧的授权树🌲 + UserAssetGrantedTreeNodeRelation.objects.filter(user=user).delete() - if only_asset_granted_nodes_qs: - only_asset_granted_nodes_q = reduce(or_, only_asset_granted_nodes_qs) - asset_perms_id = get_user_all_assetpermissions_id(user) - only_asset_granted_nodes_q &= Q(granted_by_permissions__id__in=list(asset_perms_id)) - q.append(only_asset_granted_nodes_q) + if not self.asset_perm_ids: + # 没有授权直接返回 + return - if q: - assets = Asset.objects.filter(reduce(or_, q)).distinct() - return assets + nodes = self.compute_perm_nodes_tree() + self.compute_node_assets_amount(nodes) + if not nodes: + return + self.create_mapping_nodes(nodes) + + @timeit + def compute_perm_nodes_tree(self, node_only_fields=NODE_ONLY_FIELDS) -> list: + + # 查询直接授权节点 + nodes = self.get_direct_granted_nodes().only(*node_only_fields) + nodes = list(nodes) + + # 授权的节点 key 集合 + granted_key_set = {_node.key for _node in nodes} + + def _has_ancestor_granted(node: PermNode): + """ + 判断一个节点是否有授权过的祖先节点 + """ + ancestor_keys = set(node.get_ancestor_keys()) + return ancestor_keys & granted_key_set + + key2leaf_nodes_mapper = {} + + # 给授权节点设置 granted 标识，同时去重 + for node in nodes: + node: PermNode + if _has_ancestor_granted(node): + continue + node.node_from = NodeFrom.granted + key2leaf_nodes_mapper[node.key] = node + + # 查询授权资产关联的节点设置 + def process_direct_granted_assets(): + # 查询直接授权资产 + nodes_id = {node_id_str for node_id_str, _ in self.direct_granted_asset_id_node_id_str_pairs} + # 查询授权资产关联的节点设置 2.80 + granted_asset_nodes = PermNode.objects.filter( + id__in=nodes_id + ).distinct().only(*node_only_fields) + granted_asset_nodes = list(granted_asset_nodes) + + # 给资产授权关联的节点设置 is_asset_granted 标识，同时去重 + for node in granted_asset_nodes: + if _has_ancestor_granted(node): + continue + if node.key in key2leaf_nodes_mapper: + continue + node.node_from = NodeFrom.asset + key2leaf_nodes_mapper[node.key] = node + + if not settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + process_direct_granted_assets() + + leaf_nodes = key2leaf_nodes_mapper.values() + + # 计算所有祖先节点 + ancestor_keys = set() + for node in leaf_nodes: + ancestor_keys.update(node.get_ancestor_keys()) + + # 从祖先节点 key 中去掉同时也是叶子节点的 key + ancestor_keys -= key2leaf_nodes_mapper.keys() + # 查出祖先节点 + ancestors = PermNode.objects.filter(key__in=ancestor_keys).only(*node_only_fields) + ancestors = list(ancestors) + for node in ancestors: + node.node_from = NodeFrom.child + result = [*leaf_nodes, *ancestors] + return result + + @timeit + def create_mapping_nodes(self, nodes): + user = self.user + to_create = [] + + for node in nodes: + to_create.append(UserAssetGrantedTreeNodeRelation( + user=user, + node=node, + node_key=node.key, + node_parent_key=node.parent_key, + node_from=node.node_from, + node_assets_amount=node.assets_amount, + org_id=node.org_id + )) + + UserAssetGrantedTreeNodeRelation.objects.bulk_create(to_create) + + @timeit + def _fill_direct_granted_node_assets_id_from_mem(self, nodes_key, mapper): + org_id = current_org.id + for key in nodes_key: + assets_id = PermNode.get_all_assets_id_by_node_key(org_id, key) + mapper[key].update(assets_id) + + @lazyproperty + def direct_granted_asset_id_node_id_str_pairs(self): + node_asset_pairs = Asset.nodes.through.objects.filter( + asset_id__in=self.direct_granted_asset_ids + ).annotate( + asset_id_str=output_as_string('asset_id'), + node_id_str=output_as_string('node_id') + ).values_list( + 'node_id_str', 'asset_id_str' + ) + node_asset_pairs = list(node_asset_pairs) + return node_asset_pairs + + @timeit + def compute_node_assets_amount(self, nodes: List[PermNode]): + """ + 这里计算的是一个组织的 + """ + # 直接授权了根节点，直接计算 + if len(nodes) == 1: + node = nodes[0] + if node.node_from == NodeFrom.granted and node.key.isdigit(): + with tmp_to_org(node.org): + node.granted_assets_amount = len(node.get_all_assets_id()) + return + + direct_granted_nodes_key = [] + node_id_key_mapper = {} + for node in nodes: + if node.node_from == NodeFrom.granted: + direct_granted_nodes_key.append(node.key) + node_id_key_mapper[node.id.hex] = node.key + + # 授权的节点和直接资产的映射 + nodekey_assetsid_mapper = defaultdict(set) + # 直接授权的节点，资产从完整树过来 + self._fill_direct_granted_node_assets_id_from_mem( + direct_granted_nodes_key, nodekey_assetsid_mapper + ) + + # 处理直接授权资产 + # 直接授权资产，取节点与资产的关系 + node_asset_pairs = self.direct_granted_asset_id_node_id_str_pairs + node_asset_pairs = list(node_asset_pairs) + + for node_id, asset_id in node_asset_pairs: + nkey = node_id_key_mapper[node_id] + nodekey_assetsid_mapper[nkey].add(asset_id) + + util = NodeAssetsUtil(nodes, nodekey_assetsid_mapper) + util.generate() + + for node in nodes: + assets_amount = util.get_assets_amount(node.key) + node.assets_amount = assets_amount + + def get_whole_tree_nodes(self) -> list: + node_only_fields = NODE_ONLY_FIELDS + ('value', 'full_value') + nodes = self.compute_perm_nodes_tree(node_only_fields=node_only_fields) + self.compute_node_assets_amount(nodes) + + # 查询直接授权节点的子节点 + q = Q() + for node in self.get_direct_granted_nodes().only('key'): + q |= Q(key__startswith=f'{node.key}:') + + if q: + descendant_nodes = PermNode.objects.filter(q).distinct() + else: + descendant_nodes = PermNode.objects.none() + + nodes.extend(descendant_nodes) + return nodes -def get_direct_granted_node_ids(user: User, key, asset_perms_id=None): - if asset_perms_id is None: - asset_perms_id = get_user_all_assetpermissions_id(user) +class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): - # 先查出该节点下的直接授权节点 - granted_nodes = Node.objects.filter( - Q(key__startswith=f'{key}:') | Q(key=key) - ).filter( - granted_by_permissions__id__in=asset_perms_id - ).distinct().only('id', 'key') + def get_favorite_assets(self, qs_stage: QuerySetStage = None, only=('id', )) -> AssetQuerySet: + favorite_asset_ids = FavoriteAsset.objects.filter( + user=self.user + ).values_list('asset_id', flat=True) + favorite_asset_ids = list(favorite_asset_ids) + qs_stage = qs_stage or QuerySetStage() + qs_stage.filter(id__in=favorite_asset_ids).only(*only) + assets = self.get_all_granted_assets(qs_stage) + return assets - node_ids = set() - # 根据直接授权节点查询他们的子节点 - q = Q() - for _node in granted_nodes: - q |= Q(key__startswith=f'{_node.key}:') - node_ids.add(_node.id) + def get_ungroup_assets(self) -> AssetQuerySet: + return self.get_direct_granted_assets() - if q: - descendant_ids = Node.objects.filter(q).values_list('id', flat=True).distinct() - node_ids.update(descendant_ids) - return node_ids + def get_direct_granted_assets(self) -> AssetQuerySet: + queryset = Asset.objects.order_by().filter( + granted_by_permissions__id__in=self.asset_perm_ids + ).distinct() + return queryset + + def get_direct_granted_nodes_assets(self, qs_stage: QuerySetStage = None) -> AssetQuerySet: + granted_node_ids = AssetPermission.nodes.through.objects.filter( + assetpermission_id__in=self.asset_perm_ids + ).values_list('node_id', flat=True).distinct() + granted_node_ids = list(granted_node_ids) + granted_nodes = PermNode.objects.filter(id__in=granted_node_ids).only('id', 'key') + queryset = PermNode.get_nodes_all_assets(*granted_nodes) + if qs_stage: + queryset = qs_stage.merge(queryset) + return queryset + + def get_all_granted_assets(self, qs_stage: QuerySetStage = None) -> AssetQuerySet: + nodes_assets = self.get_direct_granted_nodes_assets() + assets = self.get_direct_granted_assets() + + if qs_stage: + nodes_assets, assets = qs_stage.merge_multi_before_union(nodes_assets, assets) + queryset = nodes_assets.union(assets) + if qs_stage: + queryset = qs_stage.merge_after_union(queryset) + return queryset + + def get_node_all_assets(self, id, qs_stage: QuerySetStage = None) -> Tuple[PermNode, QuerySet]: + node = PermNode.objects.get(id=id) + granted_status = node.get_granted_status(self.user) + if granted_status == NodeFrom.granted: + assets = PermNode.get_nodes_all_assets(node) + if qs_stage: + assets = qs_stage.merge(assets) + return node, assets + elif granted_status in (NodeFrom.asset, NodeFrom.child): + node.use_granted_assets_amount() + assets = self._get_indirect_granted_node_all_assets(node, qs_stage=qs_stage) + return node, assets + else: + node.assets_amount = 0 + return node, Asset.objects.none() + + def get_node_assets(self, key) -> AssetQuerySet: + node = PermNode.objects.get(key=key) + granted_status = node.get_granted_status(self.user) + + if granted_status == NodeFrom.granted: + assets = Asset.objects.order_by().filter(nodes_id=node.id) + return assets + elif granted_status == NodeFrom.asset: + return self._get_indirect_granted_node_assets(node.id) + else: + return Asset.objects.none() + + def _get_indirect_granted_node_assets(self, id) -> AssetQuerySet: + assets = Asset.objects.order_by().filter(nodes_id=id) & self.get_direct_granted_assets() + return assets + + def _get_indirect_granted_node_all_assets(self, node, qs_stage: QuerySetStage = None) -> QuerySet: + """ + 此算法依据 `UserAssetGrantedTreeNodeRelation` 的数据查询 + 1. 查询该节点下的直接授权节点 + 2. 查询该节点下授权资产关联的节点 + """ + user = self.user + + # 查询该节点下的授权节点 + granted_nodes = UserAssetGrantedTreeNodeRelation.objects.filter( + user=user, node_from=NodeFrom.granted + ).filter( + Q(node_key__startswith=f'{node.key}:') + ).only('node_id', 'node_key') + node_assets = PermNode.get_nodes_all_assets(*granted_nodes) + + # 查询该节点下的资产授权节点 + only_asset_granted_node_ids = UserAssetGrantedTreeNodeRelation.objects.filter( + user=user, node_from=NodeFrom.asset + ).filter( + Q(node_key__startswith=f'{node.key}:') + ).values_list('node_id', flat=True) + + only_asset_granted_node_ids = list(only_asset_granted_node_ids) + if node.node_from == NodeFrom.asset: + only_asset_granted_node_ids.append(node.id) + + assets = Asset.objects.filter( + nodes__id__in=only_asset_granted_node_ids, + granted_by_permissions__id__in=self.asset_perm_ids + ).distinct().order_by() + if qs_stage: + node_assets, assets = qs_stage.merge_multi_before_union(node_assets, assets) + granted_assets = node_assets.union(assets) + granted_assets = qs_stage.merge_after_union(granted_assets) + return granted_assets -def get_node_all_granted_assets_from_perm(user: User, key, asset_perms_id=None): - """ - 此算法依据 `AssetPermission` 的数据查询 - 1. 查询该节点下的直接授权节点 - 2. 查询该节点下授权资产关联的节点 - """ - if asset_perms_id is None: - asset_perms_id = get_user_all_assetpermissions_id(user) +class UserGrantedNodesQueryUtils(UserGrantedUtilsBase): + def get_node_children(self, key): + if not key: + return self.get_top_level_nodes() - # 直接授权资产查询条件 - q = ( - Q(nodes__key__startswith=f'{key}:') | Q(nodes__key=key) - ) & Q(granted_by_permissions__id__in=asset_perms_id) + node = PermNode.objects.get(key=key) + granted_status = node.get_granted_status(self.user) + if granted_status == NodeFrom.granted: + return PermNode.objects.filter(parent_key=key) + elif granted_status in (NodeFrom.asset, NodeFrom.child): + return self.get_indirect_granted_node_children(key) + else: + return PermNode.objects.none() - node_ids = get_direct_granted_node_ids(user, key, asset_perms_id) - q |= Q(nodes__id__in=node_ids) - asset_qs = Asset.objects.filter(q).distinct() - return asset_qs + def get_indirect_granted_node_children(self, key): + """ + 获取用户授权树中未授权节点的子节点 + 只匹配在 `UserAssetGrantedTreeNodeRelation` 中存在的节点 + """ + user = self.user + nodes = PermNode.objects.filter( + granted_node_rels__user=user, + parent_key=key + ).annotate( + **PermNode.annotate_granted_node_rel_fields + ).distinct() + # 设置节点授权资产数量 + for node in nodes: + node.use_granted_assets_amount() + return nodes -def get_direct_granted_node_assets_from_perm(user: User, key, asset_perms_id=None): - node_ids = get_direct_granted_node_ids(user, key, asset_perms_id) - asset_qs = Asset.objects.filter(nodes__id__in=node_ids).distinct() - return asset_qs + def get_top_level_nodes(self): + nodes = self.get_special_nodes() + nodes.extend(self.get_indirect_granted_node_children('')) + return nodes + def get_ungrouped_node(self): + assets_util = UserGrantedAssetsQueryUtils(self.user, self.asset_perm_ids) + assets_amount = assets_util.get_direct_granted_assets().count() + return PermNode.get_ungrouped_node(assets_amount) -def count_node_all_granted_assets(user: User, key, asset_perms_id=None): - return get_node_all_granted_assets_from_perm(user, key, asset_perms_id).count() + def get_favorite_node(self): + assets_query_utils = UserGrantedAssetsQueryUtils(self.user, self.asset_perm_ids) + assets_amount = assets_query_utils.get_favorite_assets().values_list('id').count() + return PermNode.get_favorite_node(assets_amount) + def get_special_nodes(self): + nodes = [] + if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + ungrouped_node = self.get_ungrouped_node() + nodes.append(ungrouped_node) + favorite_node = self.get_favorite_node() + nodes.append(favorite_node) + return nodes -def count_direct_granted_node_assets(user: User, key, asset_perms_id=None): - return get_direct_granted_node_assets_from_perm(user, key, asset_perms_id).count() + @timeit + def get_whole_tree_nodes(self, with_special=True): + """ + 这里的 granted nodes, 是整棵树需要的node，推算出来的也算 + :param user: + :return: + """ + nodes = PermNode.objects.filter( + granted_node_rels__user=self.user + ).annotate( + **PermNode.annotate_granted_node_rel_fields + ).distinct() + key_to_node_mapper = {} + nodes_descendant_q = Q() -def get_indirect_granted_node_children(user, key=''): - """ - 获取用户授权树中未授权节点的子节点 - 只匹配在 `UserGrantedMappingNode` 中存在的节点 - """ - nodes = Node.objects.filter( - mapping_nodes__user=user, - parent_key=key - ).annotate( - _granted_assets_amount=F('mapping_nodes__assets_amount'), - _granted=F('mapping_nodes__granted') - ).distinct() + for node in nodes: + node.use_granted_assets_amount() - # 设置节点授权资产数量 - for _node in nodes: - if not is_direct_granted_by_annotate(_node): - _node.assets_amount = get_granted_assets_amount(_node) - return nodes + if node.node_from == NodeFrom.granted: + # 直接授权的节点 + # 增加查询后代节点的过滤条件 + nodes_descendant_q |= Q(key__startswith=f'{node.key}:') + key_to_node_mapper[node.key] = node - -def get_top_level_granted_nodes(user): - nodes = list(get_indirect_granted_node_children(user, key='')) - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - ungrouped_node = get_ungrouped_node(user) - nodes.insert(0, ungrouped_node) - favorite_node = get_favorite_node(user) - nodes.insert(0, favorite_node) - return nodes - - -def get_user_all_assetpermissions_id(user: User): - asset_perms_id = AssetPermission.objects.valid().filter( - Q(users=user) | Q(user_groups__users=user) - ).distinct().values_list('id', flat=True) - - # !!! 这个很重要，必须转换成 list，避免 Django 生成嵌套子查询 - asset_perms_id = list(asset_perms_id) - return asset_perms_id - - -def get_user_direct_granted_assets(user, asset_perms_id=None): - if asset_perms_id is None: - asset_perms_id = get_user_all_assetpermissions_id(user) - assets = Asset.org_objects.filter(granted_by_permissions__id__in=asset_perms_id).distinct() - return assets - - -def count_user_direct_granted_assets(user, asset_perms_id=None): - count = get_user_direct_granted_assets( - user, asset_perms_id=asset_perms_id - ).values_list('id').count() - return count - - -def get_ungrouped_node(user, asset_perms_id=None): - assets_amount = count_user_direct_granted_assets(user, asset_perms_id) - return Node( - id=UNGROUPED_NODE_KEY, - key=UNGROUPED_NODE_KEY, - value=UNGROUPED_NODE_VALUE, - assets_amount=assets_amount - ) - - -def get_favorite_node(user, asset_perms_id=None): - assets_amount = FavoriteAsset.get_user_favorite_assets( - user, asset_perms_id=asset_perms_id - ).values_list('id').count() - return Node( - id=FAVORITE_NODE_KEY, - key=FAVORITE_NODE_KEY, - value=FAVORITE_NODE_VALUE, - assets_amount=assets_amount - ) - - -def rebuild_user_tree_if_need(request, user): - """ - 升级授权树策略后，用户的数据可能还未初始化，为防止用户显示没有数据 - 先检查 MappingNode 如果没有数据，同步创建用户授权树 - """ - if is_true(request.query_params.get('rebuild_tree')) or \ - not UserGrantedMappingNode.objects.filter(user=user).exists(): - try: - rebuild_user_mapping_nodes_with_lock(user) - except lock.SomeoneIsDoingThis: - # 您的数据正在初始化，请稍等 - raise lock.SomeoneIsDoingThis( - detail=_('Please wait while your data is being initialized'), - code='rebuild_tree_conflict' + if nodes_descendant_q: + descendant_nodes = PermNode.objects.filter( + nodes_descendant_q ) + for node in descendant_nodes: + key_to_node_mapper[node.key] = node + + all_nodes = [] + if with_special: + special_nodes = self.get_special_nodes() + all_nodes.extend(special_nodes) + all_nodes.extend(key_to_node_mapper.values()) + return all_nodes diff --git a/utils/generate_fake_data/resources/assets.py b/utils/generate_fake_data/resources/assets.py index a0edc9f08..4c5cbba93 100644 --- a/utils/generate_fake_data/resources/assets.py +++ b/utils/generate_fake_data/resources/assets.py @@ -5,7 +5,6 @@ import forgery_py from .base import FakeDataGenerator from assets.models import * -from assets.utils import check_node_assets_amount class AdminUsersGenerator(FakeDataGenerator): @@ -93,4 +92,4 @@ class AssetsGenerator(FakeDataGenerator): self.set_assets_nodes(creates) def after_generate(self): - check_node_assets_amount() + pass From 50e6c963588275a8676126c777c4e63e0aa3a437 Mon Sep 17 00:00:00 2001 From: Bai Date: Fri, 5 Feb 2021 17:05:13 +0800 Subject: [PATCH 14/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20component=20s?= =?UTF-8?q?tatus=20=E8=8E=B7=E5=8F=96key=20error=20=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/models/terminal.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/terminal/models/terminal.py b/apps/terminal/models/terminal.py index b8b44dfbc..48e225cfd 100644 --- a/apps/terminal/models/terminal.py +++ b/apps/terminal/models/terminal.py @@ -6,10 +6,14 @@ from django.utils.translation import ugettext_lazy as _ from django.conf import settings from django.core.cache import cache +from common.utils import get_logger from users.models import User from .. import const +logger = get_logger(__file__) + + class ComputeStatusMixin: # system status @@ -40,7 +44,11 @@ class ComputeStatusMixin: ] system_status = [] for system_status_key in system_status_keys: - state_value = state[system_status_key] + state_value = state.get(system_status_key) + if state_value is None: + msg = 'state: {}, state_key: {}, state_value: {}' + logger.debug(msg.format(state, system_status_key, state_value)) + state_value = 0 status = getattr(self, f'_compute_{system_status_key}_status')(state_value) system_status.append(status) return system_status From 501ad698b7f5de210b3d32c1aaaa31656531f8c5 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Sun, 7 Feb 2021 10:15:39 +0800 Subject: [PATCH 15/71] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20UnionQuertSet=20(#55?= =?UTF-8?q?78)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 添加 UnionQuertSet * 跑通了 * 改变了 count 这类方法的代理模式 * 使用了老广的 Co-authored-by: xinwen --- apps/common/utils/common.py | 4 + .../user_permission_assets/mixin.py | 49 +++----- apps/perms/utils/asset/user_permission.py | 110 ++++++++++++++---- 3 files changed, 104 insertions(+), 59 deletions(-) diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index f6808b0ac..9984c8614 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -273,3 +273,7 @@ class Time: for timestamp, msg in zip(timestamps, self._msgs): logger.debug(f'TIME_IT: {msg} {timestamp-last}') last = timestamp + + +def isinstance_method(attr): + return isinstance(attr, type(Time().time)) diff --git a/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py b/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py index 0b92da278..3a1d49016 100644 --- a/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py +++ b/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py @@ -34,12 +34,12 @@ class UserAllGrantedAssetsQuerysetMixin: pagination_class = AllGrantedAssetPagination user: User - def get_union_queryset(self, qs_stage: QuerySetStage): + def get_queryset(self): if getattr(self, 'swagger_fake_view', False): return Asset.objects.none() - qs_stage.prefetch_related('platform').only(*self.only_fields) queryset = UserGrantedAssetsQueryUtils(self.user) \ - .get_all_granted_assets(qs_stage) + .get_all_granted_assets() + queryset = queryset.prefetch_related('platform').only(*self.only_fields) return queryset @@ -47,13 +47,13 @@ class UserFavoriteGrantedAssetsMixin: only_fields = serializers.AssetGrantedSerializer.Meta.only_fields user: User - def get_union_queryset(self, qs_stage: QuerySetStage): + def get_queryset(self): if getattr(self, 'swagger_fake_view', False): return Asset.objects.none() user = self.user - qs_stage.prefetch_related('platform').only(*self.only_fields) utils = UserGrantedAssetsQueryUtils(user) - assets = utils.get_favorite_assets(qs_stage=qs_stage) + assets = utils.get_favorite_assets() + assets = assets.prefetch_related('platform').only(*self.only_fields) return assets @@ -63,58 +63,35 @@ class UserGrantedNodeAssetsMixin: pagination_node: Node user: User - def get_union_queryset(self, qs_stage: QuerySetStage): + def get_queryset(self): if getattr(self, 'swagger_fake_view', False): return Asset.objects.none() node_id = self.kwargs.get("node_id") - qs_stage.prefetch_related('platform').only(*self.only_fields) + node, assets = UserGrantedAssetsQueryUtils(self.user).get_node_all_assets( - node_id, qs_stage=qs_stage + node_id ) + assets = assets.prefetch_related('platform').only(*self.only_fields) self.pagination_node = node return assets # 控制格式的 ---------------------------------------------------- -class AssetsUnionQuerysetMixin: - def get_queryset_union_prefer(self): - if hasattr(self, 'get_union_queryset'): - # 为了支持 union 查询 - queryset = Asset.objects.all().distinct() - queryset = self.filter_queryset(queryset) - qs_stage = QuerySetStage() - qs_stage.and_with_queryset(queryset) - queryset = self.get_union_queryset(qs_stage) - else: - queryset = self.filter_queryset(self.get_queryset()) - return queryset - -class AssetsSerializerFormatMixin(AssetsUnionQuerysetMixin): +class AssetsSerializerFormatMixin: serializer_class = serializers.AssetGrantedSerializer filterset_fields = ['hostname', 'ip', 'id', 'comment'] search_fields = ['hostname', 'ip', 'comment'] - def list(self, request, *args, **kwargs): - queryset = self.get_queryset_union_prefer() - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) - - -class AssetsTreeFormatMixin(AssetsUnionQuerysetMixin, SerializeToTreeNodeMixin): +class AssetsTreeFormatMixin(SerializeToTreeNodeMixin): """ 将 资产 序列化成树的结构返回 """ def list(self, request: Request, *args, **kwargs): - queryset = self.get_queryset_union_prefer() + queryset = self.filter_queryset(self.get_queryset()) if request.query_params.get('search'): # 如果用户搜索的条件不精准，会导致返回大量的无意义数据。 diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 6319f7f4c..6bd2e8b2c 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -1,5 +1,7 @@ from collections import defaultdict from typing import List, Tuple +from functools import reduce, partial +from common.utils import isinstance_method from django.core.cache import cache from django.conf import settings @@ -51,6 +53,81 @@ def get_user_all_asset_perm_ids(user) -> set: return asset_perm_ids +class UnionQuerySet(QuerySet): + after_union = ['order_by'] + not_return_qs = [ + 'query', 'get', 'create', 'get_or_create', + 'update_or_create', 'bulk_create', 'count', + 'latest', 'earliest', 'first', 'last', 'aggregate', + 'exists', 'update', 'delete', 'as_manager', 'explain', + ] + + def __init__(self, *queryset_list): + self.queryset_list = queryset_list + self.after_union_items = [] + self.before_union_items = [] + + def __execute(self): + queryset_list = [] + for qs in self.queryset_list: + for attr, args, kwargs in self.before_union_items: + qs = getattr(qs, attr)(*args, **kwargs) + queryset_list.append(qs) + union_qs = reduce(lambda x, y: x.union(y), queryset_list) + for attr, args, kwargs in self.after_union_items: + union_qs = getattr(union_qs, attr)(*args, **kwargs) + return union_qs + + def __before_union_perform(self, item, *args, **kwargs): + self.before_union_items.append((item, args, kwargs)) + return self.__clone(*self.queryset_list) + + def __after_union_perform(self, item, *args, **kwargs): + self.after_union_items.append((item, args, kwargs)) + return self.__clone(*self.queryset_list) + + def __clone(self, *queryset_list): + uqs = UnionQuerySet(*queryset_list) + uqs.after_union_items = self.after_union_items + uqs.before_union_items = self.before_union_items + return uqs + + def __getattribute__(self, item): + if item.startswith('__') or item in UnionQuerySet.__dict__ or item in [ + 'queryset_list', 'after_union_items', 'before_union_items' + ]: + return object.__getattribute__(self, item) + + if item in UnionQuerySet.not_return_qs: + return getattr(self.__execute(), item) + + origin_item = object.__getattribute__(self, 'queryset_list')[0] + origin_attr = getattr(origin_item, item, None) + if not isinstance_method(origin_attr): + return getattr(self.__execute(), item) + + if item in UnionQuerySet.after_union: + attr = partial(self.__after_union_perform, item) + else: + attr = partial(self.__before_union_perform, item) + return attr + + def __getitem__(self, item): + return self.__execute()[item] + + def __next__(self): + return next(self.__execute()) + + @classmethod + def test_it(cls): + from assets.models import Asset + assets1 = Asset.objects.filter(hostname__startswith='a') + assets2 = Asset.objects.filter(hostname__startswith='b') + + qs = cls(assets1, assets2) + return qs + + class QuerySetStage: def __init__(self): self._prefetch_related = set() @@ -541,14 +618,13 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): - def get_favorite_assets(self, qs_stage: QuerySetStage = None, only=('id', )) -> AssetQuerySet: + def get_favorite_assets(self, only=('id', )) -> QuerySet: favorite_asset_ids = FavoriteAsset.objects.filter( user=self.user ).values_list('asset_id', flat=True) favorite_asset_ids = list(favorite_asset_ids) - qs_stage = qs_stage or QuerySetStage() - qs_stage.filter(id__in=favorite_asset_ids).only(*only) - assets = self.get_all_granted_assets(qs_stage) + assets = self.get_all_granted_assets() + assets = assets.filter(id__in=favorite_asset_ids).only(*only) return assets def get_ungroup_assets(self) -> AssetQuerySet: @@ -560,39 +636,30 @@ class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): ).distinct() return queryset - def get_direct_granted_nodes_assets(self, qs_stage: QuerySetStage = None) -> AssetQuerySet: + def get_direct_granted_nodes_assets(self) -> AssetQuerySet: granted_node_ids = AssetPermission.nodes.through.objects.filter( assetpermission_id__in=self.asset_perm_ids ).values_list('node_id', flat=True).distinct() granted_node_ids = list(granted_node_ids) granted_nodes = PermNode.objects.filter(id__in=granted_node_ids).only('id', 'key') queryset = PermNode.get_nodes_all_assets(*granted_nodes) - if qs_stage: - queryset = qs_stage.merge(queryset) return queryset - def get_all_granted_assets(self, qs_stage: QuerySetStage = None) -> AssetQuerySet: + def get_all_granted_assets(self) -> QuerySet: nodes_assets = self.get_direct_granted_nodes_assets() assets = self.get_direct_granted_assets() - - if qs_stage: - nodes_assets, assets = qs_stage.merge_multi_before_union(nodes_assets, assets) - queryset = nodes_assets.union(assets) - if qs_stage: - queryset = qs_stage.merge_after_union(queryset) + queryset = UnionQuerySet(nodes_assets, assets) return queryset - def get_node_all_assets(self, id, qs_stage: QuerySetStage = None) -> Tuple[PermNode, QuerySet]: + def get_node_all_assets(self, id) -> Tuple[PermNode, QuerySet]: node = PermNode.objects.get(id=id) granted_status = node.get_granted_status(self.user) if granted_status == NodeFrom.granted: assets = PermNode.get_nodes_all_assets(node) - if qs_stage: - assets = qs_stage.merge(assets) return node, assets elif granted_status in (NodeFrom.asset, NodeFrom.child): node.use_granted_assets_amount() - assets = self._get_indirect_granted_node_all_assets(node, qs_stage=qs_stage) + assets = self._get_indirect_granted_node_all_assets(node) return node, assets else: node.assets_amount = 0 @@ -614,7 +681,7 @@ class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): assets = Asset.objects.order_by().filter(nodes_id=id) & self.get_direct_granted_assets() return assets - def _get_indirect_granted_node_all_assets(self, node, qs_stage: QuerySetStage = None) -> QuerySet: + def _get_indirect_granted_node_all_assets(self, node) -> QuerySet: """ 此算法依据 `UserAssetGrantedTreeNodeRelation` 的数据查询 1. 查询该节点下的直接授权节点 @@ -645,10 +712,7 @@ class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): nodes__id__in=only_asset_granted_node_ids, granted_by_permissions__id__in=self.asset_perm_ids ).distinct().order_by() - if qs_stage: - node_assets, assets = qs_stage.merge_multi_before_union(node_assets, assets) - granted_assets = node_assets.union(assets) - granted_assets = qs_stage.merge_after_union(granted_assets) + granted_assets = UnionQuerySet(node_assets, assets) return granted_assets From e599bca95175e099fbfe9c4e270d1c0c84b4045c Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 18 Feb 2021 15:07:49 +0800 Subject: [PATCH 16/71] =?UTF-8?q?fix:=20=E5=91=BD=E4=BB=A4=E5=AD=98?= =?UTF-8?q?=E5=82=A8=20es=20=E7=B1=BB=E5=9E=8B=E4=B8=BB=E6=9C=BA=E5=B8=A6?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=90=8D=E5=AF=86=E7=A0=81=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/serializers/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/terminal/serializers/storage.py b/apps/terminal/serializers/storage.py index 24d619540..7cc0628ba 100644 --- a/apps/terminal/serializers/storage.py +++ b/apps/terminal/serializers/storage.py @@ -157,7 +157,7 @@ def command_storage_es_host_format_validator(host): raise serializers.ValidationError(default_error_msg) if ':' not in h.netloc: raise serializers.ValidationError(default_error_msg) - _host, _port = h.netloc.split(':') + _host, _port = h.netloc.rsplit(':', maxsplit=1) if not _host: error_msg = _('Host invalid') raise serializers.ValidationError(error_msg) From 9be3cbb936e530ccb5220d5edd3e6d61ca3959d9 Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 8 Feb 2021 14:59:20 +0800 Subject: [PATCH 17/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E8=AF=A6=E6=83=85=E9=A1=B5=E6=8E=88=E6=9D=83=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E5=8A=A0=E8=BD=BD=E9=80=9F=E5=BA=A6&=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=8F=AF=E9=87=8D=E5=85=A5=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/asset.py | 2 - apps/assets/api/mixin.py | 4 +- apps/assets/api/node.py | 8 +- apps/assets/locks.py | 5 +- ...s_amount.py => 0066_auto_20210208_1802.py} | 8 +- apps/assets/models/asset.py | 2 +- apps/assets/models/node.py | 6 +- apps/assets/pagination.py | 48 ++++-- apps/assets/signals_handler/__init__.py | 3 +- .../signals_handler/node_assets_amount.py | 159 ++++++++++++++++++ ...n_nodes_tree.py => node_assets_mapping.py} | 0 apps/assets/tasks/nodes_amount.py | 33 ++++ apps/assets/utils.py | 37 +++- apps/common/utils/lock.py | 141 +++++++++++++--- apps/orgs/utils.py | 10 +- apps/perms/api/asset/asset_permission.py | 6 - .../user_permission_nodes_with_assets.py | 15 +- apps/perms/api/base.py | 2 +- ...204_1749.py => 0018_auto_20210208_1515.py} | 4 +- apps/perms/pagination.py | 36 +--- apps/perms/serializers/asset/permission.py | 11 +- apps/perms/utils/asset/user_permission.py | 18 +- 22 files changed, 434 insertions(+), 124 deletions(-) rename apps/assets/migrations/{0066_remove_node_assets_amount.py => 0066_auto_20210208_1802.py} (50%) create mode 100644 apps/assets/signals_handler/node_assets_amount.py rename apps/assets/signals_handler/{maintain_nodes_tree.py => node_assets_mapping.py} (100%) rename apps/perms/migrations/{0018_auto_20210204_1749.py => 0018_auto_20210208_1515.py} (96%) diff --git a/apps/assets/api/asset.py b/apps/assets/api/asset.py index 9d8a6bf89..2176f97aa 100644 --- a/apps/assets/api/asset.py +++ b/apps/assets/api/asset.py @@ -4,9 +4,7 @@ from assets.api import FilterAssetByNodeMixin from rest_framework.viewsets import ModelViewSet from rest_framework.generics import RetrieveAPIView from django.shortcuts import get_object_or_404 -from django.utils.decorators import method_decorator -from assets.locks import NodeTreeUpdateLock from common.utils import get_logger, get_object_or_none from common.permissions import IsOrgAdmin, IsOrgAdminOrAppUser, IsSuperUser from orgs.mixins.api import OrgBulkModelViewSet diff --git a/apps/assets/api/mixin.py b/apps/assets/api/mixin.py index 763f025f4..f7738f3f1 100644 --- a/apps/assets/api/mixin.py +++ b/apps/assets/api/mixin.py @@ -2,7 +2,7 @@ from typing import List from common.utils.common import timeit from assets.models import Node, Asset -from assets.pagination import AssetLimitOffsetPagination +from assets.pagination import NodeAssetTreePagination from common.utils import lazyproperty from assets.utils import get_node, is_query_node_all_assets @@ -81,7 +81,7 @@ class SerializeToTreeNodeMixin: class FilterAssetByNodeMixin: - pagination_class = AssetLimitOffsetPagination + pagination_class = NodeAssetTreePagination @lazyproperty def is_query_node_all_assets(self): diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index c793ad384..4ffdbfe1a 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -8,7 +8,6 @@ from rest_framework.response import Response from rest_framework.decorators import action from django.utils.translation import ugettext_lazy as _ from django.shortcuts import get_object_or_404, Http404 -from django.utils.decorators import method_decorator from django.db.models.signals import m2m_changed from common.const.http import POST @@ -25,10 +24,10 @@ from ..models import Node from ..tasks import ( update_node_assets_hardware_info_manual, test_node_assets_connectivity_manual, + check_node_assets_amount_task ) from .. import serializers from .mixin import SerializeToTreeNodeMixin -from assets.locks import NodeTreeUpdateLock logger = get_logger(__file__) @@ -54,6 +53,11 @@ class NodeViewSet(OrgModelViewSet): serializer.validated_data["key"] = child_key serializer.save() + @action(methods=[POST], detail=False, url_path='check_assets_amount_task') + def check_assets_amount_task(self, request): + task = check_node_assets_amount_task.delay(current_org.id) + return Response(data={'task': task.id}) + def perform_update(self, serializer): node = self.get_object() if node.is_org_root() and node.value != serializer.validated_data['value']: diff --git a/apps/assets/locks.py b/apps/assets/locks.py index b80db3ff8..bdab57080 100644 --- a/apps/assets/locks.py +++ b/apps/assets/locks.py @@ -15,7 +15,6 @@ class NodeTreeUpdateLock(DistributedLock): ) return name - def __init__(self, blocking=True): + def __init__(self): name = self.get_name() - super().__init__(name=name, blocking=blocking, - release_lock_on_transaction_commit=True) + super().__init__(name=name, release_on_transaction_commit=True, reentrant=True) diff --git a/apps/assets/migrations/0066_remove_node_assets_amount.py b/apps/assets/migrations/0066_auto_20210208_1802.py similarity index 50% rename from apps/assets/migrations/0066_remove_node_assets_amount.py rename to apps/assets/migrations/0066_auto_20210208_1802.py index 5d7044179..ffe7d8fb5 100644 --- a/apps/assets/migrations/0066_remove_node_assets_amount.py +++ b/apps/assets/migrations/0066_auto_20210208_1802.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1 on 2021-02-04 09:49 +# Generated by Django 3.1 on 2021-02-08 10:02 from django.db import migrations @@ -10,8 +10,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RemoveField( - model_name='node', - name='assets_amount', + migrations.AlterModelOptions( + name='asset', + options={'ordering': ['hostname'], 'verbose_name': 'Asset'}, ), ] diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 5d133f40d..a21778a42 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -353,4 +353,4 @@ class Asset(ProtocolsMixin, NodesRelationMixin, OrgModelMixin): class Meta: unique_together = [('org_id', 'hostname')] verbose_name = _("Asset") - ordering = ["hostname", "ip"] + ordering = ["hostname", ] diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index e5b53eb45..85d857a7c 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -425,11 +425,6 @@ class NodeAssetsMixin(NodeAllAssetsMappingMixin): node_ids.update(_ids) return Asset.objects.order_by().filter(nodes__id__in=node_ids).distinct() - @property - def assets_amount(self): - assets_id = self.get_all_assets_id() - return len(assets_id) - def get_all_assets_id(self): assets_id = self.get_all_assets_id_by_node_key(org_id=self.org_id, node_key=self.key) return set(assets_id) @@ -550,6 +545,7 @@ class Node(OrgModelMixin, SomeNodesMixin, FamilyMixin, NodeAssetsMixin): date_create = models.DateTimeField(auto_now_add=True) parent_key = models.CharField(max_length=64, verbose_name=_("Parent key"), db_index=True, default='') + assets_amount = models.IntegerField(default=0) objects = OrgManager.from_queryset(NodeQuerySet)() is_node = True diff --git a/apps/assets/pagination.py b/apps/assets/pagination.py index 4fd866e3d..7a55c1306 100644 --- a/apps/assets/pagination.py +++ b/apps/assets/pagination.py @@ -1,39 +1,51 @@ from rest_framework.pagination import LimitOffsetPagination from rest_framework.request import Request +from common.utils import get_logger from assets.models import Node +logger = get_logger(__name__) + + +class AssetPaginationBase(LimitOffsetPagination): + + def init_attrs(self, queryset, request: Request, view=None): + self._request = request + self._view = view + self._user = request.user + + def paginate_queryset(self, queryset, request: Request, view=None): + self.init_attrs(queryset, request, view) + return super().paginate_queryset(queryset, request, view=None) -class AssetLimitOffsetPagination(LimitOffsetPagination): - """ - 需要与 `assets.api.mixin.FilterAssetByNodeMixin` 配合使用 - """ def get_count(self, queryset): - """ - 1. 如果查询节点下的所有资产，那 count 使用 Node.assets_amount - 2. 如果有其他过滤条件使用 super - 3. 如果只查询该节点下的资产使用 super - """ exclude_query_params = { self.limit_query_param, self.offset_query_param, - 'node', 'all', 'show_current_asset', - 'node_id', 'display', 'draw', 'fields_size', + 'key', 'all', 'show_current_asset', + 'cache_policy', 'display', 'draw', + 'order', 'node', 'node_id', 'fields_size', } - for k, v in self._request.query_params.items(): if k not in exclude_query_params and v is not None: + logger.warn(f'Not hit node.assets_amount because find a unknow query_param `{k}` -> {self._request.get_full_path()}') return super().get_count(queryset) + node_assets_count = self.get_count_from_nodes(queryset) + if node_assets_count is None: + return super().get_count(queryset) + return node_assets_count + def get_count_from_nodes(self, queryset): + raise NotImplementedError + + +class NodeAssetTreePagination(AssetPaginationBase): + def get_count_from_nodes(self, queryset): is_query_all = self._view.is_query_node_all_assets if is_query_all: node = self._view.node if not node: node = Node.org_root() + logger.debug(f'Hit node.assets_amount[{node.assets_amount}] -> {self._request.get_full_path()}') return node.assets_amount - return super().get_count(queryset) - - def paginate_queryset(self, queryset, request: Request, view=None): - self._request = request - self._view = view - return super().paginate_queryset(queryset, request, view=None) + return None diff --git a/apps/assets/signals_handler/__init__.py b/apps/assets/signals_handler/__init__.py index 0c3980565..c8f332f26 100644 --- a/apps/assets/signals_handler/__init__.py +++ b/apps/assets/signals_handler/__init__.py @@ -1,2 +1,3 @@ from .common import * -from .maintain_nodes_tree import * +from .node_assets_amount import * +from .node_assets_mapping import * diff --git a/apps/assets/signals_handler/node_assets_amount.py b/apps/assets/signals_handler/node_assets_amount.py new file mode 100644 index 000000000..4501d0226 --- /dev/null +++ b/apps/assets/signals_handler/node_assets_amount.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# +from operator import add, sub +from django.db.models import Q, F +from django.dispatch import receiver +from django.db.models.signals import ( + m2m_changed +) + +from orgs.utils import ensure_in_real_or_default_org +from common.const.signals import PRE_ADD, POST_REMOVE, PRE_CLEAR +from common.utils import get_logger +from assets.models import Asset, Node, compute_parent_key +from assets.locks import NodeTreeUpdateLock + + +logger = get_logger(__file__) + + +@receiver(m2m_changed, sender=Asset.nodes.through) +def on_node_asset_change(sender, action, instance, reverse, pk_set, **kwargs): + # 不允许 `pre_clear` ，因为该信号没有 `pk_set` + # [官网](https://docs.djangoproject.com/en/3.1/ref/signals/#m2m-changed) + refused = (PRE_CLEAR,) + if action in refused: + raise ValueError + + mapper = { + PRE_ADD: add, + POST_REMOVE: sub + } + if action not in mapper: + return + + operator = mapper[action] + + if reverse: + node: Node = instance + asset_pk_set = set(pk_set) + NodeAssetsAmountUtils.update_node_assets_amount(node, asset_pk_set, operator) + else: + asset_pk = instance.id + # 与资产直接关联的节点 + node_keys = set(Node.objects.filter(id__in=pk_set).values_list('key', flat=True)) + NodeAssetsAmountUtils.update_nodes_asset_amount(node_keys, asset_pk, operator) + + +class NodeAssetsAmountUtils: + + @classmethod + def _remove_ancestor_keys(cls, ancestor_key, tree_set): + # 这里判断 `ancestor_key` 不能是空，防止数据错误导致的死循环 + # 判断是否在集合里，来区分是否已被处理过 + while ancestor_key and ancestor_key in tree_set: + tree_set.remove(ancestor_key) + ancestor_key = compute_parent_key(ancestor_key) + + @classmethod + def _is_asset_exists_in_node(cls, asset_pk, node_key): + exists = Asset.objects.filter( + Q(nodes__key__istartswith=f'{node_key}:') | Q(nodes__key=node_key) + ).filter(id=asset_pk).exists() + return exists + + @classmethod + @ensure_in_real_or_default_org + @NodeTreeUpdateLock() + def update_nodes_asset_amount(cls, node_keys, asset_pk, operator): + """ + 一个资产与多个节点关系变化时，更新计数 + + :param node_keys: 节点 id 的集合 + :param asset_pk: 资产 id + :param operator: 操作 + """ + + # 所有相关节点的祖先节点，组成一棵局部树 + ancestor_keys = set() + for key in node_keys: + ancestor_keys.update(Node.get_node_ancestor_keys(key)) + + # 相关节点可能是其他相关节点的祖先节点，如果是从相关节点里干掉 + node_keys -= ancestor_keys + + to_update_keys = [] + for key in node_keys: + # 遍历相关节点，处理它及其祖先节点 + # 查询该节点是否包含待处理资产 + exists = cls._is_asset_exists_in_node(asset_pk, key) + parent_key = compute_parent_key(key) + + if exists: + # 如果资产在该节点，那么他及其祖先节点都不用处理 + cls._remove_ancestor_keys(parent_key, ancestor_keys) + continue + else: + # 不存在，要更新本节点 + to_update_keys.append(key) + # 这里判断 `parent_key` 不能是空，防止数据错误导致的死循环 + # 判断是否在集合里，来区分是否已被处理过 + while parent_key and parent_key in ancestor_keys: + exists = cls._is_asset_exists_in_node(asset_pk, parent_key) + if exists: + cls._remove_ancestor_keys(parent_key, ancestor_keys) + break + else: + to_update_keys.append(parent_key) + ancestor_keys.remove(parent_key) + parent_key = compute_parent_key(parent_key) + + Node.objects.filter(key__in=to_update_keys).update( + assets_amount=operator(F('assets_amount'), 1) + ) + + @classmethod + @ensure_in_real_or_default_org + @NodeTreeUpdateLock() + def update_node_assets_amount(cls, node: Node, asset_pk_set: set, operator=add): + """ + 一个节点与多个资产关系变化时，更新计数 + + :param node: 节点实例 + :param asset_pk_set: 资产的`id`集合, 内部不会修改该值 + :param operator: 操作 + * -> Node + # -> Asset + + * [3] + / \ + * * [2] + / \ + * * [1] + / / \ + * [a] # # [b] + + """ + # 获取节点[1]祖先节点的 `key` 含自己，也就是[1, 2, 3]节点的`key` + ancestor_keys = node.get_ancestor_keys(with_self=True) + ancestors = Node.objects.filter(key__in=ancestor_keys).order_by('-key') + to_update = [] + for ancestor in ancestors: + # 迭代祖先节点的`key`，顺序是 [1] -> [2] -> [3] + # 查询该节点及其后代节点是否包含要操作的资产，将包含的从要操作的 + # 资产集合中去掉，他们是重复节点，无论增加或删除都不会影响节点的资产数量 + + asset_pk_set -= set(Asset.objects.filter( + id__in=asset_pk_set + ).filter( + Q(nodes__key__istartswith=f'{ancestor.key}:') | + Q(nodes__key=ancestor.key) + ).distinct().values_list('id', flat=True)) + if not asset_pk_set: + # 要操作的资产集合为空，说明都是重复资产，不用改变节点资产数量 + # 而且既然它包含了，它的祖先节点肯定也包含了，所以祖先节点都不用 + # 处理了 + break + ancestor.assets_amount = operator(F('assets_amount'), len(asset_pk_set)) + to_update.append(ancestor) + Node.objects.bulk_update(to_update, fields=('assets_amount', 'parent_key')) diff --git a/apps/assets/signals_handler/maintain_nodes_tree.py b/apps/assets/signals_handler/node_assets_mapping.py similarity index 100% rename from apps/assets/signals_handler/maintain_nodes_tree.py rename to apps/assets/signals_handler/node_assets_mapping.py diff --git a/apps/assets/tasks/nodes_amount.py b/apps/assets/tasks/nodes_amount.py index e69de29bb..a7cb46a45 100644 --- a/apps/assets/tasks/nodes_amount.py +++ b/apps/assets/tasks/nodes_amount.py @@ -0,0 +1,33 @@ +from celery import shared_task +from django.utils.translation import gettext_lazy as _ + +from orgs.models import Organization +from orgs.utils import tmp_to_org +from ops.celery.decorator import register_as_period_task +from assets.utils import check_node_assets_amount + +from common.utils.lock import AcquireFailed +from common.utils import get_logger + +logger = get_logger(__file__) + + +@shared_task +def check_node_assets_amount_task(orgid=None): + if orgid is None: + orgs = [*Organization.objects.all(), Organization.default()] + else: + orgs = [Organization.get_instance(orgid)] + + for org in orgs: + try: + with tmp_to_org(org): + check_node_assets_amount() + except AcquireFailed: + logger.error(_('The task of self-checking is already running and cannot be started repeatedly')) + + +@register_as_period_task(crontab='0 2 * * *') +@shared_task +def check_node_assets_amount_period_task(): + check_node_assets_amount_task() diff --git a/apps/assets/utils.py b/apps/assets/utils.py index 343fa704b..c9857f802 100644 --- a/apps/assets/utils.py +++ b/apps/assets/utils.py @@ -5,12 +5,45 @@ from common.utils import get_logger, dict_get_any, is_uuid, get_object_or_none, from common.http import is_true from common.struct import Stack from common.db.models import output_as_string +from orgs.utils import ensure_in_real_or_default_org, current_org -from .models import Node +from .locks import NodeTreeUpdateLock +from .models import Node, Asset logger = get_logger(__file__) +@NodeTreeUpdateLock() +@ensure_in_real_or_default_org +def check_node_assets_amount(): + logger.info(f'Check node assets amount {current_org}') + nodes = list(Node.objects.all().only('id', 'key', 'assets_amount')) + nodeid_assetid_pairs = list(Asset.nodes.through.objects.all().values_list('node_id', 'asset_id')) + + nodekey_assetids_mapper = defaultdict(set) + nodeid_nodekey_mapper = {} + for node in nodes: + nodeid_nodekey_mapper[node.id] = node.key + + for nodeid, assetid in nodeid_assetid_pairs: + if nodeid not in nodeid_nodekey_mapper: + continue + nodekey = nodeid_nodekey_mapper[nodeid] + nodekey_assetids_mapper[nodekey].add(assetid) + + util = NodeAssetsUtil(nodes, nodekey_assetids_mapper) + util.generate() + + to_updates = [] + for node in nodes: + assets_amount = util.get_assets_amount(node.key) + if node.assets_amount != assets_amount: + logger.error(f'Node[{node.key}] assets amount error {node.assets_amount} != {assets_amount}') + node.assets_amount = assets_amount + to_updates.append(node) + Node.objects.bulk_update(to_updates, fields=('assets_amount',)) + + def is_query_node_all_assets(request): request = request query_all_arg = request.query_params.get('all', 'true') @@ -104,5 +137,3 @@ class NodeAssetsUtil: util = cls(nodes, mapping) util.generate() return util - - diff --git a/apps/common/utils/lock.py b/apps/common/utils/lock.py index d7d7acbed..1a016d3d4 100644 --- a/apps/common/utils/lock.py +++ b/apps/common/utils/lock.py @@ -8,6 +8,7 @@ from django.db import transaction from common.utils import get_logger from common.utils.inspect import copy_function_args from apps.jumpserver.const import CONFIG +from common.local import thread_local logger = get_logger(__file__) @@ -16,24 +17,28 @@ class AcquireFailed(RuntimeError): pass +class LockHasTimeOut(RuntimeError): + pass + + class DistributedLock(RedisLock): - def __init__(self, name, blocking=True, expire=None, release_lock_on_transaction_commit=False, - release_raise_exc=False, auto_renewal_seconds=60*2): + def __init__(self, name, *, expire=None, release_on_transaction_commit=False, + reentrant=False, release_raise_exc=False, auto_renewal_seconds=60): """ 使用 redis 构造的分布式锁 :param name: 锁的名字，要全局唯一 - :param blocking: - 该参数只在锁作为装饰器或者 `with` 时有效。 :param expire: 锁的过期时间 - :param release_lock_on_transaction_commit: + :param release_on_transaction_commit: 是否在当前事务结束后再释放锁 :param release_raise_exc: 释放锁时，如果没有持有锁是否抛异常或静默 :param auto_renewal_seconds: 当持有一个无限期锁的时候，刷新锁的时间，具体参考 `redis_lock.Lock#auto_renewal` + :param reentrant: + 是否可重入 """ self.kwargs_copy = copy_function_args(self.__init__, locals()) redis = Redis(host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, password=CONFIG.REDIS_PASSWORD) @@ -45,28 +50,20 @@ class DistributedLock(RedisLock): auto_renewal = False super().__init__(redis_client=redis, name=name, expire=expire, auto_renewal=auto_renewal) - self._blocking = blocking - self._release_lock_on_transaction_commit = release_lock_on_transaction_commit + self._release_on_transaction_commit = release_on_transaction_commit self._release_raise_exc = release_raise_exc + self._reentrant = reentrant + self._acquired_reentrant_lock = False + self._thread_id = threading.current_thread().ident def __enter__(self): - thread_id = threading.current_thread().ident - logger.debug(f'Attempt to acquire global lock: thread {thread_id} lock {self._name}') - acquired = self.acquire(blocking=self._blocking) - if self._blocking and not acquired: - logger.debug(f'Not acquired lock, but blocking=True, thread {thread_id} lock {self._name}') - raise EnvironmentError("Lock wasn't acquired, but blocking=True") + acquired = self.acquire(blocking=True) if not acquired: - logger.debug(f'Not acquired the lock, thread {thread_id} lock {self._name}') raise AcquireFailed - logger.debug(f'Acquire lock success, thread {thread_id} lock {self._name}') return self def __exit__(self, exc_type=None, exc_value=None, traceback=None): - if self._release_lock_on_transaction_commit: - transaction.on_commit(self.release) - else: - self.release() + self.release() def __call__(self, func): @wraps(func) @@ -82,9 +79,105 @@ class DistributedLock(RedisLock): return True return False - def release(self): + def locked_by_current_thread(self): + if self.locked(): + owner_id = self.get_owner_id() + local_owner_id = getattr(thread_local, self.name, None) + + if local_owner_id and owner_id == local_owner_id: + return True + return False + + def acquire(self, blocking=True, timeout=None): + if self._reentrant: + if self.locked_by_current_thread(): + self._acquired_reentrant_lock = True + logger.debug( + f'I[{self.id}] reentry lock[{self.name}] in thread[{self._thread_id}].') + return True + + logger.debug(f'I[{self.id}] attempt acquire reentrant-lock[{self.name}].') + acquired = super().acquire(blocking=blocking, timeout=timeout) + if acquired: + logger.debug(f'I[{self.id}] acquired reentrant-lock[{self.name}] now.') + setattr(thread_local, self.name, self.id) + else: + logger.debug(f'I[{self.id}] acquired reentrant-lock[{self.name}] failed.') + return acquired + else: + logger.debug(f'I[{self.id}] attempt acquire lock[{self.name}].') + acquired = super().acquire(blocking=blocking, timeout=timeout) + logger.debug(f'I[{self.id}] acquired lock[{self.name}] {acquired}.') + return acquired + + @property + def name(self): + return self._name + + def _raise_exc_with_log(self, msg, *, exc_cls=NotAcquired): + e = exc_cls(msg) + logger.error(msg) + self._raise_exc(e) + + def _raise_exc(self, e): + if self._release_raise_exc: + raise e + + def _release_on_reentrant_locked_by_brother(self): + if self._acquired_reentrant_lock: + self._acquired_reentrant_lock = False + logger.debug(f'I[{self.id}] released reentrant-lock[{self.name}] owner[{self.get_owner_id()}] in thread[{self._thread_id}]') + return + else: + self._raise_exc_with_log(f'Reentrant-lock[{self.name}] is not acquired by me[{self.id}].') + + def _release_on_reentrant_locked_by_me(self): + logger.debug(f'I[{self.id}] release reentrant-lock[{self.name}] in thread[{self._thread_id}]') + + id = getattr(thread_local, self.name, None) + if id != self.id: + raise PermissionError(f'Reentrant-lock[{self.name}] is not locked by me[{self.id}], owner[{id}]') try: - super().release() - except AcquireFailed as e: - if self._release_raise_exc: - raise e + # 这里要保证先删除 thread_local 的标记， + delattr(thread_local, self.name) + except AttributeError: + pass + finally: + try: + # 这里处理的是边界情况， + # 判断锁是我的 -> 锁超时 -> 释放锁报错 + # 此时的报错应该被静默 + self._release_redis_lock() + except NotAcquired: + pass + + def _release_redis_lock(self): + # 最底层 api + super().release() + + def _release(self): + try: + self._release_redis_lock() + except NotAcquired as e: + logger.error(f'I[{self.id}] release lock[{self.name}] failed {e}') + self._raise_exc(e) + + def release(self): + _release = self._release + + # 处理可重入锁 + if self._reentrant: + if self.locked_by_current_thread(): + if self.locked_by_me(): + _release = self._release_on_reentrant_locked_by_me + else: + _release = self._release_on_reentrant_locked_by_brother + else: + self._raise_exc_with_log(f'Reentrant-lock[{self.name}] is not acquired in current-thread[{self._thread_id}]') + + # 处理是否在事务提交时才释放锁 + if self._release_on_transaction_commit: + logger.debug(f'I[{self.id}] release lock[{self.name}] on transaction commit ...') + transaction.on_commit(_release) + else: + _release() diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index d01ae3f77..eef12b1c6 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -186,6 +186,10 @@ def org_aware_func(org_arg_name): current_org = LocalProxy(get_current_org) -def ensure_in_real_or_default_org(): - if not current_org or current_org.is_root(): - raise ValueError('You must in a real or default org!') +def ensure_in_real_or_default_org(func): + @wraps(func) + def wrapper(*args, **kwargs): + if not current_org or current_org.is_root(): + raise ValueError('You must in a real or default org!') + return func(*args, **kwargs) + return wrapper diff --git a/apps/perms/api/asset/asset_permission.py b/apps/perms/api/asset/asset_permission.py index 5062f2099..e38a59ba6 100644 --- a/apps/perms/api/asset/asset_permission.py +++ b/apps/perms/api/asset/asset_permission.py @@ -26,12 +26,6 @@ class AssetPermissionViewSet(BasePermissionViewSet): 'node_id', 'node', 'asset_id', 'hostname', 'ip' ] - def get_queryset(self): - queryset = super().get_queryset().prefetch_related( - "nodes", "assets", "users", "user_groups", "system_users" - ) - return queryset - def filter_node(self, queryset): node_id = self.request.query_params.get('node_id') node_name = self.request.query_params.get('node') diff --git a/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py b/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py index 253a925ca..63619e9c1 100644 --- a/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py +++ b/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py @@ -14,7 +14,6 @@ from .mixin import RoleUserMixin, RoleAdminMixin from perms.utils.asset.user_permission import ( UserGrantedTreeBuildUtils, get_user_all_asset_perm_ids, UserGrantedNodesQueryUtils, UserGrantedAssetsQueryUtils, - QuerySetStage, ) from perms.models import AssetPermission, PermNode from assets.models import Asset @@ -44,10 +43,10 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): def add_favorite_resource(self, data: list, nodes_query_utils, assets_query_utils): favorite_node = nodes_query_utils.get_favorite_node() - qs_state = QuerySetStage().annotate( + favorite_assets = assets_query_utils.get_favorite_assets() + favorite_assets = favorite_assets.annotate( parent_key=Value(favorite_node.key, output_field=CharField()) ).prefetch_related('platform') - favorite_assets = assets_query_utils.get_favorite_assets(qs_stage=qs_state, only=()) data.extend(self.serialize_nodes([favorite_node], with_asset_amount=True)) data.extend(self.serialize_assets(favorite_assets)) @@ -59,13 +58,11 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): data.extend(self.serialize_nodes(nodes, with_asset_amount=True)) def add_assets(self, data: list, assets_query_utils: UserGrantedAssetsQueryUtils): - qs_stage = QuerySetStage().annotate(parent_key=F('nodes__key')).prefetch_related('platform') - if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: - all_assets = assets_query_utils.get_direct_granted_nodes_assets(qs_stage=qs_stage) + all_assets = assets_query_utils.get_direct_granted_nodes_assets() else: - all_assets = assets_query_utils.get_all_granted_assets(qs_stage=qs_stage) - + all_assets = assets_query_utils.get_all_granted_assets() + all_assets = all_assets.annotate(parent_key=F('nodes__key')).prefetch_related('platform') data.extend(self.serialize_assets(all_assets)) @tmp_to_root_org() @@ -144,8 +141,6 @@ class GrantedNodeChildrenWithAssetsAsTreeApiMixin(SerializeToTreeNodeMixin, assets = assets_query_utils.get_node_assets(key) assets = assets.prefetch_related('platform') - user = self.user - tree_nodes = self.serialize_nodes(nodes, with_asset_amount=True) tree_assets = self.serialize_assets(assets, key) return Response(data=[*tree_nodes, *tree_assets]) diff --git a/apps/perms/api/base.py b/apps/perms/api/base.py index 3d9c8672d..8c0028baf 100644 --- a/apps/perms/api/base.py +++ b/apps/perms/api/base.py @@ -45,7 +45,7 @@ class BasePermissionViewSet(OrgBulkModelViewSet): if not self.is_query_all(): queryset = queryset.filter(users=user) return queryset - groups = user.groups.all() + groups = list(user.groups.all().values_list('id', flat=True)) queryset = queryset.filter( Q(users=user) | Q(user_groups__in=groups) ).distinct() diff --git a/apps/perms/migrations/0018_auto_20210204_1749.py b/apps/perms/migrations/0018_auto_20210208_1515.py similarity index 96% rename from apps/perms/migrations/0018_auto_20210204_1749.py rename to apps/perms/migrations/0018_auto_20210208_1515.py index 00d567c2f..b5271f36f 100644 --- a/apps/perms/migrations/0018_auto_20210204_1749.py +++ b/apps/perms/migrations/0018_auto_20210208_1515.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1 on 2021-02-04 09:49 +# Generated by Django 3.1 on 2021-02-08 07:15 import assets.models.node from django.conf import settings @@ -9,8 +9,8 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('assets', '0066_remove_node_assets_amount'), migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('assets', '0065_auto_20210121_1549'), ('perms', '0017_auto_20210104_0435'), ] diff --git a/apps/perms/pagination.py b/apps/perms/pagination.py index fc5e43de7..c740830c2 100644 --- a/apps/perms/pagination.py +++ b/apps/perms/pagination.py @@ -1,37 +1,17 @@ -from rest_framework.pagination import LimitOffsetPagination +from django.conf import settings from rest_framework.request import Request -from django.db.models import Sum +from assets.pagination import AssetPaginationBase from perms.models import UserAssetGrantedTreeNodeRelation from common.utils import get_logger logger = get_logger(__name__) -class GrantedAssetPaginationBase(LimitOffsetPagination): - - def paginate_queryset(self, queryset, request: Request, view=None): - self._request = request - self._view = view - self._user = request.user - return super().paginate_queryset(queryset, request, view=None) - - def get_count(self, queryset): - exclude_query_params = { - self.limit_query_param, - self.offset_query_param, - 'key', 'all', 'show_current_asset', - 'cache_policy', 'display', 'draw', - 'order', - } - for k, v in self._request.query_params.items(): - if k not in exclude_query_params and v is not None: - logger.warn(f'Not hit node.assets_amount because find a unknow query_param `{k}` -> {self._request.get_full_path()}') - return super().get_count(queryset) - return self.get_count_from_nodes(queryset) - - def get_count_from_nodes(self, queryset): - raise NotImplementedError +class GrantedAssetPaginationBase(AssetPaginationBase): + def init_attrs(self, queryset, request: Request, view=None): + super().init_attrs(queryset, request, view) + self._user = view.user class NodeGrantedAssetPagination(GrantedAssetPaginationBase): @@ -42,11 +22,13 @@ class NodeGrantedAssetPagination(GrantedAssetPaginationBase): return node.assets_amount else: logger.warn(f'Not hit node.assets_amount[{node}] because {self._view} not has `pagination_node` -> {self._request.get_full_path()}') - return super().get_count(queryset) + return None class AllGrantedAssetPagination(GrantedAssetPaginationBase): def get_count_from_nodes(self, queryset): + if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: + return None assets_amount = sum(UserAssetGrantedTreeNodeRelation.objects.filter( user=self._user, node_parent_key='' ).values_list('node_assets_amount', flat=True)) diff --git a/apps/perms/serializers/asset/permission.py b/apps/perms/serializers/asset/permission.py index 475b83ee1..2bc723706 100644 --- a/apps/perms/serializers/asset/permission.py +++ b/apps/perms/serializers/asset/permission.py @@ -3,9 +3,12 @@ from rest_framework import serializers from django.utils.translation import ugettext_lazy as _ +from django.db.models import Prefetch from orgs.mixins.serializers import BulkOrgResourceModelSerializer from perms.models import AssetPermission, Action +from assets.models import Asset, Node, SystemUser +from users.models import User, UserGroup __all__ = [ 'AssetPermissionSerializer', @@ -68,5 +71,11 @@ class AssetPermissionSerializer(BulkOrgResourceModelSerializer): @classmethod def setup_eager_loading(cls, queryset): """ Perform necessary eager loading of data. """ - queryset = queryset.prefetch_related('users', 'user_groups', 'assets', 'nodes', 'system_users') + queryset = queryset.prefetch_related( + Prefetch('system_users', queryset=SystemUser.objects.only('id')), + Prefetch('user_groups', queryset=UserGroup.objects.only('id')), + Prefetch('users', queryset=User.objects.only('id')), + Prefetch('assets', queryset=Asset.objects.only('id')), + Prefetch('nodes', queryset=Node.objects.only('id')) + ) return queryset diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 6bd2e8b2c..7f8d53c7e 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -115,8 +115,8 @@ class UnionQuerySet(QuerySet): def __getitem__(self, item): return self.__execute()[item] - def __next__(self): - return next(self.__execute()) + def __iter__(self): + return iter(self.__execute()) @classmethod def test_it(cls): @@ -299,12 +299,12 @@ class UserGrantedTreeRefreshController: cls.remove_builed_orgs_from_users(orgs_id, users_id) @classmethod + @ensure_in_real_or_default_org def add_need_refresh_on_nodes_assets_relate_change(cls, node_ids, asset_ids): """ 1，计算与这些资产有关的授权 2，计算与这些节点以及祖先节点有关的授权 """ - ensure_in_real_or_default_org() node_ids = set(node_ids) ancestor_node_keys = set() @@ -340,8 +340,8 @@ class UserGrantedTreeRefreshController: cls.add_need_refresh_by_asset_perm_ids(perm_ids) @classmethod + @ensure_in_real_or_default_org def add_need_refresh_by_asset_perm_ids(cls, asset_perm_ids): - ensure_in_real_or_default_org() group_ids = AssetPermission.user_groups.through.objects.filter( assetpermission_id__in=asset_perm_ids @@ -429,8 +429,8 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): return asset_ids @timeit + @ensure_in_real_or_default_org def rebuild_user_granted_tree(self): - ensure_in_real_or_default_org() logger.info(f'Rebuild user:{self.user} tree in org:{current_org}') user = self.user @@ -618,13 +618,13 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): - def get_favorite_assets(self, only=('id', )) -> QuerySet: + def get_favorite_assets(self) -> QuerySet: favorite_asset_ids = FavoriteAsset.objects.filter( user=self.user ).values_list('asset_id', flat=True) favorite_asset_ids = list(favorite_asset_ids) assets = self.get_all_granted_assets() - assets = assets.filter(id__in=favorite_asset_ids).only(*only) + assets = assets.filter(id__in=favorite_asset_ids) return assets def get_ungroup_assets(self) -> AssetQuerySet: @@ -670,7 +670,7 @@ class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): granted_status = node.get_granted_status(self.user) if granted_status == NodeFrom.granted: - assets = Asset.objects.order_by().filter(nodes_id=node.id) + assets = Asset.objects.order_by().filter(nodes__id=node.id) return assets elif granted_status == NodeFrom.asset: return self._get_indirect_granted_node_assets(node.id) @@ -678,7 +678,7 @@ class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): return Asset.objects.none() def _get_indirect_granted_node_assets(self, id) -> AssetQuerySet: - assets = Asset.objects.order_by().filter(nodes_id=id) & self.get_direct_granted_assets() + assets = Asset.objects.order_by().filter(nodes__id=id).distinct() & self.get_direct_granted_assets() return assets def _get_indirect_granted_node_all_assets(self, node) -> QuerySet: From bb9790a50f9549383baeafd1891dad993a506e8a Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 23 Feb 2021 14:37:42 +0800 Subject: [PATCH 18/71] =?UTF-8?q?feat:=20=E4=B8=BArdp=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E4=B8=80=E4=B8=AAapi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/applications/models/application.py | 33 ++++ apps/applications/serializers/remote_app.py | 30 +--- apps/assets/serializers/domain.py | 2 - apps/authentication/api/connection_token.py | 168 +++++++++++++++--- apps/authentication/serializers.py | 74 +++++++- apps/authentication/urls/api_urls.py | 3 +- apps/locale/zh/LC_MESSAGES/django.po | 6 +- .../api/application/user_permission/common.py | 4 +- apps/perms/signals_handler/common.py | 24 ++- apps/perms/utils/application/permission.py | 14 +- apps/perms/utils/asset/permission.py | 20 ++- apps/users/urls/api_urls.py | 3 +- 12 files changed, 290 insertions(+), 91 deletions(-) diff --git a/apps/applications/models/application.py b/apps/applications/models/application.py index 12bc4ff0a..f7b541580 100644 --- a/apps/applications/models/application.py +++ b/apps/applications/models/application.py @@ -3,6 +3,7 @@ from django.utils.translation import ugettext_lazy as _ from orgs.mixins.models import OrgModelMixin from common.mixins import CommonModelMixin +from assets.models import Asset from .. import const @@ -35,3 +36,35 @@ class Application(CommonModelMixin, OrgModelMixin): @property def category_remote_app(self): return self.category == const.ApplicationCategoryChoices.remote_app.value + + def get_rdp_remote_app_setting(self): + from applications.serializers.attrs import get_serializer_class_by_application_type + if not self.category_remote_app: + raise ValueError(f"Not a remote app application: {self.name}") + serializer_class = get_serializer_class_by_application_type(self.type) + fields = serializer_class().get_fields() + + parameters = [self.type] + for field_name in list(fields.keys()): + if field_name in ['asset']: + continue + value = self.attrs.get(field_name) + if not value: + continue + if field_name == 'path': + value = '\"%s\"' % value + parameters.append(str(value)) + + parameters = ' '.join(parameters) + return { + 'program': '||jmservisor', + 'working_directory': '', + 'parameters': parameters + } + + def get_remote_app_asset(self): + asset_id = self.attrs.get('asset') + if not asset_id: + raise ValueError("Remote App not has asset attr") + asset = Asset.objects.filter(id=asset_id).first() + return asset diff --git a/apps/applications/serializers/remote_app.py b/apps/applications/serializers/remote_app.py index 13343ee53..d036a8c65 100644 --- a/apps/applications/serializers/remote_app.py +++ b/apps/applications/serializers/remote_app.py @@ -27,31 +27,5 @@ class RemoteAppConnectionInfoSerializer(serializers.ModelSerializer): return obj.attrs.get('asset') @staticmethod - def get_parameters(obj): - """ - 返回Guacamole需要的RemoteApp配置参数信息中的parameters参数 - """ - from .attrs import get_serializer_class_by_application_type - serializer_class = get_serializer_class_by_application_type(obj.type) - fields = serializer_class().get_fields() - - parameters = [obj.type] - for field_name in list(fields.keys()): - if field_name in ['asset']: - continue - value = obj.attrs.get(field_name) - if not value: - continue - if field_name == 'path': - value = '\"%s\"' % value - parameters.append(str(value)) - - parameters = ' '.join(parameters) - return parameters - - def get_parameter_remote_app(self, obj): - return { - 'program': '||jmservisor', - 'working_directory': '', - 'parameters': self.get_parameters(obj) - } + def get_parameter_remote_app(obj): + return obj.get_rdp_remote_app_setting() diff --git a/apps/assets/serializers/domain.py b/apps/assets/serializers/domain.py index 00df0b91d..f7b402f0e 100644 --- a/apps/assets/serializers/domain.py +++ b/apps/assets/serializers/domain.py @@ -77,8 +77,6 @@ class GatewayWithAuthSerializer(GatewaySerializer): return fields - - class DomainWithGatewaySerializer(BulkOrgResourceModelSerializer): gateways = GatewayWithAuthSerializer(many=True, read_only=True) diff --git a/apps/authentication/api/connection_token.py b/apps/authentication/api/connection_token.py index 0b24950e2..ec902a21d 100644 --- a/apps/authentication/api/connection_token.py +++ b/apps/authentication/api/connection_token.py @@ -1,40 +1,73 @@ # -*- coding: utf-8 -*- # -import uuid - from django.conf import settings from django.core.cache import cache +from django.shortcuts import get_object_or_404 +from rest_framework.serializers import ValidationError from rest_framework.response import Response -from rest_framework.generics import CreateAPIView +from rest_framework.viewsets import GenericViewSet +from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied + +from common.utils import get_logger, random_string +from common.drf.api import SerializerMixin2 +from common.permissions import IsSuperUserOrAppUser, IsValidUser, IsSuperUser -from common.utils import get_logger -from common.permissions import IsSuperUserOrAppUser from orgs.mixins.api import RootOrgViewMixin -from ..serializers import ConnectionTokenSerializer +from ..serializers import ConnectionTokenSerializer, ConnectionTokenSecretSerializer logger = get_logger(__name__) -__all__ = ['UserConnectionTokenApi'] +__all__ = ['UserConnectionTokenViewSet'] -class UserConnectionTokenApi(RootOrgViewMixin, CreateAPIView): +class UserConnectionTokenViewSet(RootOrgViewMixin, SerializerMixin2, GenericViewSet): permission_classes = (IsSuperUserOrAppUser,) - serializer_class = ConnectionTokenSerializer + serializer_classes = { + 'default': ConnectionTokenSerializer, + 'get_secret_detail': ConnectionTokenSecretSerializer + } + CACHE_KEY_PREFIX = 'CONNECTION_TOKEN_{}' - def perform_create(self, serializer): - user = serializer.validated_data['user'] - asset = serializer.validated_data['asset'] - system_user = serializer.validated_data['system_user'] - token = str(uuid.uuid4()) + @staticmethod + def check_resource_permission(user, asset, application, system_user): + from perms.utils.asset import has_asset_system_permission + from perms.utils.application import has_application_system_permission + if asset and not has_asset_system_permission(user, asset, system_user): + error = f'User not has this asset and system user permission: ' \ + f'user={user.id} system_user={system_user.id} asset={asset.id}' + raise PermissionDenied(error) + if application and not has_application_system_permission(user, application, system_user): + error = f'User not has this application and system user permission: ' \ + f'user={user.id} system_user={system_user.id} application={application.id}' + raise PermissionDenied(error) + return True + + def create_token(self, user, asset, application, system_user): + self.check_resource_permission(user, asset, application, system_user) + token = random_string(36) value = { 'user': str(user.id), 'username': user.username, - 'asset': str(asset.id), - 'hostname': asset.hostname, 'system_user': str(system_user.id), 'system_user_name': system_user.name } - cache.set(token, value, timeout=20) + + if asset: + value.update({ + 'type': 'asset', + 'asset': str(asset.id), + 'hostname': asset.hostname, + }) + elif application: + value.update({ + 'type': 'application', + 'application': application.id, + 'application_name': str(application) + }) + + key = self.CACHE_KEY_PREFIX.format(token) + cache.set(key, value, timeout=20) return token def create(self, request, *args, **kwargs): @@ -42,18 +75,107 @@ class UserConnectionTokenApi(RootOrgViewMixin, CreateAPIView): data = {'error': 'Connection token disabled'} return Response(data, status=400) - if not request.user.is_superuser: - data = {'error': 'Only super user can create token'} - return Response(data, status=403) - serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - token = self.perform_create(serializer) + + user = serializer.validated_data.get('user', None) + if not request.user.is_superuser and user: + raise PermissionDenied('Only super user can create user token') + if not request.user.is_superuser: + user = self.request.user + + asset = serializer.validated_data.get('asset') + application = serializer.validated_data.get('application') + system_user = serializer.validated_data['system_user'] + + token = self.create_token(user, asset, application, system_user) return Response({"token": token}, status=201) + @staticmethod + def _get_application_secret_detail(value): + from applications.models import Application + from perms.models import Action + application = get_object_or_404(Application, id=value.get('application')) + gateway = None + + if not application.category_remote_app: + actions = Action.NONE + remote_app = {} + asset = None + domain = application.domain + else: + remote_app = application.get_rdp_remote_app_setting() + actions = Action.CONNECT + asset = application.get_remote_app_asset() + domain = asset.domain + + if domain and domain.has_gateway(): + gateway = domain.random_gateway() + + return { + 'asset': asset, + 'application': application, + 'gateway': gateway, + 'remote_app': remote_app, + 'actions': actions + } + + @staticmethod + def _get_asset_secret_detail(value, user, system_user): + from assets.models import Asset + from perms.utils.asset import get_asset_system_users_id_with_actions_by_user + asset = get_object_or_404(Asset, id=value.get('asset')) + systemuserid_actions_mapper = get_asset_system_users_id_with_actions_by_user(user, asset) + actions = systemuserid_actions_mapper.get(system_user.id, []) + gateway = None + if asset and asset.domain and asset.domain.has_gateway(): + gateway = asset.domain.random_gateway() + return { + 'asset': asset, + 'application': None, + 'gateway': gateway, + 'remote_app': None, + 'actions': actions, + } + + @action(methods=['POST'], detail=False, permission_classes=[IsSuperUserOrAppUser], url_path='secret-info/detail') + def get_secret_detail(self, request, *args, **kwargs): + from users.models import User + from assets.models import SystemUser + + token = request.data.get('token', '') + key = self.CACHE_KEY_PREFIX.format(token) + value = cache.get(key, None) + if not value: + return Response(status=404) + user = get_object_or_404(User, id=value.get('user')) + system_user = get_object_or_404(SystemUser, id=value.get('system_user')) + data = dict(user=user, system_user=system_user) + + if value.get('type') == 'asset': + asset_detail = self._get_asset_secret_detail(value, user=user, system_user=system_user) + data['type'] = 'asset' + data.update(asset_detail) + else: + app_detail = self._get_application_secret_detail(value) + data['type'] = 'application' + data.update(app_detail) + + serializer = self.get_serializer(data) + return Response(data=serializer.data, status=200) + + def get_permissions(self): + if self.action == "create": + if self.request.data.get('user', None): + self.permission_classes = (IsSuperUser,) + else: + self.permission_classes = (IsValidUser,) + return super().get_permissions() + def get(self, request): token = request.query_params.get('token') - value = cache.get(token, None) + key = self.CACHE_KEY_PREFIX.format(token) + value = cache.get(key, None) if not value: return Response('', status=404) diff --git a/apps/authentication/serializers.py b/apps/authentication/serializers.py index d50173ff9..fe56d6fa7 100644 --- a/apps/authentication/serializers.py +++ b/apps/authentication/serializers.py @@ -4,14 +4,17 @@ from rest_framework import serializers from common.utils import get_object_or_none from users.models import User +from assets.models import Asset, SystemUser, Gateway +from applications.models import Application from users.serializers import UserProfileSerializer +from perms.serializers.asset.permission import ActionsField from .models import AccessKey, LoginConfirmSetting, SSOToken __all__ = [ 'AccessKeySerializer', 'OtpVerifySerializer', 'BearerTokenSerializer', 'MFAChallengeSerializer', 'LoginConfirmSettingSerializer', 'SSOTokenSerializer', - 'ConnectionTokenSerializer', + 'ConnectionTokenSerializer', 'ConnectionTokenSecretSerializer' ] @@ -86,9 +89,10 @@ class SSOTokenSerializer(serializers.Serializer): class ConnectionTokenSerializer(serializers.Serializer): - user = serializers.CharField(max_length=128, required=True) + user = serializers.CharField(max_length=128, required=False, allow_blank=True) system_user = serializers.CharField(max_length=128, required=True) - asset = serializers.CharField(max_length=128, required=True) + asset = serializers.CharField(max_length=128, required=False) + application = serializers.CharField(max_length=128, required=False) @staticmethod def validate_user(user_id): @@ -113,3 +117,67 @@ class ConnectionTokenSerializer(serializers.Serializer): if asset is None: raise serializers.ValidationError('asset id not exist') return asset + + @staticmethod + def validate_application(app_id): + from applications.models import Application + app = Application.objects.filter(id=app_id).first() + if app is None: + raise serializers.ValidationError('app id not exist') + return app + + def validate(self, attrs): + asset = attrs.get('asset') + application = attrs.get('application') + if not asset and not application: + raise serializers.ValidationError('asset or application required') + if asset and application: + raise serializers.ValidationError('asset and application should only one') + return super().validate(attrs) + + +class ConnectionTokenUserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ['id', 'name', 'username', 'email'] + + +class ConnectionTokenAssetSerializer(serializers.ModelSerializer): + class Meta: + model = Asset + fields = ['id', 'hostname', 'ip', 'port', 'org_id'] + + +class ConnectionTokenSystemUserSerializer(serializers.ModelSerializer): + class Meta: + model = SystemUser + fields = ['id', 'name', 'username', 'password', 'private_key'] + + +class ConnectionTokenGatewaySerializer(serializers.ModelSerializer): + class Meta: + model = Gateway + fields = ['id', 'ip', 'port', 'username', 'password', 'private_key'] + + +class ConnectionTokenRemoteAppSerializer(serializers.Serializer): + program = serializers.CharField() + working_directory = serializers.CharField() + parameters = serializers.CharField() + + +class ConnectionTokenApplicationSerializer(serializers.ModelSerializer): + class Meta: + model = Application + fields = ['id', 'name', 'category', 'type'] + + +class ConnectionTokenSecretSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=[('application', 'Application'), ('asset', 'Asset')]) + user = ConnectionTokenUserSerializer(read_only=True) + asset = ConnectionTokenAssetSerializer(read_only=True) + remote_app = ConnectionTokenRemoteAppSerializer(read_only=True) + application = ConnectionTokenApplicationSerializer(read_only=True) + system_user = ConnectionTokenSystemUserSerializer(read_only=True) + gateway = ConnectionTokenGatewaySerializer(read_only=True) + actions = ActionsField() diff --git a/apps/authentication/urls/api_urls.py b/apps/authentication/urls/api_urls.py index 3027fdad0..40eb2912e 100644 --- a/apps/authentication/urls/api_urls.py +++ b/apps/authentication/urls/api_urls.py @@ -9,6 +9,7 @@ app_name = 'authentication' router = DefaultRouter() router.register('access-keys', api.AccessKeyViewSet, 'access-key') router.register('sso', api.SSOViewSet, 'sso') +router.register('connection-token', api.UserConnectionTokenViewSet, 'connection-token') urlpatterns = [ @@ -16,8 +17,6 @@ urlpatterns = [ path('auth/', api.TokenCreateApi.as_view(), name='user-auth'), path('tokens/', api.TokenCreateApi.as_view(), name='auth-token'), path('mfa/challenge/', api.MFAChallengeApi.as_view(), name='mfa-challenge'), - path('connection-token/', - api.UserConnectionTokenApi.as_view(), name='connection-token'), path('otp/verify/', api.UserOtpVerifyApi.as_view(), name='user-otp-verify'), path('login-confirm-ticket/status/', api.TicketStatusApi.as_view(), name='login-confirm-ticket-status'), path('login-confirm-settings//', api.LoginConfirmSettingUpdateApi.as_view(), name='login-confirm-setting-update') diff --git a/apps/locale/zh/LC_MESSAGES/django.po b/apps/locale/zh/LC_MESSAGES/django.po index 7b24b53d4..6cd533e47 100644 --- a/apps/locale/zh/LC_MESSAGES/django.po +++ b/apps/locale/zh/LC_MESSAGES/django.po @@ -64,7 +64,7 @@ msgstr "名称" #: perms/serializers/application/user_permission.py:33 #: tickets/serializers/ticket/meta/ticket_type/apply_application.py:20 msgid "Category" -msgstr "种类" +msgstr "类别" #: applications/models/application.py:15 #: applications/serializers/application.py:48 assets/models/cmd_filter.py:52 @@ -3160,7 +3160,7 @@ msgstr "关闭" #: tickets/handler/apply_application.py:55 msgid "Applied category" -msgstr "申请的种类" +msgstr "申请的类别" #: tickets/handler/apply_application.py:56 msgid "Applied type" @@ -3323,7 +3323,7 @@ msgstr "受理人 (显示名称)" #: tickets/serializers/ticket/meta/ticket_type/apply_application.py:24 msgid "Category display" -msgstr "种类 (显示名称)" +msgstr "类别 (显示名称)" #: tickets/serializers/ticket/meta/ticket_type/apply_application.py:31 #: tickets/serializers/ticket/ticket.py:19 diff --git a/apps/perms/api/application/user_permission/common.py b/apps/perms/api/application/user_permission/common.py index 272f84378..999a09087 100644 --- a/apps/perms/api/application/user_permission/common.py +++ b/apps/perms/api/application/user_permission/common.py @@ -11,6 +11,7 @@ from rest_framework.generics import ( from orgs.utils import tmp_to_root_org from applications.models import Application from perms.utils.application.permission import ( + has_application_system_permission, get_application_system_users_id ) from perms.api.asset.user_permission.mixin import RoleAdminMixin, RoleUserMixin @@ -71,8 +72,7 @@ class ValidateUserApplicationPermissionApi(APIView): application = get_object_or_404(Application, id=application_id) system_user = get_object_or_404(SystemUser, id=system_user_id) - system_users_id = get_application_system_users_id(user, application) - if system_user.id in system_users_id: + if has_application_system_permission(user, application, system_user): return Response({'msg': True}, status=200) return Response({'msg': False}, status=403) diff --git a/apps/perms/signals_handler/common.py b/apps/perms/signals_handler/common.py index c714de834..804d5beaf 100644 --- a/apps/perms/signals_handler/common.py +++ b/apps/perms/signals_handler/common.py @@ -128,12 +128,10 @@ def on_asset_permission_user_groups_changed(instance, action, pk_set, model, @receiver(m2m_changed, sender=ApplicationPermission.system_users.through) def on_application_permission_system_users_changed(sender, instance: ApplicationPermission, action, reverse, pk_set, **kwargs): - if not instance.category_remote_app: - return - if reverse: raise M2MReverseNotAllowed - + if not instance.category_remote_app: + return if action != POST_ADD: return @@ -156,12 +154,12 @@ def on_application_permission_system_users_changed(sender, instance: Application @receiver(m2m_changed, sender=ApplicationPermission.users.through) def on_application_permission_users_changed(sender, instance, action, reverse, pk_set, **kwargs): - if not instance.category_remote_app: - return - if reverse: raise M2MReverseNotAllowed + if not instance.category_remote_app: + return + if action != POST_ADD: return @@ -176,12 +174,10 @@ def on_application_permission_users_changed(sender, instance, action, reverse, p @receiver(m2m_changed, sender=ApplicationPermission.user_groups.through) def on_application_permission_user_groups_changed(sender, instance, action, reverse, pk_set, **kwargs): - if not instance.category_remote_app: - return - if reverse: raise M2MReverseNotAllowed - + if not instance.category_remote_app: + return if action != POST_ADD: return @@ -196,12 +192,12 @@ def on_application_permission_user_groups_changed(sender, instance, action, reve @receiver(m2m_changed, sender=ApplicationPermission.applications.through) def on_application_permission_applications_changed(sender, instance, action, reverse, pk_set, **kwargs): - if not instance.category_remote_app: - return - if reverse: raise M2MReverseNotAllowed + if not instance.category_remote_app: + return + if action != POST_ADD: return diff --git a/apps/perms/utils/application/permission.py b/apps/perms/utils/application/permission.py index 6908db5a3..2c92e5bee 100644 --- a/apps/perms/utils/application/permission.py +++ b/apps/perms/utils/application/permission.py @@ -7,8 +7,14 @@ logger = get_logger(__file__) def get_application_system_users_id(user, application): - queryset = ApplicationPermission.objects\ - .filter(Q(users=user) | Q(user_groups__users=user), Q(applications=application))\ - .valid()\ - .values_list('system_users', flat=True) + queryset = ApplicationPermission.objects.valid()\ + .filter( + Q(users=user) | Q(user_groups__users=user), + Q(applications=application) + ).values_list('system_users', flat=True) return queryset + + +def has_application_system_permission(user, application, system_user): + system_users_id = get_application_system_users_id(user, application) + return system_user.id in system_users_id diff --git a/apps/perms/utils/asset/permission.py b/apps/perms/utils/asset/permission.py index 54272c50d..38a3b929d 100644 --- a/apps/perms/utils/asset/permission.py +++ b/apps/perms/utils/asset/permission.py @@ -4,7 +4,7 @@ from django.db.models import Q from common.utils import get_logger from perms.models import AssetPermission -from perms.hands import Asset, User, UserGroup +from perms.hands import Asset, User, UserGroup, SystemUser from perms.models.base import BasePermissionQuerySet logger = get_logger(__file__) @@ -19,10 +19,8 @@ def get_asset_system_users_id_with_actions(asset_perm_queryset: BasePermissionQu ancestor_keys = node.get_ancestor_keys(with_self=True) node_keys.update(ancestor_keys) - queryset = AssetPermission.objects.filter(id__in=asset_perms_id).filter( - Q(assets=asset) | - Q(nodes__key__in=node_keys) - ) + queryset = AssetPermission.objects.filter(id__in=asset_perms_id)\ + .filter(Q(assets=asset) | Q(nodes__key__in=node_keys)) asset_protocols = asset.protocols_as_dict.keys() values = queryset.filter( @@ -44,8 +42,14 @@ def get_asset_system_users_id_with_actions_by_user(user: User, asset: Asset): return get_asset_system_users_id_with_actions(queryset, asset) +def has_asset_system_permission(user: User, asset: Asset, system_user: SystemUser): + systemuser_actions_mapper = get_asset_system_users_id_with_actions_by_user(user, asset) + actions = systemuser_actions_mapper.get(system_user.id, []) + if actions: + return True + return False + + def get_asset_system_users_id_with_actions_by_group(group: UserGroup, asset: Asset): - queryset = AssetPermission.objects.filter( - user_groups=group - ).valid() + queryset = AssetPermission.objects.filter(user_groups=group).valid() return get_asset_system_users_id_with_actions(queryset, asset) diff --git a/apps/users/urls/api_urls.py b/apps/users/urls/api_urls.py index e6b6974c2..8b9c538bd 100644 --- a/apps/users/urls/api_urls.py +++ b/apps/users/urls/api_urls.py @@ -16,11 +16,10 @@ router.register(r'users', api.UserViewSet, 'user') router.register(r'groups', api.UserGroupViewSet, 'user-group') router.register(r'users-groups-relations', api.UserUserGroupRelationViewSet, 'users-groups-relation') router.register(r'service-account-registrations', api.ServiceAccountRegistrationViewSet, 'service-account-registration') +router.register(r'connection-token', auth_api.UserConnectionTokenViewSet, 'connection-token') urlpatterns = [ - path('connection-token/', auth_api.UserConnectionTokenApi.as_view(), - name='connection-token'), path('profile/', api.UserProfileApi.as_view(), name='user-profile'), path('profile/password/', api.UserPasswordApi.as_view(), name='user-password'), path('profile/public-key/', api.UserPublicKeyApi.as_view(), name='user-public-key'), From 83cc339d4b99592b48b314dbe0eb2d378f875590 Mon Sep 17 00:00:00 2001 From: xinwen Date: Sat, 20 Feb 2021 14:44:27 +0800 Subject: [PATCH 19/71] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E7=BB=84?= =?UTF-8?q?=E7=BB=87=E7=BB=9F=E8=AE=A1=E6=95=B0=E6=8D=AE=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E6=9B=B4=E6=96=B0=E7=AD=96=E7=95=A5=E4=B8=BA=E6=87=92?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/cache.py | 30 ++++++++++++++++++++++-------- apps/common/utils/lock.py | 1 + apps/orgs/cache.py | 5 +++++ apps/orgs/signals_handler.py | 6 +++--- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/apps/common/cache.py b/apps/common/cache.py index 8cf76154c..0e2415691 100644 --- a/apps/common/cache.py +++ b/apps/common/cache.py @@ -72,7 +72,7 @@ class Cache(metaclass=CacheBase): def get_data(self) -> dict: data = cache.get(self.key) - logger.debug(f'CACHE: get {self.key} = {data}') + logger.debug(f'Get data from cache: key={self.key} data={data}') if data is not None: data = json.loads(data) self._data = data @@ -81,7 +81,7 @@ class Cache(metaclass=CacheBase): def set_data(self, data): self._data = data to_json = json.dumps(data) - logger.info(f'CACHE: set {self.key} = {to_json}, timeout={self.timeout}') + logger.info(f'Set data to cache: key={self.key} data={to_json} timeout={self.timeout}') cache.set(self.key, to_json, timeout=self.timeout) def compute_data(self, *fields): @@ -122,6 +122,16 @@ class Cache(metaclass=CacheBase): self.set_data(data) return data + def expire_fields_with_lock(self, *fields): + with DistributedLock(name=f'{self.key}.refresh'): + data = self.get_data() + if data is not None: + logger.info(f'Expire cached fields: key={self.key} fields={fields}') + for f in fields: + data.pop(f) + self.set_data(data) + return data + def refresh(self, *fields): if not fields: # 没有指定 field 要刷新所有的值 @@ -146,10 +156,13 @@ class Cache(metaclass=CacheBase): def reload(self): self._data = None - def delete(self): - self._data = None - logger.info(f'CACHE: delete {self.key}') - cache.delete(self.key) + def expire(self, *fields): + if not fields: + self._data = None + logger.info(f'Delete cached key: key={self.key}') + cache.delete(self.key) + else: + self.expire_fields_with_lock(*fields) class CacheValueDesc: @@ -167,7 +180,8 @@ class CacheValueDesc: return self if self.field_name not in instance.data: instance.refresh(self.field_name) - value = instance.data[self.field_name] + # 防止边界情况没有值，报错 + value = instance.data.get(self.field_name) return value def compute_value(self, instance: Cache): @@ -183,5 +197,5 @@ class CacheValueDesc: new_value = compute_func() new_value = self.field_type.field_type(new_value) - logger.info(f'CACHE: compute {instance.key}.{self.field_name} = {new_value}') + logger.info(f'Compute cache field value: key={instance.key} field={self.field_name} value={new_value}') return new_value diff --git a/apps/common/utils/lock.py b/apps/common/utils/lock.py index 1a016d3d4..ffb6217e3 100644 --- a/apps/common/utils/lock.py +++ b/apps/common/utils/lock.py @@ -158,6 +158,7 @@ class DistributedLock(RedisLock): def _release(self): try: self._release_redis_lock() + logger.debug(f'I[{self.id}] released lock[{self.name}]') except NotAcquired as e: logger.error(f'I[{self.id}] release lock[{self.name}] failed {e}') self._raise_exc(e) diff --git a/apps/orgs/cache.py b/apps/orgs/cache.py index 9b1c8ffa9..f0a9cb83e 100644 --- a/apps/orgs/cache.py +++ b/apps/orgs/cache.py @@ -32,3 +32,8 @@ class OrgRelatedCache(Cache): logger.info(f'CACHE: Send refresh task {self}.{fields}') refresh_org_cache_task.delay(self, *fields) on_commit(func) + + def expire(self, *fields): + def func(): + super(OrgRelatedCache, self).expire(*fields) + on_commit(func) diff --git a/apps/orgs/signals_handler.py b/apps/orgs/signals_handler.py index 57c7f5490..7cf3a9c29 100644 --- a/apps/orgs/signals_handler.py +++ b/apps/orgs/signals_handler.py @@ -118,7 +118,7 @@ def refresh_user_amount_on_user_create_or_delete(user_id): orgs = Organization.objects.filter(m2m_org_members__user_id=user_id).distinct() for org in orgs: org_cache = OrgResourceStatisticsCache(org) - org_cache.refresh_async('users_amount') + org_cache.expire('users_amount') @receiver(post_save, sender=User) @@ -144,7 +144,7 @@ def on_org_user_changed_refresh_cache(sender, action, instance, reverse, pk_set, for org in orgs: org_cache = OrgResourceStatisticsCache(org) - org_cache.refresh_async('users_amount') + org_cache.expire('users_amount') class OrgResourceStatisticsRefreshUtil: @@ -166,7 +166,7 @@ class OrgResourceStatisticsRefreshUtil: cache_field_name = cls.model_cache_field_mapper.get(type(instance)) if cache_field_name: org_cache = OrgResourceStatisticsCache(instance.org) - org_cache.refresh_async(cache_field_name) + org_cache.expire(cache_field_name) @receiver(post_save) From a4e635bff0131aea9465f50e641f98a2de1bf51a Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Wed, 24 Feb 2021 15:31:22 +0800 Subject: [PATCH 20/71] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E4=B8=8B?= =?UTF-8?q?=E8=BD=BDrdp=E6=96=87=E4=BB=B6=E7=9A=84api=20(#5637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 添加下载rdp文件的api * perf: 优化一些权限 * perf: 优化一波token Co-authored-by: ibuler --- apps/authentication/api/connection_token.py | 90 +++++++++++++++++---- apps/authentication/serializers.py | 7 +- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/apps/authentication/api/connection_token.py b/apps/authentication/api/connection_token.py index ec902a21d..16f8907be 100644 --- a/apps/authentication/api/connection_token.py +++ b/apps/authentication/api/connection_token.py @@ -3,7 +3,7 @@ from django.conf import settings from django.core.cache import cache from django.shortcuts import get_object_or_404 -from rest_framework.serializers import ValidationError +from django.http import HttpResponse from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet from rest_framework.decorators import action @@ -15,7 +15,10 @@ from common.permissions import IsSuperUserOrAppUser, IsValidUser, IsSuperUser from orgs.mixins.api import RootOrgViewMixin -from ..serializers import ConnectionTokenSerializer, ConnectionTokenSecretSerializer +from ..serializers import ( + ConnectionTokenSerializer, ConnectionTokenSecretSerializer, + RDPFileSerializer +) logger = get_logger(__name__) __all__ = ['UserConnectionTokenViewSet'] @@ -25,7 +28,8 @@ class UserConnectionTokenViewSet(RootOrgViewMixin, SerializerMixin2, GenericView permission_classes = (IsSuperUserOrAppUser,) serializer_classes = { 'default': ConnectionTokenSerializer, - 'get_secret_detail': ConnectionTokenSecretSerializer + 'get_secret_detail': ConnectionTokenSecretSerializer, + 'get_rdp_file': RDPFileSerializer } CACHE_KEY_PREFIX = 'CONNECTION_TOKEN_{}' @@ -44,6 +48,12 @@ class UserConnectionTokenViewSet(RootOrgViewMixin, SerializerMixin2, GenericView return True def create_token(self, user, asset, application, system_user): + if not settings.CONNECTION_TOKEN_ENABLED: + raise PermissionDenied('Connection token disabled') + if not user: + user = self.request.user + if not self.request.user.is_superuser and user != self.request.user: + raise PermissionDenied('Only super user can create user token') self.check_resource_permission(user, asset, application, system_user) token = random_string(36) value = { @@ -71,26 +81,76 @@ class UserConnectionTokenViewSet(RootOrgViewMixin, SerializerMixin2, GenericView return token def create(self, request, *args, **kwargs): - if not settings.CONNECTION_TOKEN_ENABLED: - data = {'error': 'Connection token disabled'} - return Response(data, status=400) - serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) - user = serializer.validated_data.get('user', None) - if not request.user.is_superuser and user: - raise PermissionDenied('Only super user can create user token') - if not request.user.is_superuser: - user = self.request.user - asset = serializer.validated_data.get('asset') application = serializer.validated_data.get('application') system_user = serializer.validated_data['system_user'] - + user = serializer.validated_data.get('user') token = self.create_token(user, asset, application, system_user) return Response({"token": token}, status=201) + @action(methods=['POST', 'GET'], detail=False, url_path='rdp/file') + def get_rdp_file(self, request, *args, **kwargs): + options = { + 'full address:s': '', + 'username:s': '', + 'screen mode id:i': '0', + 'desktopwidth:i': '1280', + 'desktopheight:i': '800', + 'use multimon:i': '1', + 'session bpp:i': '24', + 'audiomode:i': '0', + 'disable wallpaper:i': '0', + 'disable full window drag:i': '0', + 'disable menu anims:i': '0', + 'disable themes:i': '0', + 'alternate shell:s': '', + 'shell working directory:s': '', + 'authentication level:i': '2', + 'connect to console:i': '0', + 'disable cursor setting:i': '0', + 'allow font smoothing:i': '1', + 'allow desktop composition:i': '1', + 'redirectprinters:i': '0', + 'prompt for credentials on client:i': '0', + 'autoreconnection enabled:i': '1', + 'bookmarktype:i': '3', + 'use redirection server name:i': '0', + # 'alternate shell:s:': '||MySQLWorkbench', + # 'remoteapplicationname:s': 'Firefox', + # 'remoteapplicationcmdline:s': '', + } + if self.request.method == 'GET': + data = self.request.query_params + else: + data = request.data + serializer = self.get_serializer(data=data) + serializer.is_valid(raise_exception=True) + + asset = serializer.validated_data.get('asset') + application = serializer.validated_data.get('application') + system_user = serializer.validated_data['system_user'] + user = serializer.validated_data.get('user') + height = serializer.validated_data.get('height') + width = serializer.validated_data.get('width') + token = self.create_token(user, asset, application, system_user) + + # Todo: 上线后地址是 JumpServerAddr:3389 + address = self.request.query_params.get('address') or '1.1.1.1' + options['full address:s'] = address + options['username:s'] = '{}@{}'.format(user.username, token) + options['desktopwidth:i'] = width + options['desktopheight:i'] = height + data = '' + for k, v in options.items(): + data += f'{k}:{v}\n' + response = HttpResponse(data, content_type='text/plain') + filename = "{}-{}-jumpserver.rdp".format(user.username, asset.hostname) + response['Content-Disposition'] = 'attachment; filename={}'.format(filename) + return response + @staticmethod def _get_application_secret_detail(value): from applications.models import Application @@ -165,7 +225,7 @@ class UserConnectionTokenViewSet(RootOrgViewMixin, SerializerMixin2, GenericView return Response(data=serializer.data, status=200) def get_permissions(self): - if self.action == "create": + if self.action in ["create", "get_rdp_file"]: if self.request.data.get('user', None): self.permission_classes = (IsSuperUser,) else: diff --git a/apps/authentication/serializers.py b/apps/authentication/serializers.py index fe56d6fa7..528cc7cf9 100644 --- a/apps/authentication/serializers.py +++ b/apps/authentication/serializers.py @@ -14,7 +14,7 @@ from .models import AccessKey, LoginConfirmSetting, SSOToken __all__ = [ 'AccessKeySerializer', 'OtpVerifySerializer', 'BearerTokenSerializer', 'MFAChallengeSerializer', 'LoginConfirmSettingSerializer', 'SSOTokenSerializer', - 'ConnectionTokenSerializer', 'ConnectionTokenSecretSerializer' + 'ConnectionTokenSerializer', 'ConnectionTokenSecretSerializer', 'RDPFileSerializer' ] @@ -181,3 +181,8 @@ class ConnectionTokenSecretSerializer(serializers.Serializer): system_user = ConnectionTokenSystemUserSerializer(read_only=True) gateway = ConnectionTokenGatewaySerializer(read_only=True) actions = ActionsField() + + +class RDPFileSerializer(ConnectionTokenSerializer): + width = serializers.IntegerField(default=1280) + height = serializers.IntegerField(default=800) From b03642847e78f92f5869b0adefa07a8be6c748e1 Mon Sep 17 00:00:00 2001 From: ibuler Date: Sat, 20 Feb 2021 15:46:13 +0800 Subject: [PATCH 21/71] =?UTF-8?q?perf:=20=E5=8E=BB=E6=8E=89=20data=5Ftree?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/utils/common.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index 9984c8614..8ec390558 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # import re -import data_tree from collections import OrderedDict from itertools import chain import logging @@ -11,8 +10,6 @@ from functools import wraps import time import ipaddress import psutil -from django.utils.translation import ugettext_lazy as _ -from ..exceptions import JMSException UUID_PATTERN = re.compile(r'\w{8}(-\w{4}){3}-\w{12}') From 799d1e40435a343bbd04f3869c36bc9b8cd54511 Mon Sep 17 00:00:00 2001 From: xinwen Date: Wed, 24 Feb 2021 12:06:31 +0800 Subject: [PATCH 22/71] =?UTF-8?q?feat:=20=E8=B5=84=E4=BA=A7=E6=8E=88?= =?UTF-8?q?=E6=9D=83=E8=A7=84=E5=88=99=E6=B7=BB=E5=8A=A0=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E6=9C=89=E6=95=88=E7=9A=84=E8=BF=87=E6=BB=A4=E6=9D=A1=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/drf/filters.py | 11 ++ apps/perms/api/asset/asset_permission.py | 74 +------- apps/perms/filters.py | 204 +++++++++++++++++++++++ 3 files changed, 221 insertions(+), 68 deletions(-) create mode 100644 apps/perms/filters.py diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index 7740cadba..9d8ec087e 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -6,6 +6,7 @@ from rest_framework.serializers import ValidationError from rest_framework.compat import coreapi, coreschema from django.core.cache import cache from django.core.exceptions import ImproperlyConfigured +from django_filters import rest_framework as drf_filters import logging from common import const @@ -13,6 +14,16 @@ from common import const __all__ = ["DatetimeRangeFilter", "IDSpmFilter", 'IDInFilter', "CustomFilter"] +class BaseFilterSet(drf_filters.FilterSet): + def do_nothing(self, queryset, name, value): + return queryset + + def get_query_param(self, k, default=None): + if k in self.form.data: + return self.form.cleaned_data[k] + return default + + class DatetimeRangeFilter(filters.BaseFilterBackend): def get_schema_fields(self, view): ret = [] diff --git a/apps/perms/api/asset/asset_permission.py b/apps/perms/api/asset/asset_permission.py index e38a59ba6..65fe316e6 100644 --- a/apps/perms/api/asset/asset_permission.py +++ b/apps/perms/api/asset/asset_permission.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- # -from django.db.models import Q - +from perms.filters import AssetPermissionFilter from perms.models import AssetPermission -from perms.hands import ( - Asset, Node -) +from orgs.mixins.api import OrgBulkModelViewSet from perms import serializers -from ..base import BasePermissionViewSet +from common.permissions import IsOrgAdmin __all__ = [ @@ -15,70 +12,11 @@ __all__ = [ ] -class AssetPermissionViewSet(BasePermissionViewSet): +class AssetPermissionViewSet(OrgBulkModelViewSet): """ 资产授权列表的增删改查api """ + permission_classes = (IsOrgAdmin,) model = AssetPermission serializer_class = serializers.AssetPermissionSerializer - filterset_fields = ['name'] - custom_filter_fields = BasePermissionViewSet.custom_filter_fields + [ - 'node_id', 'node', 'asset_id', 'hostname', 'ip' - ] - - def filter_node(self, queryset): - node_id = self.request.query_params.get('node_id') - node_name = self.request.query_params.get('node') - if node_id: - _nodes = Node.objects.filter(pk=node_id) - elif node_name: - _nodes = Node.objects.filter(value=node_name) - else: - return queryset - if not _nodes: - return queryset.none() - - if not self.is_query_all(): - queryset = queryset.filter(nodes__in=_nodes) - return queryset - nodes = set(_nodes) - for node in _nodes: - nodes |= set(node.get_ancestors(with_self=True)) - queryset = queryset.filter(nodes__in=nodes) - return queryset - - def filter_asset(self, queryset): - asset_id = self.request.query_params.get('asset_id') - hostname = self.request.query_params.get('hostname') - ip = self.request.query_params.get('ip') - if asset_id: - assets = Asset.objects.filter(pk=asset_id) - elif hostname: - assets = Asset.objects.filter(hostname=hostname) - elif ip: - assets = Asset.objects.filter(ip=ip) - else: - return queryset - if not assets: - return queryset.none() - if not self.is_query_all(): - queryset = queryset.filter(assets__in=assets) - return queryset - inherit_all_nodes = set() - inherit_nodes_keys = assets.all().values_list('nodes__key', flat=True) - - for key in inherit_nodes_keys: - if key is None: - continue - ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True) - inherit_all_nodes.update(ancestor_keys) - queryset = queryset.filter( - Q(assets__in=assets) | Q(nodes__key__in=inherit_all_nodes) - ).distinct() - return queryset - - def filter_queryset(self, queryset): - queryset = super().filter_queryset(queryset) - queryset = self.filter_asset(queryset) - queryset = self.filter_node(queryset) - return queryset + filterset_class = AssetPermissionFilter diff --git a/apps/perms/filters.py b/apps/perms/filters.py new file mode 100644 index 000000000..4500c8836 --- /dev/null +++ b/apps/perms/filters.py @@ -0,0 +1,204 @@ +from django_filters import rest_framework as filters +from django.db.models import QuerySet, Q + +from common.drf.filters import BaseFilterSet +from common.utils import get_object_or_none +from users.models import User, UserGroup +from assets.models import Node, Asset, SystemUser +from perms.models import AssetPermission + + +class PermissionBaseFilter(BaseFilterSet): + is_valid = filters.BooleanFilter(method='do_nothing') + user_id = filters.UUIDFilter(method='do_nothing') + username = filters.CharFilter(method='do_nothing') + system_user_id = filters.UUIDFilter(method='do_nothing') + system_user = filters.CharFilter(method='do_nothing') + user_group_id = filters.UUIDFilter(method='do_nothing') + user_group = filters.CharFilter(method='do_nothing') + all = filters.BooleanFilter(method='do_nothing') + + class Meta: + fields = ( + 'user_id', 'username', 'system_user_id', 'system_user', 'user_group_id', + 'user_group', 'name', 'all', 'is_valid', + ) + + @property + def qs(self): + qs = super().qs + qs = self.filter_valid(qs) + qs = self.filter_user(qs) + qs = self.filter_system_user(qs) + qs = self.filter_user_group(qs) + return qs + + def filter_valid(self, queryset): + is_valid = self.get_query_param('is_valid') + if is_valid is None: + return queryset + + if is_valid: + queryset = queryset.valid() + else: + queryset = queryset.invalid() + return queryset + + def filter_user(self, queryset): + is_query_all = self.get_query_param('all', True) + user_id = self.get_query_param('user_id') + username = self.get_query_param('username') + + if user_id: + user = get_object_or_none(User, pk=user_id) + elif username: + user = get_object_or_none(User, username=username) + else: + return queryset + if not user: + return queryset.none() + if is_query_all: + queryset = queryset.filter(users=user) + return queryset + groups = list(user.groups.all().values_list('id', flat=True)) + queryset = queryset.filter( + Q(users=user) | Q(user_groups__in=groups) + ).distinct() + return queryset + + def filter_system_user(self, queryset): + system_user_id = self.get_query_param('system_user_id') + system_user_name = self.get_query_param('system_user') + + if system_user_id: + system_user = get_object_or_none(SystemUser, pk=system_user_id) + elif system_user_name: + system_user = get_object_or_none(SystemUser, name=system_user_name) + else: + return queryset + if not system_user: + return queryset.none() + queryset = queryset.filter(system_users=system_user) + return queryset + + def filter_user_group(self, queryset): + user_group_id = self.get_query_param('user_group_id') + user_group_name = self.get_query_param('user_group') + + if user_group_id: + group = get_object_or_none(UserGroup, pk=user_group_id) + elif user_group_name: + group = get_object_or_none(UserGroup, name=user_group_name) + else: + return queryset + if not group: + return queryset.none() + queryset = queryset.filter(user_groups=group) + return queryset + + +class AssetPermissionFilter(PermissionBaseFilter): + is_effective = filters.BooleanFilter(method='do_nothing') + node_id = filters.UUIDFilter(method='do_nothing') + node = filters.CharFilter(method='do_nothing') + asset_id = filters.UUIDFilter(method='do_nothing') + hostname = filters.CharFilter(method='do_nothing') + ip = filters.CharFilter(method='do_nothing') + + class Meta: + model = AssetPermission + fields = ( + 'user_id', 'username', 'system_user_id', 'system_user', 'user_group_id', + 'user_group', 'node_id', 'node', 'asset_id', 'hostname', 'ip', 'name', + 'all', 'asset_id', 'is_valid', 'is_effective', + ) + + @property + def qs(self): + qs = super().qs + qs = self.filter_effective(qs) + qs = self.filter_asset(qs) + qs = self.filter_node(qs) + return qs + + def filter_node(self, queryset: QuerySet): + is_query_all = self.get_query_param('all', True) + node_id = self.get_query_param('node_id') + node_name = self.get_query_param('node') + if node_id: + _nodes = Node.objects.filter(pk=node_id) + elif node_name: + _nodes = Node.objects.filter(value=node_name) + else: + return queryset + if not _nodes: + return queryset.none() + + if not is_query_all: + queryset = queryset.filter(nodes__in=_nodes) + return queryset + nodes = set(_nodes) + for node in _nodes: + nodes |= set(node.get_ancestors(with_self=True)) + queryset = queryset.filter(nodes__in=nodes) + return queryset + + def filter_asset(self, queryset): + is_query_all = self.get_query_param('all', True) + asset_id = self.get_query_param('asset_id') + hostname = self.get_query_param('hostname') + ip = self.get_query_param('ip') + + if asset_id: + assets = Asset.objects.filter(pk=asset_id) + elif hostname: + assets = Asset.objects.filter(hostname=hostname) + elif ip: + assets = Asset.objects.filter(ip=ip) + else: + return queryset + if not assets: + return queryset.none() + if not is_query_all: + queryset = queryset.filter(assets__in=assets) + return queryset + inherit_all_nodes = set() + inherit_nodes_keys = assets.all().values_list('nodes__key', flat=True) + + for key in inherit_nodes_keys: + if key is None: + continue + ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True) + inherit_all_nodes.update(ancestor_keys) + queryset = queryset.filter( + Q(assets__in=assets) | Q(nodes__key__in=inherit_all_nodes) + ).distinct() + return queryset + + def filter_effective(self, queryset): + is_effective = self.get_query_param('is_effective') + if is_effective is None: + return queryset + + if is_effective: + have_user_q = Q(users__isnull=False) | Q(user_groups__isnull=False) + have_asset_q = Q(assets__isnull=False) | Q(nodes__isnull=False) + have_system_user_q = Q(system_users__isnull=False) + have_action_q = Q(actions__gt=0) + + queryset = queryset.filter( + have_user_q & have_asset_q & have_system_user_q & have_action_q + ) + queryset &= AssetPermission.objects.valid() + else: + not_have_user_q = Q(users__isnull=True) & Q(user_groups__isnull=True) + not_have_asset_q = Q(assets__isnull=True) & Q(nodes__isnull=True) + not_have_system_user_q = Q(system_users__isnull=True) + not_have_action_q = Q(actions=0) + + queryset = queryset.filter( + not_have_user_q | not_have_asset_q | not_have_system_user_q | + not_have_action_q + ) + queryset |= AssetPermission.objects.invalid() + return queryset From 8ec26dea4325f3e8d5251327864205e079e417e9 Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 25 Feb 2021 09:19:31 +0800 Subject: [PATCH 23/71] =?UTF-8?q?feat:=20=E9=87=8D=E7=BD=AE=20MFA=20?= =?UTF-8?q?=E5=8F=91=E4=B8=AA=E9=82=AE=E4=BB=B6=20#754?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/users/api/user.py | 2 ++ apps/users/utils.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/apps/users/api/user.py b/apps/users/api/user.py index 1ac03af5e..7e4d60646 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -16,6 +16,7 @@ from common.mixins import CommonApiMixin from common.utils import get_logger from orgs.utils import current_org from orgs.models import ROLE as ORG_ROLE, OrganizationMember +from users.utils import send_reset_mfa_mail from .. import serializers from ..serializers import UserSerializer, UserRetrieveSerializer, MiniUserSerializer, InviteSerializer from .mixins import UserQuerysetMixin @@ -201,4 +202,5 @@ class UserResetOTPApi(UserQuerysetMixin, generics.RetrieveAPIView): if user.mfa_enabled: user.reset_mfa() user.save() + send_reset_mfa_mail(user) return Response({"msg": "success"}) diff --git a/apps/users/utils.py b/apps/users/utils.py index 94669c03a..0bef4fcc2 100644 --- a/apps/users/utils.py +++ b/apps/users/utils.py @@ -235,6 +235,28 @@ def send_reset_ssh_key_mail(user): send_mail_async.delay(subject, message, recipient_list, html_message=message) +def send_reset_mfa_mail(user): + subject = _('MFA Reset') + recipient_list = [user.email] + message = _(""" + Hello %(name)s: +
+ Your MFA has been reset by site administrator. + Please login and reset your MFA. +
+ Login direct + +
+ """) % { + 'name': user.name, + 'login_url': reverse('authentication:login', external=True), + } + if settings.DEBUG: + logger.debug(message) + + send_mail_async.delay(subject, message, recipient_list, html_message=message) + + def get_user_or_pre_auth_user(request): user = request.user if user.is_authenticated: From 4c4f544f0d47cc2af0b5dcba8b05ebac4379e205 Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 25 Feb 2021 07:44:06 +0800 Subject: [PATCH 24/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=A6=81?= =?UTF-8?q?=E7=94=A8=20MFA=20=E5=90=8E=E8=BF=98=E5=8F=AF=E4=BB=A5=E7=94=A8?= =?UTF-8?q?=20MFA=20=E6=9F=A5=E7=9C=8B=E5=AF=86=E7=A0=81=E5=8C=A3=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/users/exceptions.py | 10 ++++++++++ apps/users/models/user.py | 4 ++++ 2 files changed, 14 insertions(+) create mode 100644 apps/users/exceptions.py diff --git a/apps/users/exceptions.py b/apps/users/exceptions.py new file mode 100644 index 000000000..ff873d3dc --- /dev/null +++ b/apps/users/exceptions.py @@ -0,0 +1,10 @@ +from django.utils.translation import gettext_lazy as _ +from rest_framework import status + +from common.exceptions import JMSException + + +class MFANotEnabled(JMSException): + status_code = status.HTTP_403_FORBIDDEN + default_code = 'mfa_not_enabled' + default_detail = _('MFA not enabled') diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 1d8590ed0..9cdd49ee7 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -22,6 +22,7 @@ from common.utils import date_expired_default, get_logger, lazyproperty from common import fields from common.const import choices from common.db.models import ChoiceSet +from users.exceptions import MFANotEnabled from ..signals import post_user_change_password @@ -489,6 +490,9 @@ class MFAMixin: return check_otp_code(self.otp_secret_key, code) def check_mfa(self, code): + if not self.mfa_enabled: + raise MFANotEnabled + if settings.OTP_IN_RADIUS: return self.check_radius(code) else: From d795867916ff6375b5ef4c822d79354999255fc3 Mon Sep 17 00:00:00 2001 From: xinwen Date: Wed, 24 Feb 2021 18:19:23 +0800 Subject: [PATCH 25/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=9B=B4=E6=96=B0=E4=BC=9A=E6=9F=A5=E8=AF=A2=E5=85=A8?= =?UTF-8?q?=E9=83=A8=E6=95=B0=E6=8D=AE=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/mixins/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/common/mixins/serializers.py b/apps/common/mixins/serializers.py index 020af68d7..ecdd31b1d 100644 --- a/apps/common/mixins/serializers.py +++ b/apps/common/mixins/serializers.py @@ -71,7 +71,7 @@ class BulkListSerializerMixin(object): """ List of dicts of native values <- List of dicts of primitive datatypes. """ - if not self.instance: + if self.instance is None: return super().to_internal_value(data) if html.is_html_input(data): @@ -106,7 +106,7 @@ class BulkListSerializerMixin(object): pk = item["pk"] else: raise ValidationError("id or pk not in data") - child = self.instance.get(id=pk) if self.instance else None + child = self.instance.get(id=pk) self.child.instance = child self.child.initial_data = item # raw From 8f7dcd512a0ed67774b8efe6b7e44b5380f76681 Mon Sep 17 00:00:00 2001 From: ibuler Date: Fri, 19 Feb 2021 10:50:52 +0800 Subject: [PATCH 26/71] =?UTF-8?q?perf(ops):=20ansible=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=20summary=20=E6=B1=87=E6=80=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/ops/ansible/callback.py | 6 ++++++ apps/ops/ansible/runner.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/apps/ops/ansible/callback.py b/apps/ops/ansible/callback.py index 8edbbbcb7..3264e59c2 100644 --- a/apps/ops/ansible/callback.py +++ b/apps/ops/ansible/callback.py @@ -129,6 +129,9 @@ class AdHocResultCallback(CallbackMixin, CallbackModule, CMDCallBackModule): self.gather_result("unreachable", result) super().v2_runner_on_unreachable(result) + def v2_runner_on_start(self, *args, **kwargs): + pass + def display_skipped_hosts(self): pass @@ -201,6 +204,9 @@ class CommandResultCallback(AdHocResultCallback): msg, ), color=C.COLOR_ERROR) + def v2_playbook_on_stats(self, stats): + pass + def _print_task_banner(self, task): pass diff --git a/apps/ops/ansible/runner.py b/apps/ops/ansible/runner.py index fdbed74cb..a25d681b9 100644 --- a/apps/ops/ansible/runner.py +++ b/apps/ops/ansible/runner.py @@ -6,6 +6,7 @@ import shutil from collections import namedtuple from ansible import context +from ansible.playbook import Playbook from ansible.module_utils.common.collections import ImmutableDict from ansible.executor.task_queue_manager import TaskQueueManager from ansible.vars.manager import VariableManager @@ -217,6 +218,11 @@ class AdHocRunner: variable_manager=self.variable_manager, loader=self.loader, ) + loader = DataLoader() + # used in start callback + playbook = Playbook(loader) + playbook._entries.append(play) + playbook._file_name = '__adhoc_playbook__' tqm = TaskQueueManager( inventory=self.inventory, @@ -226,7 +232,9 @@ class AdHocRunner: passwords={"conn_pass": self.options.get("password", "")} ) try: + tqm.send_callback('v2_playbook_on_start', playbook) tqm.run(play) + tqm.send_callback('v2_playbook_on_stats', tqm._stats) return self.results_callback except Exception as e: raise AnsibleError(e) From 88d8a3326ffa8ce046ef5ce691c91a0dacf7917b Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 18 Feb 2021 11:13:55 +0800 Subject: [PATCH 27/71] =?UTF-8?q?perf(ops):=20=E4=BC=98=E5=8C=96=E5=AE=9A?= =?UTF-8?q?=E6=9C=9F=E6=A3=80=E6=9F=A5=E7=A3=81=E7=9B=98=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=BC=80=E5=85=B3=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/jumpserver/conf.py | 1 + apps/jumpserver/settings/custom.py | 4 +++- apps/ops/tasks.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index edfbae2c1..022ca84b0 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -282,6 +282,7 @@ class Config(dict): 'REFERER_CHECK_ENABLED': False, 'SERVER_REPLAY_STORAGE': {}, 'CONNECTION_TOKEN_ENABLED': False, + 'DISK_CHECK_ENABLED': True, } def compatible_auth_openid_of_key(self): diff --git a/apps/jumpserver/settings/custom.py b/apps/jumpserver/settings/custom.py index 9b4417ced..e6c121f94 100644 --- a/apps/jumpserver/settings/custom.py +++ b/apps/jumpserver/settings/custom.py @@ -116,4 +116,6 @@ DATETIME_DISPLAY_FORMAT = '%Y-%m-%d %H:%M:%S' TICKETS_ENABLED = CONFIG.TICKETS_ENABLED REFERER_CHECK_ENABLED = CONFIG.REFERER_CHECK_ENABLED -CONNECTION_TOKEN_ENABLED = CONFIG.CONNECTION_TOKEN_ENABLED \ No newline at end of file +CONNECTION_TOKEN_ENABLED = CONFIG.CONNECTION_TOKEN_ENABLED +DISK_CHECK_ENABLED = CONFIG.DISK_CHECK_ENABLED + diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 8a5512f30..02cc9290e 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -132,6 +132,8 @@ def create_or_update_registered_periodic_tasks(): @shared_task @register_as_period_task(interval=3600) def check_server_performance_period(): + if not settings.DISK_CHECK_ENABLED: + return usages = get_disk_usage() uncheck_paths = ['/etc', '/boot'] From b483f78d52ee494f2b36e1d2d4d19b00a196b851 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Fri, 26 Feb 2021 16:34:15 +0800 Subject: [PATCH 28/71] =?UTF-8?q?fix(assets):=20=E7=B3=BB=E7=BB=9F?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=94=AF=E6=8C=81=20OPENSSH=20=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E7=9A=84=E7=A7=81=E9=92=A5=20(#5604)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(assets): 系统用户支持 OPENSSH 格式的私钥 * fix: 升级paramiko Co-authored-by: ibuler --- apps/assets/serializers/base.py | 4 ---- requirements/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/assets/serializers/base.py b/apps/assets/serializers/base.py index c64b767e9..2159999c7 100644 --- a/apps/assets/serializers/base.py +++ b/apps/assets/serializers/base.py @@ -41,10 +41,6 @@ class AuthSerializerMixin: def validate_private_key(self, private_key): if not private_key: return - if 'OPENSSH' in private_key: - msg = _("Not support openssh format key, using " - "ssh-keygen -t rsa -m pem to generate") - raise serializers.ValidationError(msg) password = self.initial_data.get("password") valid = validate_ssh_private_key(private_key, password) if not valid: diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 33306c712..aca226c97 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -47,7 +47,7 @@ MarkupSafe==1.1.1 mysqlclient==2.0.1 olefile==0.44 openapi-codec==1.3.2 -paramiko==2.4.2 +paramiko==2.7.2 passlib==1.7.1 Pillow==7.1.0 pyasn1==0.4.8 From a7ab7da61c931295d677365f2d8b3622dac2df52 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Fri, 26 Feb 2021 17:33:11 +0800 Subject: [PATCH 29/71] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E9=99=90?= =?UTF-8?q?=E5=88=B6=E7=94=A8=E6=88=B7=E5=8F=AA=E8=83=BD=E4=BB=8Esource?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E7=9A=84=E5=8A=9F=E8=83=BD=20(#5592)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * stash it * feat: 添加限制用户只能从source登录的功能 * fix: 修复小错误 Co-authored-by: ibuler --- apps/audits/signals_handler.py | 44 +++--- apps/authentication/backends/radius.py | 2 +- apps/authentication/errors.py | 4 +- apps/authentication/mixins.py | 65 +++++--- apps/authentication/signals_handlers.py | 2 +- apps/authentication/views/login.py | 5 +- apps/common/utils/common.py | 9 +- apps/jumpserver/conf.py | 2 + apps/jumpserver/settings/auth.py | 33 ++-- apps/settings/models.py | 2 +- apps/users/forms/__init__.py | 2 - apps/users/forms/group.py | 44 ------ apps/users/forms/profile.py | 12 ++ apps/users/forms/user.py | 199 ------------------------ apps/users/models/user.py | 13 +- apps/users/signals_handler.py | 8 + apps/users/utils.py | 16 -- apps/users/views/profile/otp.py | 3 + 18 files changed, 146 insertions(+), 319 deletions(-) delete mode 100644 apps/users/forms/group.py delete mode 100644 apps/users/forms/user.py diff --git a/apps/audits/signals_handler.py b/apps/audits/signals_handler.py index 4b527c846..c7cf24337 100644 --- a/apps/audits/signals_handler.py +++ b/apps/audits/signals_handler.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- # - from django.db.models.signals import post_save, post_delete from django.dispatch import receiver +from django.conf import settings from django.db import transaction from django.utils import timezone +from django.utils.functional import LazyObject from django.contrib.auth import BACKEND_SESSION_KEY from django.utils.translation import ugettext_lazy as _ from rest_framework.renderers import JSONRenderer @@ -34,17 +35,22 @@ MODELS_NEED_RECORD = ( ) -LOGIN_BACKEND = { - 'PublicKeyAuthBackend': _('SSH Key'), - 'RadiusBackend': User.Source.radius.label, - 'RadiusRealmBackend': User.Source.radius.label, - 'LDAPAuthorizationBackend': User.Source.ldap.label, - 'ModelBackend': _('Password'), - 'SSOAuthentication': _('SSO'), - 'CASBackend': User.Source.cas.label, - 'OIDCAuthCodeBackend': User.Source.openid.label, - 'OIDCAuthPasswordBackend': User.Source.openid.label, -} +class AuthBackendLabelMapping(LazyObject): + @staticmethod + def get_login_backends(): + backend_label_mapping = {} + for source, backends in User.SOURCE_BACKEND_MAPPING.items(): + for backend in backends: + backend_label_mapping[backend] = source.label + backend_label_mapping[settings.AUTH_BACKEND_PUBKEY] = _('SSH Key') + backend_label_mapping[settings.AUTH_BACKEND_MODEL] = _('Password') + return backend_label_mapping + + def _setup(self): + self._wrapped = self.get_login_backends() + + +AUTH_BACKEND_LABEL_MAPPING = AuthBackendLabelMapping() def create_operate_log(action, sender, resource): @@ -70,6 +76,7 @@ def create_operate_log(action, sender, resource): @receiver(post_save) def on_object_created_or_update(sender, instance=None, created=False, update_fields=None, **kwargs): + # last_login 改变是最后登录日期, 每次登录都会改变 if instance._meta.object_name == 'User' and \ update_fields and 'last_login' in update_fields: return @@ -125,14 +132,13 @@ def on_audits_log_create(sender, instance=None, **kwargs): def get_login_backend(request): - backend = request.session.get('auth_backend', '') or request.session.get(BACKEND_SESSION_KEY, '') + backend = request.session.get('auth_backend', '') or \ + request.session.get(BACKEND_SESSION_KEY, '') - backend = backend.rsplit('.', maxsplit=1)[-1] - if backend in LOGIN_BACKEND: - return LOGIN_BACKEND[backend] - else: - logger.warn(f'LOGIN_BACKEND_NOT_FOUND: {backend}') - return '' + backend_label = AUTH_BACKEND_LABEL_MAPPING.get(backend, None) + if backend_label is None: + backend_label = '' + return backend_label def generate_data(username, request): diff --git a/apps/authentication/backends/radius.py b/apps/authentication/backends/radius.py index 6798e72f2..1baf3d569 100644 --- a/apps/authentication/backends/radius.py +++ b/apps/authentication/backends/radius.py @@ -2,7 +2,7 @@ # import traceback -from django.contrib.auth import get_user_model, authenticate +from django.contrib.auth import get_user_model from radiusauth.backends import RADIUSBackend, RADIUSRealmBackend from django.conf import settings diff --git a/apps/authentication/errors.py b/apps/authentication/errors.py index 8cea830e1..679c0b748 100644 --- a/apps/authentication/errors.py +++ b/apps/authentication/errors.py @@ -18,6 +18,7 @@ reason_user_not_exist = 'user_not_exist' reason_password_expired = 'password_expired' reason_user_invalid = 'user_invalid' reason_user_inactive = 'user_inactive' +reason_backend_not_match = 'backend_not_match' reason_choices = { reason_password_failed: _('Username/password check failed'), @@ -27,7 +28,8 @@ reason_choices = { reason_user_not_exist: _("Username does not exist"), reason_password_expired: _("Password expired"), reason_user_invalid: _('Disabled or expired'), - reason_user_inactive: _("This account is inactive.") + reason_user_inactive: _("This account is inactive."), + reason_backend_not_match: _("Auth backend not match") } old_reason_choices = { '0': '-', diff --git a/apps/authentication/mixins.py b/apps/authentication/mixins.py index 0466a9ee2..e4813ef3f 100644 --- a/apps/authentication/mixins.py +++ b/apps/authentication/mixins.py @@ -9,7 +9,7 @@ from django.contrib.auth import authenticate from django.shortcuts import reverse from django.contrib.auth import BACKEND_SESSION_KEY -from common.utils import get_object_or_none, get_request_ip, get_logger +from common.utils import get_object_or_none, get_request_ip, get_logger, bulk_get from users.models import User from users.utils import ( is_block_login, clean_failed_count @@ -24,6 +24,7 @@ logger = get_logger(__name__) class AuthMixin: request = None + partial_credential_error = None def get_user_from_session(self): if self.request.session.is_empty(): @@ -75,49 +76,75 @@ class AuthMixin: return rsa_decrypt(raw_passwd, rsa_private_key) except Exception as e: logger.error(e, exc_info=True) - logger.error(f'Decrypt password faild: password[{raw_passwd}] rsa_private_key[{rsa_private_key}]') + logger.error(f'Decrypt password failed: password[{raw_passwd}] ' + f'rsa_private_key[{rsa_private_key}]') return None return raw_passwd - def check_user_auth(self, decrypt_passwd=False): - self.check_is_block() + def raise_credential_error(self, error): + raise self.partial_credential_error(error=errors.reason_password_decrypt_failed) + + def get_auth_data(self, decrypt_passwd=False): request = self.request if hasattr(request, 'data'): data = request.data else: data = request.POST - username = data.get('username', '') - password = data.get('password', '') - challenge = data.get('challenge', '') - public_key = data.get('public_key', '') - ip = self.get_request_ip() - CredentialError = partial(errors.CredentialError, username=username, ip=ip, request=request) + items = ['username', 'password', 'challenge', 'public_key'] + username, password, challenge, public_key = bulk_get(data, *items, default='') + password = password + challenge.strip() + ip = self.get_request_ip() + self.partial_credential_error = partial(errors.CredentialError, username=username, ip=ip, request=request) if decrypt_passwd: password = self.decrypt_passwd(password) if not password: - raise CredentialError(error=errors.reason_password_decrypt_failed) + self.raise_credential_error(errors.reason_password_decrypt_failed) + return username, password, public_key, ip - user = authenticate(request, - username=username, - password=password + challenge.strip(), - public_key=public_key) + def _check_only_allow_exists_user_auth(self, username): + # 仅允许预先存在的用户认证 + if settings.ONLY_ALLOW_EXIST_USER_AUTH: + exist = User.objects.filter(username=username).exists() + if not exist: + logger.error(f"Only allow exist user auth, login failed: {username}") + self.raise_credential_error(errors.reason_user_not_exist) + def _check_auth_user_is_valid(self, username, password, public_key): + user = authenticate(self.request, username=username, password=password, public_key=public_key) if not user: - raise CredentialError(error=errors.reason_password_failed) + self.raise_credential_error(errors.reason_password_failed) elif user.is_expired: - raise CredentialError(error=errors.reason_user_inactive) + self.raise_credential_error(errors.reason_user_inactive) elif not user.is_active: - raise CredentialError(error=errors.reason_user_inactive) + self.raise_credential_error(errors.reason_user_inactive) + return user + def _check_auth_source_is_valid(self, user, auth_backend): + # 限制只能从认证来源登录 + if settings.ONLY_ALLOW_AUTH_FROM_SOURCE: + auth_backends_allowed = user.SOURCE_BACKEND_MAPPING.get(user.source) + if auth_backend not in auth_backends_allowed: + self.raise_credential_error(error=errors.reason_backend_not_match) + + def check_user_auth(self, decrypt_passwd=False): + self.check_is_block() + request = self.request + username, password, public_key, ip = self.get_auth_data(decrypt_passwd=decrypt_passwd) + + self._check_only_allow_exists_user_auth(username) + user = self._check_auth_user_is_valid(username, password, public_key) + # 限制只能从认证来源登录 + + auth_backend = getattr(user, 'backend', 'django.contrib.auth.backends.ModelBackend') + self._check_auth_source_is_valid(user, auth_backend) self._check_password_require_reset_or_not(user) self._check_passwd_is_too_simple(user, password) clean_failed_count(username, ip) request.session['auth_password'] = 1 request.session['user_id'] = str(user.id) - auth_backend = getattr(user, 'backend', 'django.contrib.auth.backends.ModelBackend') request.session['auth_backend'] = auth_backend return user diff --git a/apps/authentication/signals_handlers.py b/apps/authentication/signals_handlers.py index 8174f0db7..ca06a0433 100644 --- a/apps/authentication/signals_handlers.py +++ b/apps/authentication/signals_handlers.py @@ -24,7 +24,7 @@ def on_user_auth_login_success(sender, user, request, **kwargs): @receiver(openid_user_login_success) -def on_oidc_user_login_success(sender, request, user, **kwargs): +def on_oidc_user_login_success(sender, request, user, create=False, **kwargs): request.session[BACKEND_SESSION_KEY] = 'OIDCAuthCodeBackend' post_auth_success.send(sender, user=user, request=request) diff --git a/apps/authentication/views/login.py b/apps/authentication/views/login.py index 7d7f1e8da..93a6e35b6 100644 --- a/apps/authentication/views/login.py +++ b/apps/authentication/views/login.py @@ -45,9 +45,10 @@ class UserLoginView(mixins.AuthMixin, FormView): def get(self, request, *args, **kwargs): if request.user.is_staff: - return redirect(redirect_user_first_login_or_index( - request, self.redirect_field_name) + first_login_url = redirect_user_first_login_or_index( + request, self.redirect_field_name ) + return redirect(first_login_url) request.session.set_test_cookie() return super().get(request, *args, **kwargs) diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index 8ec390558..72e7edd26 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -272,5 +272,12 @@ class Time: last = timestamp +def bulk_get(d, *keys, default=None): + values = [] + for key in keys: + values.append(d.get(key, default)) + return values + + def isinstance_method(attr): - return isinstance(attr, type(Time().time)) + return isinstance(attr, type(Time().time)) \ No newline at end of file diff --git a/apps/jumpserver/conf.py b/apps/jumpserver/conf.py index 022ca84b0..b8adea181 100644 --- a/apps/jumpserver/conf.py +++ b/apps/jumpserver/conf.py @@ -282,6 +282,8 @@ class Config(dict): 'REFERER_CHECK_ENABLED': False, 'SERVER_REPLAY_STORAGE': {}, 'CONNECTION_TOKEN_ENABLED': False, + 'ONLY_ALLOW_EXIST_USER_AUTH': False, + 'ONLY_ALLOW_AUTH_FROM_SOURCE': True, 'DISK_CHECK_ENABLED': True, } diff --git a/apps/jumpserver/settings/auth.py b/apps/jumpserver/settings/auth.py index b0190e8a6..5c3c6dcf4 100644 --- a/apps/jumpserver/settings/auth.py +++ b/apps/jumpserver/settings/auth.py @@ -2,6 +2,7 @@ # import os import ldap +from django.utils.translation import ugettext_lazy as _ from ..const import CONFIG, PROJECT_DIR @@ -94,7 +95,7 @@ CAS_IGNORE_REFERER = True CAS_LOGOUT_COMPLETELY = CONFIG.CAS_LOGOUT_COMPLETELY CAS_VERSION = CONFIG.CAS_VERSION CAS_ROOT_PROXIED_AS = CONFIG.CAS_ROOT_PROXIED_AS -CAS_CHECK_NEXT = lambda: lambda _next_page: True +CAS_CHECK_NEXT = lambda _next_page: True # SSO Auth AUTH_SSO = CONFIG.AUTH_SSO @@ -105,17 +106,29 @@ TOKEN_EXPIRATION = CONFIG.TOKEN_EXPIRATION LOGIN_CONFIRM_ENABLE = CONFIG.LOGIN_CONFIRM_ENABLE OTP_IN_RADIUS = CONFIG.OTP_IN_RADIUS -AUTHENTICATION_BACKENDS = [ - 'authentication.backends.pubkey.PublicKeyAuthBackend', - 'django.contrib.auth.backends.ModelBackend', -] + +AUTH_BACKEND_MODEL = 'django.contrib.auth.backends.ModelBackend' +AUTH_BACKEND_PUBKEY = 'authentication.backends.pubkey.PublicKeyAuthBackend' +AUTH_BACKEND_LDAP = 'authentication.backends.ldap.LDAPAuthorizationBackend' +AUTH_BACKEND_OIDC_PASSWORD = 'jms_oidc_rp.backends.OIDCAuthPasswordBackend' +AUTH_BACKEND_OIDC_CODE = 'jms_oidc_rp.backends.OIDCAuthCodeBackend' +AUTH_BACKEND_RADIUS = 'authentication.backends.radius.RadiusBackend' +AUTH_BACKEND_CAS = 'authentication.backends.cas.CASBackend' +AUTH_BACKEND_SSO = 'authentication.backends.api.SSOAuthentication' + + +AUTHENTICATION_BACKENDS = [AUTH_BACKEND_MODEL, AUTH_BACKEND_PUBKEY] if AUTH_CAS: - AUTHENTICATION_BACKENDS.insert(0, 'authentication.backends.cas.CASBackend') + AUTHENTICATION_BACKENDS.insert(0, AUTH_BACKEND_CAS) if AUTH_OPENID: - AUTHENTICATION_BACKENDS.insert(0, 'jms_oidc_rp.backends.OIDCAuthPasswordBackend') - AUTHENTICATION_BACKENDS.insert(0, 'jms_oidc_rp.backends.OIDCAuthCodeBackend') + AUTHENTICATION_BACKENDS.insert(0, AUTH_BACKEND_OIDC_PASSWORD) + AUTHENTICATION_BACKENDS.insert(0, AUTH_BACKEND_OIDC_CODE) if AUTH_RADIUS: - AUTHENTICATION_BACKENDS.insert(0, 'authentication.backends.radius.RadiusBackend') + AUTHENTICATION_BACKENDS.insert(0, AUTH_BACKEND_RADIUS) if AUTH_SSO: - AUTHENTICATION_BACKENDS.insert(0, 'authentication.backends.api.SSOAuthentication') + AUTHENTICATION_BACKENDS.insert(0, AUTH_BACKEND_SSO) + + +ONLY_ALLOW_EXIST_USER_AUTH = CONFIG.ONLY_ALLOW_EXIST_USER_AUTH +ONLY_ALLOW_AUTH_FROM_SOURCE = CONFIG.ONLY_ALLOW_AUTH_FROM_SOURCE diff --git a/apps/settings/models.py b/apps/settings/models.py index 5617b010d..9d395656c 100644 --- a/apps/settings/models.py +++ b/apps/settings/models.py @@ -90,7 +90,7 @@ class Setting(models.Model): setting = cls.objects.filter(name='AUTH_LDAP').first() if not setting: return - ldap_backend = 'authentication.backends.ldap.LDAPAuthorizationBackend' + ldap_backend = settings.AUTH_BACKEND_LABEL_MAPPING['ldap'] backends = settings.AUTHENTICATION_BACKENDS has = ldap_backend in backends if setting.cleaned_value and not has: diff --git a/apps/users/forms/__init__.py b/apps/users/forms/__init__.py index 8b4d5d888..375794fa3 100644 --- a/apps/users/forms/__init__.py +++ b/apps/users/forms/__init__.py @@ -1,5 +1,3 @@ # -*- coding: utf-8 -*- # -from .user import * -from .group import * from .profile import * diff --git a/apps/users/forms/group.py b/apps/users/forms/group.py deleted file mode 100644 index 8d026a777..000000000 --- a/apps/users/forms/group.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# -from django import forms -from django.utils.translation import gettext_lazy as _ - -from orgs.mixins.forms import OrgModelForm -from ..models import User, UserGroup - -__all__ = ['UserGroupForm'] - - -class UserGroupForm(OrgModelForm): - users = forms.ModelMultipleChoiceField( - queryset=User.objects.none(), - label=_("User"), - widget=forms.SelectMultiple( - attrs={ - 'class': 'users-select2', - 'data-placeholder': _('Select users') - } - ), - required=False, - ) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.set_fields_queryset() - - def set_fields_queryset(self): - users_field = self.fields.get('users') - if self.instance: - users_field.initial = self.instance.users.all() - users_field.queryset = self.instance.users.all() - else: - users_field.queryset = User.objects.none() - - def save(self, commit=True): - raise Exception("Save by restful api") - - class Meta: - model = UserGroup - fields = [ - 'name', 'users', 'comment', - ] diff --git a/apps/users/forms/profile.py b/apps/users/forms/profile.py index 775e33b1f..63c1078d6 100644 --- a/apps/users/forms/profile.py +++ b/apps/users/forms/profile.py @@ -12,9 +12,21 @@ __all__ = [ 'UserProfileForm', 'UserMFAForm', 'UserFirstLoginFinishForm', 'UserPasswordForm', 'UserPublicKeyForm', 'FileForm', 'UserTokenResetPasswordForm', 'UserForgotPasswordForm', + 'UserCheckPasswordForm', 'UserCheckOtpCodeForm' ] +class UserCheckPasswordForm(forms.Form): + password = forms.CharField( + label=_('Password'), widget=forms.PasswordInput, + max_length=128, strip=False + ) + + +class UserCheckOtpCodeForm(forms.Form): + otp_code = forms.CharField(label=_('MFA code'), max_length=6) + + class UserProfileForm(forms.ModelForm): username = forms.CharField(disabled=True, label=_("Username")) name = forms.CharField(disabled=True, label=_("Name")) diff --git a/apps/users/forms/user.py b/apps/users/forms/user.py deleted file mode 100644 index 299862b5e..000000000 --- a/apps/users/forms/user.py +++ /dev/null @@ -1,199 +0,0 @@ - -from django import forms -from django.utils.translation import gettext_lazy as _ - -from common.utils import validate_ssh_public_key -from orgs.mixins.forms import OrgModelForm -from ..models import User -from ..utils import ( - check_password_rules, get_current_org_members, get_source_choices -) - - -__all__ = [ - 'UserCreateForm', 'UserUpdateForm', 'UserBulkUpdateForm', - 'UserCheckOtpCodeForm', 'UserCheckPasswordForm' -] - - -class UserCreateUpdateFormMixin(OrgModelForm): - role_choices = ((i, n) for i, n in User.ROLE.choices if i != User.ROLE.APP) - password = forms.CharField( - label=_('Password'), widget=forms.PasswordInput, - max_length=128, strip=False, required=False, - ) - role = forms.ChoiceField( - choices=role_choices, required=True, - initial=User.ROLE.USER, label=_("Role") - ) - source = forms.ChoiceField( - choices=get_source_choices, required=True, - initial=User.Source.local.value, label=_("Source") - ) - public_key = forms.CharField( - label=_('ssh public key'), max_length=5000, required=False, - widget=forms.Textarea(attrs={'placeholder': _('ssh-rsa AAAA...')}), - help_text=_('Paste user id_rsa.pub here.') - ) - - class Meta: - model = User - fields = [ - 'username', 'name', 'email', 'groups', 'wechat', - 'source', 'phone', 'role', 'date_expired', - 'comment', 'mfa_level' - ] - widgets = { - 'mfa_level': forms.RadioSelect(), - 'groups': forms.SelectMultiple( - attrs={ - 'class': 'select2', - 'data-placeholder': _('Join user groups') - } - ) - } - - def __init__(self, *args, **kwargs): - self.request = kwargs.pop("request", None) - super(UserCreateUpdateFormMixin, self).__init__(*args, **kwargs) - - roles = [] - # Super admin user - if self.request.user.is_superuser: - roles.append((User.ROLE.ADMIN, User.ROLE.ADMIN.label)) - roles.append((User.ROLE.USER, User.ROLE.USER.label)) - roles.append((User.ROLE.AUDITOR, User.ROLE.AUDITOR.label)) - - # Org admin user - else: - user = kwargs.get('instance') - # Update - if user: - role = kwargs.get('instance').role - roles.append((role, User.ROLE[role])) - # Create - else: - roles.append((User.ROLE.USER, User.ROLE.USER.label)) - - field = self.fields['role'] - field.choices = set(roles) - - def clean_public_key(self): - public_key = self.cleaned_data['public_key'] - if not public_key: - return public_key - if self.instance.public_key and public_key == self.instance.public_key: - msg = _('Public key should not be the same as your old one.') - raise forms.ValidationError(msg) - - if not validate_ssh_public_key(public_key): - raise forms.ValidationError(_('Not a valid ssh public key')) - return public_key - - def clean_password(self): - password_strategy = self.data.get('password_strategy') - # 创建-不设置密码 - if password_strategy == '0': - return - password = self.data.get('password') - # 更新-密码为空 - if password_strategy is None and not password: - return - if not check_password_rules(password): - msg = _('* Your password does not meet the requirements') - raise forms.ValidationError(msg) - return password - - def save(self, commit=True): - password = self.cleaned_data.get('password') - mfa_level = self.cleaned_data.get('mfa_level') - public_key = self.cleaned_data.get('public_key') - user = super().save(commit=commit) - if password: - user.reset_password(password) - if mfa_level: - user.mfa_level = mfa_level - user.save() - if public_key: - user.public_key = public_key - user.save() - return user - - -class UserCreateForm(UserCreateUpdateFormMixin): - EMAIL_SET_PASSWORD = _('Reset link will be generated and sent to the user') - CUSTOM_PASSWORD = _('Set password') - PASSWORD_STRATEGY_CHOICES = ( - (0, EMAIL_SET_PASSWORD), - (1, CUSTOM_PASSWORD) - ) - password_strategy = forms.ChoiceField( - choices=PASSWORD_STRATEGY_CHOICES, required=True, initial=0, - widget=forms.RadioSelect(), label=_('Password strategy') - ) - - -class UserUpdateForm(UserCreateUpdateFormMixin): - pass - - -class UserBulkUpdateForm(OrgModelForm): - users = forms.ModelMultipleChoiceField( - required=True, - label=_('Select users'), - queryset=User.objects.none(), - widget=forms.SelectMultiple( - attrs={ - 'class': 'users-select2', - 'data-placeholder': _('Select users') - } - ) - ) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.set_fields_queryset() - - def set_fields_queryset(self): - users_field = self.fields['users'] - users_field.queryset = get_current_org_members() - - class Meta: - model = User - fields = ['users', 'groups', 'date_expired'] - widgets = { - "groups": forms.SelectMultiple( - attrs={ - 'class': 'select2', - 'data-placeholder': _('User group') - } - ) - } - - def save(self, commit=True): - changed_fields = [] - for field in self._meta.fields: - if self.data.get(field) is not None: - changed_fields.append(field) - - cleaned_data = {k: v for k, v in self.cleaned_data.items() - if k in changed_fields} - users = cleaned_data.pop('users', '') - groups = cleaned_data.pop('groups', []) - users = User.objects.filter(id__in=[user.id for user in users]) - users.update(**cleaned_data) - if groups: - for user in users: - user.groups.set(groups) - return users - - -class UserCheckPasswordForm(forms.Form): - password = forms.CharField( - label=_('Password'), widget=forms.PasswordInput, - max_length=128, strip=False - ) - - -class UserCheckOtpCodeForm(forms.Form): - otp_code = forms.CharField(label=_('MFA code'), max_length=6) diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 9cdd49ee7..b3b4e9d1c 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -514,6 +514,14 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser): radius = 'radius', 'Radius' cas = 'cas', 'CAS' + SOURCE_BACKEND_MAPPING = { + Source.local: [settings.AUTH_BACKEND_MODEL, settings.AUTH_BACKEND_PUBKEY], + Source.ldap: [settings.AUTH_BACKEND_LDAP], + Source.openid: [settings.AUTH_BACKEND_OIDC_PASSWORD, settings.AUTH_BACKEND_OIDC_CODE], + Source.radius: [settings.AUTH_BACKEND_RADIUS], + Source.cas: [settings.AUTH_BACKEND_CAS], + } + id = models.UUIDField(default=uuid.uuid4, primary_key=True) username = models.CharField( max_length=128, unique=True, verbose_name=_('Username') @@ -562,7 +570,8 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser): max_length=30, default='', blank=True, verbose_name=_('Created by') ) source = models.CharField( - max_length=30, default=Source.local.value, choices=Source.choices, + max_length=30, default=Source.local, + choices=Source.choices, verbose_name=_('Source') ) date_password_last_updated = models.DateTimeField( @@ -570,8 +579,6 @@ class User(AuthMixin, TokenMixin, RoleMixin, MFAMixin, AbstractUser): verbose_name=_('Date password last updated') ) - user_cache_key_prefix = '_User_{}' - def __str__(self): return '{0.name}({0.username})'.format(self) diff --git a/apps/users/signals_handler.py b/apps/users/signals_handler.py index 09320a2e1..887531f45 100644 --- a/apps/users/signals_handler.py +++ b/apps/users/signals_handler.py @@ -4,6 +4,7 @@ from django.dispatch import receiver from django_auth_ldap.backend import populate_user from django.conf import settings +from django.core.exceptions import PermissionDenied from django_cas_ng.signals import cas_user_authenticated from jms_oidc_rp.signals import openid_create_or_update_user @@ -27,6 +28,9 @@ def on_user_create(sender, user=None, **kwargs): @receiver(cas_user_authenticated) def on_cas_user_authenticated(sender, user, created, **kwargs): + if created and settings.ONLY_ALLOW_EXIST_USER_AUTH: + user.delete() + raise PermissionDenied(f'Not allow non-exist user auth: {user.username}') if created: user.source = user.Source.cas.value user.save() @@ -43,6 +47,10 @@ def on_ldap_create_user(sender, user, ldap_user, **kwargs): @receiver(openid_create_or_update_user) def on_openid_create_or_update_user(sender, request, user, created, name, username, email, **kwargs): + if created and settings.ONLY_ALLOW_EXIST_USER_AUTH: + user.delete() + raise PermissionDenied(f'Not allow non-exist user auth: {username}') + if created: logger.debug( "Receive OpenID user created signal: {}, " diff --git a/apps/users/utils.py b/apps/users/utils.py index 0bef4fcc2..960848f08 100644 --- a/apps/users/utils.py +++ b/apps/users/utils.py @@ -382,22 +382,6 @@ def get_current_org_members(exclude=()): return current_org.get_members(exclude=exclude) -def get_source_choices(): - from .models import User - choices = [ - (User.Source.local.value, User.Source.local.label), - ] - if settings.AUTH_LDAP: - choices.append((User.Source.ldap.value, User.Source.ldap.label)) - if settings.AUTH_OPENID: - choices.append((User.Source.openid.value, User.Source.openid.label)) - if settings.AUTH_RADIUS: - choices.append((User.Source.radius.value, User.Source.radius.label)) - if settings.AUTH_CAS: - choices.append((User.Source.cas.value, User.Source.cas.label)) - return choices - - def is_auth_time_valid(session, key): return True if session.get(key, 0) > time.time() else False diff --git a/apps/users/views/profile/otp.py b/apps/users/views/profile/otp.py index caed50532..f0e938170 100644 --- a/apps/users/views/profile/otp.py +++ b/apps/users/views/profile/otp.py @@ -51,6 +51,9 @@ class UserOtpEnableInstallAppView(TemplateView): return super().get_context_data(**kwargs) + + + class UserOtpEnableBindView(AuthMixin, TemplateView, FormView): template_name = 'users/user_otp_enable_bind.html' form_class = forms.UserCheckOtpCodeForm From bc3e50a529667e6b3ff211d29e5629e37c4d279f Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 1 Mar 2021 14:31:27 +0800 Subject: [PATCH 30/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E6=9B=B4=E6=94=B9=E5=BC=95=E8=B5=B7=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/settings/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/settings/models.py b/apps/settings/models.py index 9d395656c..0a986efcb 100644 --- a/apps/settings/models.py +++ b/apps/settings/models.py @@ -90,7 +90,7 @@ class Setting(models.Model): setting = cls.objects.filter(name='AUTH_LDAP').first() if not setting: return - ldap_backend = settings.AUTH_BACKEND_LABEL_MAPPING['ldap'] + ldap_backend = settings.AUTH_BACKEND_LDAP backends = settings.AUTHENTICATION_BACKENDS has = ldap_backend in backends if setting.cleaned_value and not has: From 19043d0a660d1de9fa45fa91444947bcccd657b2 Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 1 Mar 2021 11:24:25 +0800 Subject: [PATCH 31/71] =?UTF-8?q?perf:=20=E6=B7=BB=E5=8A=A0=20xrdp=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/const.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/terminal/const.py b/apps/terminal/const.py index 488726c39..9d43c450c 100644 --- a/apps/terminal/const.py +++ b/apps/terminal/const.py @@ -41,6 +41,7 @@ class TerminalTypeChoices(TextChoices): koko = 'koko', 'KoKo' guacamole = 'guacamole', 'Guacamole' omnidb = 'omnidb', 'OmniDB' + xrdp = 'xrdp', 'xrdp' @classmethod def types(cls): From 5de5fa2e964e05926179be14107b565742fdc67e Mon Sep 17 00:00:00 2001 From: ibuler Date: Mon, 1 Mar 2021 14:11:27 +0800 Subject: [PATCH 32/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E4=B8=8D=E5=88=B0=20org=20=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/drf/filters.py | 5 ++++- apps/common/utils/lock.py | 2 +- apps/jumpserver/urls.py | 2 +- apps/orgs/mixins/models.py | 7 ------- apps/orgs/models.py | 7 +++++-- apps/orgs/utils.py | 2 +- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index 9d8ec087e..f130af3b7 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -11,7 +11,10 @@ import logging from common import const -__all__ = ["DatetimeRangeFilter", "IDSpmFilter", 'IDInFilter', "CustomFilter"] +__all__ = [ + "DatetimeRangeFilter", "IDSpmFilter", 'IDInFilter', "CustomFilter", + "BaseFilterSet" +] class BaseFilterSet(drf_filters.FilterSet): diff --git a/apps/common/utils/lock.py b/apps/common/utils/lock.py index ffb6217e3..ecb37c31a 100644 --- a/apps/common/utils/lock.py +++ b/apps/common/utils/lock.py @@ -7,7 +7,7 @@ from django.db import transaction from common.utils import get_logger from common.utils.inspect import copy_function_args -from apps.jumpserver.const import CONFIG +from jumpserver.const import CONFIG from common.local import thread_local logger = get_logger(__file__) diff --git a/apps/jumpserver/urls.py b/apps/jumpserver/urls.py index 306b1d9ea..5b8202854 100644 --- a/apps/jumpserver/urls.py +++ b/apps/jumpserver/urls.py @@ -39,7 +39,7 @@ if settings.XPACK_ENABLED: apps = [ 'users', 'assets', 'perms', 'terminal', 'ops', 'audits', 'orgs', 'auth', - 'applications', 'tickets', 'settings', 'xpack' + 'applications', 'tickets', 'settings', 'xpack', 'flower', 'luna', 'koko', 'ws', 'docs', 'redocs', ] diff --git a/apps/orgs/mixins/models.py b/apps/orgs/mixins/models.py index f2f6ae49e..c0823bb74 100644 --- a/apps/orgs/mixins/models.py +++ b/apps/orgs/mixins/models.py @@ -37,13 +37,6 @@ class OrgManager(models.Manager): queryset = super(OrgManager, self).get_queryset() return filter_org_queryset(queryset) - def all(self): - if not current_org: - msg = 'You can `objects.set_current_org(org).all()` then run it' - return self - else: - return super(OrgManager, self).all() - def set_current_org(self, org): if isinstance(org, str): org = Organization.get_instance(org) diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 1fbae50ce..679dd5fbc 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -78,8 +78,11 @@ class Organization(models.Model): else: org = cls.objects.get(name=id_or_name) org.set_to_cache() - except cls.DoesNotExist: - org = cls.default() if default else None + except cls.DoesNotExist as e: + if default: + return cls.default() + else: + raise e return org def get_org_members_by_role(self, role): diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index eef12b1c6..de9e2982c 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -55,7 +55,7 @@ def _find(attr): def get_current_org(): org_id = get_current_org_id() if org_id is None: - return None + return Organization.root() org = Organization.get_instance(org_id) return org From 1036d1c1321f08a14a33bbae6d1ce9efcaaac566 Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 25 Feb 2021 14:45:21 +0800 Subject: [PATCH 33/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=8E=88?= =?UTF-8?q?=E6=9D=83=E6=A0=91=E4=B8=80=E4=BA=9B=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/node.py | 6 +- apps/common/db/models.py | 84 +++++++++ apps/common/utils/common.py | 4 - apps/common/utils/lock.py | 28 +-- .../migrations/0010_auto_20210226_1536.py | 18 ++ apps/orgs/models.py | 2 +- .../api/asset/asset_permission_relation.py | 12 +- apps/perms/api/asset/user_permission/mixin.py | 2 +- apps/perms/filters.py | 44 +++-- apps/perms/locks.py | 8 +- apps/perms/models/asset_permission.py | 14 -- apps/perms/models/base.py | 14 +- apps/perms/utils/asset/user_permission.py | 160 ++++++------------ 13 files changed, 214 insertions(+), 182 deletions(-) create mode 100644 apps/orgs/migrations/0010_auto_20210226_1536.py diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 85d857a7c..eb326859b 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -337,9 +337,9 @@ class NodeAllAssetsMappingMixin: t1 = time.time() with tmp_to_org(org_id): - nodes_id_key = Node.objects.filter(org_id=org_id) \ - .annotate(char_id=output_as_string('id')) \ - .values_list('char_id', 'key') + nodes_id_key = Node.objects.annotate( + char_id=output_as_string('id') + ).values_list('char_id', 'key') # * 直接取出全部. filter(node__org_id=org_id)(大规模下会更慢) nodes_assets_id = Asset.nodes.through.objects.all() \ diff --git a/apps/common/db/models.py b/apps/common/db/models.py index df5d6a46d..77a49a683 100644 --- a/apps/common/db/models.py +++ b/apps/common/db/models.py @@ -10,8 +10,11 @@ """ import uuid +from functools import reduce, partial +import inspect from django.db.models import * +from django.db.models import QuerySet from django.db.models.functions import Concat from django.utils.translation import ugettext_lazy as _ @@ -86,3 +89,84 @@ def concated_display(name1, name2): def output_as_string(field_name): return ExpressionWrapper(F(field_name), output_field=CharField()) + + +class UnionQuerySet(QuerySet): + after_union = ['order_by'] + not_return_qs = [ + 'query', 'get', 'create', 'get_or_create', + 'update_or_create', 'bulk_create', 'count', + 'latest', 'earliest', 'first', 'last', 'aggregate', + 'exists', 'update', 'delete', 'as_manager', 'explain', + ] + + def __init__(self, *queryset_list): + self.queryset_list = queryset_list + self.after_union_items = [] + self.before_union_items = [] + + def __execute(self): + queryset_list = [] + for qs in self.queryset_list: + for attr, args, kwargs in self.before_union_items: + qs = getattr(qs, attr)(*args, **kwargs) + queryset_list.append(qs) + union_qs = reduce(lambda x, y: x.union(y), queryset_list) + for attr, args, kwargs in self.after_union_items: + union_qs = getattr(union_qs, attr)(*args, **kwargs) + return union_qs + + def __before_union_perform(self, item, *args, **kwargs): + self.before_union_items.append((item, args, kwargs)) + return self.__clone(*self.queryset_list) + + def __after_union_perform(self, item, *args, **kwargs): + self.after_union_items.append((item, args, kwargs)) + return self.__clone(*self.queryset_list) + + def __clone(self, *queryset_list): + uqs = UnionQuerySet(*queryset_list) + uqs.after_union_items = self.after_union_items + uqs.before_union_items = self.before_union_items + return uqs + + def __getattribute__(self, item): + if item.startswith('__') or item in UnionQuerySet.__dict__ or item in [ + 'queryset_list', 'after_union_items', 'before_union_items' + ]: + return object.__getattribute__(self, item) + + if item in UnionQuerySet.not_return_qs: + return getattr(self.__execute(), item) + + origin_item = object.__getattribute__(self, 'queryset_list')[0] + origin_attr = getattr(origin_item, item, None) + if not inspect.ismethod(origin_attr): + return getattr(self.__execute(), item) + + if item in UnionQuerySet.after_union: + attr = partial(self.__after_union_perform, item) + else: + attr = partial(self.__before_union_perform, item) + return attr + + def __getitem__(self, item): + return self.__execute()[item] + + def __iter__(self): + return iter(self.__execute()) + + def __str__(self): + return str(self.__execute()) + + def __repr__(self): + return repr(self.__execute()) + + @classmethod + def test_it(cls): + from assets.models import Asset + assets1 = Asset.objects.filter(hostname__startswith='a') + assets2 = Asset.objects.filter(hostname__startswith='b') + + qs = cls(assets1, assets2) + return qs diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index 72e7edd26..f9c488e18 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -277,7 +277,3 @@ def bulk_get(d, *keys, default=None): for key in keys: values.append(d.get(key, default)) return values - - -def isinstance_method(attr): - return isinstance(attr, type(Time().time)) \ No newline at end of file diff --git a/apps/common/utils/lock.py b/apps/common/utils/lock.py index ecb37c31a..41b6376cf 100644 --- a/apps/common/utils/lock.py +++ b/apps/common/utils/lock.py @@ -93,21 +93,21 @@ class DistributedLock(RedisLock): if self.locked_by_current_thread(): self._acquired_reentrant_lock = True logger.debug( - f'I[{self.id}] reentry lock[{self.name}] in thread[{self._thread_id}].') + f'Reentry lock ok: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}') return True - logger.debug(f'I[{self.id}] attempt acquire reentrant-lock[{self.name}].') + logger.debug(f'Attempt acquire reentrant-lock: lock_id={self.id} lock={self.name} thread={self._thread_id}') acquired = super().acquire(blocking=blocking, timeout=timeout) if acquired: - logger.debug(f'I[{self.id}] acquired reentrant-lock[{self.name}] now.') + logger.debug(f'Acquired reentrant-lock ok: lock_id={self.id} lock={self.name} thread={self._thread_id}') setattr(thread_local, self.name, self.id) else: - logger.debug(f'I[{self.id}] acquired reentrant-lock[{self.name}] failed.') + logger.debug(f'Acquired reentrant-lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id}') return acquired else: - logger.debug(f'I[{self.id}] attempt acquire lock[{self.name}].') + logger.debug(f'Attempt acquire lock: lock_id={self.id} lock={self.name} thread={self._thread_id}') acquired = super().acquire(blocking=blocking, timeout=timeout) - logger.debug(f'I[{self.id}] acquired lock[{self.name}] {acquired}.') + logger.debug(f'Acquired lock: ok={acquired} lock_id={self.id} lock={self.name} thread={self._thread_id}') return acquired @property @@ -126,17 +126,17 @@ class DistributedLock(RedisLock): def _release_on_reentrant_locked_by_brother(self): if self._acquired_reentrant_lock: self._acquired_reentrant_lock = False - logger.debug(f'I[{self.id}] released reentrant-lock[{self.name}] owner[{self.get_owner_id()}] in thread[{self._thread_id}]') + logger.debug(f'Released reentrant-lock: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}') return else: - self._raise_exc_with_log(f'Reentrant-lock[{self.name}] is not acquired by me[{self.id}].') + self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}') def _release_on_reentrant_locked_by_me(self): - logger.debug(f'I[{self.id}] release reentrant-lock[{self.name}] in thread[{self._thread_id}]') + logger.debug(f'Release reentrant-lock locked by me: lock_id={self.id} lock={self.name} thread={self._thread_id}') id = getattr(thread_local, self.name, None) if id != self.id: - raise PermissionError(f'Reentrant-lock[{self.name}] is not locked by me[{self.id}], owner[{id}]') + raise PermissionError(f'Reentrant-lock is not locked by me: lock_id={self.id} owner_id={self.get_owner_id()} lock={self.name} thread={self._thread_id}') try: # 这里要保证先删除 thread_local 的标记， delattr(thread_local, self.name) @@ -158,9 +158,9 @@ class DistributedLock(RedisLock): def _release(self): try: self._release_redis_lock() - logger.debug(f'I[{self.id}] released lock[{self.name}]') + logger.debug(f'Released lock: lock_id={self.id} lock={self.name} thread={self._thread_id}') except NotAcquired as e: - logger.error(f'I[{self.id}] release lock[{self.name}] failed {e}') + logger.error(f'Release lock failed: lock_id={self.id} lock={self.name} thread={self._thread_id} error: {e}') self._raise_exc(e) def release(self): @@ -174,11 +174,11 @@ class DistributedLock(RedisLock): else: _release = self._release_on_reentrant_locked_by_brother else: - self._raise_exc_with_log(f'Reentrant-lock[{self.name}] is not acquired in current-thread[{self._thread_id}]') + self._raise_exc_with_log(f'Reentrant-lock is not acquired: lock_id={self.id} lock={self.name} thread={self._thread_id}') # 处理是否在事务提交时才释放锁 if self._release_on_transaction_commit: - logger.debug(f'I[{self.id}] release lock[{self.name}] on transaction commit ...') + logger.debug(f'Release lock on transaction commit ... :lock_id={self.id} lock={self.name} thread={self._thread_id}') transaction.on_commit(_release) else: _release() diff --git a/apps/orgs/migrations/0010_auto_20210226_1536.py b/apps/orgs/migrations/0010_auto_20210226_1536.py new file mode 100644 index 000000000..b9abe32e7 --- /dev/null +++ b/apps/orgs/migrations/0010_auto_20210226_1536.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2021-02-26 07:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orgs', '0009_auto_20201023_1628'), + ] + + operations = [ + migrations.AlterField( + model_name='organizationmember', + name='role', + field=models.CharField(choices=[('Admin', 'Organization administrator'), ('Auditor', 'Organization auditor'), ('User', 'User')], db_index=True, default='User', max_length=16, verbose_name='Role'), + ), + ] diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 679dd5fbc..4d3850181 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -424,7 +424,7 @@ class OrganizationMember(models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True) org = models.ForeignKey(Organization, related_name='m2m_org_members', on_delete=models.CASCADE, verbose_name=_('Organization')) user = models.ForeignKey('users.User', related_name='m2m_org_members', on_delete=models.CASCADE, verbose_name=_('User')) - role = models.CharField(max_length=16, choices=ROLE.choices, default=ROLE.USER, verbose_name=_("Role")) + role = models.CharField(db_index=True, max_length=16, choices=ROLE.choices, default=ROLE.USER, verbose_name=_("Role")) date_created = models.DateTimeField(auto_now_add=True, verbose_name=_("Date created")) date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) diff --git a/apps/perms/api/asset/asset_permission_relation.py b/apps/perms/api/asset/asset_permission_relation.py index 4deebeb61..ed3c30964 100644 --- a/apps/perms/api/asset/asset_permission_relation.py +++ b/apps/perms/api/asset/asset_permission_relation.py @@ -13,6 +13,7 @@ from orgs.utils import current_org from common.permissions import IsOrgAdmin from perms import serializers from perms import models +from perms.utils.asset.user_permission import UserGrantedAssetsQueryUtils __all__ = [ 'AssetPermissionUserRelationViewSet', 'AssetPermissionUserGroupRelationViewSet', @@ -103,15 +104,8 @@ class AssetPermissionAllAssetListApi(generics.ListAPIView): def get_queryset(self): pk = self.kwargs.get("pk") - perm = get_object_or_404(models.AssetPermission, pk=pk) - - asset_q = Q(granted_by_permissions=perm) - granted_node_keys = Node.objects.filter(granted_by_permissions=perm).distinct().values_list('key', flat=True) - for key in granted_node_keys: - asset_q |= Q(nodes__key__startswith=f'{key}:') - asset_q |= Q(nodes__key=key) - - assets = Asset.objects.filter(asset_q).only(*self.serializer_class.Meta.only_fields).distinct() + query_utils = UserGrantedAssetsQueryUtils(None, asset_perm_ids=[pk]) + assets = query_utils.get_all_granted_assets() return assets diff --git a/apps/perms/api/asset/user_permission/mixin.py b/apps/perms/api/asset/user_permission/mixin.py index 5c5db7729..3b8733b3b 100644 --- a/apps/perms/api/asset/user_permission/mixin.py +++ b/apps/perms/api/asset/user_permission/mixin.py @@ -13,7 +13,7 @@ from perms.utils.asset.user_permission import UserGrantedTreeRefreshController class PermBaseMixin: user: User - def get(self, request, *args, **kwargs): + def get(self, request: Request, *args, **kwargs): force = is_true(request.query_params.get('rebuild_tree')) controller = UserGrantedTreeRefreshController(self.user) controller.refresh_if_need(force) diff --git a/apps/perms/filters.py b/apps/perms/filters.py index 4500c8836..de7d12ebe 100644 --- a/apps/perms/filters.py +++ b/apps/perms/filters.py @@ -1,6 +1,7 @@ from django_filters import rest_framework as filters from django.db.models import QuerySet, Q +from common.db.models import UnionQuerySet from common.drf.filters import BaseFilterSet from common.utils import get_object_or_none from users.models import User, UserGroup @@ -134,13 +135,15 @@ class AssetPermissionFilter(PermissionBaseFilter): if not _nodes: return queryset.none() + node = _nodes.get() + if not is_query_all: - queryset = queryset.filter(nodes__in=_nodes) + queryset = queryset.filter(nodes=node) return queryset - nodes = set(_nodes) - for node in _nodes: - nodes |= set(node.get_ancestors(with_self=True)) - queryset = queryset.filter(nodes__in=nodes) + nodeids = node.get_ancestors(with_self=True).values_list('id', flat=True) + nodeids = list(nodeids) + + queryset = queryset.filter(nodes__in=nodeids) return queryset def filter_asset(self, queryset): @@ -159,21 +162,26 @@ class AssetPermissionFilter(PermissionBaseFilter): return queryset if not assets: return queryset.none() - if not is_query_all: - queryset = queryset.filter(assets__in=assets) - return queryset - inherit_all_nodes = set() - inherit_nodes_keys = assets.all().values_list('nodes__key', flat=True) + asset = assets.get() - for key in inherit_nodes_keys: - if key is None: - continue + if not is_query_all: + queryset = queryset.filter(assets=asset) + return queryset + inherit_all_nodekeys = set() + inherit_nodekeys = asset.nodes.values_list('key', flat=True) + + for key in inherit_nodekeys: ancestor_keys = Node.get_node_ancestor_keys(key, with_self=True) - inherit_all_nodes.update(ancestor_keys) - queryset = queryset.filter( - Q(assets__in=assets) | Q(nodes__key__in=inherit_all_nodes) - ).distinct() - return queryset + inherit_all_nodekeys.update(ancestor_keys) + + inherit_all_nodeids = Node.objects.filter(key__in=inherit_all_nodekeys).values_list('id', flat=True) + inherit_all_nodeids = list(inherit_all_nodeids) + + qs1 = queryset.filter(assets=asset).distinct() + qs2 = queryset.filter(nodes__id__in=inherit_all_nodeids).distinct() + + qs = UnionQuerySet(qs1, qs2) + return qs def filter_effective(self, queryset): is_effective = self.get_query_param('is_effective') diff --git a/apps/perms/locks.py b/apps/perms/locks.py index e1bd67f09..a6ffa6b98 100644 --- a/apps/perms/locks.py +++ b/apps/perms/locks.py @@ -2,10 +2,10 @@ from common.utils.lock import DistributedLock class UserGrantedTreeRebuildLock(DistributedLock): - name_template = 'perms.user.asset.node.tree.rebuid..' + name_template = 'perms.user.asset.node.tree.rebuid.' - def __init__(self, org_id, user_id): + def __init__(self, user_id): name = self.name_template.format( - org_id=org_id, user_id=user_id + user_id=user_id ) - super().__init__(name=name) + super().__init__(name=name, release_on_transaction_commit=True) diff --git a/apps/perms/models/asset_permission.py b/apps/perms/models/asset_permission.py index e5a03c879..fb54ad1a6 100644 --- a/apps/perms/models/asset_permission.py +++ b/apps/perms/models/asset_permission.py @@ -193,25 +193,11 @@ class PermNode(Node): node_from = '' granted_assets_amount = 0 - # 提供可以设置 资产数量的字段 - _assets_amount = None - annotate_granted_node_rel_fields = { 'granted_assets_amount': F('granted_node_rels__node_assets_amount'), 'node_from': F('granted_node_rels__node_from') } - @property - def assets_amount(self): - _assets_amount = getattr(self, '_assets_amount') - if isinstance(_assets_amount, int): - return _assets_amount - return super().assets_amount - - @assets_amount.setter - def assets_amount(self, value): - self._assets_amount = value - def use_granted_assets_amount(self): self.assets_amount = self.granted_assets_amount diff --git a/apps/perms/models/base.py b/apps/perms/models/base.py index f5adeb838..7651a06e8 100644 --- a/apps/perms/models/base.py +++ b/apps/perms/models/base.py @@ -8,6 +8,7 @@ from django.db.models import Q from django.utils import timezone from orgs.mixins.models import OrgModelMixin +from common.db.models import UnionQuerySet from common.utils import date_expired_default, lazyproperty from orgs.mixins.models import OrgManager @@ -100,10 +101,15 @@ class BasePermission(OrgModelMixin): from users.models import User users_id = self.users.all().values_list('id', flat=True) groups_id = self.user_groups.all().values_list('id', flat=True) - users = User.objects.filter( - Q(id__in=users_id) | Q(groups__id__in=groups_id) - ).distinct() - return users + + users_id = list(users_id) + groups_id = list(groups_id) + + qs1 = User.objects.filter(id__in=users_id).distinct() + qs2 = User.objects.filter(groups__id__in=groups_id).distinct() + + qs = UnionQuerySet(qs1, qs2) + return qs @lazyproperty def users_amount(self): diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 7f8d53c7e..c6872c6e1 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -1,18 +1,16 @@ from collections import defaultdict from typing import List, Tuple -from functools import reduce, partial -from common.utils import isinstance_method from django.core.cache import cache from django.conf import settings from django.db.models import Q, QuerySet -from common.db.models import output_as_string -from common.utils.common import lazyproperty, timeit, Time +from common.db.models import output_as_string, UnionQuerySet +from common.utils.common import lazyproperty, timeit from assets.utils import NodeAssetsUtil from common.utils import get_logger from common.decorator import on_transaction_commit -from orgs.utils import tmp_to_org, current_org, ensure_in_real_or_default_org +from orgs.utils import tmp_to_org, current_org, ensure_in_real_or_default_org, tmp_to_root_org from assets.models import ( Asset, FavoriteAsset, AssetQuerySet, NodeQuerySet ) @@ -50,84 +48,10 @@ def get_user_all_asset_perm_ids(user) -> set: asset_perm_ids = AssetPermission.objects.filter( id__in=asset_perm_ids).valid().values_list('id', flat=True) + asset_perm_ids = set(asset_perm_ids) return asset_perm_ids -class UnionQuerySet(QuerySet): - after_union = ['order_by'] - not_return_qs = [ - 'query', 'get', 'create', 'get_or_create', - 'update_or_create', 'bulk_create', 'count', - 'latest', 'earliest', 'first', 'last', 'aggregate', - 'exists', 'update', 'delete', 'as_manager', 'explain', - ] - - def __init__(self, *queryset_list): - self.queryset_list = queryset_list - self.after_union_items = [] - self.before_union_items = [] - - def __execute(self): - queryset_list = [] - for qs in self.queryset_list: - for attr, args, kwargs in self.before_union_items: - qs = getattr(qs, attr)(*args, **kwargs) - queryset_list.append(qs) - union_qs = reduce(lambda x, y: x.union(y), queryset_list) - for attr, args, kwargs in self.after_union_items: - union_qs = getattr(union_qs, attr)(*args, **kwargs) - return union_qs - - def __before_union_perform(self, item, *args, **kwargs): - self.before_union_items.append((item, args, kwargs)) - return self.__clone(*self.queryset_list) - - def __after_union_perform(self, item, *args, **kwargs): - self.after_union_items.append((item, args, kwargs)) - return self.__clone(*self.queryset_list) - - def __clone(self, *queryset_list): - uqs = UnionQuerySet(*queryset_list) - uqs.after_union_items = self.after_union_items - uqs.before_union_items = self.before_union_items - return uqs - - def __getattribute__(self, item): - if item.startswith('__') or item in UnionQuerySet.__dict__ or item in [ - 'queryset_list', 'after_union_items', 'before_union_items' - ]: - return object.__getattribute__(self, item) - - if item in UnionQuerySet.not_return_qs: - return getattr(self.__execute(), item) - - origin_item = object.__getattribute__(self, 'queryset_list')[0] - origin_attr = getattr(origin_item, item, None) - if not isinstance_method(origin_attr): - return getattr(self.__execute(), item) - - if item in UnionQuerySet.after_union: - attr = partial(self.__after_union_perform, item) - else: - attr = partial(self.__before_union_perform, item) - return attr - - def __getitem__(self, item): - return self.__execute()[item] - - def __iter__(self): - return iter(self.__execute()) - - @classmethod - def test_it(cls): - from assets.models import Asset - assets1 = Asset.objects.filter(hostname__startswith='a') - assets2 = Asset.objects.filter(hostname__startswith='b') - - qs = cls(assets1, assets2) - return qs - - class QuerySetStage: def __init__(self): self._prefetch_related = set() @@ -273,11 +197,11 @@ class UserGrantedTreeRefreshController: ret = p.execute() builded_orgs_id = {org_id.decode() for org_id in ret[0]} ids = orgs_id - builded_orgs_id - orgs = [] + orgs = set() if Organization.DEFAULT_ID in ids: ids.remove(Organization.DEFAULT_ID) - orgs.append(Organization.default()) - orgs.extend(Organization.objects.filter(id__in=ids)) + orgs.add(Organization.default()) + orgs.update(Organization.objects.filter(id__in=ids)) logger.info(f'Need rebuild orgs are {orgs}, builed orgs are {ret[0]}, all orgs are {orgs_id}') return orgs @@ -292,7 +216,7 @@ class UserGrantedTreeRefreshController: key = cls.key_template.format(user_id=user_id) p.srem(key, *org_ids) p.execute() - logger.info(f'Remove orgs from users builded tree, users:{users_id} orgs:{orgs_id}') + logger.info(f'Remove orgs from users builded tree: users:{users_id} orgs:{orgs_id}') @classmethod def add_need_refresh_orgs_for_users(cls, orgs_id, users_id): @@ -364,29 +288,37 @@ class UserGrantedTreeRefreshController: @lazyproperty def orgs_id(self): - ret = [org.id for org in self.orgs] + ret = {org.id for org in self.orgs} return ret @lazyproperty def orgs(self): - orgs = [*self.user.orgs.all(), Organization.default()] + orgs = {*self.user.orgs.all().distinct(), Organization.default()} return orgs @timeit def refresh_if_need(self, force=False): user = self.user - exists = UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exists() - if force or not exists: - orgs = self.orgs - self.set_all_orgs_as_builed() - else: - orgs = self.get_need_refresh_orgs_and_fill_up() + with UserGrantedTreeRebuildLock(user_id=user.id): + with tmp_to_root_org(): + orgids = self.orgs_id.copy() + orgids.remove(Organization.default().id) + orgids.add('') # 添加 default - for org in orgs: - with tmp_to_org(org): - utils = UserGrantedTreeBuildUtils(user) - utils.rebuild_user_granted_tree() + UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=orgids).delete() + exists = UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exists() + + if force or not exists: + orgs = self.orgs + self.set_all_orgs_as_builed() + else: + orgs = self.get_need_refresh_orgs_and_fill_up() + + for org in orgs: + with tmp_to_org(org): + utils = UserGrantedTreeBuildUtils(user) + utils.rebuild_user_granted_tree() class UserGrantedUtilsBase: @@ -394,7 +326,7 @@ class UserGrantedUtilsBase: def __init__(self, user, asset_perm_ids=None): self.user = user - self._asset_perm_ids = asset_perm_ids + self._asset_perm_ids = asset_perm_ids and set(asset_perm_ids) @lazyproperty def asset_perm_ids(self) -> set: @@ -431,24 +363,26 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): @timeit @ensure_in_real_or_default_org def rebuild_user_granted_tree(self): - logger.info(f'Rebuild user:{self.user} tree in org:{current_org}') + """ + 注意：调用该方法一定要被 `UserGrantedTreeRebuildLock` 锁住 + """ + + logger.info(f'Rebuild user tree: user={self.user} org={current_org}') user = self.user - org_id = current_org.id - with UserGrantedTreeRebuildLock(org_id, user.id): - # 先删除旧的授权树🌲 - UserAssetGrantedTreeNodeRelation.objects.filter(user=user).delete() + # 先删除旧的授权树🌲 + UserAssetGrantedTreeNodeRelation.objects.filter(user=user).delete() - if not self.asset_perm_ids: - # 没有授权直接返回 - return + if not self.asset_perm_ids: + # 没有授权直接返回 + return - nodes = self.compute_perm_nodes_tree() - self.compute_node_assets_amount(nodes) - if not nodes: - return - self.create_mapping_nodes(nodes) + nodes = self.compute_perm_nodes_tree() + self.compute_node_assets_amount(nodes) + if not nodes: + return + self.create_mapping_nodes(nodes) @timeit def compute_perm_nodes_tree(self, node_only_fields=NODE_ONLY_FIELDS) -> list: @@ -587,6 +521,8 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): node_asset_pairs = list(node_asset_pairs) for node_id, asset_id in node_asset_pairs: + if node_id not in node_id_key_mapper: + continue nkey = node_id_key_mapper[node_id] nodekey_assetsid_mapper[nkey].add(asset_id) @@ -695,6 +631,10 @@ class UserGrantedAssetsQueryUtils(UserGrantedUtilsBase): ).filter( Q(node_key__startswith=f'{node.key}:') ).only('node_id', 'node_key') + + for n in granted_nodes: + n.id = n.node_id + node_assets = PermNode.get_nodes_all_assets(*granted_nodes) # 查询该节点下的资产授权节点 From 6f3ead3c425ec0eb04992ac8b72d1d3d85439204 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Mon, 1 Mar 2021 18:40:07 +0800 Subject: [PATCH 34/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E7=94=A8=E6=88=B7=E7=94=9F=E6=88=90=E5=AF=86=E7=A0=81?= =?UTF-8?q?=E7=9A=84=E5=A4=8D=E6=9D=82=E5=BA=A6=20(#5648)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 优化系统用户生成密码的复杂度 * perf: 修改 common.random_string Co-authored-by: ibuler Co-authored-by: Bai --- apps/assets/models/base.py | 6 +++--- apps/common/utils/__init__.py | 1 + apps/common/utils/common.py | 10 ++-------- apps/common/utils/random.py | 30 +++++++++++++++++++++++++++++- apps/users/models/user.py | 4 ++-- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 094f029bc..9fd4836f7 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -11,7 +11,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from django.conf import settings -from common.utils.common import timeit +from common.utils import random_string from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, lazyproperty ) @@ -205,8 +205,8 @@ class AuthMixin: self.save() @staticmethod - def gen_password(): - return str(uuid.uuid4()) + def gen_password(length=36): + return random_string(length, special_char=True) @staticmethod def gen_key(username): diff --git a/apps/common/utils/__init__.py b/apps/common/utils/__init__.py index 01850f0cf..8b4576221 100644 --- a/apps/common/utils/__init__.py +++ b/apps/common/utils/__init__.py @@ -7,3 +7,4 @@ from .encode import * from .http import * from .ipip import * from .crypto import * +from .random import * diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index f9c488e18..d1ecf8579 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -7,6 +7,8 @@ import logging import datetime import uuid from functools import wraps +import string +import random import time import ipaddress import psutil @@ -191,14 +193,6 @@ def with_cache(func): return wrapper -def random_string(length): - import string - import random - charset = string.ascii_letters + string.digits - s = [random.choice(charset) for i in range(length)] - return ''.join(s) - - logger = get_logger(__name__) diff --git a/apps/common/utils/random.py b/apps/common/utils/random.py index f32147b6d..055966947 100644 --- a/apps/common/utils/random.py +++ b/apps/common/utils/random.py @@ -1,8 +1,13 @@ # -*- coding: utf-8 -*- # -import socket import struct import random +import socket +import string +import secrets + + +string_punctuation = '!#$%&()*+,-.:;<=>?@[]^_{}~' def random_datetime(date_start, date_end): @@ -14,6 +19,29 @@ def random_ip(): return socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff))) +def random_string(length, lower=True, upper=True, digit=True, special_char=False): + chars = string.ascii_letters + if digit: + chars += string.digits + + while True: + password = list(random.choice(chars) for i in range(length)) + if upper and not any(c.upper() for c in password): + continue + if lower and not any(c.lower() for c in password): + continue + if digit and not any(c.isdigit() for c in password): + continue + break + + if special_char: + spc = random.choice(string_punctuation) + i = random.choice(range(len(password))) + password[i] = spc + + password = ''.join(password) + return password + # def strTimeProp(start, end, prop, fmt): # time_start = time.mktime(time.strptime(start, fmt)) diff --git a/apps/users/models/user.py b/apps/users/models/user.py index b3b4e9d1c..50099253d 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -18,7 +18,7 @@ from django.shortcuts import reverse from orgs.utils import current_org from orgs.models import OrganizationMember, Organization -from common.utils import date_expired_default, get_logger, lazyproperty +from common.utils import date_expired_default, get_logger, lazyproperty, random_string from common import fields from common.const import choices from common.db.models import ChoiceSet @@ -387,7 +387,7 @@ class TokenMixin: cache_key = '%s_%s' % (self.id, remote_addr) token = cache.get(cache_key) if not token: - token = uuid.uuid4().hex + token = random_string(36) cache.set(token, self.id, expiration) cache.set('%s_%s' % (self.id, remote_addr), token, expiration) date_expired = timezone.now() + timezone.timedelta(seconds=expiration) From 51c9a89b1feb958210cbfe80b57f8fa40a948866 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 2 Mar 2021 14:48:47 +0800 Subject: [PATCH 35/71] =?UTF-8?q?fix:=20=E7=94=A8=E6=88=B7=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E6=8E=88=E6=9D=83=E8=B5=84=E4=BA=A7=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E7=B3=BB=E7=BB=9F=E7=94=A8=E6=88=B7=E6=85=A2?= =?UTF-8?q?=20(#5663)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: xinwen --- apps/perms/utils/asset/permission.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/apps/perms/utils/asset/permission.py b/apps/perms/utils/asset/permission.py index 38a3b929d..c9b1bf51e 100644 --- a/apps/perms/utils/asset/permission.py +++ b/apps/perms/utils/asset/permission.py @@ -6,20 +6,19 @@ from common.utils import get_logger from perms.models import AssetPermission from perms.hands import Asset, User, UserGroup, SystemUser from perms.models.base import BasePermissionQuerySet +from perms.utils.asset.user_permission import get_user_all_asset_perm_ids logger = get_logger(__file__) -def get_asset_system_users_id_with_actions(asset_perm_queryset: BasePermissionQuerySet, asset: Asset): - asset_perms_id = set(asset_perm_queryset.values_list('id', flat=True)) - +def get_asset_system_users_id_with_actions(asset_perm_ids, asset: Asset): nodes = asset.get_nodes() node_keys = set() for node in nodes: ancestor_keys = node.get_ancestor_keys(with_self=True) node_keys.update(ancestor_keys) - queryset = AssetPermission.objects.filter(id__in=asset_perms_id)\ + queryset = AssetPermission.objects.filter(id__in=asset_perm_ids)\ .filter(Q(assets=asset) | Q(nodes__key__in=node_keys)) asset_protocols = asset.protocols_as_dict.keys() @@ -36,10 +35,8 @@ def get_asset_system_users_id_with_actions(asset_perm_queryset: BasePermissionQu def get_asset_system_users_id_with_actions_by_user(user: User, asset: Asset): - queryset = AssetPermission.objects.filter( - Q(users=user) | Q(user_groups__users=user) - ).valid() - return get_asset_system_users_id_with_actions(queryset, asset) + asset_perm_ids = get_user_all_asset_perm_ids(user) + return get_asset_system_users_id_with_actions(asset_perm_ids, asset) def has_asset_system_permission(user: User, asset: Asset, system_user: SystemUser): @@ -51,5 +48,7 @@ def has_asset_system_permission(user: User, asset: Asset, system_user: SystemUse def get_asset_system_users_id_with_actions_by_group(group: UserGroup, asset: Asset): - queryset = AssetPermission.objects.filter(user_groups=group).valid() - return get_asset_system_users_id_with_actions(queryset, asset) + asset_perm_ids = AssetPermission.objects.filter( + user_groups=group + ).valid().values_list('id', flat=True).distinct() + return get_asset_system_users_id_with_actions(asset_perm_ids, asset) From a56ac7b34ead5d2ddbbb47c474352eb6c7551df8 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 2 Mar 2021 14:57:48 +0800 Subject: [PATCH 36/71] =?UTF-8?q?perf(orgs):=20=E9=BB=98=E8=AE=A4=E7=BB=84?= =?UTF-8?q?=E7=BB=87=E6=94=B9=E4=B8=BA=E5=AE=9E=E4=BD=93=E7=BB=84=E7=BB=87?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E6=94=AF=E6=8C=81=E5=85=A8=E5=B1=80=E7=BB=84?= =?UTF-8?q?=E7=BB=87=20(#5617)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(orgs): 默认组织改为实体组织 * perf: 添加获取当前组织信息的api * perf: 资产列表在 root 组织下的表现 * fix: 修复 root 组织引起的问题 * perf: 优化OrgModelMixin save; org_root获取; org_roles获取; UserCanUseCurrentOrg权限类 Co-authored-by: ibuler Co-authored-by: Bai --- apps/assets/api/node.py | 27 +- apps/assets/models/node.py | 39 +- apps/assets/tasks/nodes_amount.py | 12 +- apps/common/permissions.py | 9 +- apps/locale/zh/LC_MESSAGES/django.mo | Bin 69926 -> 69720 bytes apps/locale/zh/LC_MESSAGES/django.po | 340 +++++++++--------- apps/orgs/api.py | 15 +- apps/orgs/caches.py | 6 +- .../migrations/0010_auto_20210219_1241.py | 56 +++ apps/orgs/mixins/models.py | 25 +- apps/orgs/models.py | 65 ++-- apps/orgs/serializers.py | 9 +- apps/orgs/urls/api_urls.py | 16 +- apps/orgs/utils.py | 12 +- apps/terminal/models/session.py | 2 +- apps/users/api/mixins.py | 2 +- apps/users/api/profile.py | 5 +- apps/users/api/user.py | 8 +- apps/users/models/user.py | 31 +- 19 files changed, 381 insertions(+), 298 deletions(-) create mode 100644 apps/orgs/migrations/0010_auto_20210219_1241.py diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index 4ffdbfe1a..f964b7704 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -127,9 +127,13 @@ class NodeChildrenApi(generics.ListCreateAPIView): def get_object(self): pk = self.kwargs.get('pk') or self.request.query_params.get('id') key = self.request.query_params.get("key") + if not pk and not key: - node = Node.org_root() self.is_initial = True + if current_org.is_root(): + node = None + else: + node = Node.org_root() return node if pk: node = get_object_or_404(Node, pk=pk) @@ -137,16 +141,26 @@ class NodeChildrenApi(generics.ListCreateAPIView): node = get_object_or_404(Node, key=key) return node + def get_org_root_queryset(self, query_all): + if query_all: + return Node.objects.all() + else: + return Node.org_root_nodes() + def get_queryset(self): query_all = self.request.query_params.get("all", "0") == "all" - if not self.instance: - return Node.objects.none() + + if self.is_initial and current_org.is_root(): + return self.get_org_root_queryset(query_all) if self.is_initial: with_self = True else: with_self = False + if not self.instance: + return Node.objects.none() + if query_all: queryset = self.instance.get_all_children(with_self=with_self) else: @@ -178,7 +192,7 @@ class NodeChildrenAsTreeApi(SerializeToTreeNodeMixin, NodeChildrenApi): def get_assets(self): include_assets = self.request.query_params.get('assets', '0') == '1' - if not include_assets: + if not self.instance or not include_assets: return [] assets = self.instance.get_assets().only( "id", "hostname", "ip", "os", "platform_id", @@ -240,7 +254,10 @@ class NodeRemoveAssetsApi(generics.UpdateAPIView): node.assets.remove(*assets) # 把孤儿资产添加到 root 节点 - orphan_assets = Asset.objects.filter(id__in=[a.id for a in assets], nodes__isnull=True).distinct() + orphan_assets = Asset.objects.filter( + id__in=[a.id for a in assets], + nodes__isnull=True + ).distinct() Node.org_root().assets.add(*orphan_assets) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index eb326859b..27a2a7df6 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -40,7 +40,7 @@ def compute_parent_key(key): class NodeQuerySet(models.QuerySet): def delete(self): raise NotImplementedError - +# class FamilyMixin: __parents = None @@ -446,8 +446,9 @@ class SomeNodesMixin: @classmethod def default_node(cls): - with tmp_to_org(Organization.default()): - defaults = {'value': cls.default_value} + default_org = Organization.default() + with tmp_to_org(default_org): + defaults = {'value': default_org.name} try: obj, created = cls.objects.get_or_create( defaults=defaults, key=cls.default_key, @@ -482,25 +483,34 @@ class SomeNodesMixin: @classmethod def create_org_root_node(cls): - # 如果使用current_org 在set_current_org时会死循环 ori_org = get_current_org() with transaction.atomic(): - if not ori_org.is_real(): - return cls.default_node() key = cls.get_next_org_root_node_key() root = cls.objects.create(key=key, value=ori_org.name) return root @classmethod - def org_root(cls): - root = cls.objects.filter(parent_key='')\ - .filter(key__regex=r'^[0-9]+$')\ - .exclude(key__startswith='-')\ + def org_root_nodes(cls): + nodes = cls.objects.filter(parent_key='') \ + .filter(key__regex=r'^[0-9]+$') \ + .exclude(key__startswith='-') \ .order_by('key') - if root: - return root[0] + return nodes + + @classmethod + def org_root(cls): + org_roots = cls.org_root_nodes() + if org_roots: + return org_roots[0] + ori_org = get_current_org() + # 如果使用current_org 在set_current_org时会死循环 + if ori_org.is_root(): + root = cls.default_node() + elif ori_org.is_default(): + root = cls.default_node() else: - return cls.create_org_root_node() + root = cls.create_org_root_node() + return root @classmethod def initial_some_nodes(cls): @@ -519,9 +529,6 @@ class SomeNodesMixin: if not node_key1: logger.info("Not found node that `key` = 1") return - if not node_key1.org.is_real(): - logger.info("Org is not real for node that `key` = 1") - return with transaction.atomic(): with tmp_to_org(node_key1.org): diff --git a/apps/assets/tasks/nodes_amount.py b/apps/assets/tasks/nodes_amount.py index a7cb46a45..2f8f0592d 100644 --- a/apps/assets/tasks/nodes_amount.py +++ b/apps/assets/tasks/nodes_amount.py @@ -13,18 +13,20 @@ logger = get_logger(__file__) @shared_task -def check_node_assets_amount_task(orgid=None): - if orgid is None: - orgs = [*Organization.objects.all(), Organization.default()] +def check_node_assets_amount_task(org_id=None): + if org_id is None: + orgs = Organization.objects.all() else: - orgs = [Organization.get_instance(orgid)] + orgs = [Organization.get_instance(org_id)] for org in orgs: try: with tmp_to_org(org): check_node_assets_amount() except AcquireFailed: - logger.error(_('The task of self-checking is already running and cannot be started repeatedly')) + error = _('The task of self-checking is already running ' + 'and cannot be started repeatedly') + logger.error(error) @register_as_period_task(crontab='0 2 * * *') diff --git a/apps/common/permissions.py b/apps/common/permissions.py index 43779204c..7098cdb84 100644 --- a/apps/common/permissions.py +++ b/apps/common/permissions.py @@ -110,12 +110,17 @@ class PermissionsMixin(UserPassesTestMixin): return True -class UserCanUpdatePassword: +class UserCanUseCurrentOrg(permissions.BasePermission): + def has_permission(self, request, view): + return current_org.can_use_by(request.user) + + +class UserCanUpdatePassword(permissions.BasePermission): def has_permission(self, request, view): return request.user.can_update_password() -class UserCanUpdateSSHKey: +class UserCanUpdateSSHKey(permissions.BasePermission): def has_permission(self, request, view): return request.user.can_update_ssh_key() diff --git a/apps/locale/zh/LC_MESSAGES/django.mo b/apps/locale/zh/LC_MESSAGES/django.mo index ea2503cb993c1688fe3ba25e7386495b773ed7b6..f5ee34c63e32671c880b408d202411d9787f3a82 100644 GIT binary patch delta 19439 zcmYk@2XqzH+Q#uoqb5KSLVy6F6B3G{N^cTc=+Zk9y3&PEJb+Yb!l4QZkq!dEP^C!~ zt~6=VK|n?69hLI^pUmsGoVE7-?fve)XXcy}^{)8Z|NIhv&(#qBIS$t$KgY?1@p&94 zeSqVPtgNWxENS96MSLA63ENOU(#&zDVS(n3GaNT!4j;!U_P*n^q+MJK#|fnTWh=+| zjQTaL9cMI_rSm1+g2$+z)z)#c`Z^w`b9=`*NyU(kjuVD)og60(w#Oj+05jqM%!Z>d zAI>(nU?k=9m;+y;KZbO6oLCIU;#d#WZzvYU1(=8Vo&98@3EaRE7}Uja3SnhDk6rO9 zR_f|FnK9=Fj*|hSF%Zj|HOE7VDRgdsS>>ZhYNwixxSZ0N!HtK$&@ zI*E&@4i8WR`Sx_{bD-kUsCX6B0Pkb!M5u>%1Zskrm==>z6iu=qb6Em^_x)R>_zqa4Rxy?A|DHmZy&c`0BT`@ z$haOSn2e4h4;H|(SQgu0X`F!-aX;q7H>d$}^mQl7gF2xodx!brsfs#*TB!aFQRBUj zI>ElElNlxT{?D*Ll9__K#|Kdl-38RMa2GYO??V&%G4Zfdqi zk5JXecc0!IsD(vh5EjNjEQ1=j25P7EQ2pMwcsI;Xxi4y? zGf?v+p-wotKj*Iw2MEYBs2$!y4SWxE3!b3{NH@TZ=R^&ZAG2Vr#cN`A%JnfLet=qN zKdT>!>Nn2HUJn@^(Lz+i<(Ln@MLqpzP&-dI(EXgqjA|c)nxHJE9=X{LgNYA8-GYgz zllc<){&H5K+GiZ(Zq!qVj3y|7I)SpNBd(2_sJYnz^HT1CI-yCZ1~2l}ip~ z5X$kWe$!B&^>eW(E<&BeA&Z|wO?(;k(B4Fkp4O*iw9}wr?m%H?9@Oho1T|2748gXj zFRc%;FiyruOvYF|i8@i=ME8VBpvJF>I{NykTh=C#^H)V50$SN1)J}$@;-6Z43hE`}(EFY?KRl$Y_F!sC!%=welXQ zhiVXNfKjN4rl20qFU-YOzYNv?TZ^AS^*>|ftEh2rqE7As=0(pdGCG>vBi#uKqZ*V& zoj^_0L-jstp>3_cH~LZ@Y7R$@GYYlf38-ghwz(CnP`-#dsqCXtPtxNQBg5YuPH9|& z%`i8m&eA~1K+`|sH44&+VNY|!t|=?1hb+RSO9bBF^(mp zqiTeDx?7_<^zl9l4E5+mnm|N%GZ}N@4pfhG7Qc>~>JDm(7nu5ZjduH0LX8-QG1wS2 z-U!sqoPc_2rl5X5FG7vCWHj$b9x~q$&|&^)-bXbE9^*caxlwy6h}wH8)CBb~4BtmB ztT$?$MATU>LQcS0hT8h~R(~4R{`nYoqNgEXtlJ?3^)%!(^P(m$jGC~Fm8+o^(hxO4 zH`IbhqXwRe`s7%M>bDQo{ut_HZ=p`?zQ+PDPz`-PbypgQT3941UJUhEmB(;wfO=6s zz|@^%KFW#chYL{?EJoepRj37?LXC43wLs5fGCGgZ!ou^=Glwd;PJ_vzgF}Ifui^Z zbt|Gib0@Bb+EE;8f~Kg~u`OyLJyCDNNOLw8ru+@+1Lw4P%j%zDUgAMh+!HA=h4U{- zpb7yU$%m!~buu$i4_A`83v*DufZFL3)I|T7fm7Xnxv?Pe;;4Jx%xsO?NJrF0dU(j_ zD2AYp>~kwGMNPaLHPA)W+wc~(fb7%UKVwUw`gKO#xmyL*f|vnIpiZm;mcn|daYte^oQ_fW3r1pw z8L12PIMHM>5h#xuFb?(5H9;+?73v5(Svi@YL4>$9m1}=ofu?%WqT~Pz~L_N#{F%~DH+HXgl3AIPUoQpT!K2vRaV|?^}A5F;1CwZ zqo{>HK`rbhY8>A=?n9Xo%TNyYkkL`aTSar!PCKFQ;V>+Uld&A8pmy{ps^2rzLSJDB zW}NFzoCj53097At$54aBUJLs1KfLfyLZ$h;n>IT;Pu7d6mu)O$Mz)gcM>@FZLPJ`AON1$7dS z%vY!dr=9QKqCnJ<=RhqizmNngA)I>SHaGVZU!(5EL zD8I%??6J`Onx2U5Deu9Wm}`;ytaL)XRdcZt?!{sF8r$Q?UvmD1$($ym*Y0mDh}pk# z|J100gDHQ6F?bWhG4o=+im?c;$3)D5QA^xgSQE8@4yf_RV|n}rE8s0mi+Pebe;sk* zB=@MwpzdvL)I-z8;vb=&0T1TEnHEn*_1}wnEl;4Hoy({RZlj)whp1cn49DPGe1>Db z_P7%cUF!b&oq{!}*oejP3F_A5TgDd?*2lIu6W_(VsAs3pa(A2<%ucx)>ZDp@HtdgD z;6&8hG#j^)NTeAei(6g3IPBME@M|&9~@gLN~6S2}=aU5#kmR9bGc_|M=-Kqts9j`}Cdb6^-nRa-hXG6yYh6X9c95Tm=m?ak*E`zg&KIC#g`%jINQy=R)56m&!RSV z2eptitKDZM7!xQ*VvOb=PbQYYM$`m1QSavi)Q;bvc9vz0`<6suY07b^TQwXr;1pE* zxu|hgp*~MGp~m|Gbpq#5{eHvr%FwVk`Aj;4ny^CfvWF_9^IR6WVEtAs3RSL8Szt0k2A0jF2I5qu+Cjj zH0st=!9c8OHpZ%y+oQ%`hVc9d;2r$8F`F)2-79I6XZsf<53Ijg1LP7NJq`{DU<31R&M0{wZnA;G~iauf~PPu z-b8&MJwe^WwBNb|W z(_kUy<&U{dTRBR~XSegQN`o8U@%J6J-{CmB@EY!+{>zf{LXqZYIqyd;A1R;f0_Aya9@{tsQNzG2nV7*i1wm(^bGa> z`*QAjX!D`sWl`hTL~S(Q;t81g`+uMXMw*k&x#m*T)18cZEB2svc*4pzP)GgPOn1aB z=RrMmWvm=$wnr^w;1S+`4LF5>R=Ut!h1%(6RQ*oW&W>97j+Otxg2Z$F&z-Ous(lMH z!5m`s&K=eGdjL$K50hv$@ikoFn4J)CZfw~rNWp*?ZQ0@Ano{dC{PepCOi`v*c)R$Kh zvSE+2os4#R9yReJ)Icv$9RiNK_ck+X;6kW{yld7mTbcU`*m~tqpLq*g?bx;d#W%1tTP;(q=;+a;SkD7S7#Wz@d%TL^Y4ZPnfPNH`9 zGpfTwi~nt9pHuEaGNLBPZska`7#1O30qbBFRJ+v}g2||H_oL=Lc8dG29bL4*pQ!Q+ zGwo@&LpXLIo)7h!c~Ap?Vf9IoZB!f>Ni?G)Nj7p7WbeAo@#n6 z{)M^H+=Ml0x7W)4Kf5OsjPDRHXyq2EW$u4ZGTQM1)C4P0J6n%Bfs?3Duu)atLaiNQ}h77LP+ce9ci`S{<+$^E+e7Xk{C)9A3x57=FS1$y629u?1?v zPG(;-5jEiia~i7sY%70-TJUm>Q?SXjdR>Qdx`T`hsy-CfIm=2=X=?07-)u~+T}EhpzduMRQvZW z-VrrXA9Dn1!YP;qXQA3JMPFR!u}m@s5J|a}?H~oo5`G24oJQemI0&b0?^Z+Hq6VL>;Z1U=Bnby~pb3S^O*1 z4%b?Iv$+%Xt+*f4;eE`S`uDdQ|;p z)N6YfL-8o;q^@IL{2R0J{5cW7xPK~@LJd?EH9!MYhmIB>X7TY>o{xE`Uyj=G0o2hS zNA`_xBbec}~RCov8+&}=iwTyO3`jdK{){*smNSosaA zf2KcOBanr9oG3E-;3??_oOWhc)DibWbsTHX!1R8jmJ%lM3h?h_k+_v%qE5AaWpx=GhAT!*|huTmKYMl4( zbN(8rDFJ=pv_>6WFVsLIt>FYrN7-xfMHXLyx|b?R-&VrD#Q<;&(x)Bt~4`2}i#Hx^I#(5(+M^P446Cs`S_v1X`w60E+j z$1;hi0Vi04&n-R=b#Iqh{Cm`ZKVW9Of_k6tp%z&1kvmZ=MpJ$db>#g}w{{Y00c%k0 zJsZhrz};4H#=L}D&<#|>yQri4%gWgwy8{(M?W`E8T{SZv)xM3DhhY%q@t6_4t{!Kx z1=gBdF#`?uqjq>4wUDdkZSyhe;e3VKQI;pJ;i!e=F=H@<^1D{w0CghGQsw^lA)^lC ztieocFduso-)QBWPu-5CQ45MUTcQ@w8TF9%K#kWQHSs76#m})IE<-Kg2&Vr1e?u7p z56oAnTax~n8;?XyP|V7uF@SP))If1o-^uKbI=K%~Cpij><0RBPTdaORrvCS_r^#qX zcTo??bF7BhpSvAeqZW{Wny{}q-r`@OAMxd=307Nt3+hC6S@{s^7u7K<|Nfl!Ka9YA z0{-a#!acI|s2$`&4Oq_(wftbVPrkCU|TO zUSSkvzrWm1uwrIo%uakDYJe%Izl2s{7Tk~OcLpQzHELtI|8^Hz993Tt^P}ej3rxVq z1U6tCzQqa{_mBH9jz)b4%*Beh8{fshFak@xa&3fq*m`3toR8)3A?j8adF?K!3R3ns z4awxC;saF2aj1vzThva^UQ*g6 zoyclT{r=ycn&Dnp#WgEGz#KI6b$n7gTf+47fawKNO85oSq%q^&e9!9l0 zYvtRh1wKRFidUHW{)hOxD~>@mC~H7>Y=+~^-s*#R-ZP&ohTFP z$P1cfP$%(TfR86N(}jSZ#!pZyPcqk=J5bNUVJrWFy65+;9FWG{KnSXTF0-gv4)sY` z%WQ<&Xqz-1AEy?X&IEL?ms!Pb^C0R3j-u}AHS-Uve`582TRbSOyMS<1zk>J~mPfrU z`^9rH?~6!oM=u!ov0Ue#6P3H8?K_p zyMbzd*RA(Bf4Lcc6}TPJVqY3$LQOaVwSd{EfmWghT8kQJGd93|s0lOiuTS5>Ff4>| zsLzwWxD=;jPAnDV!v`1dUqdpQpcm?+aU@3Lax9G}QJ--BncQ}HQLj&FRQr0Ug$zV3 zz=PR*_$i0_zW*|_yYPLeg&#uAcT%<|bCZk)c#b-HzbtOUP}G3AQ9Cb)dN!g^?JA>w zO2s4dIs6gqTqcvB#7BxV(3?~_!ZV&+FOsgACLbuySyB-V%XI*^lCHT$KJxf5r(Bv$ zSK7_8ctK*F$*15BQhgh*9Ui6a7u1a;KVJj5m$y=tFrUfZrd1+6-_YYDY70>F8!ZZu zej-gEb_Vrzp=%eZ59M@<(MMM_{)EZ2)zyr;Vx&@T(V2%CDYvG6oVD3(eJ2v{>cfT!9iP!Sl2}nJ?G3IP8rG58tfZN=xI@hhY~ig?H6|#6QaEWnwN<x|idsI3odEBGYB3#7lY2~?8n&WB zSdv&UvFmt~*3F20V)cI#FGGF>by>*gA)kphu~vVXSVQtXhz;`Qs~(#0JuUye4v|?$ z?Ic>ABwvOUL;eqIla0E$l%uV@m-2ktbR(@IKaX)c*!Q%pjnlmgsuyWmifA2r zwjuH}X)*Z;v?xsc-~Xq-UX6nqo!Eb`{y51uQD`%< zoTR3tHP(MOZFOBGHi_7J?|>Si>2FawO*G0otwwm$8I-1zz98Kt+8jq=4f@U|>2qWo z=?(csHm)yLrks}a-)jhcYk8m5D4pYbYpL2Exu~lcmY{7_(sz{a;Amn)tlt!3`N^MGW3Hv7k(7TSdC7P8N&OFf zRK-WTb^47#Hfm4?x{cNFTdT`JelM}Ir2ORPdxLA0_c`yaS1U4iQ)^S0y8m9;h|lwm zuN4`#g&xzW9ZLF+l%1OS-tDzY*N!LGpHAg*BCSSK+YS2>D@*J+NtXw&TfPAG`rN%v z`iR&(Z|r-ap87=py>b!lO!*0cUU+~}dsy#V#3oqVuZVw4T1wqGQUmg@N&2xg3Uyr| zo}Kn*@qfg|kbWhFkvb7iB;MVZZ=M}wVhB8z_B-W$7F*-J_g?8XkE!9S z*=azuk!ooji#v$@d#$0Ys{@uG4JGwwv>?(s^~K$86X!$nH>vk{6Kh9Q*+{7wE$R|k zMT?c>+Yl>G8b|6vJPm0f@wwCwAf_uGH+X-l9qLoo`)ln2o~hK&A+E1GUHSzPO1~e8 z$I&)7`8K4w#B1Ygk}kdxQ?EU^*76GcOx}++!$}z}egXTF`jTFf@{x2M^KrZr<6;xu zQ2v&*%f?n+GIJ-87Lb0Rte-~N)z8{|M0_Fnt)#DsucSPP{O2_J8Bd3M6eH^`)jODKGt| z<2U$$w?^HF*eB%15#cxI|6kk4e?ln-J$032Rc)<)zjs>QaL;?x{6wUJ?XME~UugLp zhgh#J_&Mnh+SIppsl)Ldl?r*u#WU95L^IH;9Ztid#LE+}jLW^j^}~bi*^KIc&0Ds9f%N66D@VHM?NUFY zN{G!-i~5~J|9g#~{DS(1jNyaN|5KSltb=!Z{g@W7{xd_8jWE|@|D%`wlI~3Ted->Q zFG-5DcAr?g)|7LSI#PF!bje$xK><%gdS9f}m{v2%>uQea(9cHr4zp1HkT(C2j+2^F zeusE(+BPQf7nd{M+W$>nR}&1vyH-vO@;?hGFDD)JbG(-t6!I*!`XV-eEaj$*6;9jx zq;sT9#L`oLj?|L;D(f?rx<0gh*XCY^Ut69kr>C{+huesMNNk49vy0e6AAW4s`Zqv> zI^=g#F@lDD$tU32{|wlHavkr+`0%2;iMFCe6s_OEE7a)E(VxldszrPd&hb8quhb%u z(mK+1BD3i^g8Tq$sn%~vPl@T;LwzMT>ukZow5x7yCt91fwE5UOreS!Y?&PXi>poW7 z$;vNr38}qzSHlR;Bx?O>-I=lS63vWNh@~N&B2}PH*N4Q1kuQXoDeDihBjhLGCfc&-?_o_+dGh~Wrzz_iO*%qbT@Ubk9B*|| zjQ9oV0I}lK-S_5b9O?0=cAK?0hD9y^EB2-JNh>GQYZ1Nlj{~S8tc8%IKp#=7SpX( zf4%DZ)P~xEnca|c!eT$*RAO~)l$JQj+WbXbHS(Wf7Sd4aT44)(mpK|yw-(poK4N+G zvu3?@PK%!s97vi)`iF8oI_pp47}Cy^q$Xj}0r8D%)`(BpH2g?N%KB+bf>RbPo>3|A j*YT@<-8lO99pirAF)=0jQiafz@%Qg$PWk%HxRCz?wd6xj delta 19643 zcmYk@2YgQF-^cM2VkE>!LP(6nju3ms-lKMHN{A8ENT}VL+9RkNMeS8o?7fPr)ml|b z?b=$}qNwWM^ZDld`n-C-Uf1)!e!pv;>zs2Z@$YkaneUnvzMdO_zH=Qu$5T5_E_^?a z+ur7nI;xRl;`-%>Z6P(KNI0HL5&N&(;eBe0QuuT`oNsIk30246-j>YVl zgb}#PJcfCRA7V~S-_>z^F&~z~!dM*JqsC3bNL+_`Sl_uor6`GK7>&7O9j73^hnFz{ zZ(!qYj*}USc6S_qtc)44uG!M;g4%FD48ma;jMFh4F1Pk|n346J9jf3F)PS?76S#w! z@G)xOUzYcYb5ABcYD3vE1Ph_&sfK>o7*k_wOponR^K`>3*atn@*+?q7qFLtWsEOC0 zZoy70ghx?F_X`fdY&{&O0!~2nJAhj7TP%WSu`s?yU08Te$4P@_k;lWS+>`UyJ(^8I z6C6d|g6}aa`t)*l9)!B0T&R2@)Dc&-xDM)y+Mu41UZ|7!7z1&dwJ$+k*e2Ao@>MU+ zUjwg@&`CT-4S0u|D6qHN9*N3VM&+BJCWyn7g-{RgG}HpiF&%C}&ASKncAY`p(qB;X zyzx*8qmnh=?N}VOp<1X3TcIY5w|F?}C}&!{67v%8vHTU(Nj^be{2O)TP9OIq)1ull zqfXG1n@RyHk?4<2u{gFzKb(ZR!cS2fTY_nE6KbNJ)_xQ<&w13im#ACi^mQD*7MwuT zxNNA6tZ>K$1=DSE8_(W#Z3L&2_jJoMWarrB9_DkmLHBYlRx$ z88u%V>I6rjP9{m}{aq2ljW(Wfn3^nw2p<{pz5uyotr_ zERHq%p+`FzN~ID`LGAc!EP|&{SNt4x#DAkE&M?5el3=qS>eiG;y%jA`8}5tR@L1G5 z)35}3Q77{C0M1_nu9C=vf1+L&-+}IeIZ;nMIvMfUm{o=bp@kP6OKhKIK`ZYT6iUD zfeomQ9YSsV6zUdU#76iE)xXw9?!1jr8)}Xj^!~S11-qj<4nTfua)zOf>OAU)$4#t- zZ!NAq)NyJMcS4O@jCxzvU?grpox~-}-$yO{E9#+rjvhU&zQf!r&5fFVSdR2lYd1G#0`Im=_OWDZGU`(ZJ#E2~|U#WJ}c1cR<~;_~D$t8b*@P&c>lOFcp>e zTK;p?SMCaHKZLp^CoR5&8h;Zt?mp^7pIZJ6YF?iaZr&fu5@#R5`RiUaCZP#GL>>7E z)D9=3u6!Zt-mXV2umg1^2T;$zcc>%1h3fYQYM!^K{{AD~*DesV6Bk3xTiZiL_qG*k z#{*Fh(l-P#i|74aBz9O`H%qBcAe^~@|b4`Ef}JE)V2813Gwa>$<=9;XVG z^)z(E+?e)b_s9yMHc-ss(iT_7669-OGaQH|@et~0|3F=FdTy;Y7L2;U+^7u}!(3Py z{nWE96+P9lUY~#})}?oG=-LM`Sw|ud?m!*p59TXWd+1pA5DTHMsW|G|V^9mU#BA6Z zwXsChJfEN*$JH2y$*4;|Y3+LgQ`p{F6!C+>hS)Xm6mMxqujfm*Pt#SKs!X@gqe zL)3<+pf)lW^^LI#HSSwf|4XQoed3{_BYR~DXPn#7AGOnLsErk|d^yx(RRcrtebhHW zKTNrDj3E95Q{yVs{A*FScnfNSS5fnL9#GK^|3)pCZ@l~0U^!IBj;NjYLcJFWsQ2X) z)RE7@a9oet=(ngVyNue<&!`RlVZKJq`wo5eK4h5S9(5KoJL*};g?c|?FbB3b6EK)~ z8fpW}P#fQgx&Ox{L<@e#nRM?5r8a1E;>c|G678+skOw<+5 zNA+KgrExtL!|PZI{U^CEToueu+#a>yXw9{8hbg}g-=m_5UZDo0o8s;? z7%LFxL~W=AYJqmBho>v*UJkZ+45lZZgX*^!weV`o?=}yk);TqW`>!kfk%V^i3?nh! zRQFbtLM_|?bw$lk3v@uej@?lk8H9QpCYXz{5b+Mw7tS^FiM9U^!^vl#X77K6Y3`S7 zJ=D|vkvSQ4G7C`;*Lw39<|O_Zb)|1n3;9lW&4wCR2n%3&)IINL#-c8ymxqe3WFYD! zK1Lnc0*g1G7Cw%e=nm>_NT1|xAOedM$Dqc=V@Vu^df1XNBW^`)^efcD$1wms=cr_$ zas&PFG3o?fpa!I!;SR`zdOJc;KTe`i{To|*Yl}ZXfAYOiCyOya#Huwvs zy#Ft$XrMF8eFy_l3l_%A7>zpGYL;(dwnI(a1&d>U)W#N}HogM&P;bCeco^0HIqD=+ zed_*jc5+hDH(fncLmX=7BQXHSTl;L(#x|R~PzxPIUC{~D_zS3wJVreOzhf5s6E!~V zZ1OT~R#hE$EMRa1!b*`406x@DO!D&K!5+ z{-_P-K+PYH+F;Zi-v3}Kl}YIJYlfw;59)`@BGd-Aq6Qp6UFmt$gg>D+_Or#mS^F#0 zEpX<#Uqoq98!wL9SZUNeRXkMmRMy3^*aCG_6D^;Fy3z%xd$<)N@oOxPPf%A>V4gd! z1Ztz@F%avb7H((lovl62VoxF!UEv7S6-`AgINRbcu?X=t)I`@%{cl_QAE;;JFVuqR zK66hp6m_NLP#09)tcStGO^}UxoNiQf?*^e3PC`w%26aoeqfY27>dLO79-hb6{uYCX zL*~0D5oMM~ZMY`t7S%%?c`MY$I;F(i|DM)i0O}}5q3+!z)DmuC{TH|w5{3Gvtb=-pV^O!zgXM4*=3{;57!_UFBh=BpMjd(jh3@ZirBDmC!p=C- zyn^w>6+U;rL6>6%;=|Yp|H4|>W|8}>EWo0~=P(BUM$a%R6&AY#He(^;fF^wZA}} z)LYcJOkcRKr6<%X5vZrX2xh}FsC(8B^I#n2#c`;IXeH`lJ%gI~hQ+^QII*+Vy;TvY zD=v>(xG8F#4wzQ&e-A2}sGoHhh3SYVo3l_?G#_Je1!{x7>)aCxMNOO+m5)M=uW2^4 z_7>LO8FgWa=+FAj3@Un97GMwj0;4csz5BDE5^90IsMmBT>WU|#u53Q){oaga@C@o! zrTWsn(jZj-Jg9j}qP|zEU?$dgno-dSe1IC*AN|mSQ8>lo9jJ%sD^$PBsEHn7dVGu8 zP?`;{xv?B^In*uei)nBwX2e?Ioc2EZ_!AuO;sNZnpXY-1f&_2y-a&qHlD~Dxy(M!G zyBl1K8OSH2Znd1aDA7fr(=ZM=Ljv7!Ji((9x#-6CBe*qT4U8obgf#LX< zwTFJqFC)Yi@dSR1+jPzKkGfxle`5d*f#0}>qmC@v;_9e}sfpRa?1g%m23tJRoNUfD zSD^ZBLS5+2l$bOBj*9N}1=Ir9&Bv(M?Ult@j=BAEqb4eh>K|=!j9JTUXzk6-&KOAl zKB)CZV^Q7nNmPp9R_uURQBQUG<8H@+n3i}17RIsWI*cGbk7|F8P4Nxti>M*ztPPGv zz5i2D5A8b3A3%?Ga*B$s^apErg37H7I^`~$-zR z92LE0ai|G~U`8BePC;#8p0)2V_gnkdsAu4u0pEK?{!N>*h_dgX~ zSzgpnuSnE_HBndE6}7S9sENj)`X`}o-DjwYlQ9z>GS8Yf%qJK?`&-lhton2Rb5PNQ zh0RiCj9J%ghT2Gb%#2;lf#w){8n+(H;}+D3Jv4pJyBiHf&0Eys8mMv2(W6RRDw^;E)WW^3!w`!{ zp+EU4mY;)#h(EXXBbb@^ENa6yE&t4XXJ-7~T{tIdo$&8De=S_x8Y);rHM6nh+o7(k z3#$Jx%YS0=6x4)X%P+I|3v)Xbru{J1$496OFLik}VrN{3dd<@P;7*(m)gFnOrwnS|+ScCO+S_`l=xL9D0osi-Snh`M*%P&+<} zn&_N)-F$4mGJP+)7nap5fErf;HE$h@J)N!6#~g}UXuQSKES`^geU_p&m~8n2sELo7 z=TYB$H%#Y}`xjF_)V$5jw#Yb-^MPA&`dY&<)B+yM&omcde)8*3KOIk7KHZP*#KC45 zYD4+V5@r>wMZTfM(=kx*{{ku{Y1ojG;OD=^zoV`=;MOXnwf93k zONporOtie$;)ND}fq7})sJ!0)GgS1jTt)rVx`)Nk|BAb@3Rs@FHx|OBm>rLy#$87( z_`rN&`uyZ>Jd>FX)jyZTg)rsc|B72f4b()9EN*MYqV8Q^Gr=5gjzNu^gnG^9pe|&; zzc&KPV9!!Nr%qUDl9F3WMEwBf5#owVOx?=Gy^9kzc-&lL#Ew_J8 z)CCqojVozZKzfzd9@lU7?K0v*mZ&2fM-FD}R zM75Vhy{@$}2Y4 zqmI5IYJ3;eGnHUYQ=avmRa7)!mw6a<0^gz@p6^i``PuvpHQ`%p&wR&i&ubP#{jyof z;%=yo4M43k19jr_(W8l%Q;{1{9rs!MllcI%kbhyOx$E}Nj@n3m)NeNBQ5%j$UFjgy zYd95k5{EGie?)!bKEKQPD^cK{`xi?MR6H0pU>52NWsT*}So|0@LEwG2Uv1R5_NWUP zf@$#+bBg6>p-y12#b4a#{L_-yPC{3*&pcz^HeaBwEZqaw5LExds1vJ3 zDA)``Z6u%RDM3Z|sETz+L=7Ba9g;Li+fTbsPPe~XQ7D2bx{2qTigz{ zp)M(M{|8Xfk$F%%UV@rv3;N-1i;titI%!@)^}mCf@Fi;Dw7KGcRwqxv<*2>bvu z;uy@M_dm%J8&DJNvG^ov!b|2e)P$LyxEm~t8HuCK7_$LtTx*L5p)TlSizk?~(WBRF zsU>!xHgLrJ9<}42QT=?Mx({b4>LH9qZJ-URUw70(6D&X7;svOaTxPB}w?F0ntKlFC zUD*lLgtt)>Jw<)-{E0fkw7Nf>-)blLfftu$C2I5J}Uo#(}`oDBz{{82<`)hVS%s@j) zvx?ckd>{R3?~1y@zNn3iG$)udF^K#E)JBrc?Whg!Gf!a3@BbIA!$Z`O{AQ+q;SPvE zO%#o4FNeLcsl~g@bEpkHGM}Ty{eyZ)egAal3&23)Pz=)hU!01*yKABr=!x2?$DD31 zz-;8#SiB##z)6eGVH)D=sCn*K`&%>hOZVjbQ70LSp5jyrQqh8~tV37S1Ori5GzIm3 zd$Bt1K=uC57jBkBtykkU=dahO0|{N}N2quVYJo+l zf$PjYn2Y!%YD4!?0rmDZ@8c0(-?k2<1>s0C(P`vNRN zybSdXcGCP6a}Z~Kuy|HY0ow z>)|4-gm+N4DEC|UXF(aPOxzJG;yet)v*s_TXD!`7{Cg9Y!}6@}BvH{sM^QWa$>PTt zPVD^89T#{z_Xm#eWyB=Z*UXl!Xc@A zQvRx)joRsE)RF&%I>IukeVq3&7Te)QjKOri?gF(@8*hZc*cr28f;k?w(RrBi_y0;u z>_qMGYt$_{g*xI}nDUfcd+Ic9KY!FhSyAl;a0EtM`#RLax7peco8Mad#WX(dzyDt& zp(B50rb_FcM4(v$^-#Wt+W7!;lsN^p(RmhcLf!NI7GFSZ@Rr4o%~xqX?u5SSd{Vv= zvzob3Cs7z{V>IesCs=;E=|!EuLev7u<}PbLZ0#p3e-(8B_bmU+LuD$7H0ga(UYl9w zNmR#Fe(uR+MlF;VgRr#4^)QyWJ?aGZp*HdZ>O^l?{M6#tW_o|O-IK#AMNkV?L_K76 zElxyDJjUV$sEOC07TSs0*dbKE3s?nz#oQQ~!JW6h`9A7IW04c~IIF4Xhr|Zdgqu+t z+3R+2j+cx|xAYq7`{NE4)cYSC;N$3v zr82I>P8f<#CZCjVurSmD6;NNHjZnWrC1M#|g8HWW9@XzD>h1B#?Dh{qZKMWj15Gfy z5C5YT7SQ|OH;cRTS*V@QK~1#8+=80mDC+3Xqx#)OP52Ay%AcX0jhCo?{#kueexnLS z)^qry*||z3ABC?PXAs3Zr%Ddbcq+Xq`ph!jjEc>XR5bm)x4f$Mtn$XewyynBOm2F zWdgY$P(L~JIY8-4oQ~3tG5SQ|dE7)_eOk~KMJeru&S&Va{nkS?5tp zqEw;i)5x1tEhKUpQEnCbX0?_a)b$BKJ!tF65BHv~7Sh6N+f5)lj#7yp$&|~KyA*wT zTAv!!gUBtl_;=!})@OkE)LX84{eVu)zl2u4)g52&lyw?fdBjC6K0-X7K5>+F)Mr!Pr+iFlL+%iL>*7rBlp2Lwlp$N6k?qO+NLfOC zB0UO||M&m-Uw>~L)$HW{`x!{T%9IS0?v{X)%K+iL!w5E7?{!8f!3i4n3Cwlpx@%*UOe;(`P|4*(2 zeU?&Qk^cyS)KW;q>LiIP4QCi;gj;;{HTe4Ng$b@ z!S|SCGo?E92Ur8Q(x#6;^+V*!QSwp$%0J`{lKc0wo>-sGSe!DHGKkp%C>QDX9v*bNIQ^;Lp?!=uzHV66twb&9QJ>6O zdaR+|fm}3Y0wtDw8p=ZQ^Jq^Xr%w~y=sjFF$fvybV%_|nX|&HJ-zlY)?@J71+$qgX z-(1w&Q}i2g9sGx)4?jgyJ_m8V)s?tJJr#XMQ213L<#QPaQ2J6{Q6eb%eCOkMht@0A z;~(NJlmj-m+P1KEPs&2dabo>)r2jJOosAtp{&V{8qAVx3mN=36Ec(5{mE>ws{%8GL zc?0T)MZ^v($Iss+1o5@8vki>WQ=+rfjo#hWB*+Fu&&X)~A{GkNRN+ml2JmWTb4S^rNL4 zC7f|H@Jo#ImTC}I>KV21WcYRU|3AB^e@v8x-d& zo1TB-M>eV(eoA>npN7^iWjg*(S^5p6Z8-IsxCv`xIQ=@&)}Q!OtciIkanxVXHk#5b zHUEx1lf+;K=u?I=igJ{A7Nr=SrqZs@ZF8^nEl=((?I$SuWb}4!SU9&2(H_cD%Z9p* zjvu3zc$YVfDL9APd(8EQ7C(A*#3U?Az5@BGxW?<#C?w!lTTxTo@D^;8->;(jP_B4e zGzzPl)z+v@`vJ25{fr`hMSBy?fiM5lxRu-o-gS+lTL1l@6;{{`b1ip@QTk)LD{*Vu zo>4DB$!q;aTfg?ip_C73dqVli8`(I&=RHRMMAVF4pHkPSC8o#JHp4#5Lfnr&Zz!iJ zEr?5z??d0Fl#j_xwElll*QYrK;4c=ZB>6w{iC0mMrRG1vYFyB>!rBYj`lX0lGFJ$F zpHePS_yyfbPx~cG8|q)!m``ZyN8d`e_G(;db()-B)~`SAA^#z{nYPY;a=-cT%VsT0 z9Hc`%>IY~TsfnrgzzzSIurqN3@ACITBEKTrh8{)eT>`JuqJN|Nk-9#$$tU6*@9pomh~5)vfPD>(h}wL%oBVgcR&Wt*Z6z zW362*evQj0oxRCT!aS2`^`&=L<_ag939FJzOF2)eOq)I*k{?dJ0A3^3-(n}JPsDBX zYfXIw?eW&1bjqh3`PI})QNE$yabkVyx&4YdS4kYF!Jp!#z6@XEQA#U1t|jhHeU*1m z)1aI=sXZbyg_f3-7EJR2xt-p{O+#A$OiL-EGqlvknv@FE|NUGb)@Kam1by{s$P9;Y zg0)2we@;0{t~h<3c>S8?^`xeCxAi!UMXY`gKcx3Li#Idsb4KYO1N7-meLXgzd_li| z@LO`-C?8QDh%YJAZ2SpwL6lX*x5&4}lBxOcMHX0x-6T6wFJm!%oxV)0&sgv2W?`NS z^q66-DSzV7@(DS8c3@_=>YTCMF`Q1Wi_Ov&CtIJ_v{k1*1+!3w(bg8*Ut6yt2+Te6Jo@64$OZl6)A%pctZxm(UwojX9E4nRt#PPrq{d&YE_6Q#u8$URF zXz%!bJ;H|%7?Kbk7dtpMJU%hJTaWnuy~5-B#}AH=?H50yN8GlAq!n2k#3uF)AJ8*A zu}8n2#k=?J(Y!}q>cBJSKeE+_TJ>h_r92RZ~DC7_D_GfY1f0b zNe_~zYFdC#}D~a?FFZ)9&q^d2iR&2m5By|KV4&9xk6w=exU;9*#TkXj0O> TRY~_|ZMrvg?Y0a5ObGlxi%*E1 diff --git a/apps/locale/zh/LC_MESSAGES/django.po b/apps/locale/zh/LC_MESSAGES/django.po index 6cd533e47..c8368dce6 100644 --- a/apps/locale/zh/LC_MESSAGES/django.po +++ b/apps/locale/zh/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: JumpServer 0.3.3\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-01-26 17:25+0800\n" +"POT-Creation-Date: 2021-02-20 15:17+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: ibuler \n" "Language-Team: JumpServer team\n" @@ -32,8 +32,8 @@ msgstr "远程应用" msgid "Custom" msgstr "自定义" -#: applications/models/application.py:10 assets/models/asset.py:149 -#: assets/models/base.py:234 assets/models/cluster.py:18 +#: applications/models/application.py:10 assets/models/asset.py:142 +#: assets/models/base.py:235 assets/models/cluster.py:18 #: assets/models/cmd_filter.py:21 assets/models/domain.py:21 #: assets/models/group.py:20 assets/models/label.py:18 ops/mixin.py:24 #: orgs/models.py:23 perms/models/base.py:48 settings/models.py:29 @@ -78,7 +78,7 @@ msgstr "类别" msgid "Type" msgstr "类型" -#: applications/models/application.py:19 assets/models/asset.py:198 +#: applications/models/application.py:19 assets/models/asset.py:191 #: assets/models/domain.py:27 assets/models/domain.py:55 msgid "Domain" msgstr "网域" @@ -89,8 +89,8 @@ msgstr "" # msgid "Date created" # msgstr "创建日期" -#: applications/models/application.py:23 assets/models/asset.py:154 -#: assets/models/asset.py:230 assets/models/base.py:239 +#: applications/models/application.py:23 assets/models/asset.py:147 +#: assets/models/asset.py:223 assets/models/base.py:240 #: assets/models/cluster.py:29 assets/models/cmd_filter.py:23 #: assets/models/cmd_filter.py:57 assets/models/domain.py:22 #: assets/models/domain.py:56 assets/models/group.py:23 @@ -125,16 +125,16 @@ msgstr "主机" #: applications/serializers/attrs/application_type/mysql_workbench.py:22 #: applications/serializers/attrs/application_type/oracle.py:11 #: applications/serializers/attrs/application_type/pgsql.py:11 -#: assets/models/asset.py:195 assets/models/domain.py:53 +#: assets/models/asset.py:188 assets/models/domain.py:53 msgid "Port" msgstr "端口" #: applications/serializers/attrs/application_category/remote_app.py:33 -#: assets/models/asset.py:363 assets/models/authbook.py:26 +#: assets/models/asset.py:355 assets/models/authbook.py:26 #: assets/models/gathered_user.py:14 assets/serializers/admin_user.py:32 #: assets/serializers/asset_user.py:47 assets/serializers/asset_user.py:84 #: assets/serializers/system_user.py:191 audits/models.py:38 -#: perms/models/asset_permission.py:96 templates/index.html:82 +#: perms/models/asset_permission.py:99 templates/index.html:82 #: terminal/backends/command/models.py:19 #: terminal/backends/command/serializers.py:13 terminal/models/session.py:39 #: users/templates/users/user_asset_permission.html:40 @@ -161,7 +161,7 @@ msgstr "目标URL" #: applications/serializers/attrs/application_type/custom.py:21 #: applications/serializers/attrs/application_type/mysql_workbench.py:30 #: applications/serializers/attrs/application_type/vmware_client.py:26 -#: assets/models/base.py:235 assets/models/gathered_user.py:15 +#: assets/models/base.py:236 assets/models/gathered_user.py:15 #: audits/models.py:99 authentication/forms.py:11 #: authentication/templates/authentication/login.html:101 #: ops/models/adhoc.py:148 users/forms/profile.py:19 users/models/user.py:515 @@ -178,7 +178,7 @@ msgstr "用户名" #: applications/serializers/attrs/application_type/custom.py:25 #: applications/serializers/attrs/application_type/mysql_workbench.py:34 #: applications/serializers/attrs/application_type/vmware_client.py:30 -#: assets/models/base.py:236 assets/serializers/asset_user.py:71 +#: assets/models/base.py:237 assets/serializers/asset_user.py:71 #: audits/signals_handler.py:42 authentication/forms.py:13 #: authentication/templates/authentication/login.html:109 #: settings/serializers/settings.py:84 users/forms/user.py:22 @@ -204,7 +204,7 @@ msgid "Target url" msgstr "目标URL" #: applications/serializers/attrs/application_type/mysql_workbench.py:18 -#: assets/models/asset.py:190 assets/models/domain.py:52 +#: assets/models/asset.py:183 assets/models/domain.py:52 #: assets/serializers/asset_user.py:46 settings/serializers/settings.py:103 #: users/templates/users/_granted_assets.html:26 #: users/templates/users/user_asset_permission.html:156 @@ -219,15 +219,15 @@ msgstr "删除失败，存在关联资产" msgid "Number required" msgstr "需要为数字" -#: assets/api/node.py:67 +#: assets/api/node.py:60 msgid "You can't update the root node name" msgstr "不能修改根节点名称" -#: assets/api/node.py:74 +#: assets/api/node.py:67 msgid "You can't delete the root node ({})" msgstr "不能删除根节点 ({})" -#: assets/api/node.py:77 +#: assets/api/node.py:70 msgid "Deletion failed and the node contains children or assets" msgstr "删除失败，节点包含子节点或资产" @@ -239,137 +239,137 @@ msgstr "不能移除资产的管理用户账号" msgid "Latest version could not be delete" msgstr "最新版本的不能被删除" -#: assets/models/asset.py:150 xpack/plugins/cloud/providers/base.py:17 +#: assets/models/asset.py:143 xpack/plugins/cloud/providers/base.py:17 msgid "Base" msgstr "基础" -#: assets/models/asset.py:151 +#: assets/models/asset.py:144 msgid "Charset" msgstr "编码" -#: assets/models/asset.py:152 tickets/models/ticket.py:40 +#: assets/models/asset.py:145 tickets/models/ticket.py:40 msgid "Meta" msgstr "元数据" -#: assets/models/asset.py:153 +#: assets/models/asset.py:146 msgid "Internal" msgstr "内部的" -#: assets/models/asset.py:173 assets/models/asset.py:197 +#: assets/models/asset.py:166 assets/models/asset.py:190 #: assets/serializers/asset.py:66 msgid "Platform" msgstr "系统平台" -#: assets/models/asset.py:191 assets/serializers/asset_user.py:45 +#: assets/models/asset.py:184 assets/serializers/asset_user.py:45 #: assets/serializers/gathered_user.py:20 settings/serializers/settings.py:102 #: users/templates/users/_granted_assets.html:25 #: users/templates/users/user_asset_permission.html:157 msgid "Hostname" msgstr "主机名" -#: assets/models/asset.py:194 assets/models/domain.py:54 +#: assets/models/asset.py:187 assets/models/domain.py:54 #: assets/models/user.py:120 terminal/serializers/session.py:29 #: terminal/serializers/storage.py:69 msgid "Protocol" msgstr "协议" -#: assets/models/asset.py:196 assets/serializers/asset.py:68 +#: assets/models/asset.py:189 assets/serializers/asset.py:68 #: perms/serializers/asset/user_permission.py:41 msgid "Protocols" msgstr "协议组" -#: assets/models/asset.py:199 assets/models/user.py:115 -#: perms/models/asset_permission.py:97 +#: assets/models/asset.py:192 assets/models/user.py:115 +#: perms/models/asset_permission.py:100 #: xpack/plugins/change_auth_plan/models.py:56 #: xpack/plugins/gathered_user/models.py:24 msgid "Nodes" msgstr "节点" -#: assets/models/asset.py:200 assets/models/cmd_filter.py:22 +#: assets/models/asset.py:193 assets/models/cmd_filter.py:22 #: assets/models/domain.py:57 assets/models/label.py:22 #: authentication/models.py:46 msgid "Is active" msgstr "激活" -#: assets/models/asset.py:203 assets/models/cluster.py:19 +#: assets/models/asset.py:196 assets/models/cluster.py:19 #: assets/models/user.py:66 templates/_nav.html:44 #: xpack/plugins/cloud/models.py:143 xpack/plugins/cloud/serializers.py:137 msgid "Admin user" msgstr "管理用户" -#: assets/models/asset.py:206 +#: assets/models/asset.py:199 msgid "Public IP" msgstr "公网IP" -#: assets/models/asset.py:207 +#: assets/models/asset.py:200 msgid "Asset number" msgstr "资产编号" -#: assets/models/asset.py:210 +#: assets/models/asset.py:203 msgid "Vendor" msgstr "制造商" -#: assets/models/asset.py:211 +#: assets/models/asset.py:204 msgid "Model" msgstr "型号" -#: assets/models/asset.py:212 +#: assets/models/asset.py:205 msgid "Serial number" msgstr "序列号" -#: assets/models/asset.py:214 +#: assets/models/asset.py:207 msgid "CPU model" msgstr "CPU型号" -#: assets/models/asset.py:215 +#: assets/models/asset.py:208 msgid "CPU count" msgstr "CPU数量" -#: assets/models/asset.py:216 +#: assets/models/asset.py:209 msgid "CPU cores" msgstr "CPU核数" -#: assets/models/asset.py:217 +#: assets/models/asset.py:210 msgid "CPU vcpus" msgstr "CPU总数" -#: assets/models/asset.py:218 +#: assets/models/asset.py:211 msgid "Memory" msgstr "内存" -#: assets/models/asset.py:219 +#: assets/models/asset.py:212 msgid "Disk total" msgstr "硬盘大小" -#: assets/models/asset.py:220 +#: assets/models/asset.py:213 msgid "Disk info" msgstr "硬盘信息" -#: assets/models/asset.py:222 +#: assets/models/asset.py:215 msgid "OS" msgstr "操作系统" -#: assets/models/asset.py:223 +#: assets/models/asset.py:216 msgid "OS version" msgstr "系统版本" -#: assets/models/asset.py:224 +#: assets/models/asset.py:217 msgid "OS arch" msgstr "系统架构" -#: assets/models/asset.py:225 +#: assets/models/asset.py:218 msgid "Hostname raw" msgstr "主机名原始" -#: assets/models/asset.py:227 templates/_nav.html:46 +#: assets/models/asset.py:220 templates/_nav.html:46 msgid "Labels" msgstr "标签管理" -#: assets/models/asset.py:228 assets/models/base.py:242 +#: assets/models/asset.py:221 assets/models/base.py:243 #: assets/models/cluster.py:28 assets/models/cmd_filter.py:26 #: assets/models/cmd_filter.py:60 assets/models/group.py:21 #: common/db/models.py:67 common/mixins/models.py:49 orgs/models.py:24 -#: orgs/models.py:427 perms/models/base.py:54 users/models/user.py:558 +#: orgs/models.py:412 perms/models/base.py:54 users/models/user.py:558 #: users/serializers/group.py:35 users/templates/users/user_detail.html:97 #: xpack/plugins/change_auth_plan/models.py:81 xpack/plugins/cloud/models.py:58 #: xpack/plugins/cloud/models.py:156 xpack/plugins/gathered_user/models.py:30 @@ -378,12 +378,12 @@ msgstr "创建者" # msgid "Created by" # msgstr "创建者" -#: assets/models/asset.py:229 assets/models/base.py:240 +#: assets/models/asset.py:222 assets/models/base.py:241 #: assets/models/cluster.py:26 assets/models/domain.py:24 #: assets/models/gathered_user.py:19 assets/models/group.py:22 #: assets/models/label.py:25 common/db/models.py:69 common/mixins/models.py:50 #: ops/models/adhoc.py:38 ops/models/command.py:29 orgs/models.py:25 -#: orgs/models.py:425 perms/models/base.py:55 users/models/group.py:18 +#: orgs/models.py:410 perms/models/base.py:55 users/models/group.py:18 #: users/templates/users/user_group_detail.html:58 #: xpack/plugins/cloud/models.py:61 xpack/plugins/cloud/models.py:159 msgid "Date created" @@ -405,21 +405,21 @@ msgstr "版本" msgid "AuthBook" msgstr "" -#: assets/models/base.py:237 xpack/plugins/change_auth_plan/models.py:72 +#: assets/models/base.py:238 xpack/plugins/change_auth_plan/models.py:72 #: xpack/plugins/change_auth_plan/models.py:197 #: xpack/plugins/change_auth_plan/models.py:292 msgid "SSH private key" msgstr "SSH密钥" -#: assets/models/base.py:238 xpack/plugins/change_auth_plan/models.py:75 +#: assets/models/base.py:239 xpack/plugins/change_auth_plan/models.py:75 #: xpack/plugins/change_auth_plan/models.py:193 #: xpack/plugins/change_auth_plan/models.py:288 msgid "SSH public key" msgstr "SSH公钥" -#: assets/models/base.py:241 assets/models/gathered_user.py:20 +#: assets/models/base.py:242 assets/models/gathered_user.py:20 #: common/db/models.py:70 common/mixins/models.py:51 ops/models/adhoc.py:39 -#: orgs/models.py:426 +#: orgs/models.py:411 msgid "Date updated" msgstr "更新日期" @@ -556,9 +556,9 @@ msgstr "默认资产组" #: assets/models/label.py:15 audits/models.py:36 audits/models.py:56 #: audits/models.py:69 audits/serializers.py:81 authentication/models.py:44 -#: authentication/models.py:95 orgs/models.py:18 orgs/models.py:423 -#: perms/models/asset_permission.py:173 perms/models/base.py:49 -#: templates/index.html:78 terminal/backends/command/models.py:18 +#: authentication/models.py:95 orgs/models.py:18 orgs/models.py:408 +#: perms/models/base.py:49 templates/index.html:78 +#: terminal/backends/command/models.py:18 #: terminal/backends/command/serializers.py:12 terminal/models/session.py:37 #: tickets/models/comment.py:17 users/forms/group.py:15 #: users/models/user.py:158 users/models/user.py:665 @@ -575,31 +575,31 @@ msgstr "默认资产组" msgid "User" msgstr "用户" -#: assets/models/label.py:19 assets/models/node.py:413 settings/models.py:30 +#: assets/models/label.py:19 assets/models/node.py:545 settings/models.py:30 msgid "Value" msgstr "值" -#: assets/models/node.py:143 +#: assets/models/node.py:152 msgid "New node" msgstr "新节点" -#: assets/models/node.py:316 users/templates/users/_granted_assets.html:130 +#: assets/models/node.py:450 users/templates/users/_granted_assets.html:130 msgid "empty" msgstr "空" -#: assets/models/node.py:412 perms/models/asset_permission.py:148 +#: assets/models/node.py:544 perms/models/asset_permission.py:156 msgid "Key" msgstr "键" -#: assets/models/node.py:414 +#: assets/models/node.py:546 msgid "Full value" msgstr "全称" -#: assets/models/node.py:417 perms/models/asset_permission.py:152 +#: assets/models/node.py:549 perms/models/asset_permission.py:157 msgid "Parent key" msgstr "ssh私钥" -#: assets/models/node.py:426 assets/serializers/system_user.py:190 +#: assets/models/node.py:557 assets/serializers/system_user.py:190 #: users/templates/users/user_asset_permission.html:41 #: users/templates/users/user_asset_permission.html:73 #: users/templates/users/user_asset_permission.html:158 @@ -668,7 +668,7 @@ msgstr "用户组" #: assets/models/user.py:221 audits/models.py:39 #: perms/models/application_permission.py:31 -#: perms/models/asset_permission.py:98 templates/_nav.html:45 +#: perms/models/asset_permission.py:101 templates/_nav.html:45 #: terminal/backends/command/models.py:20 #: terminal/backends/command/serializers.py:14 terminal/models/session.py:41 #: users/templates/users/_granted_assets.html:27 @@ -727,7 +727,7 @@ msgstr "硬件信息" msgid "Org name" msgstr "组织名称" -#: assets/serializers/asset.py:162 assets/serializers/asset.py:201 +#: assets/serializers/asset.py:162 assets/serializers/asset.py:194 msgid "Connectivity" msgstr "连接" @@ -873,11 +873,6 @@ msgstr "更新节点资产硬件信息: {}" msgid "Gather assets users" msgstr "收集资产上的用户" -#: assets/tasks/nodes_amount.py:21 -msgid "" -"The task of self-checking is already running and cannot be started repeatedly" -msgstr "自检程序已经在运行，不能重复启动" - #: assets/tasks/push_system_user.py:184 #: assets/tasks/system_user_connectivity.py:89 msgid "System user is dynamic: {}" @@ -1081,7 +1076,7 @@ msgstr "用户代理" #: authentication/templates/authentication/_mfa_confirm_modal.html:14 #: authentication/templates/authentication/login_otp.html:6 #: users/forms/profile.py:52 users/models/user.py:539 -#: users/serializers/user.py:232 users/templates/users/user_detail.html:77 +#: users/serializers/profile.py:102 users/templates/users/user_detail.html:77 #: users/templates/users/user_profile.html:87 msgid "MFA" msgstr "多因子认证" @@ -1355,7 +1350,7 @@ msgid "Show" msgstr "显示" #: authentication/templates/authentication/_access_key_modal.html:66 -#: users/models/user.py:443 users/serializers/user.py:229 +#: users/models/user.py:443 users/serializers/profile.py:99 #: users/templates/users/user_profile.html:94 #: users/templates/users/user_profile.html:163 #: users/templates/users/user_profile.html:166 @@ -1364,7 +1359,7 @@ msgid "Disable" msgstr "禁用" #: authentication/templates/authentication/_access_key_modal.html:67 -#: users/models/user.py:444 users/serializers/user.py:230 +#: users/models/user.py:444 users/serializers/profile.py:100 #: users/templates/users/user_profile.html:92 #: users/templates/users/user_profile.html:170 msgid "Enable" @@ -1605,7 +1600,7 @@ msgstr "不能包含特殊字符" msgid "

Flow service unavailable, check it

" msgstr "" -#: jumpserver/views/other.py:26 +#: jumpserver/views/other.py:27 msgid "" "
Luna is a separately deployed program, you need to deploy Luna, koko, " "configure nginx for url distribution,
If you see this page, " @@ -1614,11 +1609,11 @@ msgstr "" "
Luna是单独部署的一个程序，你需要部署luna，koko,
如果你看到了" "这个页面，证明你访问的不是nginx监听的端口，祝你好运
" -#: jumpserver/views/other.py:77 +#: jumpserver/views/other.py:78 msgid "Websocket server run on port: {}, you should proxy it on nginx" msgstr "Websocket 服务运行在端口: {}, 请检查nginx是否代理是否设置" -#: jumpserver/views/other.py:91 +#: jumpserver/views/other.py:92 msgid "" "
Koko is a separately deployed program, you need to deploy Koko, " "configure nginx for url distribution,
If you see this page, " @@ -1795,8 +1790,8 @@ msgstr "组织包含未删除的资源" msgid "The current organization cannot be deleted" msgstr "当前组织不能被删除" -#: orgs/mixins/models.py:56 orgs/mixins/serializers.py:25 orgs/models.py:41 -#: orgs/models.py:422 orgs/serializers.py:100 +#: orgs/mixins/models.py:53 orgs/mixins/serializers.py:25 orgs/models.py:39 +#: orgs/models.py:407 orgs/serializers.py:100 #: tickets/serializers/ticket/ticket.py:81 msgid "Organization" msgstr "组织" @@ -1809,7 +1804,11 @@ msgstr "组织管理员" msgid "Organization auditor" msgstr "组织审计员" -#: orgs/models.py:424 users/forms/user.py:27 users/models/user.py:527 +#: orgs/models.py:33 +msgid "GLOBAL" +msgstr "全局组织" + +#: orgs/models.py:409 users/forms/user.py:27 users/models/user.py:527 #: users/templates/users/_select_user_modal.html:15 #: users/templates/users/user_detail.html:73 #: users/templates/users/user_list.html:16 @@ -1817,7 +1816,7 @@ msgstr "组织审计员" msgid "Role" msgstr "角色" -#: perms/const.py:7 perms/utils/asset/user_permission.py:28 +#: perms/const.py:7 perms/models/asset_permission.py:189 msgid "Ungrouped" msgstr "未分组" @@ -1841,47 +1840,52 @@ msgstr "应用程序" msgid "Application permission" msgstr "应用管理" -#: perms/models/asset_permission.py:34 settings/serializers/settings.py:107 +#: perms/models/asset_permission.py:37 settings/serializers/settings.py:107 msgid "All" msgstr "全部" -#: perms/models/asset_permission.py:35 +#: perms/models/asset_permission.py:38 msgid "Connect" msgstr "连接" -#: perms/models/asset_permission.py:36 +#: perms/models/asset_permission.py:39 msgid "Upload file" msgstr "上传文件" -#: perms/models/asset_permission.py:37 +#: perms/models/asset_permission.py:40 msgid "Download file" msgstr "下载文件" -#: perms/models/asset_permission.py:38 +#: perms/models/asset_permission.py:41 msgid "Upload download" msgstr "上传下载" -#: perms/models/asset_permission.py:39 +#: perms/models/asset_permission.py:42 msgid "Clipboard copy" msgstr "剪贴板复制" -#: perms/models/asset_permission.py:40 +#: perms/models/asset_permission.py:43 msgid "Clipboard paste" msgstr "剪贴板粘贴" -#: perms/models/asset_permission.py:41 +#: perms/models/asset_permission.py:44 msgid "Clipboard copy paste" msgstr "剪贴板复制粘贴" -#: perms/models/asset_permission.py:99 perms/serializers/asset/permission.py:60 +#: perms/models/asset_permission.py:102 +#: perms/serializers/asset/permission.py:60 msgid "Actions" msgstr "动作" -#: perms/models/asset_permission.py:103 templates/_nav.html:78 +#: perms/models/asset_permission.py:106 templates/_nav.html:78 #: users/templates/users/_user_detail_nav_header.html:31 msgid "Asset permission" msgstr "资产授权" +#: perms/models/asset_permission.py:191 +msgid "Favorite" +msgstr "收藏夹" + #: perms/models/base.py:50 templates/_nav.html:21 users/forms/user.py:168 #: users/models/group.py:31 users/models/user.py:523 #: users/templates/users/_select_user_modal.html:16 @@ -1912,11 +1916,11 @@ msgid "" "permission type. ({})" msgstr "应用列表中包含与授权类型不同的应用。({})" -#: perms/serializers/asset/permission.py:58 users/serializers/user.py:80 +#: perms/serializers/asset/permission.py:58 users/serializers/user.py:65 msgid "Is expired" msgstr "是否过期" -#: perms/serializers/asset/permission.py:59 users/serializers/user.py:79 +#: perms/serializers/asset/permission.py:59 users/serializers/user.py:64 msgid "Is valid" msgstr "账户是否有效" @@ -1932,14 +1936,6 @@ msgstr "用户组数量" msgid "System users amount" msgstr "系统用户数量" -#: perms/utils/asset/user_permission.py:30 -msgid "Favorite" -msgstr "收藏夹" - -#: perms/utils/asset/user_permission.py:522 -msgid "Please wait while your data is being initialized" -msgstr "数据正在初始化，请稍等" - #: settings/api/common.py:24 msgid "Test mail sent to {}, please check" msgstr "邮件已经发送{}, 请检查" @@ -3054,7 +3050,7 @@ msgstr "索引" msgid "Doc type" msgstr "文档类型" -#: terminal/serializers/terminal.py:44 terminal/serializers/terminal.py:52 +#: terminal/serializers/terminal.py:47 terminal/serializers/terminal.py:55 msgid "Not found" msgstr "没有发现" @@ -3530,8 +3526,8 @@ msgid "Public key should not be the same as your old one." msgstr "不能和原来的密钥相同" #: users/forms/profile.py:137 users/forms/user.py:90 -#: users/serializers/user.py:192 users/serializers/user.py:277 -#: users/serializers/user.py:335 +#: users/serializers/profile.py:74 users/serializers/profile.py:147 +#: users/serializers/profile.py:160 msgid "Not a valid ssh public key" msgstr "SSH密钥不合法" @@ -3555,15 +3551,15 @@ msgstr "添加到用户组" msgid "* Your password does not meet the requirements" msgstr "* 您的密码不符合要求" -#: users/forms/user.py:124 users/serializers/user.py:35 +#: users/forms/user.py:124 users/serializers/user.py:20 msgid "Reset link will be generated and sent to the user" msgstr "生成重置密码链接，通过邮件发送给用户" -#: users/forms/user.py:125 users/serializers/user.py:36 +#: users/forms/user.py:125 users/serializers/user.py:21 msgid "Set password" msgstr "设置密码" -#: users/forms/user.py:132 users/serializers/user.py:43 +#: users/forms/user.py:132 users/serializers/user.py:28 #: xpack/plugins/change_auth_plan/models.py:61 #: xpack/plugins/change_auth_plan/serializers.py:30 msgid "Password strategy" @@ -3605,75 +3601,75 @@ msgstr "管理员" msgid "Administrator is the super user of system" msgstr "Administrator是初始的超级管理员" -#: users/serializers/user.py:45 -msgid "MFA level for display" -msgstr "多因子认证等级（显示名称）" - -#: users/serializers/user.py:46 -msgid "Login blocked" -msgstr "登录被阻塞" - -#: users/serializers/user.py:47 -msgid "Can update" -msgstr "是否可更新" - -#: users/serializers/user.py:48 -msgid "Can delete" -msgstr "是否可删除" - -#: users/serializers/user.py:49 users/serializers/user.py:85 -msgid "Organization role name" -msgstr "组织角色名称" - -#: users/serializers/user.py:78 users/serializers/user.py:248 -msgid "Is first login" -msgstr "首次登录" - -#: users/serializers/user.py:81 -msgid "Avatar url" -msgstr "头像路径" - -#: users/serializers/user.py:83 -msgid "Groups name" -msgstr "用户组名" - -#: users/serializers/user.py:84 -msgid "Source name" -msgstr "用户来源名" - -#: users/serializers/user.py:86 -msgid "Super role name" -msgstr "超级角色名称" - -#: users/serializers/user.py:87 -msgid "Total role name" -msgstr "汇总角色名称" - -#: users/serializers/user.py:88 -msgid "MFA enabled" -msgstr "是否开启多因子认证" - -#: users/serializers/user.py:89 -msgid "MFA force enabled" -msgstr "强制启用多因子认证" - -#: users/serializers/user.py:112 -msgid "Role limit to {}" -msgstr "角色只能为 {}" - -#: users/serializers/user.py:124 users/serializers/user.py:301 -msgid "Password does not match security rules" -msgstr "密码不满足安全规则" - -#: users/serializers/user.py:293 +#: users/serializers/profile.py:32 msgid "The old password is incorrect" msgstr "旧密码错误" -#: users/serializers/user.py:307 +#: users/serializers/profile.py:40 users/serializers/user.py:109 +msgid "Password does not match security rules" +msgstr "密码不满足安全规则" + +#: users/serializers/profile.py:46 msgid "The newly set password is inconsistent" msgstr "两次密码不一致" -#: users/serializers_v2/user.py:36 +#: users/serializers/profile.py:118 users/serializers/user.py:63 +msgid "Is first login" +msgstr "首次登录" + +#: users/serializers/user.py:30 +msgid "MFA level for display" +msgstr "多因子认证等级（显示名称）" + +#: users/serializers/user.py:31 +msgid "Login blocked" +msgstr "登录被阻塞" + +#: users/serializers/user.py:32 +msgid "Can update" +msgstr "是否可更新" + +#: users/serializers/user.py:33 +msgid "Can delete" +msgstr "是否可删除" + +#: users/serializers/user.py:34 users/serializers/user.py:70 +msgid "Organization role name" +msgstr "组织角色名称" + +#: users/serializers/user.py:66 +msgid "Avatar url" +msgstr "头像路径" + +#: users/serializers/user.py:68 +msgid "Groups name" +msgstr "用户组名" + +#: users/serializers/user.py:69 +msgid "Source name" +msgstr "用户来源名" + +#: users/serializers/user.py:71 +msgid "Super role name" +msgstr "超级角色名称" + +#: users/serializers/user.py:72 +msgid "Total role name" +msgstr "汇总角色名称" + +#: users/serializers/user.py:73 +msgid "MFA enabled" +msgstr "是否开启多因子认证" + +#: users/serializers/user.py:74 +msgid "MFA force enabled" +msgstr "强制启用多因子认证" + +#: users/serializers/user.py:97 +msgid "Role limit to {}" +msgstr "角色只能为 {}" + +#: users/serializers/user.py:210 msgid "name not unique" msgstr "名称重复" @@ -4891,3 +4887,11 @@ msgstr "旗舰版" #: xpack/plugins/license/models.py:77 msgid "Community edition" msgstr "社区版" + +#~ msgid "" +#~ "The task of self-checking is already running and cannot be started " +#~ "repeatedly" +#~ msgstr "自检程序已经在运行，不能重复启动" + +#~ msgid "Please wait while your data is being initialized" +#~ msgstr "数据正在初始化，请稍等" diff --git a/apps/orgs/api.py b/apps/orgs/api.py index 1a6545f0b..618ebe9ed 100644 --- a/apps/orgs/api.py +++ b/apps/orgs/api.py @@ -5,14 +5,17 @@ from django.utils.translation import ugettext as _ from rest_framework import status from rest_framework.views import Response from rest_framework_bulk import BulkModelViewSet +from rest_framework.generics import RetrieveAPIView +from rest_framework.exceptions import PermissionDenied -from common.permissions import IsSuperUserOrAppUser +from common.permissions import IsSuperUserOrAppUser, IsValidUser, UserCanUseCurrentOrg from common.drf.api import JMSBulkRelationModelViewSet from .models import Organization, ROLE from .serializers import ( OrgSerializer, OrgReadSerializer, OrgRetrieveSerializer, OrgMemberSerializer, - OrgMemberAdminSerializer, OrgMemberUserSerializer + OrgMemberAdminSerializer, OrgMemberUserSerializer, + CurrentOrgSerializer ) from users.models import User, UserGroup from assets.models import Asset, Domain, AdminUser, SystemUser, Label @@ -129,3 +132,11 @@ class OrgMemberUserRelationBulkViewSet(JMSBulkRelationModelViewSet): objs = list(queryset.all().prefetch_related('user', 'org')) queryset.delete() self.send_m2m_changed_signal(objs, action='post_remove') + + +class CurrentOrgDetailApi(RetrieveAPIView): + serializer_class = CurrentOrgSerializer + permission_classes = (IsValidUser, UserCanUseCurrentOrg) + + def get_object(self): + return current_org diff --git a/apps/orgs/caches.py b/apps/orgs/caches.py index 462b13594..b7e086d6a 100644 --- a/apps/orgs/caches.py +++ b/apps/orgs/caches.py @@ -33,12 +33,12 @@ class OrgResourceStatisticsCache(OrgRelatedCache): return self.org def compute_users_amount(self): - if self.org.is_real(): + if self.org.is_root(): + users_amount = User.objects.all().count() + else: users_amount = OrganizationMember.objects.values( 'user_id' ).filter(org_id=self.org.id).distinct().count() - else: - users_amount = User.objects.all().distinct().count() return users_amount def compute_assets_amount(self): diff --git a/apps/orgs/migrations/0010_auto_20210219_1241.py b/apps/orgs/migrations/0010_auto_20210219_1241.py new file mode 100644 index 000000000..70a4463f1 --- /dev/null +++ b/apps/orgs/migrations/0010_auto_20210219_1241.py @@ -0,0 +1,56 @@ +# Generated by Django 3.1 on 2021-02-19 04:41 + +import time +from django.db import migrations + + +default_id = '00000000-0000-0000-0000-000000000001' + + +def add_default_org(apps, schema_editor): + org_cls = apps.get_model('orgs', 'Organization') + defaults = {'name': 'DEFAULT', 'id': default_id} + org_cls.objects.get_or_create(defaults=defaults, id=default_id) + + +def migrate_default_org_id(apps, schema_editor): + org_app_models = [ + ('applications', ['Application']), + ('assets', [ + 'AdminUser', 'Asset', 'AuthBook', 'CommandFilter', + 'CommandFilterRule', 'Domain', 'Gateway', 'GatheredUser', + 'Label', 'Node', 'SystemUser' + ]), + ('audits', ['FTPLog', 'OperateLog']), + ('ops', ['AdHoc', 'AdHocExecution', 'CommandExecution', 'Task']), + ('perms', ['ApplicationPermission', 'AssetPermission', 'UserAssetGrantedTreeNodeRelation']), + ('terminal', ['Session', 'Command']), + ('tickets', ['Ticket']), + ('users', ['UserGroup']), + ('xpack', [ + 'Account', 'SyncInstanceDetail', 'SyncInstanceTask', 'SyncInstanceTaskExecution', + 'ChangeAuthPlan', 'ChangeAuthPlanExecution', 'ChangeAuthPlanTask', + 'GatherUserTask', 'GatherUserTaskExecution', + ]), + ] + print("") + for app, models_name in org_app_models: + for model_name in models_name: + t_start = time.time() + print("Migrate model org id: {}".format(model_name), end='') + model_cls = apps.get_model(app, model_name) + model_cls.objects.filter(org_id='').update(org_id=default_id) + interval = round((time.time() - t_start) * 1000, 2) + print(" done, use {} ms".format(interval)) + + +class Migration(migrations.Migration): + + dependencies = [ + ('orgs', '0009_auto_20201023_1628'), + ] + + operations = [ + migrations.RunPython(add_default_org), + migrations.RunPython(migrate_default_org_id) + ] diff --git a/apps/orgs/mixins/models.py b/apps/orgs/mixins/models.py index c0823bb74..dd967757f 100644 --- a/apps/orgs/mixins/models.py +++ b/apps/orgs/mixins/models.py @@ -23,14 +23,11 @@ class OrgManager(models.Manager): def all_group_by_org(self): from ..models import Organization orgs = list(Organization.objects.all()) - orgs.append(Organization.default()) querysets = {} for org in orgs: - if org.is_real(): - org_id = org.id - else: - org_id = '' - querysets[org] = super(OrgManager, self).get_queryset().filter(org_id=org_id) + org_id = org.id + queryset = super(OrgManager, self).get_queryset().filter(org_id=org_id) + querysets[org] = queryset return querysets def get_queryset(self): @@ -53,12 +50,11 @@ class OrgModelMixin(models.Model): def save(self, *args, **kwargs): org = get_current_org() - if org is None: - return super().save(*args, **kwargs) - if org.is_real() or org.is_system(): + if org.is_root(): + if not self.org_id: + raise ValidationError('Please save in a organization') + else: self.org_id = org.id - elif org.is_default(): - self.org_id = '' return super().save(*args, **kwargs) @property @@ -78,10 +74,7 @@ class OrgModelMixin(models.Model): name = self.name elif hasattr(self, 'hostname'): name = self.hostname - if self.org.is_real(): - return name + self.sep + self.org_name - else: - return name + return name + self.sep + self.org_name def validate_unique(self, exclude=None): """ @@ -89,7 +82,7 @@ class OrgModelMixin(models.Model): failed. Form 提交时会使用这个检验 """ - self.org_id = current_org.id if current_org.is_real() else '' + self.org_id = current_org.id if exclude and 'org_id' in exclude: exclude.remove('org_id') unique_checks, date_checks = self._get_unique_checks(exclude=exclude) diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 4d3850181..2c9e85591 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -30,11 +30,9 @@ class Organization(models.Model): orgs = None CACHE_PREFIX = 'JMS_ORG_{}' ROOT_ID = '00000000-0000-0000-0000-000000000000' - ROOT_NAME = 'ROOT' - DEFAULT_ID = 'DEFAULT' + ROOT_NAME = _('GLOBAL') + DEFAULT_ID = '00000000-0000-0000-0000-000000000001' DEFAULT_NAME = 'DEFAULT' - SYSTEM_ID = '00000000-0000-0000-0000-000000000002' - SYSTEM_NAME = 'SYSTEM' _user_admin_orgs = None class Meta: @@ -69,8 +67,6 @@ class Organization(models.Model): return cls.default() elif id_or_name in [cls.ROOT_ID, cls.ROOT_NAME]: return cls.root() - elif id_or_name in [cls.SYSTEM_ID, cls.SYSTEM_NAME]: - return cls.system() try: if is_uuid(id_or_name): @@ -87,7 +83,7 @@ class Organization(models.Model): def get_org_members_by_role(self, role): from users.models import User - if self.is_real(): + if not self.is_root(): return self.members.filter(m2m_org_members__role=role) users = User.objects.filter(role=role) return users @@ -105,20 +101,14 @@ class Organization(models.Model): return self.get_org_members_by_role(ROLE.AUDITOR) def org_id(self): - if self.is_real(): - return self.id - elif self.is_root(): - return self.ROOT_ID - else: - return '' + return self.id def get_members(self, exclude=()): from users.models import User - if self.is_real(): - members = self.members.exclude(m2m_org_members__role__in=exclude) - else: + if self.is_root(): members = User.objects.exclude(role__in=exclude) - + else: + members = self.members.exclude(m2m_org_members__role__in=exclude) return members.exclude(role=User.ROLE.APP).distinct() def can_admin_by(self, user): @@ -129,20 +119,19 @@ class Organization(models.Model): return False def can_audit_by(self, user): - if user.is_super_auditor: + if user.is_superuser or user.is_super_auditor: return True if self.auditors.filter(id=user.id).exists(): return True return False - def can_user_by(self, user): + def can_use_by(self, user): + if user.is_superuser or user.is_super_auditor: + return True if self.users.filter(id=user.id).exists(): return True return False - def is_real(self): - return self.id not in (self.DEFAULT_NAME, self.ROOT_ID, self.SYSTEM_ID) - @classmethod def get_user_orgs_by_role(cls, user, role): if not isinstance(role, (tuple, list)): @@ -165,7 +154,7 @@ class Organization(models.Model): if user.is_anonymous: return cls.objects.none() if user.is_superuser: - return [*cls.objects.all(), cls.default()] + return [cls.root(), *cls.objects.all()] return cls.get_user_orgs_by_role(user, ROLE.ADMIN) @classmethod @@ -182,7 +171,7 @@ class Organization(models.Model): if user.is_anonymous: return cls.objects.none() if user.is_super_auditor: - return [*cls.objects.all(), cls.default()] + return [cls.root(), *cls.objects.all()] return cls.get_user_orgs_by_role(user, ROLE.AUDITOR) @classmethod @@ -190,29 +179,24 @@ class Organization(models.Model): if user.is_anonymous: return cls.objects.none() if user.is_superuser or user.is_super_auditor: - return [*cls.objects.all(), cls.default()] + return [cls.root(), *cls.objects.all()] return cls.get_user_orgs_by_role(user, (ROLE.AUDITOR, ROLE.ADMIN)) @classmethod def default(cls): - return cls(id=cls.DEFAULT_ID, name=cls.DEFAULT_NAME) + defaults = dict(name=cls.DEFAULT_NAME, id=cls.DEFAULT_ID) + obj, created = cls.objects.get_or_create(defaults=defaults, id=cls.DEFAULT_ID) + return obj @classmethod def root(cls): return cls(id=cls.ROOT_ID, name=cls.ROOT_NAME) - @classmethod - def system(cls): - return cls(id=cls.SYSTEM_ID, name=cls.SYSTEM_NAME) - def is_root(self): - return self.id is self.ROOT_ID + return self.id == self.ROOT_ID def is_default(self): - return self.id is self.DEFAULT_ID - - def is_system(self): - return self.id is self.SYSTEM_ID + return str(self.id) == self.DEFAULT_ID def change_to(self): from .utils import set_current_org @@ -282,7 +266,7 @@ class UserRoleMapper(dict): self[ROLE.AUDITOR] = self.auditors -class OrgMemeberManager(models.Manager): +class OrgMemberManager(models.Manager): def remove_users(self, org, users): from users.models import User @@ -337,8 +321,11 @@ class OrgMemeberManager(models.Manager): _user = _user.id oms_add.append(self.model(org_id=org.id, user_id=_user, role=_role)) - send = partial(signals.m2m_changed.send, sender=self.model, instance=org, reverse=False, - model=User, pk_set=_users2pks_if_need(users, admins, auditors), using=self.db) + pk_set = _users2pks_if_need(users, admins, auditors) + send = partial( + signals.m2m_changed.send, sender=self.model, instance=org, reverse=False, + model=User, pk_set=pk_set, using=self.db + ) send(action='pre_add') self.bulk_create(oms_add, ignore_conflicts=True) @@ -429,7 +416,7 @@ class OrganizationMember(models.Model): date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) - objects = OrgMemeberManager() + objects = OrgMemberManager() class Meta: unique_together = [('org', 'user', 'role')] diff --git a/apps/orgs/serializers.py b/apps/orgs/serializers.py index e167aa525..482c622e1 100644 --- a/apps/orgs/serializers.py +++ b/apps/orgs/serializers.py @@ -38,7 +38,8 @@ class OrgSerializer(ModelSerializer): list_serializer_class = AdaptedBulkListSerializer fields_mini = ['id', 'name'] fields_small = fields_mini + [ - 'created_by', 'date_created', 'comment', 'resource_statistics' + 'is_default', 'is_root', 'comment', + 'created_by', 'date_created', 'resource_statistics' ] fields_m2m = ['users', 'admins', 'auditors'] @@ -127,3 +128,9 @@ class OrgRetrieveSerializer(OrgReadSerializer): class Meta(OrgReadSerializer.Meta): pass + + +class CurrentOrgSerializer(ModelSerializer): + class Meta: + model = Organization + fields = ['id', 'name', 'is_default', 'is_root', 'comment'] diff --git a/apps/orgs/urls/api_urls.py b/apps/orgs/urls/api_urls.py index d8533bb87..61fad175e 100644 --- a/apps/orgs/urls/api_urls.py +++ b/apps/orgs/urls/api_urls.py @@ -1,28 +1,26 @@ # -*- coding: utf-8 -*- # -from django.urls import re_path -from rest_framework.routers import DefaultRouter +from django.urls import path from rest_framework_bulk.routes import BulkRouter -from common import api as capi from .. import api app_name = 'orgs' -router = DefaultRouter() -bulk_router = BulkRouter() +router = BulkRouter() router.register(r'orgs', api.OrgViewSet, 'org') -bulk_router.register(r'org-member-relation', api.OrgMemberRelationBulkViewSet, 'org-member-relation') +router.register(r'org-member-relation', api.OrgMemberRelationBulkViewSet, 'org-member-relation') router.register(r'orgs/(?P[0-9a-zA-Z\-]{36})/membership/admins', api.OrgMemberAdminRelationBulkViewSet, 'membership-admins') router.register(r'orgs/(?P[0-9a-zA-Z\-]{36})/membership/users', api.OrgMemberUserRelationBulkViewSet, 'membership-users'), -old_version_urlpatterns = [ - re_path('(?Porg)/.*', capi.redirect_plural_name_api) + +urlpatterns = [ + path('orgs/current/', api.CurrentOrgDetailApi.as_view(), name='current-org-detail'), ] -urlpatterns = router.urls + bulk_router.urls + old_version_urlpatterns +urlpatterns += router.urls diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index de9e2982c..56f22ac86 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -68,12 +68,8 @@ def get_current_org_id(): def construct_org_mapper(): orgs = Organization.objects.all() org_mapper = {str(org.id): org for org in orgs} - default_org = Organization.default() org_mapper.update({ - '': default_org, - Organization.DEFAULT_ID: default_org, Organization.ROOT_ID: Organization.root(), - Organization.SYSTEM_ID: Organization.system() }) return org_mapper @@ -137,11 +133,9 @@ def get_org_filters(): _current_org = get_current_org() if _current_org is None: return kwargs - - if _current_org.is_real(): - kwargs['org_id'] = _current_org.id - elif _current_org.is_default(): - kwargs["org_id"] = '' + if _current_org.is_root(): + return kwargs + kwargs['org_id'] = _current_org.id return kwargs diff --git a/apps/terminal/models/session.py b/apps/terminal/models/session.py index b35a618bc..b440d2f1e 100644 --- a/apps/terminal/models/session.py +++ b/apps/terminal/models/session.py @@ -168,7 +168,7 @@ class Session(OrgModelMixin): from common.utils.random import random_datetime, random_ip org = get_current_org() - if not org or not org.is_real(): + if not org or org.is_root(): Organization.default().change_to() i = 0 users = User.objects.all()[:100] diff --git a/apps/users/api/mixins.py b/apps/users/api/mixins.py index 0e81bd8d2..23425aa0e 100644 --- a/apps/users/api/mixins.py +++ b/apps/users/api/mixins.py @@ -8,7 +8,7 @@ from orgs.utils import current_org class UserQuerysetMixin: def get_queryset(self): - if self.request.query_params.get('all') or not current_org.is_real(): + if self.request.query_params.get('all') or current_org.is_root(): queryset = User.objects.exclude(role=User.ROLE.APP) else: queryset = utils.get_current_org_members() diff --git a/apps/users/api/profile.py b/apps/users/api/profile.py index 473ad3819..a9123047a 100644 --- a/apps/users/api/profile.py +++ b/apps/users/api/profile.py @@ -2,6 +2,7 @@ import uuid from rest_framework import generics +from common.permissions import IsOrgAdmin from rest_framework.permissions import IsAuthenticated from django.conf import settings @@ -23,7 +24,7 @@ __all__ = [ class UserResetPasswordApi(UserQuerysetMixin, generics.UpdateAPIView): queryset = User.objects.all() serializer_class = serializers.UserSerializer - permission_classes = (IsAuthenticated,) + permission_classes = (IsOrgAdmin,) def perform_update(self, serializer): # Note: we are not updating the user object here. @@ -37,7 +38,7 @@ class UserResetPasswordApi(UserQuerysetMixin, generics.UpdateAPIView): class UserResetPKApi(UserQuerysetMixin, generics.UpdateAPIView): serializer_class = serializers.UserSerializer - permission_classes = (IsAuthenticated,) + permission_classes = (IsOrgAdmin,) def perform_update(self, serializer): from ..utils import send_reset_ssh_key_mail diff --git a/apps/users/api/user.py b/apps/users/api/user.py index 7e4d60646..488c699c3 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -48,7 +48,7 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): queryset = super().get_queryset().prefetch_related( 'groups' ) - if current_org.is_real(): + if not current_org.is_root(): # 为在列表中计算用户在真实组织里的角色 queryset = queryset.prefetch_related( Prefetch( @@ -67,7 +67,7 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): @staticmethod def set_users_to_org(users, org_roles, update=False): # 只有真实存在的组织才真正关联用户 - if not current_org or not current_org.is_real(): + if not current_org or current_org.is_root(): return for user, roles in zip(users, org_roles): if update and roles is None: @@ -94,7 +94,7 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): return super().get_permissions() def perform_destroy(self, instance): - if current_org.is_real(): + if not current_org.is_root(): instance.remove() else: return super().perform_destroy(instance) @@ -150,7 +150,7 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): data = request.data if not isinstance(data, list): data = [request.data] - if not current_org or not current_org.is_real(): + if not current_org or current_org.is_root(): error = {"error": "Not a valid org"} return Response(error, status=400) diff --git a/apps/users/models/user.py b/apps/users/models/user.py index 50099253d..bbacd998b 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -169,20 +169,21 @@ class RoleMixin: def org_roles(self): from orgs.models import ROLE as ORG_ROLE - if not current_org.is_real(): - # 不是真实的组织，取 User 本身的角色 + if current_org.is_root(): + # root 组织, 取 User 本身的角色 if self.is_superuser: - return [ORG_ROLE.ADMIN] + roles = [ORG_ROLE.ADMIN] + elif self.is_super_auditor: + roles = [ORG_ROLE.AUDITOR] else: - return [ORG_ROLE.USER] - - # 是真实组织，取 OrganizationMember 中的角色 - roles = [ - org_member.role - for org_member in self.m2m_org_members.all() - if org_member.org_id == current_org.id - ] - roles.sort() + roles = [ORG_ROLE.USER] + else: + # 是真实组织, 取 OrganizationMember 中的角色 + roles = [ + org_member.role for org_member in self.m2m_org_members.all() + if org_member.org_id == current_org.id + ] + roles.sort() return roles @lazyproperty @@ -202,7 +203,7 @@ class RoleMixin: def current_org_roles(self): from orgs.models import OrganizationMember, ROLE as ORG_ROLE - if not current_org.is_real(): + if current_org.is_root(): if self.is_superuser: return [ORG_ROLE.ADMIN] else: @@ -297,7 +298,7 @@ class RoleMixin: @lazyproperty def can_user_current_org(self): - return current_org.can_user_by(self) + return current_org.can_use_by(self) @lazyproperty def can_admin_or_audit_current_org(self): @@ -325,7 +326,7 @@ class RoleMixin: return app, access_key def remove(self): - if not current_org.is_real(): + if current_org.is_root(): return org = Organization.get_instance(current_org.id) OrganizationMember.objects.remove_users(org, [self]) From f548b4bd2b07ed04533356a312b8ace14ef84e8b Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 2 Mar 2021 19:18:25 +0800 Subject: [PATCH 37/71] =?UTF-8?q?feat:=20serializer=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=80=BC=EF=BC=8C=E5=89=8D=E7=AB=AF=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E8=B0=83=E7=94=A8=20(#5666)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: 优化默认值 Co-authored-by: ibuler --- apps/common/drf/metadata.py | 10 ++++---- apps/common/mixins/serializers.py | 40 +++++++++++++++++++++++++++++-- apps/users/serializers/user.py | 11 +++++---- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/apps/common/drf/metadata.py b/apps/common/drf/metadata.py index 7d2332006..04c365d97 100644 --- a/apps/common/drf/metadata.py +++ b/apps/common/drf/metadata.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from collections import OrderedDict +import datetime from django.core.exceptions import PermissionDenied from django.http import Http404 @@ -21,7 +22,7 @@ class SimpleMetadataWithFilters(SimpleMetadata): attrs = [ 'read_only', 'label', 'help_text', 'min_length', 'max_length', - 'min_value', 'max_value', "write_only" + 'min_value', 'max_value', "write_only", ] def determine_actions(self, request, view): @@ -59,9 +60,10 @@ class SimpleMetadataWithFilters(SimpleMetadata): field_info['type'] = self.label_lookup[field] field_info['required'] = getattr(field, 'required', False) - default = getattr(field, 'default', False) - if default and isinstance(default, (str, int)): - field_info['default'] = default + default = getattr(field, 'default', None) + if default is not None and default != empty: + if isinstance(default, (str, int, bool, datetime.datetime, list)): + field_info['default'] = default for attr in self.attrs: value = getattr(field, attr, None) diff --git a/apps/common/mixins/serializers.py b/apps/common/mixins/serializers.py index ecdd31b1d..3c310c9e9 100644 --- a/apps/common/mixins/serializers.py +++ b/apps/common/mixins/serializers.py @@ -2,7 +2,7 @@ # from collections import Iterable -from django.db.models import Prefetch, F +from django.db.models import Prefetch, F, NOT_PROVIDED from django.core.exceptions import ObjectDoesNotExist from rest_framework.utils import html from rest_framework.settings import api_settings @@ -228,7 +228,43 @@ class SizedModelFieldsMixin(BaseDynamicFieldsPlugin): return fields_to_drop +class DefaultValueFieldsMixin: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_fields_default_value() + + def set_fields_default_value(self): + if not hasattr(self, 'Meta'): + return + if not hasattr(self.Meta, 'model'): + return + model = self.Meta.model + for name, serializer_field in self.fields.items(): + if serializer_field.default != empty or serializer_field.required: + continue + model_field = getattr(model, name, None) + if model_field is None: + continue + if not hasattr(model_field, 'field') \ + or not hasattr(model_field.field, 'default') \ + or model_field.field.default == NOT_PROVIDED: + continue + if name == 'id': + continue + default = model_field.field.default + + if callable(default): + default = default() + if default == '': + continue + # print(f"Set default value: {name}: {default}") + serializer_field.default = default + + class DynamicFieldsMixin: + """ + 可以控制显示不同的字段，mini 最少，small 不包含关系 + """ dynamic_fields_plugins = [QueryFieldsMixin, SizedModelFieldsMixin] def __init__(self, *args, **kwargs): @@ -256,7 +292,7 @@ class EagerLoadQuerySetFields: return queryset -class CommonSerializerMixin(DynamicFieldsMixin): +class CommonSerializerMixin(DynamicFieldsMixin, DefaultValueFieldsMixin): pass diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index 9eb10bc5a..239b80af1 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -24,15 +24,17 @@ class UserSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer): (1, CUSTOM_PASSWORD) ) password_strategy = serializers.ChoiceField( - choices=PASSWORD_STRATEGY_CHOICES, required=False, initial=0, - label=_('Password strategy'), write_only=True + choices=PASSWORD_STRATEGY_CHOICES, required=False, + label=_('Password strategy'), write_only=True, default=0 ) mfa_level_display = serializers.ReadOnlyField(source='get_mfa_level_display', label=_('MFA level for display')) login_blocked = serializers.SerializerMethodField(label=_('Login blocked')) can_update = serializers.SerializerMethodField(label=_('Can update')) can_delete = serializers.SerializerMethodField(label=_('Can delete')) - org_roles = serializers.ListField(label=_('Organization role name'), allow_null=True, required=False, - child=serializers.ChoiceField(choices=ORG_ROLE.choices)) + org_roles = serializers.ListField( + label=_('Organization role name'), allow_null=True, required=False, + child=serializers.ChoiceField(choices=ORG_ROLE.choices), default=["User"] + ) key_prefix_block = "_LOGIN_BLOCK_{}" class Meta: @@ -72,6 +74,7 @@ class UserSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer): 'total_role_display': {'label': _('Total role name')}, 'mfa_enabled': {'label': _('MFA enabled')}, 'mfa_force_enabled': {'label': _('MFA force enabled')}, + 'role': {'default': "User"}, } def __init__(self, *args, **kwargs): From 1870fc97d5428a20a5d8fb5f0b6ecb75ae6a8024 Mon Sep 17 00:00:00 2001 From: xinwen Date: Tue, 2 Mar 2021 19:45:44 +0800 Subject: [PATCH 38/71] =?UTF-8?q?refactor:=20=E9=80=82=E9=85=8D=E6=96=B0?= =?UTF-8?q?=E7=9A=84=20default=20=E7=BB=84=E7=BB=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0010_auto_20210219_1241.py | 26 ++++++- apps/orgs/signals_handler/__init__.py | 2 + apps/orgs/signals_handler/cache.py | 78 +++++++++++++++++++ .../common.py} | 77 ++---------------- apps/perms/utils/asset/user_permission.py | 11 +-- 5 files changed, 113 insertions(+), 81 deletions(-) create mode 100644 apps/orgs/signals_handler/__init__.py create mode 100644 apps/orgs/signals_handler/cache.py rename apps/orgs/{signals_handler.py => signals_handler/common.py} (57%) diff --git a/apps/orgs/migrations/0010_auto_20210219_1241.py b/apps/orgs/migrations/0010_auto_20210219_1241.py index 70a4463f1..14a9b2ba0 100644 --- a/apps/orgs/migrations/0010_auto_20210219_1241.py +++ b/apps/orgs/migrations/0010_auto_20210219_1241.py @@ -1,6 +1,8 @@ # Generated by Django 3.1 on 2021-02-19 04:41 import time +import sys + from django.db import migrations @@ -38,12 +40,33 @@ def migrate_default_org_id(apps, schema_editor): for model_name in models_name: t_start = time.time() print("Migrate model org id: {}".format(model_name), end='') + sys.stdout.flush() + model_cls = apps.get_model(app, model_name) model_cls.objects.filter(org_id='').update(org_id=default_id) interval = round((time.time() - t_start) * 1000, 2) print(" done, use {} ms".format(interval)) +def add_all_user_to_default_org(apps, schema_editor): + User = apps.get_model('users', 'User') + Organization = apps.get_model('orgs', 'Organization') + + users = User.objects.all() + default_org = Organization.objects.get(id=default_id) + + t_start = time.time() + count = users.count() + print(f'{count} users to add') + + batch_size = 1000 + for i in range(0, count, batch_size): + default_org.members.add(*users[i:i+batch_size]) + print(f'Add {i+1}-{i+batch_size} users') + interval = round((time.time() - t_start) * 1000, 2) + print(f'done, use {interval} ms') + + class Migration(migrations.Migration): dependencies = [ @@ -52,5 +75,6 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(add_default_org), - migrations.RunPython(migrate_default_org_id) + migrations.RunPython(migrate_default_org_id), + migrations.RunPython(add_all_user_to_default_org) ] diff --git a/apps/orgs/signals_handler/__init__.py b/apps/orgs/signals_handler/__init__.py new file mode 100644 index 000000000..267008aac --- /dev/null +++ b/apps/orgs/signals_handler/__init__.py @@ -0,0 +1,2 @@ +from . import common +from . import cache diff --git a/apps/orgs/signals_handler/cache.py b/apps/orgs/signals_handler/cache.py new file mode 100644 index 000000000..c3c23efb4 --- /dev/null +++ b/apps/orgs/signals_handler/cache.py @@ -0,0 +1,78 @@ +from django.db.models.signals import m2m_changed +from django.db.models.signals import post_save, pre_delete +from django.dispatch import receiver + +from orgs.models import Organization, OrganizationMember +from assets.models import Node +from perms.models import (AssetPermission, ApplicationPermission) +from users.models import UserGroup, User +from applications.models import Application +from assets.models import Asset, AdminUser, SystemUser, Domain, Gateway +from common.const.signals import POST_PREFIX +from orgs.caches import OrgResourceStatisticsCache + + +def refresh_user_amount_on_user_create_or_delete(user_id): + orgs = Organization.objects.filter(m2m_org_members__user_id=user_id).distinct() + for org in orgs: + org_cache = OrgResourceStatisticsCache(org) + org_cache.expire('users_amount') + + +@receiver(post_save, sender=User) +def on_user_create_refresh_cache(sender, instance, created, **kwargs): + if created: + refresh_user_amount_on_user_create_or_delete(instance.id) + + +@receiver(pre_delete, sender=User) +def on_user_delete_refresh_cache(sender, instance, **kwargs): + refresh_user_amount_on_user_create_or_delete(instance.id) + + +@receiver(m2m_changed, sender=OrganizationMember) +def on_org_user_changed_refresh_cache(sender, action, instance, reverse, pk_set, **kwargs): + if not action.startswith(POST_PREFIX): + return + + if reverse: + orgs = Organization.objects.filter(id__in=pk_set) + else: + orgs = [instance] + + for org in orgs: + org_cache = OrgResourceStatisticsCache(org) + org_cache.expire('users_amount') + + +class OrgResourceStatisticsRefreshUtil: + model_cache_field_mapper = { + ApplicationPermission: 'app_perms_amount', + AssetPermission: 'asset_perms_amount', + Application: 'applications_amount', + Gateway: 'gateways_amount', + Domain: 'domains_amount', + SystemUser: 'system_users_amount', + AdminUser: 'admin_users_amount', + Node: 'nodes_amount', + Asset: 'assets_amount', + UserGroup: 'groups_amount', + } + + @classmethod + def refresh_if_need(cls, instance): + cache_field_name = cls.model_cache_field_mapper.get(type(instance)) + if cache_field_name: + org_cache = OrgResourceStatisticsCache(instance.org) + org_cache.expire(cache_field_name) + + +@receiver(post_save) +def on_post_save_refresh_org_resource_statistics_cache(sender, instance, created, **kwargs): + if created: + OrgResourceStatisticsRefreshUtil.refresh_if_need(instance) + + +@receiver(pre_delete) +def on_pre_delete_refresh_org_resource_statistics_cache(sender, instance, **kwargs): + OrgResourceStatisticsRefreshUtil.refresh_if_need(instance) diff --git a/apps/orgs/signals_handler.py b/apps/orgs/signals_handler/common.py similarity index 57% rename from apps/orgs/signals_handler.py rename to apps/orgs/signals_handler/common.py index 7cf3a9c29..f32549d29 100644 --- a/apps/orgs/signals_handler.py +++ b/apps/orgs/signals_handler/common.py @@ -4,18 +4,15 @@ from collections import defaultdict from functools import partial from django.db.models.signals import m2m_changed -from django.db.models.signals import post_save, pre_delete +from django.db.models.signals import post_save from django.dispatch import receiver from orgs.utils import tmp_to_org -from .models import Organization, OrganizationMember -from .hands import set_current_org, Node, get_current_org +from orgs.models import Organization, OrganizationMember +from orgs.hands import set_current_org, Node, get_current_org from perms.models import (AssetPermission, ApplicationPermission) from users.models import UserGroup, User -from applications.models import Application -from assets.models import Asset, AdminUser, SystemUser, Domain, Gateway -from common.const.signals import PRE_REMOVE, POST_REMOVE, POST_PREFIX -from .caches import OrgResourceStatisticsCache +from common.const.signals import PRE_REMOVE, POST_REMOVE @receiver(post_save, sender=Organization) @@ -111,70 +108,8 @@ def on_org_user_changed(action, instance, reverse, pk_set, **kwargs): _clear_users_from_org(org, leaved_users) -# 缓存相关 -# ----------------------------------------------------- - -def refresh_user_amount_on_user_create_or_delete(user_id): - orgs = Organization.objects.filter(m2m_org_members__user_id=user_id).distinct() - for org in orgs: - org_cache = OrgResourceStatisticsCache(org) - org_cache.expire('users_amount') - - @receiver(post_save, sender=User) def on_user_create_refresh_cache(sender, instance, created, **kwargs): if created: - refresh_user_amount_on_user_create_or_delete(instance.id) - - -@receiver(pre_delete, sender=User) -def on_user_delete_refresh_cache(sender, instance, **kwargs): - refresh_user_amount_on_user_create_or_delete(instance.id) - - -@receiver(m2m_changed, sender=OrganizationMember) -def on_org_user_changed_refresh_cache(sender, action, instance, reverse, pk_set, **kwargs): - if not action.startswith(POST_PREFIX): - return - - if reverse: - orgs = Organization.objects.filter(id__in=pk_set) - else: - orgs = [instance] - - for org in orgs: - org_cache = OrgResourceStatisticsCache(org) - org_cache.expire('users_amount') - - -class OrgResourceStatisticsRefreshUtil: - model_cache_field_mapper = { - ApplicationPermission: 'app_perms_amount', - AssetPermission: 'asset_perms_amount', - Application: 'applications_amount', - Gateway: 'gateways_amount', - Domain: 'domains_amount', - SystemUser: 'system_users_amount', - AdminUser: 'admin_users_amount', - Node: 'nodes_amount', - Asset: 'assets_amount', - UserGroup: 'groups_amount', - } - - @classmethod - def refresh_if_need(cls, instance): - cache_field_name = cls.model_cache_field_mapper.get(type(instance)) - if cache_field_name: - org_cache = OrgResourceStatisticsCache(instance.org) - org_cache.expire(cache_field_name) - - -@receiver(post_save) -def on_post_save_refresh_org_resource_statistics_cache(sender, instance, created, **kwargs): - if created: - OrgResourceStatisticsRefreshUtil.refresh_if_need(instance) - - -@receiver(pre_delete) -def on_pre_delete_refresh_org_resource_statistics_cache(sender, instance, **kwargs): - OrgResourceStatisticsRefreshUtil.refresh_if_need(instance) + default_org = Organization.default() + default_org.members.add(instance) diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index c6872c6e1..518148a1b 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -198,9 +198,6 @@ class UserGrantedTreeRefreshController: builded_orgs_id = {org_id.decode() for org_id in ret[0]} ids = orgs_id - builded_orgs_id orgs = set() - if Organization.DEFAULT_ID in ids: - ids.remove(Organization.DEFAULT_ID) - orgs.add(Organization.default()) orgs.update(Organization.objects.filter(id__in=ids)) logger.info(f'Need rebuild orgs are {orgs}, builed orgs are {ret[0]}, all orgs are {orgs_id}') return orgs @@ -293,7 +290,7 @@ class UserGrantedTreeRefreshController: @lazyproperty def orgs(self): - orgs = {*self.user.orgs.all().distinct(), Organization.default()} + orgs = {*self.user.orgs.all().distinct()} return orgs @timeit @@ -302,11 +299,7 @@ class UserGrantedTreeRefreshController: with UserGrantedTreeRebuildLock(user_id=user.id): with tmp_to_root_org(): - orgids = self.orgs_id.copy() - orgids.remove(Organization.default().id) - orgids.add('') # 添加 default - - UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=orgids).delete() + UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=self.orgs_id).delete() exists = UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exists() if force or not exists: From e6b17da57dd2ef030d29894fde7e9340b324776d Mon Sep 17 00:00:00 2001 From: ibuler Date: Wed, 3 Mar 2021 10:44:14 +0800 Subject: [PATCH 39/71] =?UTF-8?q?perf:=20=E5=8E=BB=E6=8E=89pycrypto?= =?UTF-8?q?=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: 显示添加pycryptodome --- requirements/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index aca226c97..f160614c1 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -52,8 +52,8 @@ passlib==1.7.1 Pillow==7.1.0 pyasn1==0.4.8 pycparser==2.19 -pycrypto==2.6.1 -pycryptodomex==3.9.9 +pycryptodome==3.10.1 +pycryptodomex==3.10.1 pyotp==2.2.6 PyNaCl==1.2.1 python-dateutil==2.6.1 From 1d15f7125e6b2795f9c2b960df68cb3c1661e801 Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 3 Mar 2021 11:20:40 +0800 Subject: [PATCH 40/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96org=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E9=80=BB=E8=BE=91=20-=20=E9=87=87=E7=94=A8redis?= =?UTF-8?q?=E8=AE=A2=E9=98=85=E6=9C=BA=E5=88=B6=E5=AE=9E=E7=8E=B0orgs=5Fma?= =?UTF-8?q?pping=E6=95=B0=E6=8D=AE=E7=9A=84=E7=BB=B4=E6=8A=A4;=E5=88=A0?= =?UTF-8?q?=E9=99=A4get=5Forg=5Fby=5Fid=E7=AD=89=E6=96=B9=E6=B3=95;?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: 优化get_instance接口 --- .../migrations/0010_auto_20210226_1536.py | 18 ----- apps/orgs/mixins/models.py | 7 +- apps/orgs/models.py | 80 ++++++++----------- apps/orgs/signals_handler/common.py | 53 +++++++++++- apps/orgs/utils.py | 43 +--------- apps/terminal/const.py | 2 +- .../migrations/0032_auto_20210302_1853.py | 18 +++++ apps/tickets/api/assignee.py | 4 +- apps/tickets/serializers/ticket/ticket.py | 6 +- 9 files changed, 113 insertions(+), 118 deletions(-) delete mode 100644 apps/orgs/migrations/0010_auto_20210226_1536.py create mode 100644 apps/terminal/migrations/0032_auto_20210302_1853.py diff --git a/apps/orgs/migrations/0010_auto_20210226_1536.py b/apps/orgs/migrations/0010_auto_20210226_1536.py deleted file mode 100644 index b9abe32e7..000000000 --- a/apps/orgs/migrations/0010_auto_20210226_1536.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1 on 2021-02-26 07:36 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('orgs', '0009_auto_20201023_1628'), - ] - - operations = [ - migrations.AlterField( - model_name='organizationmember', - name='role', - field=models.CharField(choices=[('Admin', 'Organization administrator'), ('Auditor', 'Organization auditor'), ('User', 'User')], db_index=True, default='User', max_length=16, verbose_name='Role'), - ), - ] diff --git a/apps/orgs/mixins/models.py b/apps/orgs/mixins/models.py index dd967757f..66edc37e0 100644 --- a/apps/orgs/mixins/models.py +++ b/apps/orgs/mixins/models.py @@ -7,8 +7,7 @@ from django.core.exceptions import ValidationError from common.utils import get_logger from ..utils import ( - set_current_org, get_current_org, current_org, - filter_org_queryset, get_org_by_id, get_org_name_by_id + set_current_org, get_current_org, current_org, filter_org_queryset ) from ..models import Organization @@ -59,11 +58,11 @@ class OrgModelMixin(models.Model): @property def org(self): - return get_org_by_id(self.org_id) + return Organization.get_instance(self.org_id) @property def org_name(self): - return get_org_name_by_id(self.org_id) + return self.org.name @property def fullname(self, attr=None): diff --git a/apps/orgs/models.py b/apps/orgs/models.py index 2c9e85591..65e7bd886 100644 --- a/apps/orgs/models.py +++ b/apps/orgs/models.py @@ -24,63 +24,53 @@ class Organization(models.Model): created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by')) date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date created')) comment = models.TextField(default='', blank=True, verbose_name=_('Comment')) - members = models.ManyToManyField('users.User', related_name='orgs', through='orgs.OrganizationMember', - through_fields=('org', 'user')) + members = models.ManyToManyField('users.User', related_name='orgs', through='orgs.OrganizationMember', through_fields=('org', 'user')) - orgs = None - CACHE_PREFIX = 'JMS_ORG_{}' ROOT_ID = '00000000-0000-0000-0000-000000000000' ROOT_NAME = _('GLOBAL') DEFAULT_ID = '00000000-0000-0000-0000-000000000001' DEFAULT_NAME = 'DEFAULT' - _user_admin_orgs = None + orgs_mapping = None class Meta: verbose_name = _("Organization") def __str__(self): - return self.name - - def set_to_cache(self): - if self.__class__.orgs is None: - self.__class__.orgs = {} - self.__class__.orgs[str(self.id)] = self - - def expire_cache(self): - self.__class__.orgs.pop(str(self.id), None) + return str(self.name) @classmethod - def get_instance_from_cache(cls, oid): - if not cls.orgs or not isinstance(cls.orgs, dict): - return None - return cls.orgs.get(str(oid)) - - @classmethod - def get_instance(cls, id_or_name, default=False): - cached = cls.get_instance_from_cache(id_or_name) - if cached: - return cached - - if id_or_name is None: - return cls.default() if default else None - elif id_or_name in [cls.DEFAULT_ID, cls.DEFAULT_NAME, '']: - return cls.default() - elif id_or_name in [cls.ROOT_ID, cls.ROOT_NAME]: - return cls.root() - - try: - if is_uuid(id_or_name): - org = cls.objects.get(id=id_or_name) - else: - org = cls.objects.get(name=id_or_name) - org.set_to_cache() - except cls.DoesNotExist as e: - if default: - return cls.default() - else: - raise e + def get_instance(cls, id_or_name, default=None): + assert default is None or isinstance(default, cls), ( + '`default` must be None or `Organization` instance' + ) + org = cls.get_instance_from_memory(id_or_name) + org = org or default return org + @classmethod + def get_instance_from_memory(cls, id_or_name): + if not isinstance(cls.orgs_mapping, dict): + cls.orgs_mapping = cls.construct_orgs_mapping() + return cls.orgs_mapping.get(str(id_or_name)) + + @classmethod + def construct_orgs_mapping(cls): + orgs_mapping = {} + for org in cls.objects.all(): + orgs_mapping[str(org.id)] = org + orgs_mapping[str(org.name)] = org + root_org = cls.root() + orgs_mapping.update({ + root_org.id: root_org, + 'GLOBAL': root_org, + '全局组织': root_org + }) + return orgs_mapping + + @classmethod + def expire_orgs_mapping(cls): + cls.orgs_mapping = None + def get_org_members_by_role(self, role): from users.models import User if not self.is_root(): @@ -184,7 +174,7 @@ class Organization(models.Model): @classmethod def default(cls): - defaults = dict(name=cls.DEFAULT_NAME, id=cls.DEFAULT_ID) + defaults = dict(id=cls.DEFAULT_ID, name=cls.DEFAULT_NAME) obj, created = cls.objects.get_or_create(defaults=defaults, id=cls.DEFAULT_ID) return obj @@ -411,7 +401,7 @@ class OrganizationMember(models.Model): id = models.UUIDField(default=uuid.uuid4, primary_key=True) org = models.ForeignKey(Organization, related_name='m2m_org_members', on_delete=models.CASCADE, verbose_name=_('Organization')) user = models.ForeignKey('users.User', related_name='m2m_org_members', on_delete=models.CASCADE, verbose_name=_('User')) - role = models.CharField(db_index=True, max_length=16, choices=ROLE.choices, default=ROLE.USER, verbose_name=_("Role")) + role = models.CharField(max_length=16, choices=ROLE.choices, default=ROLE.USER, verbose_name=_("Role")) date_created = models.DateTimeField(auto_now_add=True, verbose_name=_("Date created")) date_updated = models.DateTimeField(auto_now=True, verbose_name=_("Date updated")) created_by = models.CharField(max_length=128, null=True, verbose_name=_('Created by')) diff --git a/apps/orgs/signals_handler/common.py b/apps/orgs/signals_handler/common.py index f32549d29..f7b6f8e78 100644 --- a/apps/orgs/signals_handler/common.py +++ b/apps/orgs/signals_handler/common.py @@ -1,11 +1,13 @@ # -*- coding: utf-8 -*- # +import threading from collections import defaultdict from functools import partial -from django.db.models.signals import m2m_changed -from django.db.models.signals import post_save from django.dispatch import receiver +from django.utils.functional import LazyObject +from django.db.models.signals import m2m_changed +from django.db.models.signals import post_save, post_delete from orgs.utils import tmp_to_org from orgs.models import Organization, OrganizationMember @@ -13,10 +15,51 @@ from orgs.hands import set_current_org, Node, get_current_org from perms.models import (AssetPermission, ApplicationPermission) from users.models import UserGroup, User from common.const.signals import PRE_REMOVE, POST_REMOVE +from common.signals import django_ready +from common.utils import get_logger +from common.utils.connection import RedisPubSub + + +logger = get_logger(__file__) + + +def get_orgs_mapping_for_memory_pub_sub(): + return RedisPubSub('fm.orgs_mapping') + + +class OrgsMappingForMemoryPubSub(LazyObject): + def _setup(self): + self._wrapped = get_orgs_mapping_for_memory_pub_sub() + + +orgs_mapping_for_memory_pub_sub = OrgsMappingForMemoryPubSub() + + +def expire_orgs_mapping_for_memory(): + orgs_mapping_for_memory_pub_sub.publish('expire_orgs_mapping') + + +@receiver(django_ready) +def subscribe_orgs_mapping_expire(sender, **kwargs): + logger.debug("Start subscribe for expire orgs mapping from memory") + + def keep_subscribe(): + subscribe = orgs_mapping_for_memory_pub_sub.subscribe() + for message in subscribe.listen(): + if message['type'] != 'message': + continue + Organization.expire_orgs_mapping() + logger.debug('Expire orgs mapping') + + t = threading.Thread(target=keep_subscribe) + t.daemon = True + t.start() @receiver(post_save, sender=Organization) def on_org_create_or_update(sender, instance=None, created=False, **kwargs): + # 必须放到最开始, 因为下面调用Node.save方法时会获取当前组织的org_id(即instance.org_id), 如果不过期会找不到 + expire_orgs_mapping_for_memory() if instance: old_org = get_current_org() set_current_org(instance) @@ -26,8 +69,10 @@ def on_org_create_or_update(sender, instance=None, created=False, **kwargs): node_root.save() set_current_org(old_org) - if instance and not created: - instance.expire_cache() + +@receiver(post_delete, sender=Organization) +def on_org_delete(sender, **kwargs): + expire_orgs_mapping_for_memory() def _remove_users(model, users, org): diff --git a/apps/orgs/utils.py b/apps/orgs/utils.py index 56f22ac86..b2f6d3737 100644 --- a/apps/orgs/utils.py +++ b/apps/orgs/utils.py @@ -30,7 +30,7 @@ def get_org_from_request(request): oid = Organization.DEFAULT_ID elif oid.lower() == "root": oid = Organization.ROOT_ID - org = Organization.get_instance(oid, True) + org = Organization.get_instance(oid, default=Organization.default()) return org @@ -54,9 +54,7 @@ def _find(attr): def get_current_org(): org_id = get_current_org_id() - if org_id is None: - return Organization.root() - org = Organization.get_instance(org_id) + org = Organization.get_instance(org_id, default=Organization.root()) return org @@ -65,43 +63,6 @@ def get_current_org_id(): return org_id -def construct_org_mapper(): - orgs = Organization.objects.all() - org_mapper = {str(org.id): org for org in orgs} - org_mapper.update({ - Organization.ROOT_ID: Organization.root(), - }) - return org_mapper - - -def set_org_mapper(org_mapper): - setattr(thread_local, 'org_mapper', org_mapper) - - -def get_org_mapper(): - org_mapper = _find('org_mapper') - if org_mapper is None: - org_mapper = construct_org_mapper() - set_org_mapper(org_mapper) - return org_mapper - - -def get_org_by_id(org_id): - org_id = str(org_id) - org_mapper = get_org_mapper() - org = org_mapper.get(org_id) - return org - - -def get_org_name_by_id(org_id): - org = get_org_by_id(org_id) - if org: - org_name = org.name - else: - org_name = 'Not Found' - return org_name - - def get_current_org_id_for_serializer(): org_id = get_current_org_id() if org_id == Organization.DEFAULT_ID: diff --git a/apps/terminal/const.py b/apps/terminal/const.py index 9d43c450c..6e3a38027 100644 --- a/apps/terminal/const.py +++ b/apps/terminal/const.py @@ -41,7 +41,7 @@ class TerminalTypeChoices(TextChoices): koko = 'koko', 'KoKo' guacamole = 'guacamole', 'Guacamole' omnidb = 'omnidb', 'OmniDB' - xrdp = 'xrdp', 'xrdp' + xrdp = 'xrdp', 'Xrdp' @classmethod def types(cls): diff --git a/apps/terminal/migrations/0032_auto_20210302_1853.py b/apps/terminal/migrations/0032_auto_20210302_1853.py new file mode 100644 index 000000000..df94e9b2a --- /dev/null +++ b/apps/terminal/migrations/0032_auto_20210302_1853.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2021-03-02 10:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('terminal', '0031_auto_20210113_1356'), + ] + + operations = [ + migrations.AlterField( + model_name='terminal', + name='type', + field=models.CharField(choices=[('koko', 'KoKo'), ('guacamole', 'Guacamole'), ('omnidb', 'OmniDB'), ('xrdp', 'Xrdp')], default='koko', max_length=64, verbose_name='type'), + ), + ] diff --git a/apps/tickets/api/assignee.py b/apps/tickets/api/assignee.py index 5b223ffcd..d95729085 100644 --- a/apps/tickets/api/assignee.py +++ b/apps/tickets/api/assignee.py @@ -5,7 +5,7 @@ from rest_framework import viewsets from common.permissions import IsValidUser from common.exceptions import JMSException from users.models import User -from orgs.utils import get_org_by_id +from orgs.models import Organization from .. import serializers @@ -17,7 +17,7 @@ class AssigneeViewSet(viewsets.ReadOnlyModelViewSet): def get_org(self): org_id = self.request.query_params.get('org_id') - org = get_org_by_id(org_id) + org = Organization.get_instance(org_id) if not org: error = ('The organization `{}` does not exist'.format(org_id)) raise JMSException(error) diff --git a/apps/tickets/serializers/ticket/ticket.py b/apps/tickets/serializers/ticket/ticket.py index c28b9a9f1..fd776caea 100644 --- a/apps/tickets/serializers/ticket/ticket.py +++ b/apps/tickets/serializers/ticket/ticket.py @@ -3,8 +3,8 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from common.drf.serializers import MethodSerializer -from orgs.utils import get_org_by_id from orgs.mixins.serializers import OrgResourceModelSerializerMixin +from orgs.models import Organization from users.models import User from tickets.models import Ticket from .meta import type_serializer_classes_mapping @@ -104,7 +104,7 @@ class TicketApplySerializer(TicketSerializer): @staticmethod def validate_org_id(org_id): - org = get_org_by_id(org_id) + org = Organization.get_instance(org_id) if not org: error = _('The organization `{}` does not exist'.format(org_id)) raise serializers.ValidationError(error) @@ -113,7 +113,7 @@ class TicketApplySerializer(TicketSerializer): def validate_assignees(self, assignees): org_id = self.initial_data.get('org_id') self.validate_org_id(org_id) - org = get_org_by_id(org_id) + org = Organization.get_instance(org_id) admins = User.get_super_and_org_admins(org) valid_assignees = list(set(assignees) & set(admins)) if not valid_assignees: From 56328e112a6474f076a326a1e300941c0a8d3f8f Mon Sep 17 00:00:00 2001 From: Bai Date: Wed, 3 Mar 2021 15:51:35 +0800 Subject: [PATCH 41/71] =?UTF-8?q?perf:=20=E7=A7=BB=E9=99=A4=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E5=88=9B=E5=BB=BA=E6=97=B6=E5=AF=B9=E4=BA=8EAuditor?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=9A=84=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/users/api/relation.py | 9 --------- apps/users/serializers/group.py | 4 ---- apps/users/serializers/user.py | 11 ----------- 3 files changed, 24 deletions(-) diff --git a/apps/users/api/relation.py b/apps/users/api/relation.py index 050d9e4e6..c3da7816e 100644 --- a/apps/users/api/relation.py +++ b/apps/users/api/relation.py @@ -28,12 +28,3 @@ class UserUserGroupRelationViewSet(JMSBulkRelationModelViewSet): return False else: return True - - def perform_create(self, serializer): - validated_data = [] - for item in serializer.validated_data: - if item['user'].role == User.ROLE.AUDITOR: - continue - validated_data.append(item) - serializer._validated_data = validated_data - return super().perform_create(serializer) diff --git a/apps/users/serializers/group.py b/apps/users/serializers/group.py index b7a6d204c..41d2282a8 100644 --- a/apps/users/serializers/group.py +++ b/apps/users/serializers/group.py @@ -53,7 +53,3 @@ class UserGroupSerializer(BulkOrgResourceModelSerializer): Prefetch('users', queryset=User.objects.only('id')) ).annotate(users_amount=Count('users')) return queryset - - def validate_users(self, users): - users = [user for user in users if user.role != User.ROLE.AUDITOR] - return users diff --git a/apps/users/serializers/user.py b/apps/users/serializers/user.py index 239b80af1..44264bdb8 100644 --- a/apps/users/serializers/user.py +++ b/apps/users/serializers/user.py @@ -113,17 +113,6 @@ class UserSerializer(CommonBulkSerializerMixin, serializers.ModelSerializer): raise serializers.ValidationError(msg) return password - def validate_groups(self, groups): - """ - 审计员不能加入到组中 - """ - role = self.initial_data.get('role') - if self.instance: - role = role or self.instance.role - if role == User.ROLE.AUDITOR: - return [] - return groups - @staticmethod def change_password_to_raw(attrs): password = attrs.pop('password', None) From 09bdff4a6730e06b0b947f570dc77494d286a2b0 Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 4 Mar 2021 10:34:38 +0800 Subject: [PATCH 42/71] =?UTF-8?q?fix:=20=E7=BC=93=E5=AD=98=E6=A1=86?= =?UTF-8?q?=E6=9E=B6=20expire=5Ffields=20=E5=8F=AF=E8=83=BD=E6=8A=A5?= =?UTF-8?q?=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/common/cache.py b/apps/common/cache.py index 0e2415691..0bed4fa30 100644 --- a/apps/common/cache.py +++ b/apps/common/cache.py @@ -128,7 +128,7 @@ class Cache(metaclass=CacheBase): if data is not None: logger.info(f'Expire cached fields: key={self.key} fields={fields}') for f in fields: - data.pop(f) + data.pop(f, None) self.set_data(data) return data From d7e7c62c7a8eebab2270cb94810cefb521cd9510 Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 4 Mar 2021 10:33:35 +0800 Subject: [PATCH 43/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E8=A1=A8?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E8=BF=81=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/migrations/0010_auto_20210219_1241.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/orgs/migrations/0010_auto_20210219_1241.py b/apps/orgs/migrations/0010_auto_20210219_1241.py index 14a9b2ba0..73d4b8173 100644 --- a/apps/orgs/migrations/0010_auto_20210219_1241.py +++ b/apps/orgs/migrations/0010_auto_20210219_1241.py @@ -29,11 +29,6 @@ def migrate_default_org_id(apps, schema_editor): ('terminal', ['Session', 'Command']), ('tickets', ['Ticket']), ('users', ['UserGroup']), - ('xpack', [ - 'Account', 'SyncInstanceDetail', 'SyncInstanceTask', 'SyncInstanceTaskExecution', - 'ChangeAuthPlan', 'ChangeAuthPlanExecution', 'ChangeAuthPlanTask', - 'GatherUserTask', 'GatherUserTaskExecution', - ]), ] print("") for app, models_name in org_app_models: From 91a26abf9e25f64b7df18fceb1120dee93e4324e Mon Sep 17 00:00:00 2001 From: ibuler Date: Thu, 4 Mar 2021 11:17:54 +0800 Subject: [PATCH 44/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96default?= =?UTF-8?q?=E8=8A=82=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/models/node.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 27a2a7df6..68aa300d3 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -529,6 +529,10 @@ class SomeNodesMixin: if not node_key1: logger.info("Not found node that `key` = 1") return + if node_key1.org_id == '': + node_key1.org_id = str(Organization.default().id) + node_key1.save() + return with transaction.atomic(): with tmp_to_org(node_key1.org): From 78bf6f5817126a41255b066900774796df5f0e56 Mon Sep 17 00:00:00 2001 From: xinwen Date: Wed, 3 Mar 2021 15:36:42 +0800 Subject: [PATCH 45/71] =?UTF-8?q?refactor:=20=E8=8E=B7=E5=8F=96=E6=8E=88?= =?UTF-8?q?=E6=9D=83=E6=A0=91=E6=88=96=E8=80=85=E8=B5=84=E4=BA=A7=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E6=97=B6=E9=81=BF=E5=85=8D=E8=AF=BB=E6=97=B6=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/perms/pagination.py | 8 +++- apps/perms/utils/asset/user_permission.py | 50 ++++++++++++----------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/apps/perms/pagination.py b/apps/perms/pagination.py index c740830c2..248958a3e 100644 --- a/apps/perms/pagination.py +++ b/apps/perms/pagination.py @@ -29,8 +29,12 @@ class AllGrantedAssetPagination(GrantedAssetPaginationBase): def get_count_from_nodes(self, queryset): if settings.PERM_SINGLE_ASSET_TO_UNGROUP_NODE: return None - assets_amount = sum(UserAssetGrantedTreeNodeRelation.objects.filter( + values = UserAssetGrantedTreeNodeRelation.objects.filter( user=self._user, node_parent_key='' - ).values_list('node_assets_amount', flat=True)) + ).values_list('node_assets_amount', flat=True) + if not values: + return None + + assets_amount = sum(values) logger.debug(f'Hit all assets amount {assets_amount} -> {self._request.get_full_path()}') return assets_amount diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 518148a1b..14205e12e 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -1,5 +1,6 @@ from collections import defaultdict from typing import List, Tuple +import time from django.core.cache import cache from django.conf import settings @@ -185,11 +186,16 @@ class UserGrantedTreeRefreshController: return {org_id.decode() for org_id in org_ids} def set_all_orgs_as_builed(self): - orgs_id = [str(org_id) for org_id in self.orgs_id] - self.client.sadd(self.key, *orgs_id) + self.client.sadd(self.key, *self.orgs_id) + + def have_need_refresh_orgs(self): + builded_org_ids = self.client.smembers(self.key) + builded_org_ids = {org_id.decode() for org_id in builded_org_ids} + have = self.orgs_id - builded_org_ids + return have def get_need_refresh_orgs_and_fill_up(self): - orgs_id = set(str(org_id) for org_id in self.orgs_id) + orgs_id = self.orgs_id with self.client.pipeline() as p: p.smembers(self.key) @@ -197,8 +203,7 @@ class UserGrantedTreeRefreshController: ret = p.execute() builded_orgs_id = {org_id.decode() for org_id in ret[0]} ids = orgs_id - builded_orgs_id - orgs = set() - orgs.update(Organization.objects.filter(id__in=ids)) + orgs = {*Organization.objects.filter(id__in=ids)} logger.info(f'Need rebuild orgs are {orgs}, builed orgs are {ret[0]}, all orgs are {orgs_id}') return orgs @@ -285,7 +290,7 @@ class UserGrantedTreeRefreshController: @lazyproperty def orgs_id(self): - ret = {org.id for org in self.orgs} + ret = {str(org.id) for org in self.orgs} return ret @lazyproperty @@ -297,21 +302,24 @@ class UserGrantedTreeRefreshController: def refresh_if_need(self, force=False): user = self.user - with UserGrantedTreeRebuildLock(user_id=user.id): - with tmp_to_root_org(): - UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=self.orgs_id).delete() - exists = UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exists() + with tmp_to_root_org(): + UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=self.orgs_id).delete() - if force or not exists: - orgs = self.orgs - self.set_all_orgs_as_builed() - else: - orgs = self.get_need_refresh_orgs_and_fill_up() + if force or self.have_need_refresh_orgs(): + with UserGrantedTreeRebuildLock(user_id=user.id): + if force: + orgs = self.orgs + self.set_all_orgs_as_builed() + else: + orgs = self.get_need_refresh_orgs_and_fill_up() - for org in orgs: - with tmp_to_org(org): - utils = UserGrantedTreeBuildUtils(user) - utils.rebuild_user_granted_tree() + for org in orgs: + with tmp_to_org(org): + t_start = time.time() + logger.info(f'Rebuild user tree: user={self.user} org={current_org}') + utils = UserGrantedTreeBuildUtils(user) + utils.rebuild_user_granted_tree() + logger.info(f'Rebuild user tree ok: cost={time.time() - t_start} user={self.user} org={current_org}') class UserGrantedUtilsBase: @@ -353,15 +361,11 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): asset_ids = list(asset_ids) return asset_ids - @timeit @ensure_in_real_or_default_org def rebuild_user_granted_tree(self): """ 注意：调用该方法一定要被 `UserGrantedTreeRebuildLock` 锁住 """ - - logger.info(f'Rebuild user tree: user={self.user} org={current_org}') - user = self.user # 先删除旧的授权树🌲 From ab23a357f7d85ef1076fe14b482d98eae7baaabe Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 4 Mar 2021 10:31:15 +0800 Subject: [PATCH 46/71] =?UTF-8?q?feat:=20=E6=8E=A8=E9=80=81=E5=8A=A8?= =?UTF-8?q?=E6=80=81=E7=B3=BB=E7=BB=9F=E7=94=A8=E6=88=B7=EF=BC=8C=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E4=B8=8A=E7=94=A8=E6=88=B7=E7=9A=84=20comment=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=98=AF=20userdisplayname?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/tasks/push_system_user.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/assets/tasks/push_system_user.py b/apps/assets/tasks/push_system_user.py index 7506d1fe2..be337baea 100644 --- a/apps/assets/tasks/push_system_user.py +++ b/apps/assets/tasks/push_system_user.py @@ -32,11 +32,19 @@ def _dump_args(args: dict): def get_push_unixlike_system_user_tasks(system_user, username=None): + comment = system_user.name + if username is None: username = system_user.username + + if system_user.username_same_with_user: + from users.models import User + user = User.objects.filter(username=username).only('name', 'username').first() + if user: + comment = f'{system_user.name}[{str(user)}]' + password = system_user.password public_key = system_user.public_key - comment = system_user.name groups = _split_by_comma(system_user.system_groups) From c1bf85482488e9cfd3ee76fcc8308bc8e5e78a2f Mon Sep 17 00:00:00 2001 From: Bai Date: Mon, 1 Mar 2021 15:58:05 +0800 Subject: [PATCH 47/71] =?UTF-8?q?perf:=20=E6=B7=BB=E5=8A=A0=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E5=8C=85(pyvmomi=3D=3D7.0.1)(termcolor=3D=3D1.1.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f160614c1..fcd414935 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -108,3 +108,5 @@ openpyxl==3.0.5 pyexcel==0.6.6 pyexcel-xlsx==0.6.0 data-tree==0.0.1 +pyvmomi==7.0.1 +termcolor==1.1.0 From 24fb8b2a8967111872297c96dd35cc0421a0b87f Mon Sep 17 00:00:00 2001 From: xinwen Date: Thu, 4 Mar 2021 18:17:42 +0800 Subject: [PATCH 48/71] =?UTF-8?q?feat:=20=E7=AE=A1=E7=90=86=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E8=AF=A6=E6=83=85=E9=A1=B5=E6=B7=BB=E5=8A=A0=E8=AE=A4?= =?UTF-8?q?=E8=AF=81=E6=96=B9=E5=BC=8F=E4=B8=8E=E7=A7=98=E9=92=A5=E6=8C=87?= =?UTF-8?q?=E7=BA=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/assets/api/admin_user.py | 4 ++++ apps/assets/models/base.py | 15 +++++++++++++++ apps/assets/serializers/admin_user.py | 8 +++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/apps/assets/api/admin_user.py b/apps/assets/api/admin_user.py index bdbae55dd..5ad648635 100644 --- a/apps/assets/api/admin_user.py +++ b/apps/assets/api/admin_user.py @@ -33,6 +33,10 @@ class AdminUserViewSet(OrgBulkModelViewSet): search_fields = filterset_fields serializer_class = serializers.AdminUserSerializer permission_classes = (IsOrgAdmin,) + serializer_classes = { + 'default': serializers.AdminUserSerializer, + 'retrieve': serializers.AdminUserDetailSerializer, + } def get_queryset(self): queryset = super().get_queryset() diff --git a/apps/assets/models/base.py b/apps/assets/models/base.py index 9fd4836f7..404c7a991 100644 --- a/apps/assets/models/base.py +++ b/apps/assets/models/base.py @@ -11,10 +11,12 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ from django.conf import settings +from common.db.models import ChoiceSet from common.utils import random_string from common.utils import ( ssh_key_string_to_obj, ssh_key_gen, get_logger, lazyproperty ) +from common.utils.encode import ssh_pubkey_gen from common.validators import alphanumeric from common import fields from orgs.mixins.models import OrgModelMixin @@ -106,6 +108,19 @@ class AuthMixin: username = '' _prefer = 'system_user' + @property + def ssh_key_fingerprint(self): + if self.public_key: + public_key = self.public_key + elif self.private_key: + public_key = ssh_pubkey_gen(self.private_key, self.password) + else: + return '' + + public_key_obj = sshpubkeys.SSHKey(public_key) + fingerprint = public_key_obj.hash_md5() + return fingerprint + @property def private_key_obj(self): if self.private_key: diff --git a/apps/assets/serializers/admin_user.py b/apps/assets/serializers/admin_user.py index b8b086205..21eca51d0 100644 --- a/apps/assets/serializers/admin_user.py +++ b/apps/assets/serializers/admin_user.py @@ -3,8 +3,6 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from common.drf.serializers import AdaptedBulkListSerializer - from ..models import Node, AdminUser from orgs.mixins.serializers import BulkOrgResourceModelSerializer @@ -17,7 +15,6 @@ class AdminUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): """ class Meta: - list_serializer_class = AdaptedBulkListSerializer model = AdminUser fields = [ 'id', 'name', 'username', 'password', 'private_key', 'public_key', @@ -33,6 +30,11 @@ class AdminUserSerializer(AuthSerializerMixin, BulkOrgResourceModelSerializer): } +class AdminUserDetailSerializer(AdminUserSerializer): + class Meta(AdminUserSerializer.Meta): + fields = AdminUserSerializer.Meta.fields + ['ssh_key_fingerprint'] + + class AdminUserAuthSerializer(AuthSerializer): class Meta: From 840e5e8863d70b9e41c7f71b50bd36dd6016897a Mon Sep 17 00:00:00 2001 From: xinwen Date: Fri, 5 Mar 2021 15:14:07 +0800 Subject: [PATCH 49/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20default=20?= =?UTF-8?q?=E7=BB=84=E7=BB=87=E8=BF=81=E7=A7=BB=E8=84=9A=E6=9C=AC=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/migrations/0010_auto_20210219_1241.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/orgs/migrations/0010_auto_20210219_1241.py b/apps/orgs/migrations/0010_auto_20210219_1241.py index 73d4b8173..38c79f552 100644 --- a/apps/orgs/migrations/0010_auto_20210219_1241.py +++ b/apps/orgs/migrations/0010_auto_20210219_1241.py @@ -66,6 +66,7 @@ class Migration(migrations.Migration): dependencies = [ ('orgs', '0009_auto_20201023_1628'), + ('perms', '0018_auto_20210208_1515'), ] operations = [ From 7f42e59714b9c3e48da0480df4b2b9d62ee067f6 Mon Sep 17 00:00:00 2001 From: xinwen Date: Fri, 5 Mar 2021 14:29:54 +0800 Subject: [PATCH 50/71] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=94=9F?= =?UTF-8?q?=E6=88=90=E5=81=87=E6=95=B0=E6=8D=AE=E8=84=9A=E6=9C=AC=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/generate_fake_data/resources/assets.py | 8 ++------ utils/generate_fake_data/resources/base.py | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/utils/generate_fake_data/resources/assets.py b/utils/generate_fake_data/resources/assets.py index 4c5cbba93..4279e6c5a 100644 --- a/utils/generate_fake_data/resources/assets.py +++ b/utils/generate_fake_data/resources/assets.py @@ -65,13 +65,9 @@ class AssetsGenerator(FakeDataGenerator): self.nodes_id = list(Node.objects.all().values_list('id', flat=True)) def set_assets_nodes(self, assets): - assets_id = [asset.id for asset in assets] - objs = [] - for asset_id in assets_id: + for asset in assets: nodes_id_add_to = random.sample(self.nodes_id, 3) - objs_add = [Asset.nodes.through(asset_id=asset_id, node_id=nid) for nid in nodes_id_add_to] - objs.extend(objs_add) - Asset.nodes.through.objects.bulk_create(objs, ignore_conflicts=True) + asset.nodes.add(*nodes_id_add_to) def do_generate(self, batch, batch_size): assets = [] diff --git a/utils/generate_fake_data/resources/base.py b/utils/generate_fake_data/resources/base.py index ba45f602b..723baa6dd 100644 --- a/utils/generate_fake_data/resources/base.py +++ b/utils/generate_fake_data/resources/base.py @@ -15,7 +15,7 @@ class FakeDataGenerator: seed() def switch_org(self, org_id): - o = Organization.get_instance(org_id, default=True) + o = Organization.get_instance(org_id, default=Organization.default()) if o: o.change_to() print('Current org is: {}'.format(o)) From 3e7e01418dabf9456167fa119658e1f7ed19f6f5 Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 22 Feb 2021 18:35:53 +0800 Subject: [PATCH 51/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E5=91=BD?= =?UTF-8?q?=E4=BB=A4=E8=AE=B0=E5=BD=95=E6=85=A2=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/drf/exc_handlers.py | 5 +- apps/common/drf/metadata.py | 2 + .../user_permission_assets/mixin.py | 2 +- apps/perms/utils/asset/user_permission.py | 116 ------- apps/terminal/api/command.py | 16 +- apps/terminal/api/storage.py | 51 ++- apps/terminal/backends/command/es.py | 293 +++++++++++++++--- apps/terminal/filters.py | 82 +++++ apps/terminal/models/storage.py | 20 ++ 9 files changed, 427 insertions(+), 160 deletions(-) create mode 100644 apps/terminal/filters.py diff --git a/apps/common/drf/exc_handlers.py b/apps/common/drf/exc_handlers.py index 93ff84146..b99a53547 100644 --- a/apps/common/drf/exc_handlers.py +++ b/apps/common/drf/exc_handlers.py @@ -19,7 +19,10 @@ def extract_object_name(exc, index=0): `No User matches the given query.` 提取 `User`，`index=1` """ - (msg, *_) = exc.args + if exc.args: + (msg, *others) = exc.args + else: + return gettext('Object') return gettext(msg.split(sep=' ', maxsplit=index + 1)[index]) diff --git a/apps/common/drf/metadata.py b/apps/common/drf/metadata.py index 04c365d97..cc2903d2f 100644 --- a/apps/common/drf/metadata.py +++ b/apps/common/drf/metadata.py @@ -97,6 +97,8 @@ class SimpleMetadataWithFilters(SimpleMetadata): fields = view.filterset_fields elif hasattr(view, 'get_filterset_fields'): fields = view.get_filterset_fields(request) + elif hasattr(view, 'filterset_class'): + fields = view.filterset_class.Meta.fields if isinstance(fields, dict): fields = list(fields.keys()) diff --git a/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py b/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py index 3a1d49016..d7a5c23dc 100644 --- a/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py +++ b/apps/perms/api/asset/user_permission/user_permission_assets/mixin.py @@ -7,7 +7,7 @@ from common.utils import get_logger from perms.pagination import NodeGrantedAssetPagination, AllGrantedAssetPagination from assets.models import Asset, Node from perms import serializers -from perms.utils.asset.user_permission import UserGrantedAssetsQueryUtils, QuerySetStage +from perms.utils.asset.user_permission import UserGrantedAssetsQueryUtils logger = get_logger(__name__) diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 14205e12e..0741f42e0 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -53,122 +53,6 @@ def get_user_all_asset_perm_ids(user) -> set: return asset_perm_ids -class QuerySetStage: - def __init__(self): - self._prefetch_related = set() - self._only = () - self._filters = [] - self._querysets_and = [] - self._querysets_or = [] - self._order_by = None - self._annotate = [] - self._before_union_merge_funs = set() - self._after_union_merge_funs = set() - - def annotate(self, *args, **kwargs): - self._annotate.append((args, kwargs)) - self._before_union_merge_funs.add(self._merge_annotate) - return self - - def prefetch_related(self, *lookups): - self._prefetch_related.update(lookups) - self._before_union_merge_funs.add(self._merge_prefetch_related) - return self - - def only(self, *fields): - self._only = fields - self._before_union_merge_funs.add(self._merge_only) - return self - - def order_by(self, *field_names): - self._order_by = field_names - self._after_union_merge_funs.add(self._merge_order_by) - return self - - def filter(self, *args, **kwargs): - self._filters.append((args, kwargs)) - self._before_union_merge_funs.add(self._merge_filters) - return self - - def and_with_queryset(self, qs: QuerySet): - assert isinstance(qs, QuerySet), f'Must be `QuerySet`' - self._order_by = qs.query.order_by - self._after_union_merge_funs.add(self._merge_order_by) - self._querysets_and.append(qs.order_by()) - self._before_union_merge_funs.add(self._merge_querysets_and) - return self - - def or_with_queryset(self, qs: QuerySet): - assert isinstance(qs, QuerySet), f'Must be `QuerySet`' - self._order_by = qs.query.order_by - self._after_union_merge_funs.add(self._merge_order_by) - self._querysets_or.append(qs.order_by()) - self._before_union_merge_funs.add(self._merge_querysets_or) - return self - - def merge_multi_before_union(self, *querysets): - ret = [] - for qs in querysets: - qs = self.merge_before_union(qs) - ret.append(qs) - return ret - - def _merge_only(self, qs: QuerySet): - if self._only: - qs = qs.only(*self._only) - return qs - - def _merge_filters(self, qs: QuerySet): - if self._filters: - for args, kwargs in self._filters: - qs = qs.filter(*args, **kwargs) - return qs - - def _merge_querysets_and(self, qs: QuerySet): - if self._querysets_and: - for qs_and in self._querysets_and: - qs &= qs_and - return qs - - def _merge_annotate(self, qs: QuerySet): - if self._annotate: - for args, kwargs in self._annotate: - qs = qs.annotate(*args, **kwargs) - return qs - - def _merge_querysets_or(self, qs: QuerySet): - if self._querysets_or: - for qs_or in self._querysets_or: - qs |= qs_or - return qs - - def _merge_prefetch_related(self, qs: QuerySet): - if self._prefetch_related: - qs = qs.prefetch_related(*self._prefetch_related) - return qs - - def _merge_order_by(self, qs: QuerySet): - if self._order_by is not None: - qs = qs.order_by(*self._order_by) - return qs - - def merge_before_union(self, qs: QuerySet) -> QuerySet: - assert isinstance(qs, QuerySet), f'Must be `QuerySet`' - for fun in self._before_union_merge_funs: - qs = fun(qs) - return qs - - def merge_after_union(self, qs: QuerySet) -> QuerySet: - for fun in self._after_union_merge_funs: - qs = fun(qs) - return qs - - def merge(self, qs: QuerySet) -> QuerySet: - qs = self.merge_before_union(qs) - qs = self.merge_after_union(qs) - return qs - - class UserGrantedTreeRefreshController: key_template = 'perms.user.node_tree.builded_orgs.user_id:{user_id}' diff --git a/apps/terminal/api/command.py b/apps/terminal/api/command.py index cd01df1b3..e75ce491e 100644 --- a/apps/terminal/api/command.py +++ b/apps/terminal/api/command.py @@ -8,11 +8,14 @@ from rest_framework import viewsets from rest_framework import generics from rest_framework.fields import DateTimeField from rest_framework.response import Response -from rest_framework import status +from rest_framework.decorators import action from django.template import loader +from terminal.models import CommandStorage +from terminal.filters import CommandFilter from orgs.utils import current_org from common.permissions import IsOrgAdminOrAppUser, IsOrgAuditor, IsAppUser +from common.const.http import GET from common.utils import get_logger from terminal.utils import send_command_alert_mail from terminal.serializers import InsecureCommandAlertSerializer @@ -89,7 +92,7 @@ class CommandQueryMixin: return date_from_st, date_to_st -class CommandViewSet(CommandQueryMixin, viewsets.ModelViewSet): +class CommandViewSet(viewsets.ModelViewSet): """接受app发送来的command log, 格式如下 { "user": "admin", @@ -103,7 +106,16 @@ class CommandViewSet(CommandQueryMixin, viewsets.ModelViewSet): """ command_store = get_command_storage() + permission_classes = [IsOrgAdminOrAppUser | IsOrgAuditor] serializer_class = SessionCommandSerializer + filterset_class = CommandFilter + ordering_fields = ('timestamp', ) + + def get_queryset(self): + command_storage_id = self.request.query_params.get('command_storage_id') + storage = CommandStorage.objects.get(id=command_storage_id) + qs = storage.get_command_queryset() + return qs def create(self, request, *args, **kwargs): serializer = self.serializer_class(data=request.data, many=True) diff --git a/apps/terminal/api/storage.py b/apps/terminal/api/storage.py index be3002060..b2bd69d28 100644 --- a/apps/terminal/api/storage.py +++ b/apps/terminal/api/storage.py @@ -3,9 +3,15 @@ from rest_framework import viewsets, generics, status from rest_framework.response import Response +from rest_framework.request import Request +from rest_framework.decorators import action from django.utils.translation import ugettext_lazy as _ +from django_filters import utils +from terminal import const +from common.const.http import GET from common.permissions import IsSuperUser +from terminal.filters import CommandStorageFilter, CommandFilter, CommandFilterForStorageTree from ..models import CommandStorage, ReplayStorage from ..serializers import CommandStorageSerializer, ReplayStorageSerializer @@ -30,11 +36,52 @@ class BaseStorageViewSetMixin: class CommandStorageViewSet(BaseStorageViewSetMixin, viewsets.ModelViewSet): - filterset_fields = ('name', 'type',) - search_fields = filterset_fields + search_fields = ('name', 'type',) queryset = CommandStorage.objects.all() serializer_class = CommandStorageSerializer permission_classes = (IsSuperUser,) + filterset_class = CommandStorageFilter + + @action(methods=[GET], detail=False, filterset_class=CommandFilterForStorageTree) + def tree(self, request: Request): + storage_qs = self.get_queryset().exclude(name='null') + storages_with_count = [] + for storage in storage_qs: + command_qs = storage.get_command_queryset() + filterset = CommandFilter( + data=request.query_params, queryset=command_qs, + request=request + ) + if not filterset.is_valid(): + raise utils.translate_validation(filterset.errors) + command_qs = filterset.qs + if storage.type == const.CommandStorageTypeChoices.es: + command_count = command_qs.count(limit_to_max_result_window=False) + else: + command_count = command_qs.count() + storages_with_count.append((storage, command_count)) + + root = { + 'id': 'root', + 'name': _('Command storages'), + 'title': _('Command storages'), + 'pId': '', + 'isParent': True, + 'open': True, + } + + nodes = [ + { + 'id': storage.id, + 'name': f'{storage.name}({storage.type})({command_count})', + 'title': f'{storage.name}({storage.type})', + 'pId': 'root', + 'isParent': False, + 'open': False, + } for storage, command_count in storages_with_count + ] + nodes.append(root) + return Response(data=nodes) class ReplayStorageViewSet(BaseStorageViewSetMixin, viewsets.ModelViewSet): diff --git a/apps/terminal/backends/command/es.py b/apps/terminal/backends/command/es.py index 43bd52c02..1137f6ec8 100644 --- a/apps/terminal/backends/command/es.py +++ b/apps/terminal/backends/command/es.py @@ -1,53 +1,270 @@ # -*- coding: utf-8 -*- # - from datetime import datetime -from jms_storage.es import ESStorage +from functools import reduce, partial +from itertools import groupby +import pytz +from uuid import UUID +import inspect + +from django.db.models import QuerySet as DJQuerySet +from elasticsearch import Elasticsearch +from elasticsearch.helpers import bulk + +from common.utils.common import lazyproperty from common.utils import get_logger -from .base import CommandBase from .models import AbstractSessionCommand logger = get_logger(__file__) -class CommandStore(ESStorage, CommandBase): - def __init__(self, params): - super().__init__(params) +class CommandStore(): + def __init__(self, config): + hosts = config.get("HOSTS") + kwargs = config.get("OTHER", {}) + self.index = config.get("INDEX") or 'jumpserver' + self.doc_type = config.get("DOC_TYPE") or 'command_store' + self.es = Elasticsearch(hosts=hosts, **kwargs) - def filter(self, date_from=None, date_to=None, - user=None, asset=None, system_user=None, - input=None, session=None, risk_level=None, org_id=None): + @staticmethod + def make_data(command): + data = dict( + user=command["user"], asset=command["asset"], + system_user=command["system_user"], input=command["input"], + output=command["output"], risk_level=command["risk_level"], + session=command["session"], timestamp=command["timestamp"], + org_id=command["org_id"] + ) + data["date"] = datetime.fromtimestamp(command['timestamp'], tz=pytz.UTC) + return data - if date_from is not None: - if isinstance(date_from, float): - date_from = datetime.fromtimestamp(date_from) - if date_to is not None: - if isinstance(date_to, float): - date_to = datetime.fromtimestamp(date_to) - - try: - data = super().filter(date_from=date_from, date_to=date_to, - user=user, asset=asset, system_user=system_user, - input=input, session=session, - risk_level=risk_level, org_id=org_id) - except Exception as e: - logger.error(e, exc_info=True) - return [] - else: - return AbstractSessionCommand.from_multi_dict( - [item["_source"] for item in data["hits"] if item] + def bulk_save(self, command_set, raise_on_error=True): + actions = [] + for command in command_set: + data = dict( + _index=self.index, + _type=self.doc_type, + _source=self.make_data(command), ) + actions.append(data) + return bulk(self.es, actions, index=self.index, raise_on_error=raise_on_error) - def count(self, date_from=None, date_to=None, user=None, asset=None, - system_user=None, input=None, session=None): + def save(self, command): + """ + 保存命令到数据库 + """ + data = self.make_data(command) + return self.es.index(index=self.index, doc_type=self.doc_type, body=data) + + def filter(self, query: dict, from_=None, size=None, sort=None): + body = self.get_query_body(**query) + + data = self.es.search( + index=self.index, doc_type=self.doc_type, body=body, from_=from_, size=size, + sort=sort + ) + + return AbstractSessionCommand.from_multi_dict( + [item['_source'] for item in data['hits']['hits'] if item] + ) + + def count(self, **query): + body = self.get_query_body(**query) + data = self.es.count(index=self.index, doc_type=self.doc_type, body=body) + return data["count"] + + def __getattr__(self, item): + return getattr(self.es, item) + + def all(self): + """返回所有数据""" + raise NotImplementedError("Not support") + + def ping(self): try: - count = super().count( - date_from=date_from, date_to=date_to, user=user, asset=asset, - system_user=system_user, input=input, session=session - ) - except Exception as e: - logger.error(e, exc_info=True) - return 0 - else: - return count + return self.es.ping() + except Exception: + return False + + @staticmethod + def get_query_body(**kwargs): + new_kwargs = {} + for k, v in kwargs.items(): + new_kwargs[k] = str(v) if isinstance(v, UUID) else v + kwargs = new_kwargs + + exact_fields = {} + match_fields = {'session', 'input', 'org_id', 'risk_level', 'user', 'asset', 'system_user'} + + match = {} + exact = {} + + for k, v in kwargs.items(): + if k in exact_fields: + exact[k] = v + elif k in match_fields: + match[k] = v + + # 处理时间 + timestamp__gte = kwargs.get('timestamp__gte') + timestamp__lte = kwargs.get('timestamp__lte') + timestamp_range = {} + + if timestamp__gte: + timestamp_range['gte'] = timestamp__gte + if timestamp__lte: + timestamp_range['lte'] = timestamp__lte + + # 处理组织 + must_not = [] + org_id = match.get('org_id') + if org_id == '': + match.pop('org_id') + must_not.append({'wildcard': {'org_id': '*'}}) + + # 构建 body + body = { + 'query': { + 'bool': { + 'must': [ + {'match': {k: v}} for k, v in match.items() + ], + 'must_not': must_not, + 'filter': [ + { + 'term': {k: v} + } for k, v in exact.items() + ] + [ + { + 'range': { + 'timestamp': timestamp_range + } + } + ] + } + }, + } + return body + + +class QuerySet(DJQuerySet): + _method_calls = None + _storage = None + _command_store_config = None + _slice = None # (from_, size) + default_days_ago = 5 + max_result_window = 10000 + + def __init__(self, command_store_config): + self._method_calls = [] + self._command_store_config = command_store_config + self._storage = CommandStore(command_store_config) + + @lazyproperty + def _grouped_method_calls(self): + _method_calls = {k: list(v) for k, v in groupby(self._method_calls, lambda x: x[0])} + return _method_calls + + @lazyproperty + def _filter_kwargs(self): + _method_calls = self._grouped_method_calls + filter_calls = _method_calls.get('filter') + if not filter_calls: + return {} + names, multi_args, multi_kwargs = zip(*filter_calls) + kwargs = reduce(lambda x, y: {**x, **y}, multi_kwargs, {}) + + striped_kwargs = {} + for k, v in kwargs.items(): + k = k.replace('__exact', '') + k = k.replace('__startswith', '') + k = k.replace('__icontains', '') + striped_kwargs[k] = v + return striped_kwargs + + @lazyproperty + def _sort(self): + order_by = self._grouped_method_calls.get('order_by') + if order_by: + for call in reversed(order_by): + fields = call[1] + if fields: + field = fields[-1] + + if field.startswith('-'): + direction = 'desc' + else: + direction = 'asc' + field = field.lstrip('-+') + sort = f'{field}:{direction}' + return sort + + def __execute(self): + _filter_kwargs = self._filter_kwargs + _sort = self._sort + from_, size = self._slice or (None, None) + data = self._storage.filter(_filter_kwargs, from_=from_, size=size, sort=_sort) + return data + + def __stage_method_call(self, item, *args, **kwargs): + _clone = self.__clone() + _clone._method_calls.append((item, args, kwargs)) + return _clone + + def __clone(self): + uqs = QuerySet(self._command_store_config) + uqs._method_calls = self._method_calls.copy() + uqs._slice = self._slice + return uqs + + def count(self, limit_to_max_result_window=True): + filter_kwargs = self._filter_kwargs + count = self._storage.count(**filter_kwargs) + if limit_to_max_result_window: + count = min(count, self.max_result_window) + return count + + def __getattribute__(self, item): + if any(( + item.startswith('__'), + item in QuerySet.__dict__, + )): + return object.__getattribute__(self, item) + + origin_attr = object.__getattribute__(self, item) + if not inspect.ismethod(origin_attr): + return origin_attr + + attr = partial(self.__stage_method_call, item) + return attr + + def __getitem__(self, item): + max_window = self.max_result_window + if isinstance(item, slice): + if self._slice is None: + clone = self.__clone() + from_ = item.start or 0 + if item.stop is None: + size = 10 + else: + size = item.stop - from_ + + if from_ + size > max_window: + if from_ >= max_window: + from_ = max_window + size = 0 + else: + size = max_window - from_ + clone._slice = (from_, size) + return clone + return self.__execute()[item] + + def __repr__(self): + return self.__execute().__repr__() + + def __iter__(self): + return iter(self.__execute()) + + def __len__(self): + return self.count() diff --git a/apps/terminal/filters.py b/apps/terminal/filters.py new file mode 100644 index 000000000..caed19a9c --- /dev/null +++ b/apps/terminal/filters.py @@ -0,0 +1,82 @@ +from django_filters import rest_framework as filters +from django.db.models import QuerySet + +from orgs.utils import current_org +from terminal.models import Command, CommandStorage + + +class CommandFilter(filters.FilterSet): + date_from = filters.DateTimeFilter(method='do_nothing') + date_to = filters.DateTimeFilter(method='do_nothing') + session_id = filters.CharFilter(field_name='session') + command_storage_id = filters.UUIDFilter(method='do_nothing') + user = filters.CharFilter(lookup_expr='startswith') + input = filters.CharFilter(lookup_expr='icontains') + + class Meta: + model = Command + fields = [ + 'asset', 'system_user', 'user', 'session', 'risk_level', 'input', + 'date_from', 'date_to', 'session_id', 'risk_level', 'command_storage_id', + ] + + def do_nothing(self, queryset, name, value): + return queryset + + @property + def qs(self): + qs = super().qs + qs = qs.filter(org_id=self.get_org_id()) + qs = self.filter_by_timestamp(qs) + return qs + + def filter_by_timestamp(self, qs: QuerySet): + date_from = self.form.cleaned_data.get('date_from') + date_to = self.form.cleaned_data.get('date_to') + + filters = {} + if date_from: + date_from = date_from.timestamp() + filters['timestamp__gte'] = date_from + + if date_to: + date_to = date_to.timestamp() + filters['timestamp__lte'] = date_to + + qs = qs.filter(**filters) + return qs + + @staticmethod + def get_org_id(): + if current_org.is_default(): + org_id = '' + else: + org_id = current_org.id + return org_id + + +class CommandFilterForStorageTree(CommandFilter): + asset = filters.CharFilter(method='do_nothing') + system_user = filters.CharFilter(method='do_nothing') + session = filters.CharFilter(method='do_nothing') + risk_level = filters.NumberFilter(method='do_nothing') + + class Meta: + model = CommandStorage + fields = [ + 'asset', 'system_user', 'user', 'session', 'risk_level', 'input', + 'date_from', 'date_to', 'session_id', 'risk_level', 'command_storage_id', + ] + + +class CommandStorageFilter(filters.FilterSet): + real = filters.BooleanFilter(method='filter_real') + + class Meta: + model = CommandStorage + fields = ['real', 'name', 'type'] + + def filter_real(self, queryset, name, value): + if value: + queryset = queryset.exclude(name='null') + return queryset diff --git a/apps/terminal/models/storage.py b/apps/terminal/models/storage.py index b74feae40..3f574985c 100644 --- a/apps/terminal/models/storage.py +++ b/apps/terminal/models/storage.py @@ -1,16 +1,24 @@ from __future__ import unicode_literals import os +from importlib import import_module + import jms_storage from django.db import models from django.utils.translation import ugettext_lazy as _ from django.conf import settings from common.mixins import CommonModelMixin +from common.utils import get_logger from common.fields.model import EncryptJsonDictTextField +from terminal.backends import TYPE_ENGINE_MAPPING from .terminal import Terminal +from .command import Command from .. import const +logger = get_logger(__file__) + + class CommandStorage(CommonModelMixin): name = models.CharField(max_length=128, verbose_name=_("Name"), unique=True) type = models.CharField( @@ -50,6 +58,18 @@ class CommandStorage(CommonModelMixin): def is_use(self): return Terminal.objects.filter(command_storage=self.name).exists() + def get_command_queryset(self): + if self.type_server: + qs = Command.objects.all() + else: + if self.type not in TYPE_ENGINE_MAPPING: + logger.error(f'Command storage `{self.type}` not support') + return Command.objects.none() + engine_mod = import_module(TYPE_ENGINE_MAPPING[self.type]) + qs = engine_mod.QuerySet(self.config) + qs.model = Command + return qs + class ReplayStorage(CommonModelMixin): name = models.CharField(max_length=128, verbose_name=_("Name"), unique=True) From 935947c97a9df45aee121db8cc41f30340c77d77 Mon Sep 17 00:00:00 2001 From: xinwen Date: Fri, 5 Mar 2021 10:50:23 +0800 Subject: [PATCH 52/71] =?UTF-8?q?fix:=20=E7=94=A8=E6=88=B7=E8=AF=A6?= =?UTF-8?q?=E6=83=85=E9=A1=B5=E7=9A=84=E8=B5=84=E4=BA=A7=E6=8E=88=E6=9D=83?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E6=85=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/perms/filters.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/perms/filters.py b/apps/perms/filters.py index de7d12ebe..c52c11c5a 100644 --- a/apps/perms/filters.py +++ b/apps/perms/filters.py @@ -58,12 +58,19 @@ class PermissionBaseFilter(BaseFilterSet): return queryset if not user: return queryset.none() - if is_query_all: + + if not is_query_all: queryset = queryset.filter(users=user) return queryset groups = list(user.groups.all().values_list('id', flat=True)) + + user_asset_perm_ids = AssetPermission.objects.filter(users=user).distinct().values_list('id', flat=True) + group_asset_perm_ids = AssetPermission.objects.filter(user_groups__in=groups).distinct().values_list('id', flat=True) + + asset_perm_ids = {*user_asset_perm_ids, *group_asset_perm_ids} + queryset = queryset.filter( - Q(users=user) | Q(user_groups__in=groups) + id__in=asset_perm_ids ).distinct() return queryset From 0aa2c2016f3fd37cca7b99456e9abdaff976f25d Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Mon, 8 Mar 2021 10:08:51 +0800 Subject: [PATCH 53/71] =?UTF-8?q?perf(project):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E5=91=BD=E5=90=8D=E7=9A=84=E9=A3=8E=E6=A0=BC=20(#5693)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: 修改错误的地 perf: 优化写错的几个 Co-authored-by: ibuler --- apps/applications/api/mixin.py | 4 +- apps/assets/api/node.py | 4 +- apps/assets/api/system_user.py | 12 +-- apps/assets/backends/base.py | 2 +- apps/assets/backends/db.py | 6 +- apps/assets/models/favorite_asset.py | 2 +- apps/assets/models/node.py | 78 +++++++++---------- apps/assets/models/user.py | 8 +- apps/assets/signals_handler/common.py | 12 +-- .../signals_handler/node_assets_mapping.py | 6 +- apps/assets/tasks/push_system_user.py | 10 +-- apps/authentication/api/connection_token.py | 4 +- apps/common/api.py | 4 +- apps/common/const/__init__.py | 2 +- apps/common/drf/filters.py | 8 +- apps/orgs/signals_handler/common.py | 10 +-- .../api/application/user_permission/common.py | 10 +-- apps/perms/api/asset/user_group_permission.py | 24 +++--- .../perms/api/asset/user_permission/common.py | 16 ++-- .../user_permission_nodes_with_assets.py | 16 ++-- apps/perms/api/system_user_permission.py | 4 +- apps/perms/models/application_permission.py | 6 +- apps/perms/models/asset_permission.py | 8 +- apps/perms/models/base.py | 12 +-- apps/perms/signals_handler/common.py | 34 ++++---- apps/perms/utils/application/permission.py | 6 +- .../utils/application/user_permission.py | 6 +- apps/perms/utils/asset/permission.py | 13 ++-- apps/perms/utils/asset/user_permission.py | 52 ++++++------- apps/terminal/api/terminal.py | 10 +-- apps/terminal/models/session.py | 4 +- apps/tickets/handler/apply_application.py | 16 ++-- apps/tickets/handler/apply_asset.py | 16 ++-- .../meta/ticket_type/apply_application.py | 28 +++---- .../ticket/meta/ticket_type/apply_asset.py | 28 +++---- apps/users/api/user.py | 4 +- apps/users/models/user.py | 8 +- utils/generate_fake_data/resources/assets.py | 12 +-- utils/generate_fake_data/resources/perms.py | 34 ++++---- utils/generate_fake_data/resources/users.py | 6 +- 40 files changed, 272 insertions(+), 273 deletions(-) diff --git a/apps/applications/api/mixin.py b/apps/applications/api/mixin.py index dd8bc64c2..f79902b72 100644 --- a/apps/applications/api/mixin.py +++ b/apps/applications/api/mixin.py @@ -77,8 +77,8 @@ class SerializeApplicationToTreeNodeMixin: @staticmethod def filter_organizations(applications): - organizations_id = set(applications.values_list('org_id', flat=True)) - organizations = [Organization.get_instance(org_id) for org_id in organizations_id] + organization_ids = set(applications.values_list('org_id', flat=True)) + organizations = [Organization.get_instance(org_id) for org_id in organization_ids] return organizations def serialize_applications_with_org(self, applications): diff --git a/apps/assets/api/node.py b/apps/assets/api/node.py index f964b7704..09508e6e3 100644 --- a/apps/assets/api/node.py +++ b/apps/assets/api/node.py @@ -223,8 +223,8 @@ class NodeAddChildrenApi(generics.UpdateAPIView): def put(self, request, *args, **kwargs): instance = self.get_object() - nodes_id = request.data.get("nodes") - children = Node.objects.filter(id__in=nodes_id) + node_ids = request.data.get("nodes") + children = Node.objects.filter(id__in=node_ids) for node in children: node.parent = instance return Response("OK") diff --git a/apps/assets/api/system_user.py b/apps/assets/api/system_user.py index af7590579..27baaa017 100644 --- a/apps/assets/api/system_user.py +++ b/apps/assets/api/system_user.py @@ -87,13 +87,13 @@ class SystemUserTaskApi(generics.CreateAPIView): permission_classes = (IsOrgAdmin,) serializer_class = serializers.SystemUserTaskSerializer - def do_push(self, system_user, assets_id=None): - if assets_id is None: + def do_push(self, system_user, asset_ids=None): + if asset_ids is None: task = push_system_user_to_assets_manual.delay(system_user) else: username = self.request.query_params.get('username') task = push_system_user_to_assets.delay( - system_user.id, assets_id, username=username + system_user.id, asset_ids, username=username ) return task @@ -114,9 +114,9 @@ class SystemUserTaskApi(generics.CreateAPIView): system_user = self.get_object() if action == 'push': assets = [asset] if asset else assets - assets_id = [asset.id for asset in assets] - assets_id = assets_id if assets_id else None - task = self.do_push(system_user, assets_id) + asset_ids = [asset.id for asset in assets] + asset_ids = asset_ids if asset_ids else None + task = self.do_push(system_user, asset_ids) else: task = self.do_test(system_user) data = getattr(serializer, '_data', {}) diff --git a/apps/assets/backends/base.py b/apps/assets/backends/base.py index d5ce1f903..3b27a57af 100644 --- a/apps/assets/backends/base.py +++ b/apps/assets/backends/base.py @@ -40,7 +40,7 @@ class BaseBackend: return values @staticmethod - def make_assets_as_id(assets): + def make_assets_as_ids(assets): if not assets: return [] if isinstance(assets[0], Asset): diff --git a/apps/assets/backends/db.py b/apps/assets/backends/db.py index 35be8b9fd..bf31e04c8 100644 --- a/apps/assets/backends/db.py +++ b/apps/assets/backends/db.py @@ -69,9 +69,9 @@ class DBBackend(BaseBackend): self.queryset = self.queryset.filter(union_id=union_id) def _filter_assets(self, assets): - assets_id = self.make_assets_as_id(assets) - if assets_id: - self.queryset = self.queryset.filter(asset_id__in=assets_id) + asset_ids = self.make_assets_as_ids(assets) + if asset_ids: + self.queryset = self.queryset.filter(asset_id__in=asset_ids) def _filter_node(self, node): pass diff --git a/apps/assets/models/favorite_asset.py b/apps/assets/models/favorite_asset.py index 3abc69c8c..c5e6db484 100644 --- a/apps/assets/models/favorite_asset.py +++ b/apps/assets/models/favorite_asset.py @@ -16,5 +16,5 @@ class FavoriteAsset(CommonModelMixin): unique_together = ('user', 'asset') @classmethod - def get_user_favorite_assets_id(cls, user): + def get_user_favorite_asset_ids(cls, user): return cls.objects.filter(user=user).values_list('asset', flat=True) diff --git a/apps/assets/models/node.py b/apps/assets/models/node.py index 68aa300d3..9ff9c03bf 100644 --- a/apps/assets/models/node.py +++ b/apps/assets/models/node.py @@ -263,38 +263,38 @@ class NodeAllAssetsMappingMixin: orgid_nodekey_assetsid_mapping = defaultdict(dict) @classmethod - def get_node_all_assets_id_mapping(cls, org_id): - _mapping = cls.get_node_all_assets_id_mapping_from_memory(org_id) + def get_node_all_asset_ids_mapping(cls, org_id): + _mapping = cls.get_node_all_asset_ids_mapping_from_memory(org_id) if _mapping: return _mapping - _mapping = cls.get_node_all_assets_id_mapping_from_cache_or_generate_to_cache(org_id) - cls.set_node_all_assets_id_mapping_to_memory(org_id, mapping=_mapping) + _mapping = cls.get_node_all_asset_ids_mapping_from_cache_or_generate_to_cache(org_id) + cls.set_node_all_asset_ids_mapping_to_memory(org_id, mapping=_mapping) return _mapping # from memory @classmethod - def get_node_all_assets_id_mapping_from_memory(cls, org_id): + def get_node_all_asset_ids_mapping_from_memory(cls, org_id): mapping = cls.orgid_nodekey_assetsid_mapping.get(org_id, {}) return mapping @classmethod - def set_node_all_assets_id_mapping_to_memory(cls, org_id, mapping): + def set_node_all_asset_ids_mapping_to_memory(cls, org_id, mapping): cls.orgid_nodekey_assetsid_mapping[org_id] = mapping @classmethod - def expire_node_all_assets_id_mapping_from_memory(cls, org_id): + def expire_node_all_asset_ids_mapping_from_memory(cls, org_id): org_id = str(org_id) cls.orgid_nodekey_assetsid_mapping.pop(org_id, None) # get order: from memory -> (from cache -> to generate) @classmethod - def get_node_all_assets_id_mapping_from_cache_or_generate_to_cache(cls, org_id): - mapping = cls.get_node_all_assets_id_mapping_from_cache(org_id) + def get_node_all_asset_ids_mapping_from_cache_or_generate_to_cache(cls, org_id): + mapping = cls.get_node_all_asset_ids_mapping_from_cache(org_id) if mapping: return mapping - lock_key = f'KEY_LOCK_GENERATE_ORG_{org_id}_NODE_ALL_ASSETS_ID_MAPPING' + lock_key = f'KEY_LOCK_GENERATE_ORG_{org_id}_NODE_ALL_ASSET_ids_MAPPING' logger.info(f'Thread[{threading.get_ident()}] acquiring lock[{lock_key}] ...') with DistributedLock(lock_key): logger.info(f'Thread[{threading.get_ident()}] acquire lock[{lock_key}] ok') @@ -303,67 +303,67 @@ class NodeAllAssetsMappingMixin: # 这里最好先判断内存中有没有，防止同一进程的多个线程重复从 cache 中获取数据， # 但逻辑过于繁琐，直接判断 cache 吧 - _mapping = cls.get_node_all_assets_id_mapping_from_cache(org_id) + _mapping = cls.get_node_all_asset_ids_mapping_from_cache(org_id) if _mapping: return _mapping - _mapping = cls.generate_node_all_assets_id_mapping(org_id) - cls.set_node_all_assets_id_mapping_to_cache(org_id=org_id, mapping=_mapping) + _mapping = cls.generate_node_all_asset_ids_mapping(org_id) + cls.set_node_all_asset_ids_mapping_to_cache(org_id=org_id, mapping=_mapping) return _mapping @classmethod - def get_node_all_assets_id_mapping_from_cache(cls, org_id): - cache_key = cls._get_cache_key_for_node_all_assets_id_mapping(org_id) + def get_node_all_asset_ids_mapping_from_cache(cls, org_id): + cache_key = cls._get_cache_key_for_node_all_asset_ids_mapping(org_id) mapping = cache.get(cache_key) return mapping @classmethod - def set_node_all_assets_id_mapping_to_cache(cls, org_id, mapping): - cache_key = cls._get_cache_key_for_node_all_assets_id_mapping(org_id) + def set_node_all_asset_ids_mapping_to_cache(cls, org_id, mapping): + cache_key = cls._get_cache_key_for_node_all_asset_ids_mapping(org_id) cache.set(cache_key, mapping, timeout=None) @classmethod - def expire_node_all_assets_id_mapping_from_cache(cls, org_id): - cache_key = cls._get_cache_key_for_node_all_assets_id_mapping(org_id) + def expire_node_all_asset_ids_mapping_from_cache(cls, org_id): + cache_key = cls._get_cache_key_for_node_all_asset_ids_mapping(org_id) cache.delete(cache_key) @staticmethod - def _get_cache_key_for_node_all_assets_id_mapping(org_id): - return 'ASSETS_ORG_NODE_ALL_ASSETS_ID_MAPPING_{}'.format(org_id) + def _get_cache_key_for_node_all_asset_ids_mapping(org_id): + return 'ASSETS_ORG_NODE_ALL_ASSET_ids_MAPPING_{}'.format(org_id) @classmethod - def generate_node_all_assets_id_mapping(cls, org_id): + def generate_node_all_asset_ids_mapping(cls, org_id): from .asset import Asset t1 = time.time() with tmp_to_org(org_id): - nodes_id_key = Node.objects.annotate( + node_ids_key = Node.objects.annotate( char_id=output_as_string('id') ).values_list('char_id', 'key') # * 直接取出全部. filter(node__org_id=org_id)(大规模下会更慢) - nodes_assets_id = Asset.nodes.through.objects.all() \ + nodes_asset_ids = Asset.nodes.through.objects.all() \ .annotate(char_node_id=output_as_string('node_id')) \ .annotate(char_asset_id=output_as_string('asset_id')) \ .values_list('char_node_id', 'char_asset_id') node_id_ancestor_keys_mapping = { node_id: cls.get_node_ancestor_keys(node_key, with_self=True) - for node_id, node_key in nodes_id_key + for node_id, node_key in node_ids_key } nodeid_assetsid_mapping = defaultdict(set) - for node_id, asset_id in nodes_assets_id: + for node_id, asset_id in nodes_asset_ids: nodeid_assetsid_mapping[node_id].add(asset_id) t2 = time.time() mapping = defaultdict(set) - for node_id, node_key in nodes_id_key: - assets_id = nodeid_assetsid_mapping[node_id] + for node_id, node_key in node_ids_key: + asset_ids = nodeid_assetsid_mapping[node_id] node_ancestor_keys = node_id_ancestor_keys_mapping[node_id] for ancestor_key in node_ancestor_keys: - mapping[ancestor_key].update(assets_id) + mapping[ancestor_key].update(asset_ids) t3 = time.time() logger.debug('t1-t2(DB Query): {} s, t3-t2(Generate mapping): {} s'.format(t2-t1, t3-t2)) @@ -407,10 +407,10 @@ class NodeAssetsMixin(NodeAllAssetsMappingMixin): return self.get_all_assets().valid() @classmethod - def get_nodes_all_assets_ids_by_keys(cls, nodes_keys): + def get_nodes_all_asset_ids_by_keys(cls, nodes_keys): nodes = Node.objects.filter(key__in=nodes_keys) - assets_ids = cls.get_nodes_all_assets(*nodes).values_list('id', flat=True) - return assets_ids + asset_ids = cls.get_nodes_all_assets(*nodes).values_list('id', flat=True) + return asset_ids @classmethod def get_nodes_all_assets(cls, *nodes): @@ -425,16 +425,16 @@ class NodeAssetsMixin(NodeAllAssetsMappingMixin): node_ids.update(_ids) return Asset.objects.order_by().filter(nodes__id__in=node_ids).distinct() - def get_all_assets_id(self): - assets_id = self.get_all_assets_id_by_node_key(org_id=self.org_id, node_key=self.key) - return set(assets_id) + def get_all_asset_ids(self): + asset_ids = self.get_all_asset_ids_by_node_key(org_id=self.org_id, node_key=self.key) + return set(asset_ids) @classmethod - def get_all_assets_id_by_node_key(cls, org_id, node_key): + def get_all_asset_ids_by_node_key(cls, org_id, node_key): org_id = str(org_id) - nodekey_assetsid_mapping = cls.get_node_all_assets_id_mapping(org_id) - assets_id = nodekey_assetsid_mapping.get(node_key, []) - return set(assets_id) + nodekey_assetsid_mapping = cls.get_node_all_asset_ids_mapping(org_id) + asset_ids = nodekey_assetsid_mapping.get(node_key, []) + return set(asset_ids) class SomeNodesMixin: diff --git a/apps/assets/models/user.py b/apps/assets/models/user.py index 885543796..dd576b077 100644 --- a/apps/assets/models/user.py +++ b/apps/assets/models/user.py @@ -198,10 +198,10 @@ class SystemUser(BaseUser): def get_all_assets(self): from assets.models import Node nodes_keys = self.nodes.all().values_list('key', flat=True) - assets_ids = set(self.assets.all().values_list('id', flat=True)) - nodes_assets_ids = Node.get_nodes_all_assets_ids_by_keys(nodes_keys) - assets_ids.update(nodes_assets_ids) - assets = Asset.objects.filter(id__in=assets_ids) + asset_ids = set(self.assets.all().values_list('id', flat=True)) + nodes_asset_ids = Node.get_nodes_all_asset_ids_by_keys(nodes_keys) + asset_ids.update(nodes_asset_ids) + assets = Asset.objects.filter(id__in=asset_ids) return assets @classmethod diff --git a/apps/assets/signals_handler/common.py b/apps/assets/signals_handler/common.py index 6625e493e..af6a7895c 100644 --- a/apps/assets/signals_handler/common.py +++ b/apps/assets/signals_handler/common.py @@ -82,13 +82,13 @@ def on_system_user_assets_change(instance, action, model, pk_set, **kwargs): return logger.debug("System user assets change signal recv: {}".format(instance)) if model == Asset: - system_users_id = [instance.id] - assets_id = pk_set + system_user_ids = [instance.id] + asset_ids = pk_set else: - system_users_id = pk_set - assets_id = [instance.id] - for system_user_id in system_users_id: - push_system_user_to_assets.delay(system_user_id, assets_id) + system_user_ids = pk_set + asset_ids = [instance.id] + for system_user_id in system_user_ids: + push_system_user_to_assets.delay(system_user_id, asset_ids) @receiver(m2m_changed, sender=SystemUser.users.through) diff --git a/apps/assets/signals_handler/node_assets_mapping.py b/apps/assets/signals_handler/node_assets_mapping.py index d6615c885..4e2b0d07b 100644 --- a/apps/assets/signals_handler/node_assets_mapping.py +++ b/apps/assets/signals_handler/node_assets_mapping.py @@ -22,7 +22,7 @@ logger = get_logger(__file__) def get_node_assets_mapping_for_memory_pub_sub(): - return RedisPubSub('fm.node_all_assets_id_memory_mapping') + return RedisPubSub('fm.node_all_asset_ids_memory_mapping') class NodeAssetsMappingForMemoryPubSub(LazyObject): @@ -42,7 +42,7 @@ def expire_node_assets_mapping_for_memory(org_id): "Expire node assets id mapping from cache of org={}, pid={}" "".format(org_id, os.getpid()) ) - Node.expire_node_all_assets_id_mapping_from_cache(org_id) + Node.expire_node_all_asset_ids_mapping_from_cache(org_id) @receiver(post_save, sender=Node) @@ -78,7 +78,7 @@ def subscribe_node_assets_mapping_expire(sender, **kwargs): if message["type"] != "message": continue org_id = message['data'].decode() - Node.expire_node_all_assets_id_mapping_from_memory(org_id) + Node.expire_node_all_asset_ids_mapping_from_memory(org_id) logger.debug( "Expire node assets id mapping from memory of org={}, pid={}" "".format(str(org_id), os.getpid()) diff --git a/apps/assets/tasks/push_system_user.py b/apps/assets/tasks/push_system_user.py index be337baea..b56cfcdd8 100644 --- a/apps/assets/tasks/push_system_user.py +++ b/apps/assets/tasks/push_system_user.py @@ -233,18 +233,18 @@ def push_system_user_util(system_user, assets, task_name, username=None): print(_("Hosts count: {}").format(len(_assets))) id_asset_map = {_asset.id: _asset for _asset in _assets} - assets_id = id_asset_map.keys() + asset_ids = id_asset_map.keys() no_special_auth = [] special_auth_set = set() - auth_books = AuthBook.objects.filter(username__in=usernames, asset_id__in=assets_id) + auth_books = AuthBook.objects.filter(username__in=usernames, asset_id__in=asset_ids) for auth_book in auth_books: special_auth_set.add((auth_book.username, auth_book.asset_id)) for _username in usernames: no_special_assets = [] - for asset_id in assets_id: + for asset_id in asset_ids: if (_username, asset_id) not in special_auth_set: no_special_assets.append(id_asset_map[asset_id]) if no_special_assets: @@ -289,12 +289,12 @@ def push_system_user_a_asset_manual(system_user, asset, username=None): @shared_task(queue="ansible") @tmp_to_root_org() -def push_system_user_to_assets(system_user_id, assets_id, username=None): +def push_system_user_to_assets(system_user_id, asset_ids, username=None): """ 推送系统用户到指定的若干资产上 """ system_user = SystemUser.objects.get(id=system_user_id) - assets = get_objects(Asset, assets_id) + assets = get_objects(Asset, asset_ids) task_name = _("Push system users to assets: {}").format(system_user.name) return push_system_user_util(system_user, assets, task_name, username=username) diff --git a/apps/authentication/api/connection_token.py b/apps/authentication/api/connection_token.py index 16f8907be..ae9e4a2de 100644 --- a/apps/authentication/api/connection_token.py +++ b/apps/authentication/api/connection_token.py @@ -183,9 +183,9 @@ class UserConnectionTokenViewSet(RootOrgViewMixin, SerializerMixin2, GenericView @staticmethod def _get_asset_secret_detail(value, user, system_user): from assets.models import Asset - from perms.utils.asset import get_asset_system_users_id_with_actions_by_user + from perms.utils.asset import get_asset_system_user_ids_with_actions_by_user asset = get_object_or_404(Asset, id=value.get('asset')) - systemuserid_actions_mapper = get_asset_system_users_id_with_actions_by_user(user, asset) + systemuserid_actions_mapper = get_asset_system_user_ids_with_actions_by_user(user, asset) actions = systemuserid_actions_mapper.get(system_user.id, []) gateway = None if asset and asset.domain and asset.domain.has_gateway(): diff --git a/apps/common/api.py b/apps/common/api.py index 06c162539..5bf20f027 100644 --- a/apps/common/api.py +++ b/apps/common/api.py @@ -13,7 +13,7 @@ from rest_framework.viewsets import GenericViewSet from common.permissions import IsValidUser from .http import HttpResponseTemporaryRedirect -from .const import KEY_CACHE_RESOURCES_ID +from .const import KEY_CACHE_RESOURCE_IDS from .utils import get_logger from .mixins import CommonApiMixin @@ -93,7 +93,7 @@ class ResourcesIDCacheApi(APIView): spm = str(uuid.uuid4()) resources = request.data.get('resources') if resources is not None: - cache_key = KEY_CACHE_RESOURCES_ID.format(spm) + cache_key = KEY_CACHE_RESOURCE_IDS.format(spm) cache.set(cache_key, resources, 300) return Response({'spm': spm}) diff --git a/apps/common/const/__init__.py b/apps/common/const/__init__.py index 65d445eec..84a07a11a 100644 --- a/apps/common/const/__init__.py +++ b/apps/common/const/__init__.py @@ -7,7 +7,7 @@ create_success_msg = _("%(name)s was created successfully") update_success_msg = _("%(name)s was updated successfully") FILE_END_GUARD = ">>> Content End <<<" celery_task_pre_key = "CELERY_" -KEY_CACHE_RESOURCES_ID = "RESOURCES_ID_{}" +KEY_CACHE_RESOURCE_IDS = "RESOURCE_IDS_{}" # AD User AccountDisable # https://blog.csdn.net/bytxl/article/details/17763975 diff --git a/apps/common/drf/filters.py b/apps/common/drf/filters.py index f130af3b7..609cb5ab0 100644 --- a/apps/common/drf/filters.py +++ b/apps/common/drf/filters.py @@ -108,11 +108,11 @@ class IDSpmFilter(filters.BaseFilterBackend): spm = request.query_params.get('spm') if not spm: return queryset - cache_key = const.KEY_CACHE_RESOURCES_ID.format(spm) - resources_id = cache.get(cache_key) - if resources_id is None or not isinstance(resources_id, list): + cache_key = const.KEY_CACHE_RESOURCE_IDS.format(spm) + resource_ids = cache.get(cache_key) + if resource_ids is None or not isinstance(resource_ids, list): return queryset - queryset = queryset.filter(id__in=resources_id) + queryset = queryset.filter(id__in=resource_ids) return queryset diff --git a/apps/orgs/signals_handler/common.py b/apps/orgs/signals_handler/common.py index f7b6f8e78..419bccdeb 100644 --- a/apps/orgs/signals_handler/common.py +++ b/apps/orgs/signals_handler/common.py @@ -91,14 +91,14 @@ def _remove_users(model, users, org): f'{m2m_field_name}__org_id': org.id }) - object_id_users_id_map = defaultdict(set) + object_id_user_ids_map = defaultdict(set) m2m_field_attr_name = f'{m2m_field_name}_id' for relation in relations: object_id = getattr(relation, m2m_field_attr_name) - object_id_users_id_map[object_id].add(relation.user_id) + object_id_user_ids_map[object_id].add(relation.user_id) - objects = model.objects.filter(id__in=object_id_users_id_map.keys()) + objects = model.objects.filter(id__in=object_id_user_ids_map.keys()) send_m2m_change_signal = partial( m2m_changed.send, sender=m2m_model, reverse=reverse, model=User, using=model.objects.db @@ -107,7 +107,7 @@ def _remove_users(model, users, org): for obj in objects: send_m2m_change_signal( instance=obj, - pk_set=object_id_users_id_map[obj.id], + pk_set=object_id_user_ids_map[obj.id], action=PRE_REMOVE ) @@ -116,7 +116,7 @@ def _remove_users(model, users, org): for obj in objects: send_m2m_change_signal( instance=obj, - pk_set=object_id_users_id_map[obj.id], + pk_set=object_id_user_ids_map[obj.id], action=POST_REMOVE ) diff --git a/apps/perms/api/application/user_permission/common.py b/apps/perms/api/application/user_permission/common.py index 999a09087..4e7d0b5f2 100644 --- a/apps/perms/api/application/user_permission/common.py +++ b/apps/perms/api/application/user_permission/common.py @@ -12,7 +12,7 @@ from orgs.utils import tmp_to_root_org from applications.models import Application from perms.utils.application.permission import ( has_application_system_permission, - get_application_system_users_id + get_application_system_user_ids ) from perms.api.asset.user_permission.mixin import RoleAdminMixin, RoleUserMixin from common.permissions import IsOrgAdminOrAppUser @@ -32,14 +32,14 @@ class GrantedApplicationSystemUsersMixin(ListAPIView): only_fields = serializers.ApplicationSystemUserSerializer.Meta.only_fields user: None - def get_application_system_users_id(self, application): - return get_application_system_users_id(self.user, application) + def get_application_system_user_ids(self, application): + return get_application_system_user_ids(self.user, application) def get_queryset(self): application_id = self.kwargs.get('application_id') application = get_object_or_404(Application, id=application_id) - system_users_id = self.get_application_system_users_id(application) - system_users = SystemUser.objects.filter(id__in=system_users_id)\ + system_user_ids = self.get_application_system_user_ids(application) + system_users = SystemUser.objects.filter(id__in=system_user_ids)\ .only(*self.only_fields).order_by('priority') return system_users diff --git a/apps/perms/api/asset/user_group_permission.py b/apps/perms/api/asset/user_group_permission.py index c1b025cb3..e8cea21b5 100644 --- a/apps/perms/api/asset/user_group_permission.py +++ b/apps/perms/api/asset/user_group_permission.py @@ -12,7 +12,7 @@ from perms.models import AssetPermission from assets.models import Asset, Node from perms.api.asset import user_permission as uapi from perms import serializers -from perms.utils.asset.permission import get_asset_system_users_id_with_actions_by_group +from perms.utils.asset.permission import get_asset_system_user_ids_with_actions_by_group from assets.api.mixin import SerializeToTreeNodeMixin from users.models import UserGroup @@ -41,12 +41,12 @@ class UserGroupGrantedAssetsApi(ListAPIView): def get_queryset(self): user_group_id = self.kwargs.get('pk', '') - asset_perms_id = list(AssetPermission.objects.valid().filter( + asset_perm_ids = list(AssetPermission.objects.valid().filter( user_groups__id=user_group_id ).distinct().values_list('id', flat=True)) granted_node_keys = Node.objects.filter( - granted_by_permissions__id__in=asset_perms_id, + granted_by_permissions__id__in=asset_perm_ids, ).distinct().values_list('key', flat=True) granted_q = Q() @@ -54,7 +54,7 @@ class UserGroupGrantedAssetsApi(ListAPIView): granted_q |= Q(nodes__key__startswith=f'{_key}:') granted_q |= Q(nodes__key=_key) - granted_q |= Q(granted_by_permissions__id__in=asset_perms_id) + granted_q |= Q(granted_by_permissions__id__in=asset_perm_ids) assets = Asset.objects.filter( granted_q @@ -89,12 +89,12 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView): ) return assets else: - asset_perms_id = list(AssetPermission.objects.valid().filter( + asset_perm_ids = list(AssetPermission.objects.valid().filter( user_groups__id=user_group_id ).distinct().values_list('id', flat=True)) granted_node_keys = Node.objects.filter( - granted_by_permissions__id__in=asset_perms_id, + granted_by_permissions__id__in=asset_perm_ids, key__startswith=f'{node.key}:' ).distinct().values_list('key', flat=True) @@ -104,7 +104,7 @@ class UserGroupGrantedNodeAssetsApi(ListAPIView): granted_node_q |= Q(nodes__key=_key) granted_asset_q = ( - Q(granted_by_permissions__id__in=asset_perms_id) & + Q(granted_by_permissions__id__in=asset_perm_ids) & ( Q(nodes__key__startswith=f'{node.key}:') | Q(nodes__key=node.key) @@ -148,16 +148,16 @@ class UserGroupGrantedNodeChildrenAsTreeApi(SerializeToTreeNodeMixin, ListAPIVie group_id = self.kwargs.get('pk') node_key = self.request.query_params.get('key', None) - asset_perms_id = list(AssetPermission.objects.valid().filter( + asset_perm_ids = list(AssetPermission.objects.valid().filter( user_groups__id=group_id ).distinct().values_list('id', flat=True)) granted_keys = Node.objects.filter( - granted_by_permissions__id__in=asset_perms_id + granted_by_permissions__id__in=asset_perm_ids ).values_list('key', flat=True) asset_granted_keys = Node.objects.filter( - assets__granted_by_permissions__id__in=asset_perms_id + assets__granted_by_permissions__id__in=asset_perm_ids ).values_list('key', flat=True) if node_key is None: @@ -188,5 +188,5 @@ class UserGroupGrantedNodeChildrenAsTreeApi(SerializeToTreeNodeMixin, ListAPIVie class UserGroupGrantedAssetSystemUsersApi(UserGroupMixin, uapi.UserGrantedAssetSystemUsersForAdminApi): - def get_asset_system_users_id_with_actions(self, asset): - return get_asset_system_users_id_with_actions_by_group(self.group, asset) + def get_asset_system_user_ids_with_actions(self, asset): + return get_asset_system_user_ids_with_actions_by_group(self.group, asset) diff --git a/apps/perms/api/asset/user_permission/common.py b/apps/perms/api/asset/user_permission/common.py index 8746beb2a..88e4f069e 100644 --- a/apps/perms/api/asset/user_permission/common.py +++ b/apps/perms/api/asset/user_permission/common.py @@ -10,7 +10,7 @@ from rest_framework.generics import ( ) from orgs.utils import tmp_to_root_org -from perms.utils.asset.permission import get_asset_system_users_id_with_actions_by_user +from perms.utils.asset.permission import get_asset_system_user_ids_with_actions_by_user from common.permissions import IsOrgAdminOrAppUser, IsOrgAdmin, IsValidUser from common.utils import get_logger, lazyproperty @@ -53,7 +53,7 @@ class GetUserAssetPermissionActionsApi(RetrieveAPIView): asset = get_object_or_404(Asset, id=asset_id) system_user = get_object_or_404(SystemUser, id=system_id) - system_users_actions = get_asset_system_users_id_with_actions_by_user(self.get_user(), asset) + system_users_actions = get_asset_system_user_ids_with_actions_by_user(self.get_user(), asset) actions = system_users_actions.get(system_user.id) return {"actions": actions} @@ -84,7 +84,7 @@ class ValidateUserAssetPermissionApi(APIView): asset = get_object_or_404(Asset, id=asset_id) system_user = get_object_or_404(SystemUser, id=system_id) - system_users_actions = get_asset_system_users_id_with_actions_by_user(self.get_user(), asset) + system_users_actions = get_asset_system_user_ids_with_actions_by_user(self.get_user(), asset) actions = system_users_actions.get(system_user.id) if actions is None: return Response({'msg': False}, status=403) @@ -111,15 +111,15 @@ class UserGrantedAssetSystemUsersForAdminApi(ListAPIView): user_id = self.kwargs.get('pk') return User.objects.get(id=user_id) - def get_asset_system_users_id_with_actions(self, asset): - return get_asset_system_users_id_with_actions_by_user(self.user, asset) + def get_asset_system_user_ids_with_actions(self, asset): + return get_asset_system_user_ids_with_actions_by_user(self.user, asset) def get_queryset(self): asset_id = self.kwargs.get('asset_id') asset = get_object_or_404(Asset, id=asset_id) - system_users_with_actions = self.get_asset_system_users_id_with_actions(asset) - system_users_id = system_users_with_actions.keys() - system_users = SystemUser.objects.filter(id__in=system_users_id)\ + system_users_with_actions = self.get_asset_system_user_ids_with_actions(asset) + system_user_ids = system_users_with_actions.keys() + system_users = SystemUser.objects.filter(id__in=system_user_ids)\ .only(*self.serializer_class.Meta.only_fields) \ .order_by('priority') system_users = list(system_users) diff --git a/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py b/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py index 63619e9c1..f66ed3c3f 100644 --- a/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py +++ b/apps/perms/api/asset/user_permission/user_permission_nodes_with_assets.py @@ -52,8 +52,8 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): data.extend(self.serialize_assets(favorite_assets)) @timeit - def add_node_filtered_by_system_user(self, data: list, user, asset_perms_id): - utils = UserGrantedTreeBuildUtils(user, asset_perms_id) + def add_node_filtered_by_system_user(self, data: list, user, asset_perm_ids): + utils = UserGrantedTreeBuildUtils(user, asset_perm_ids) nodes = utils.get_whole_tree_nodes() data.extend(self.serialize_nodes(nodes, with_asset_amount=True)) @@ -77,23 +77,23 @@ class MyGrantedNodesWithAssetsAsTreeApi(SerializeToTreeNodeMixin, ListAPIView): user = request.user data = [] - asset_perms_id = get_user_all_asset_perm_ids(user) + asset_perm_ids = get_user_all_asset_perm_ids(user) system_user_id = request.query_params.get('system_user') if system_user_id: - asset_perms_id = list(AssetPermission.objects.valid().filter( - id__in=asset_perms_id, system_users__id=system_user_id, actions__gt=0 + asset_perm_ids = list(AssetPermission.objects.valid().filter( + id__in=asset_perm_ids, system_users__id=system_user_id, actions__gt=0 ).values_list('id', flat=True).distinct()) - nodes_query_utils = UserGrantedNodesQueryUtils(user, asset_perms_id) - assets_query_utils = UserGrantedAssetsQueryUtils(user, asset_perms_id) + nodes_query_utils = UserGrantedNodesQueryUtils(user, asset_perm_ids) + assets_query_utils = UserGrantedAssetsQueryUtils(user, asset_perm_ids) self.add_ungrouped_resource(data, nodes_query_utils, assets_query_utils) self.add_favorite_resource(data, nodes_query_utils, assets_query_utils) if system_user_id: # 有系统用户筛选的需要重新计算树结构 - self.add_node_filtered_by_system_user(data, user, asset_perms_id) + self.add_node_filtered_by_system_user(data, user, asset_perm_ids) else: all_nodes = nodes_query_utils.get_whole_tree_nodes(with_special=False) data.extend(self.serialize_nodes(all_nodes, with_asset_amount=True)) diff --git a/apps/perms/api/system_user_permission.py b/apps/perms/api/system_user_permission.py index 17ddfc786..48d440baa 100644 --- a/apps/perms/api/system_user_permission.py +++ b/apps/perms/api/system_user_permission.py @@ -16,9 +16,9 @@ class SystemUserPermission(generics.ListAPIView): def get_queryset(self): user = self.request.user - asset_perms_id = get_user_all_asset_perm_ids(user) + asset_perm_ids = get_user_all_asset_perm_ids(user) queryset = SystemUser.objects.filter( - granted_by_permissions__id__in=asset_perms_id + granted_by_permissions__id__in=asset_perm_ids ).distinct() return queryset diff --git a/apps/perms/models/application_permission.py b/apps/perms/models/application_permission.py index 44cb38323..d16b154ae 100644 --- a/apps/perms/models/application_permission.py +++ b/apps/perms/models/application_permission.py @@ -65,9 +65,9 @@ class ApplicationPermission(BasePermission): return self.system_users.count() def get_all_users(self): - users_id = self.users.all().values_list('id', flat=True) - user_groups_id = self.user_groups.all().values_list('id', flat=True) + user_ids = self.users.all().values_list('id', flat=True) + user_group_ids = self.user_groups.all().values_list('id', flat=True) users = User.objects.filter( - Q(id__in=users_id) | Q(groups__id__in=user_groups_id) + Q(id__in=user_ids) | Q(groups__id__in=user_group_ids) ) return users diff --git a/apps/perms/models/asset_permission.py b/apps/perms/models/asset_permission.py index fb54ad1a6..f7551306e 100644 --- a/apps/perms/models/asset_permission.py +++ b/apps/perms/models/asset_permission.py @@ -137,10 +137,10 @@ class AssetPermission(BasePermission): def get_all_assets(self): from assets.models import Node nodes_keys = self.nodes.all().values_list('key', flat=True) - assets_ids = set(self.assets.all().values_list('id', flat=True)) - nodes_assets_ids = Node.get_nodes_all_assets_ids_by_keys(nodes_keys) - assets_ids.update(nodes_assets_ids) - assets = Asset.objects.filter(id__in=assets_ids) + asset_ids = set(self.assets.all().values_list('id', flat=True)) + nodes_asset_ids = Node.get_nodes_all_asset_ids_by_keys(nodes_keys) + asset_ids.update(nodes_asset_ids) + assets = Asset.objects.filter(id__in=asset_ids) return assets diff --git a/apps/perms/models/base.py b/apps/perms/models/base.py index 7651a06e8..05f780e8f 100644 --- a/apps/perms/models/base.py +++ b/apps/perms/models/base.py @@ -99,14 +99,14 @@ class BasePermission(OrgModelMixin): def get_all_users(self): from users.models import User - users_id = self.users.all().values_list('id', flat=True) - groups_id = self.user_groups.all().values_list('id', flat=True) + user_ids = self.users.all().values_list('id', flat=True) + group_ids = self.user_groups.all().values_list('id', flat=True) - users_id = list(users_id) - groups_id = list(groups_id) + user_ids = list(user_ids) + group_ids = list(group_ids) - qs1 = User.objects.filter(id__in=users_id).distinct() - qs2 = User.objects.filter(groups__id__in=groups_id).distinct() + qs1 = User.objects.filter(id__in=user_ids).distinct() + qs2 = User.objects.filter(groups__id__in=group_ids).distinct() qs = UnionQuerySet(qs1, qs2) return qs diff --git a/apps/perms/signals_handler/common.py b/apps/perms/signals_handler/common.py index 804d5beaf..7399346db 100644 --- a/apps/perms/signals_handler/common.py +++ b/apps/perms/signals_handler/common.py @@ -27,15 +27,15 @@ def on_user_groups_change(sender, instance, action, reverse, pk_set, **kwargs): if not reverse: # 一个用户添加了多个用户组 - users_id = [instance.id] + user_ids = [instance.id] system_users = SystemUser.objects.filter(groups__id__in=pk_set).distinct() else: # 一个用户组添加了多个用户 - users_id = pk_set + user_ids = pk_set system_users = SystemUser.objects.filter(groups__id=instance.pk).distinct() for system_user in system_users: - system_user.users.add(*users_id) + system_user.users.add(*user_ids) @receiver(m2m_changed, sender=AssetPermission.nodes.through) @@ -139,17 +139,17 @@ def on_application_permission_system_users_changed(sender, instance: Application logger.debug("Application permission system_users change signal received") attrs = instance.applications.all().values_list('attrs', flat=True) - assets_id = [attr['asset'] for attr in attrs if attr.get('asset')] - if not assets_id: + asset_ids = [attr['asset'] for attr in attrs if attr.get('asset')] + if not asset_ids: return for system_user in system_users: - system_user.assets.add(*assets_id) + system_user.assets.add(*asset_ids) if system_user.username_same_with_user: - users_id = instance.users.all().values_list('id', flat=True) - groups_id = instance.user_groups.all().values_list('id', flat=True) - system_user.groups.add(*groups_id) - system_user.users.add(*users_id) + user_ids = instance.users.all().values_list('id', flat=True) + group_ids = instance.user_groups.all().values_list('id', flat=True) + system_user.groups.add(*group_ids) + system_user.users.add(*user_ids) @receiver(m2m_changed, sender=ApplicationPermission.users.through) @@ -164,12 +164,12 @@ def on_application_permission_users_changed(sender, instance, action, reverse, p return logger.debug("Application permission users change signal received") - users_id = User.objects.filter(pk__in=pk_set).values_list('id', flat=True) + user_ids = User.objects.filter(pk__in=pk_set).values_list('id', flat=True) system_users = instance.system_users.all() for system_user in system_users: if system_user.username_same_with_user: - system_user.users.add(*users_id) + system_user.users.add(*user_ids) @receiver(m2m_changed, sender=ApplicationPermission.user_groups.through) @@ -182,12 +182,12 @@ def on_application_permission_user_groups_changed(sender, instance, action, reve return logger.debug("Application permission user groups change signal received") - groups_id = UserGroup.objects.filter(pk__in=pk_set).values_list('id', flat=True) + group_ids = UserGroup.objects.filter(pk__in=pk_set).values_list('id', flat=True) system_users = instance.system_users.all() for system_user in system_users: if system_user.username_same_with_user: - system_user.groups.add(*groups_id) + system_user.groups.add(*group_ids) @receiver(m2m_changed, sender=ApplicationPermission.applications.through) @@ -202,11 +202,11 @@ def on_application_permission_applications_changed(sender, instance, action, rev return attrs = Application.objects.filter(id__in=pk_set).values_list('attrs', flat=True) - assets_id = [attr['asset'] for attr in attrs if attr.get('asset')] - if not assets_id: + asset_ids = [attr['asset'] for attr in attrs if attr.get('asset')] + if not asset_ids: return system_users = instance.system_users.all() for system_user in system_users: - system_user.assets.add(*assets_id) + system_user.assets.add(*asset_ids) diff --git a/apps/perms/utils/application/permission.py b/apps/perms/utils/application/permission.py index 2c92e5bee..4e4c8113d 100644 --- a/apps/perms/utils/application/permission.py +++ b/apps/perms/utils/application/permission.py @@ -6,7 +6,7 @@ from perms.models import ApplicationPermission logger = get_logger(__file__) -def get_application_system_users_id(user, application): +def get_application_system_user_ids(user, application): queryset = ApplicationPermission.objects.valid()\ .filter( Q(users=user) | Q(user_groups__users=user), @@ -16,5 +16,5 @@ def get_application_system_users_id(user, application): def has_application_system_permission(user, application, system_user): - system_users_id = get_application_system_users_id(user, application) - return system_user.id in system_users_id + system_user_ids = get_application_system_user_ids(user, application) + return system_user.id in system_user_ids diff --git a/apps/perms/utils/application/user_permission.py b/apps/perms/utils/application/user_permission.py index 4308b474b..524d5cd42 100644 --- a/apps/perms/utils/application/user_permission.py +++ b/apps/perms/utils/application/user_permission.py @@ -3,7 +3,7 @@ from perms.models import ApplicationPermission from applications.models import Application -def get_user_all_applicationpermissions_id(user): +def get_user_all_applicationpermission_ids(user): application_perm_ids = ApplicationPermission.objects.valid().filter( Q(users=user) | Q(user_groups__users=user) ).distinct().values_list('id', flat=True) @@ -11,8 +11,8 @@ def get_user_all_applicationpermissions_id(user): def get_user_granted_all_applications(user): - application_perms_id = get_user_all_applicationpermissions_id(user) + application_perm_ids = get_user_all_applicationpermission_ids(user) applications = Application.objects.filter( - granted_by_permissions__id__in=application_perms_id + granted_by_permissions__id__in=application_perm_ids ).distinct() return applications diff --git a/apps/perms/utils/asset/permission.py b/apps/perms/utils/asset/permission.py index c9b1bf51e..197abc9d9 100644 --- a/apps/perms/utils/asset/permission.py +++ b/apps/perms/utils/asset/permission.py @@ -5,13 +5,12 @@ from django.db.models import Q from common.utils import get_logger from perms.models import AssetPermission from perms.hands import Asset, User, UserGroup, SystemUser -from perms.models.base import BasePermissionQuerySet from perms.utils.asset.user_permission import get_user_all_asset_perm_ids logger = get_logger(__file__) -def get_asset_system_users_id_with_actions(asset_perm_ids, asset: Asset): +def get_asset_system_user_ids_with_actions(asset_perm_ids, asset: Asset): nodes = asset.get_nodes() node_keys = set() for node in nodes: @@ -34,21 +33,21 @@ def get_asset_system_users_id_with_actions(asset_perm_ids, asset: Asset): return system_users_actions -def get_asset_system_users_id_with_actions_by_user(user: User, asset: Asset): +def get_asset_system_user_ids_with_actions_by_user(user: User, asset: Asset): asset_perm_ids = get_user_all_asset_perm_ids(user) - return get_asset_system_users_id_with_actions(asset_perm_ids, asset) + return get_asset_system_user_ids_with_actions(asset_perm_ids, asset) def has_asset_system_permission(user: User, asset: Asset, system_user: SystemUser): - systemuser_actions_mapper = get_asset_system_users_id_with_actions_by_user(user, asset) + systemuser_actions_mapper = get_asset_system_user_ids_with_actions_by_user(user, asset) actions = systemuser_actions_mapper.get(system_user.id, []) if actions: return True return False -def get_asset_system_users_id_with_actions_by_group(group: UserGroup, asset: Asset): +def get_asset_system_user_ids_with_actions_by_group(group: UserGroup, asset: Asset): asset_perm_ids = AssetPermission.objects.filter( user_groups=group ).valid().values_list('id', flat=True).distinct() - return get_asset_system_users_id_with_actions(asset_perm_ids, asset) + return get_asset_system_user_ids_with_actions(asset_perm_ids, asset) diff --git a/apps/perms/utils/asset/user_permission.py b/apps/perms/utils/asset/user_permission.py index 0741f42e0..e638c7d64 100644 --- a/apps/perms/utils/asset/user_permission.py +++ b/apps/perms/utils/asset/user_permission.py @@ -70,43 +70,43 @@ class UserGrantedTreeRefreshController: return {org_id.decode() for org_id in org_ids} def set_all_orgs_as_builed(self): - self.client.sadd(self.key, *self.orgs_id) + self.client.sadd(self.key, *self.org_ids) def have_need_refresh_orgs(self): builded_org_ids = self.client.smembers(self.key) builded_org_ids = {org_id.decode() for org_id in builded_org_ids} - have = self.orgs_id - builded_org_ids + have = self.org_ids - builded_org_ids return have def get_need_refresh_orgs_and_fill_up(self): - orgs_id = self.orgs_id + org_ids = self.org_ids with self.client.pipeline() as p: p.smembers(self.key) - p.sadd(self.key, *orgs_id) + p.sadd(self.key, *org_ids) ret = p.execute() - builded_orgs_id = {org_id.decode() for org_id in ret[0]} - ids = orgs_id - builded_orgs_id + builded_org_ids = {org_id.decode() for org_id in ret[0]} + ids = org_ids - builded_org_ids orgs = {*Organization.objects.filter(id__in=ids)} - logger.info(f'Need rebuild orgs are {orgs}, builed orgs are {ret[0]}, all orgs are {orgs_id}') + logger.info(f'Need rebuild orgs are {orgs}, builed orgs are {ret[0]}, all orgs are {org_ids}') return orgs @classmethod @on_transaction_commit - def remove_builed_orgs_from_users(cls, orgs_id, users_id): + def remove_builed_orgs_from_users(cls, org_ids, user_ids): client = cls.get_redis_client() - org_ids = [str(org_id) for org_id in orgs_id] + org_ids = [str(org_id) for org_id in org_ids] with client.pipeline() as p: - for user_id in users_id: + for user_id in user_ids: key = cls.key_template.format(user_id=user_id) p.srem(key, *org_ids) p.execute() - logger.info(f'Remove orgs from users builded tree: users:{users_id} orgs:{orgs_id}') + logger.info(f'Remove orgs from users builded tree: users:{user_ids} orgs:{org_ids}') @classmethod - def add_need_refresh_orgs_for_users(cls, orgs_id, users_id): - cls.remove_builed_orgs_from_users(orgs_id, users_id) + def add_need_refresh_orgs_for_users(cls, org_ids, user_ids): + cls.remove_builed_orgs_from_users(org_ids, user_ids) @classmethod @ensure_in_real_or_default_org @@ -127,15 +127,15 @@ class UserGrantedTreeRefreshController: ancestor_id = PermNode.objects.filter(key__in=ancestor_node_keys).values_list('id', flat=True) node_ids.update(ancestor_id) - assets_related_perms_id = AssetPermission.nodes.through.objects.filter( + assets_related_perm_ids = AssetPermission.nodes.through.objects.filter( node_id__in=node_ids ).values_list('assetpermission_id', flat=True) - asset_perm_ids.update(assets_related_perms_id) + asset_perm_ids.update(assets_related_perm_ids) - nodes_related_perms_id = AssetPermission.assets.through.objects.filter( + nodes_related_perm_ids = AssetPermission.assets.through.objects.filter( asset_id__in=asset_ids ).values_list('assetpermission_id', flat=True) - asset_perm_ids.update(nodes_related_perms_id) + asset_perm_ids.update(nodes_related_perm_ids) cls.add_need_refresh_by_asset_perm_ids(asset_perm_ids) @@ -173,7 +173,7 @@ class UserGrantedTreeRefreshController: ) @lazyproperty - def orgs_id(self): + def org_ids(self): ret = {str(org.id) for org in self.orgs} return ret @@ -187,7 +187,7 @@ class UserGrantedTreeRefreshController: user = self.user with tmp_to_root_org(): - UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=self.orgs_id).delete() + UserAssetGrantedTreeNodeRelation.objects.filter(user=user).exclude(org_id__in=self.org_ids).delete() if force or self.have_need_refresh_orgs(): with UserGrantedTreeRebuildLock(user_id=user.id): @@ -295,10 +295,10 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): # 查询授权资产关联的节点设置 def process_direct_granted_assets(): # 查询直接授权资产 - nodes_id = {node_id_str for node_id_str, _ in self.direct_granted_asset_id_node_id_str_pairs} + node_ids = {node_id_str for node_id_str, _ in self.direct_granted_asset_id_node_id_str_pairs} # 查询授权资产关联的节点设置 2.80 granted_asset_nodes = PermNode.objects.filter( - id__in=nodes_id + id__in=node_ids ).distinct().only(*node_only_fields) granted_asset_nodes = list(granted_asset_nodes) @@ -350,11 +350,11 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): UserAssetGrantedTreeNodeRelation.objects.bulk_create(to_create) @timeit - def _fill_direct_granted_node_assets_id_from_mem(self, nodes_key, mapper): + def _fill_direct_granted_node_asset_ids_from_mem(self, nodes_key, mapper): org_id = current_org.id for key in nodes_key: - assets_id = PermNode.get_all_assets_id_by_node_key(org_id, key) - mapper[key].update(assets_id) + asset_ids = PermNode.get_all_asset_ids_by_node_key(org_id, key) + mapper[key].update(asset_ids) @lazyproperty def direct_granted_asset_id_node_id_str_pairs(self): @@ -379,7 +379,7 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): node = nodes[0] if node.node_from == NodeFrom.granted and node.key.isdigit(): with tmp_to_org(node.org): - node.granted_assets_amount = len(node.get_all_assets_id()) + node.granted_assets_amount = len(node.get_all_asset_ids()) return direct_granted_nodes_key = [] @@ -392,7 +392,7 @@ class UserGrantedTreeBuildUtils(UserGrantedUtilsBase): # 授权的节点和直接资产的映射 nodekey_assetsid_mapper = defaultdict(set) # 直接授权的节点，资产从完整树过来 - self._fill_direct_granted_node_assets_id_from_mem( + self._fill_direct_granted_node_asset_ids_from_mem( direct_granted_nodes_key, nodekey_assetsid_mapper ) diff --git a/apps/terminal/api/terminal.py b/apps/terminal/api/terminal.py index 444061035..91e9d4d07 100644 --- a/apps/terminal/api/terminal.py +++ b/apps/terminal/api/terminal.py @@ -86,13 +86,13 @@ class StatusViewSet(viewsets.ModelViewSet): return Response(serializer.data, status=201) def handle_sessions(self): - sessions_id = self.request.data.get('sessions', []) + session_ids = self.request.data.get('sessions', []) # guacamole 上报的 session 是字符串 # "[53cd3e47-210f-41d8-b3c6-a184f3, 53cd3e47-210f-41d8-b3c6-a184f4]" - if isinstance(sessions_id, str): - sessions_id = sessions_id[1:-1].split(',') - sessions_id = [sid.strip() for sid in sessions_id if sid.strip()] - Session.set_sessions_active(sessions_id) + if isinstance(session_ids, str): + session_ids = session_ids[1:-1].split(',') + session_ids = [sid.strip() for sid in session_ids if sid.strip()] + Session.set_sessions_active(session_ids) def get_queryset(self): terminal_id = self.kwargs.get("terminal", None) diff --git a/apps/terminal/models/session.py b/apps/terminal/models/session.py index b440d2f1e..8c759a74d 100644 --- a/apps/terminal/models/session.py +++ b/apps/terminal/models/session.py @@ -137,8 +137,8 @@ class Session(OrgModelMixin): return name, None @classmethod - def set_sessions_active(cls, sessions_id): - data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in sessions_id} + def set_sessions_active(cls, session_ids): + data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in session_ids} cache.set_many(data, timeout=5*60) @classmethod diff --git a/apps/tickets/handler/apply_application.py b/apps/tickets/handler/apply_application.py index 83f62b417..311fca3e3 100644 --- a/apps/tickets/handler/apply_application.py +++ b/apps/tickets/handler/apply_application.py @@ -26,11 +26,11 @@ class Handler(BaseHandler): def _construct_meta_display_of_approve(self): meta_display_fields = ['approve_applications_display', 'approve_system_users_display'] - approve_applications_id = self.ticket.meta.get('approve_applications', []) - approve_system_users_id = self.ticket.meta.get('approve_system_users', []) + approve_application_ids = self.ticket.meta.get('approve_applications', []) + approve_system_user_ids = self.ticket.meta.get('approve_system_users', []) with tmp_to_org(self.ticket.org_id): - approve_applications = Application.objects.filter(id__in=approve_applications_id) - system_users = SystemUser.objects.filter(id__in=approve_system_users_id) + approve_applications = Application.objects.filter(id__in=approve_application_ids) + system_users = SystemUser.objects.filter(id__in=approve_system_user_ids) approve_applications_display = [str(application) for application in approve_applications] approve_system_users_display = [str(system_user) for system_user in system_users] meta_display_values = [approve_applications_display, approve_system_users_display] @@ -89,8 +89,8 @@ class Handler(BaseHandler): apply_category = self.ticket.meta.get('apply_category') apply_type = self.ticket.meta.get('apply_type') approve_permission_name = self.ticket.meta.get('approve_permission_name', '') - approved_applications_id = self.ticket.meta.get('approve_applications', []) - approve_system_users_id = self.ticket.meta.get('approve_system_users', []) + approved_application_ids = self.ticket.meta.get('approve_applications', []) + approve_system_user_ids = self.ticket.meta.get('approve_system_users', []) approve_date_start = self.ticket.meta.get('approve_date_start') approve_date_expired = self.ticket.meta.get('approve_date_expired') permission_created_by = '{}:{}'.format( @@ -121,7 +121,7 @@ class Handler(BaseHandler): with tmp_to_org(self.ticket.org_id): application_permission = ApplicationPermission.objects.create(**permissions_data) application_permission.users.add(self.ticket.applicant) - application_permission.applications.set(approved_applications_id) - application_permission.system_users.set(approve_system_users_id) + application_permission.applications.set(approved_application_ids) + application_permission.system_users.set(approve_system_user_ids) return application_permission diff --git a/apps/tickets/handler/apply_asset.py b/apps/tickets/handler/apply_asset.py index 71507e7ae..9af310294 100644 --- a/apps/tickets/handler/apply_asset.py +++ b/apps/tickets/handler/apply_asset.py @@ -27,11 +27,11 @@ class Handler(BaseHandler): ] approve_actions = self.ticket.meta.get('approve_actions', Action.NONE) approve_actions_display = Action.value_to_choices_display(approve_actions) - approve_assets_id = self.ticket.meta.get('approve_assets', []) - approve_system_users_id = self.ticket.meta.get('approve_system_users', []) + approve_asset_ids = self.ticket.meta.get('approve_assets', []) + approve_system_user_ids = self.ticket.meta.get('approve_system_users', []) with tmp_to_org(self.ticket.org_id): - assets = Asset.objects.filter(id__in=approve_assets_id) - system_users = SystemUser.objects.filter(id__in=approve_system_users_id) + assets = Asset.objects.filter(id__in=approve_asset_ids) + system_users = SystemUser.objects.filter(id__in=approve_system_user_ids) approve_assets_display = [str(asset) for asset in assets] approve_system_users_display = [str(system_user) for system_user in system_users] meta_display_values = [ @@ -91,8 +91,8 @@ class Handler(BaseHandler): return asset_permission approve_permission_name = self.ticket.meta.get('approve_permission_name', ) - approve_assets_id = self.ticket.meta.get('approve_assets', []) - approve_system_users_id = self.ticket.meta.get('approve_system_users', []) + approve_asset_ids = self.ticket.meta.get('approve_assets', []) + approve_system_user_ids = self.ticket.meta.get('approve_system_users', []) approve_actions = self.ticket.meta.get('approve_actions', Action.NONE) approve_date_start = self.ticket.meta.get('approve_date_start') approve_date_expired = self.ticket.meta.get('approve_date_expired') @@ -124,7 +124,7 @@ class Handler(BaseHandler): with tmp_to_org(self.ticket.org_id): asset_permission = AssetPermission.objects.create(**permission_data) asset_permission.users.add(self.ticket.applicant) - asset_permission.assets.set(approve_assets_id) - asset_permission.system_users.set(approve_system_users_id) + asset_permission.assets.set(approve_asset_ids) + asset_permission.system_users.set(approve_system_user_ids) return asset_permission diff --git a/apps/tickets/serializers/ticket/meta/ticket_type/apply_application.py b/apps/tickets/serializers/ticket/meta/ticket_type/apply_application.py index 4b5bb00e2..3a047d980 100644 --- a/apps/tickets/serializers/ticket/meta/ticket_type/apply_application.py +++ b/apps/tickets/serializers/ticket/meta/ticket_type/apply_application.py @@ -98,10 +98,10 @@ class ApproveSerializer(serializers.Serializer): apply_type = self.root.instance.meta.get('apply_type') queries = Q(type=apply_type) queries &= Q(id__in=approve_applications) - applications_id = Application.objects.filter(queries).values_list('id', flat=True) - applications_id = [str(application_id) for application_id in applications_id] - if applications_id: - return applications_id + application_ids = Application.objects.filter(queries).values_list('id', flat=True) + application_ids = [str(application_id) for application_id in application_ids] + if application_ids: + return application_ids raise serializers.ValidationError(_( 'No `Application` are found under Organization `{}`'.format(self.root.instance.org_name) @@ -116,10 +116,10 @@ class ApproveSerializer(serializers.Serializer): protocol = SystemUser.get_protocol_by_application_type(apply_type) queries = Q(protocol=protocol) queries &= Q(id__in=approve_system_users) - system_users_id = SystemUser.objects.filter(queries).values_list('id', flat=True) - system_users_id = [str(system_user_id) for system_user_id in system_users_id] - if system_users_id: - return system_users_id + system_user_ids = SystemUser.objects.filter(queries).values_list('id', flat=True) + system_user_ids = [str(system_user_id) for system_user_id in system_user_ids] + if system_user_ids: + return system_user_ids raise serializers.ValidationError(_( 'No `SystemUser` are found under Organization `{}`'.format(self.root.instance.org_name) @@ -146,9 +146,9 @@ class ApplyApplicationSerializer(ApplySerializer, ApproveSerializer): queries &= Q(type=apply_type) with tmp_to_org(self.root.instance.org_id): - applications_id = Application.objects.filter(queries).values_list('id', flat=True)[:5] - applications_id = [str(application_id) for application_id in applications_id] - return applications_id + application_ids = Application.objects.filter(queries).values_list('id', flat=True)[:5] + application_ids = [str(application_id) for application_id in application_ids] + return application_ids def get_recommend_system_users(self, value): if not isinstance(self.root.instance, Ticket): @@ -167,6 +167,6 @@ class ApplyApplicationSerializer(ApplySerializer, ApproveSerializer): queries &= Q(protocol=protocol) with tmp_to_org(self.root.instance.org_id): - system_users_id = SystemUser.objects.filter(queries).values_list('id', flat=True)[:5] - system_users_id = [str(system_user_id) for system_user_id in system_users_id] - return system_users_id + system_user_ids = SystemUser.objects.filter(queries).values_list('id', flat=True)[:5] + system_user_ids = [str(system_user_id) for system_user_id in system_user_ids] + return system_user_ids diff --git a/apps/tickets/serializers/ticket/meta/ticket_type/apply_asset.py b/apps/tickets/serializers/ticket/meta/ticket_type/apply_asset.py index acdd217e1..e6bc65c29 100644 --- a/apps/tickets/serializers/ticket/meta/ticket_type/apply_asset.py +++ b/apps/tickets/serializers/ticket/meta/ticket_type/apply_asset.py @@ -99,10 +99,10 @@ class ApproveSerializer(serializers.Serializer): return [] with tmp_to_org(self.root.instance.org_id): - assets_id = Asset.objects.filter(id__in=approve_assets).values_list('id', flat=True) - assets_id = [str(asset_id) for asset_id in assets_id] - if assets_id: - return assets_id + asset_ids = Asset.objects.filter(id__in=approve_assets).values_list('id', flat=True) + asset_ids = [str(asset_id) for asset_id in asset_ids] + if asset_ids: + return asset_ids raise serializers.ValidationError(_( 'No `Asset` are found under Organization `{}`'.format(self.root.instance.org_name) @@ -115,10 +115,10 @@ class ApproveSerializer(serializers.Serializer): with tmp_to_org(self.root.instance.org_id): queries = Q(protocol__in=SystemUser.ASSET_CATEGORY_PROTOCOLS) queries &= Q(id__in=approve_system_users) - system_users_id = SystemUser.objects.filter(queries).values_list('id', flat=True) - system_users_id = [str(system_user_id) for system_user_id in system_users_id] - if system_users_id: - return system_users_id + system_user_ids = SystemUser.objects.filter(queries).values_list('id', flat=True) + system_user_ids = [str(system_user_id) for system_user_id in system_user_ids] + if system_user_ids: + return system_user_ids raise serializers.ValidationError(_( 'No `SystemUser` are found under Organization `{}`'.format(self.root.instance.org_name) @@ -144,9 +144,9 @@ class ApplyAssetSerializer(ApplySerializer, ApproveSerializer): if not queries: return [] with tmp_to_org(self.root.instance.org_id): - assets_id = Asset.objects.filter(queries).values_list('id', flat=True)[:5] - assets_id = [str(asset_id) for asset_id in assets_id] - return assets_id + asset_ids = Asset.objects.filter(queries).values_list('id', flat=True)[:5] + asset_ids = [str(asset_id) for asset_id in asset_ids] + return asset_ids def get_recommend_system_users(self, value): if not isinstance(self.root.instance, Ticket): @@ -163,6 +163,6 @@ class ApplyAssetSerializer(ApplySerializer, ApproveSerializer): queries &= Q(protocol__in=SystemUser.ASSET_CATEGORY_PROTOCOLS) with tmp_to_org(self.root.instance.org_id): - system_users_id = SystemUser.objects.filter(queries).values_list('id', flat=True)[:5] - system_users_id = [str(system_user_id) for system_user_id in system_users_id] - return system_users_id + system_user_ids = SystemUser.objects.filter(queries).values_list('id', flat=True)[:5] + system_user_ids = [str(system_user_id) for system_user_id in system_user_ids] + return system_user_ids diff --git a/apps/users/api/user.py b/apps/users/api/user.py index 488c699c3..7f7d91deb 100644 --- a/apps/users/api/user.py +++ b/apps/users/api/user.py @@ -123,10 +123,10 @@ class UserViewSet(CommonApiMixin, UserQuerysetMixin, BulkModelViewSet): def perform_bulk_update(self, serializer): # TODO: 需要测试 - users_ids = [ + user_ids = [ d.get("id") or d.get("pk") for d in serializer.validated_data ] - users = current_org.get_members().filter(id__in=users_ids) + users = current_org.get_members().filter(id__in=user_ids) for user in users: self.check_object_permissions(self.request, user) return super().perform_bulk_update(serializer) diff --git a/apps/users/models/user.py b/apps/users/models/user.py index bbacd998b..c7f52a02c 100644 --- a/apps/users/models/user.py +++ b/apps/users/models/user.py @@ -346,11 +346,11 @@ class RoleMixin: @classmethod def get_super_and_org_admins(cls, org=None): super_admins = cls.get_super_admins() - super_admins_id = list(super_admins.values_list('id', flat=True)) + super_admin_ids = list(super_admins.values_list('id', flat=True)) org_admins = cls.get_org_admins(org) - org_admins_id = list(org_admins.values_list('id', flat=True)) - admins_id = set(org_admins_id + super_admins_id) - admins = User.objects.filter(id__in=admins_id) + org_admin_ids = list(org_admins.values_list('id', flat=True)) + admin_ids = set(org_admin_ids + super_admin_ids) + admins = User.objects.filter(id__in=admin_ids) return admins diff --git a/utils/generate_fake_data/resources/assets.py b/utils/generate_fake_data/resources/assets.py index 4279e6c5a..c6817e440 100644 --- a/utils/generate_fake_data/resources/assets.py +++ b/utils/generate_fake_data/resources/assets.py @@ -57,16 +57,16 @@ class NodesGenerator(FakeDataGenerator): class AssetsGenerator(FakeDataGenerator): resource = 'asset' - admin_users_id: list - nodes_id: list + admin_user_ids: list + node_ids: list def pre_generate(self): - self.admin_users_id = list(AdminUser.objects.all().values_list('id', flat=True)) - self.nodes_id = list(Node.objects.all().values_list('id', flat=True)) + self.admin_user_ids = list(AdminUser.objects.all().values_list('id', flat=True)) + self.node_ids = list(Node.objects.all().values_list('id', flat=True)) def set_assets_nodes(self, assets): for asset in assets: - nodes_id_add_to = random.sample(self.nodes_id, 3) + nodes_id_add_to = random.sample(self.node_ids, 3) asset.nodes.add(*nodes_id_add_to) def do_generate(self, batch, batch_size): @@ -79,7 +79,7 @@ class AssetsGenerator(FakeDataGenerator): data = dict( ip=ip, hostname=hostname, - admin_user_id=choice(self.admin_users_id), + admin_user_id=choice(self.admin_user_ids), created_by='Fake', org_id=self.org.id ) diff --git a/utils/generate_fake_data/resources/perms.py b/utils/generate_fake_data/resources/perms.py index e76e3618f..953712a5d 100644 --- a/utils/generate_fake_data/resources/perms.py +++ b/utils/generate_fake_data/resources/perms.py @@ -10,46 +10,46 @@ from perms.models import * class AssetPermissionGenerator(FakeDataGenerator): resource = 'asset_permission' - users_id: list - user_groups_id: list - assets_id: list - nodes_id: list - system_users_id: list + user_ids: list + user_group_ids: list + asset_ids: list + node_ids: list + system_user_ids: list def pre_generate(self): - self.nodes_id = list(Node.objects.all().values_list('id', flat=True)) - self.assets_id = list(Asset.objects.all().values_list('id', flat=True)) - self.system_users_id = list(SystemUser.objects.all().values_list('id', flat=True)) - self.users_id = list(User.objects.all().values_list('id', flat=True)) - self.user_groups_id = list(UserGroup.objects.all().values_list('id', flat=True)) + self.node_ids = list(Node.objects.all().values_list('id', flat=True)) + self.asset_ids = list(Asset.objects.all().values_list('id', flat=True)) + self.system_user_ids = list(SystemUser.objects.all().values_list('id', flat=True)) + self.user_ids = list(User.objects.all().values_list('id', flat=True)) + self.user_group_ids = list(UserGroup.objects.all().values_list('id', flat=True)) def set_users(self, perms): through = AssetPermission.users.through - choices = self.users_id + choices = self.user_ids relation_name = 'user_id' self.set_relations(perms, through, relation_name, choices) def set_user_groups(self, perms): through = AssetPermission.user_groups.through - choices = self.user_groups_id + choices = self.user_group_ids relation_name = 'usergroup_id' self.set_relations(perms, through, relation_name, choices) def set_assets(self, perms): through = AssetPermission.assets.through - choices = self.assets_id + choices = self.asset_ids relation_name = 'asset_id' self.set_relations(perms, through, relation_name, choices) def set_nodes(self, perms): through = AssetPermission.nodes.through - choices = self.nodes_id + choices = self.node_ids relation_name = 'node_id' self.set_relations(perms, through, relation_name, choices) def set_system_users(self, perms): through = AssetPermission.system_users.through - choices = self.system_users_id + choices = self.system_user_ids relation_name = 'systemuser_id' self.set_relations(perms, through, relation_name, choices) @@ -59,8 +59,8 @@ class AssetPermissionGenerator(FakeDataGenerator): for perm in perms: if choice_count is None: choice_count = choice(range(8)) - resources_id = sample(choices, choice_count) - for rid in resources_id: + resource_ids = sample(choices, choice_count) + for rid in resource_ids: data = {'assetpermission_id': perm.id} data[relation_name] = rid relations.append(through(**data)) diff --git a/utils/generate_fake_data/resources/users.py b/utils/generate_fake_data/resources/users.py index 0d8cbab9a..887c7240c 100644 --- a/utils/generate_fake_data/resources/users.py +++ b/utils/generate_fake_data/resources/users.py @@ -21,11 +21,11 @@ class UserGroupGenerator(FakeDataGenerator): class UserGenerator(FakeDataGenerator): resource = 'user' roles: list - groups_id: list + group_ids: list def pre_generate(self): self.roles = list(dict(User.ROLE.choices).keys()) - self.groups_id = list(UserGroup.objects.all().values_list('id', flat=True)) + self.group_ids = list(UserGroup.objects.all().values_list('id', flat=True)) def set_org(self, users): relations = [] @@ -39,7 +39,7 @@ class UserGenerator(FakeDataGenerator): def set_groups(self, users): relations = [] for i in users: - groups_to_join = sample(self.groups_id, 3) + groups_to_join = sample(self.group_ids, 3) _relations = [User.groups.through(user_id=i.id, usergroup_id=gid) for gid in groups_to_join] relations.extend(_relations) User.groups.through.objects.bulk_create(relations, ignore_conflicts=True) From 19e2a5b9f92ea4e89591fdbf01208d84632006f1 Mon Sep 17 00:00:00 2001 From: wojiushixiaobai <296015668@qq.com> Date: Sat, 6 Mar 2021 17:09:36 +0800 Subject: [PATCH 54/71] =?UTF-8?q?fix(terminal):=20=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E5=BD=95=E5=83=8F=E6=8E=A5=E5=8F=A3=E8=BF=94=E5=9B=9E=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/terminal/api/session.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/terminal/api/session.py b/apps/terminal/api/session.py index 805b9b4a1..2a749aac5 100644 --- a/apps/terminal/api/session.py +++ b/apps/terminal/api/session.py @@ -76,8 +76,7 @@ class SessionViewSet(OrgBulkModelViewSet): session = self.get_object() local_path, url = utils.get_session_replay_url(session) if local_path is None: - error = url - return HttpResponse(error) + return Response({"error": url}, status=404) file = self.prepare_offline_file(session, local_path) response = FileResponse(file) @@ -167,7 +166,7 @@ class SessionReplayViewSet(AsyncApiMixin, viewsets.ViewSet): if not local_path: local_path, url = download_session_replay(session) if not local_path: - return Response({"error": url}) + return Response({"error": url}, status=404) data = self.get_replay_data(session, url) return Response(data) From 15b0ad9c12c1763d91410bde36cfefd801745714 Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 8 Mar 2021 17:19:59 +0800 Subject: [PATCH 55/71] =?UTF-8?q?refactor:=20=E6=B8=85=E7=90=86=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=20`orgs.mixins.api.OrgMembershipModelViewSetMixin`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/orgs/mixins/api.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/apps/orgs/mixins/api.py b/apps/orgs/mixins/api.py index f40f9f7fa..bfcd4b7af 100644 --- a/apps/orgs/mixins/api.py +++ b/apps/orgs/mixins/api.py @@ -1,17 +1,15 @@ # -*- coding: utf-8 -*- # -from django.shortcuts import get_object_or_404 from rest_framework.viewsets import ModelViewSet, GenericViewSet from rest_framework_bulk import BulkModelViewSet from common.mixins import CommonApiMixin, RelationMixin from orgs.utils import current_org from ..utils import set_to_root_org -from ..models import Organization __all__ = [ - 'RootOrgViewMixin', 'OrgMembershipModelViewSetMixin', 'OrgModelViewSet', - 'OrgBulkModelViewSet', 'OrgQuerySetMixin', 'OrgGenericViewSet', 'OrgRelationMixin' + 'RootOrgViewMixin', 'OrgModelViewSet', 'OrgBulkModelViewSet', 'OrgQuerySetMixin', + 'OrgGenericViewSet', 'OrgRelationMixin' ] @@ -62,27 +60,6 @@ class OrgBulkModelViewSet(CommonApiMixin, OrgQuerySetMixin, BulkModelViewSet): return False -class OrgMembershipModelViewSetMixin: - org = None - membership_class = None - lookup_field = 'user' - lookup_url_kwarg = 'user_id' - http_method_names = ['get', 'post', 'delete', 'head', 'options'] - - def dispatch(self, request, *args, **kwargs): - self.org = get_object_or_404(Organization, pk=kwargs.get('org_id')) - return super().dispatch(request, *args, **kwargs) - - def get_serializer_context(self): - context = super().get_serializer_context() - context['org'] = self.org - return context - - def get_queryset(self): - queryset = self.membership_class.objects.filter(organization=self.org) - return queryset - - class OrgRelationMixin(RelationMixin): def get_queryset(self): queryset = super().get_queryset() From 886393c539233164d00184a3463a12610b884327 Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 8 Mar 2021 16:55:21 +0800 Subject: [PATCH 56/71] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E5=AF=BC=E5=85=A5=E6=97=B6=E5=B0=86=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F(csv,=20excel)=E8=A7=A3=E6=9E=90=E4=B8=BAJson?= =?UTF-8?q?=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/drf/api.py | 30 +++++++++++++----------------- apps/common/drf/parsers/base.py | 6 ++++-- apps/common/mixins/api.py | 13 +++++++++++-- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/apps/common/drf/api.py b/apps/common/drf/api.py index febd4467e..6a3c3c3e2 100644 --- a/apps/common/drf/api.py +++ b/apps/common/drf/api.py @@ -3,39 +3,35 @@ from rest_framework_bulk import BulkModelViewSet from ..mixins.api import ( SerializerMixin2, QuerySetMixin, ExtraFilterFieldsMixin, PaginatedResponseMixin, - RelationMixin, AllowBulkDestoryMixin + RelationMixin, AllowBulkDestoryMixin, ParseToJsonMixin, ) -class JmsGenericViewSet(SerializerMixin2, - QuerySetMixin, - ExtraFilterFieldsMixin, - PaginatedResponseMixin, +class CommonMixin(SerializerMixin2, + QuerySetMixin, + ExtraFilterFieldsMixin, + PaginatedResponseMixin, + ParseToJsonMixin): + pass + + +class JmsGenericViewSet(CommonMixin, GenericViewSet): pass -class JMSModelViewSet(SerializerMixin2, - QuerySetMixin, - ExtraFilterFieldsMixin, - PaginatedResponseMixin, +class JMSModelViewSet(CommonMixin, ModelViewSet): pass -class JMSBulkModelViewSet(SerializerMixin2, - QuerySetMixin, - ExtraFilterFieldsMixin, - PaginatedResponseMixin, +class JMSBulkModelViewSet(CommonMixin, AllowBulkDestoryMixin, BulkModelViewSet): pass -class JMSBulkRelationModelViewSet(SerializerMixin2, - QuerySetMixin, - ExtraFilterFieldsMixin, - PaginatedResponseMixin, +class JMSBulkRelationModelViewSet(CommonMixin, RelationMixin, AllowBulkDestoryMixin, BulkModelViewSet): diff --git a/apps/common/drf/parsers/base.py b/apps/common/drf/parsers/base.py index c8619b34e..4bfde5ed3 100644 --- a/apps/common/drf/parsers/base.py +++ b/apps/common/drf/parsers/base.py @@ -22,6 +22,7 @@ class BaseFileParser(BaseParser): FILE_CONTENT_MAX_LENGTH = 1024 * 1024 * 10 serializer_cls = None + serializer_fields = None def check_content_length(self, meta): content_length = int(meta.get('CONTENT_LENGTH', meta.get('HTTP_CONTENT_LENGTH', 0))) @@ -45,7 +46,7 @@ class BaseFileParser(BaseParser): def convert_to_field_names(self, column_titles): fields_map = {} - fields = self.serializer_cls().fields + fields = self.serializer_fields fields_map.update({v.label: k for k, v in fields.items()}) fields_map.update({k: k for k, _ in fields.items()}) field_names = [ @@ -89,7 +90,7 @@ class BaseFileParser(BaseParser): 构建json数据后的行数据处理 """ new_row_data = {} - serializer_fields = self.serializer_cls().fields + serializer_fields = self.serializer_fields for k, v in row_data.items(): if isinstance(v, list) or isinstance(v, dict) or isinstance(v, str) and k.strip() and v.strip(): # 解决类似disk_info为字符串的'{}'的问题 @@ -117,6 +118,7 @@ class BaseFileParser(BaseParser): view = parser_context['view'] meta = view.request.META self.serializer_cls = view.get_serializer_class() + self.serializer_fields = self.serializer_cls().fields except Exception as e: logger.debug(e, exc_info=True) raise ParseError('The resource does not support imports!') diff --git a/apps/common/mixins/api.py b/apps/common/mixins/api.py index 4aa10deec..8fbaf39c5 100644 --- a/apps/common/mixins/api.py +++ b/apps/common/mixins/api.py @@ -11,13 +11,16 @@ from django.core.cache import cache from django.http import JsonResponse from rest_framework.response import Response from rest_framework.settings import api_settings +from rest_framework.decorators import action +from rest_framework.request import Request +from common.const.http import POST from common.drf.filters import IDSpmFilter, CustomFilter, IDInFilter from ..utils import lazyproperty __all__ = [ 'JSONResponseMixin', 'CommonApiMixin', 'AsyncApiMixin', 'RelationMixin', - 'SerializerMixin2', 'QuerySetMixin', 'ExtraFilterFieldsMixin' + 'SerializerMixin2', 'QuerySetMixin', 'ExtraFilterFieldsMixin', 'ParseToJsonMixin', ] @@ -32,6 +35,12 @@ class JSONResponseMixin(object): # ---------------------- +class ParseToJsonMixin: + @action(methods=[POST], detail=False, url_path='render-to-json') + def render_to_json(self, request: Request): + return Response(data=request.data) + + class SerializerMixin: """ 根据用户请求动作的不同，获取不同的 `serializer_class `""" @@ -98,7 +107,7 @@ class PaginatedResponseMixin: return Response(serializer.data) -class CommonApiMixin(SerializerMixin, ExtraFilterFieldsMixin): +class CommonApiMixin(SerializerMixin, ExtraFilterFieldsMixin, ParseToJsonMixin): pass From ccb0509d85f53af81133b0af60da39f1e5a27cba Mon Sep 17 00:00:00 2001 From: xinwen Date: Mon, 8 Mar 2021 16:55:21 +0800 Subject: [PATCH 57/71] =?UTF-8?q?feat:=20=E6=89=B9=E9=87=8F=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E8=A7=A3=E6=9E=90=E4=B8=BAJson=E7=9A=84=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=B7=BB=E5=8A=A0=20`title`=20=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/drf/api.py | 4 ++-- apps/common/drf/parsers/base.py | 15 ++++++++++++--- apps/common/mixins/api.py | 17 +++++++++++++---- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/apps/common/drf/api.py b/apps/common/drf/api.py index 6a3c3c3e2..22b1321d5 100644 --- a/apps/common/drf/api.py +++ b/apps/common/drf/api.py @@ -3,7 +3,7 @@ from rest_framework_bulk import BulkModelViewSet from ..mixins.api import ( SerializerMixin2, QuerySetMixin, ExtraFilterFieldsMixin, PaginatedResponseMixin, - RelationMixin, AllowBulkDestoryMixin, ParseToJsonMixin, + RelationMixin, AllowBulkDestoryMixin, RenderToJsonMixin, ) @@ -11,7 +11,7 @@ class CommonMixin(SerializerMixin2, QuerySetMixin, ExtraFilterFieldsMixin, PaginatedResponseMixin, - ParseToJsonMixin): + RenderToJsonMixin): pass diff --git a/apps/common/drf/parsers/base.py b/apps/common/drf/parsers/base.py index 4bfde5ed3..9b9379b3e 100644 --- a/apps/common/drf/parsers/base.py +++ b/apps/common/drf/parsers/base.py @@ -112,11 +112,13 @@ class BaseFileParser(BaseParser): return data def parse(self, stream, media_type=None, parser_context=None): - parser_context = parser_context or {} + assert parser_context is not None, '`parser_context` should not be `None`' + + view = parser_context['view'] + request = view.request try: - view = parser_context['view'] - meta = view.request.META + meta = request.META self.serializer_cls = view.get_serializer_class() self.serializer_fields = self.serializer_cls().fields except Exception as e: @@ -130,6 +132,13 @@ class BaseFileParser(BaseParser): rows = self.generate_rows(stream_data) column_titles = self.get_column_titles(rows) field_names = self.convert_to_field_names(column_titles) + + # 给 `common.mixins.api.RenderToJsonMixin` 提供，暂时只能耦合 + column_title_field_pairs = list(zip(column_titles, field_names)) + if not hasattr(request, 'jms_context'): + request.jms_context = {} + request.jms_context['column_title_field_pairs'] = column_title_field_pairs + data = self.generate_data(field_names, rows) return data except Exception as e: diff --git a/apps/common/mixins/api.py b/apps/common/mixins/api.py index 8fbaf39c5..7078b70b7 100644 --- a/apps/common/mixins/api.py +++ b/apps/common/mixins/api.py @@ -20,7 +20,7 @@ from ..utils import lazyproperty __all__ = [ 'JSONResponseMixin', 'CommonApiMixin', 'AsyncApiMixin', 'RelationMixin', - 'SerializerMixin2', 'QuerySetMixin', 'ExtraFilterFieldsMixin', 'ParseToJsonMixin', + 'SerializerMixin2', 'QuerySetMixin', 'ExtraFilterFieldsMixin', 'RenderToJsonMixin', ] @@ -35,10 +35,19 @@ class JSONResponseMixin(object): # ---------------------- -class ParseToJsonMixin: +class RenderToJsonMixin: @action(methods=[POST], detail=False, url_path='render-to-json') def render_to_json(self, request: Request): - return Response(data=request.data) + data = { + 'title': (), + 'data': request.data, + } + + jms_context = getattr(request, 'jms_context', {}) + column_title_field_pairs = jms_context.get('column_title_field_pairs', ()) + data['title'] = column_title_field_pairs + + return Response(data=data) class SerializerMixin: @@ -107,7 +116,7 @@ class PaginatedResponseMixin: return Response(serializer.data) -class CommonApiMixin(SerializerMixin, ExtraFilterFieldsMixin, ParseToJsonMixin): +class CommonApiMixin(SerializerMixin, ExtraFilterFieldsMixin, RenderToJsonMixin): pass From c4eacbabc6fc31dbfa9d2d8cccd35228537fc671 Mon Sep 17 00:00:00 2001 From: xinwen Date: Tue, 9 Mar 2021 13:57:58 +0800 Subject: [PATCH 58/71] =?UTF-8?q?refactor:=20=E9=87=8D=E6=9E=84=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E6=A1=86=E6=9E=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/common/cache.py | 164 ++++++++++++++++++++++++------------------- apps/orgs/cache.py | 39 ---------- apps/orgs/caches.py | 44 +++++++++++- 3 files changed, 135 insertions(+), 112 deletions(-) delete mode 100644 apps/orgs/cache.py diff --git a/apps/common/cache.py b/apps/common/cache.py index 0bed4fa30..b16d1e7dd 100644 --- a/apps/common/cache.py +++ b/apps/common/cache.py @@ -1,13 +1,24 @@ -import json -from django.core.cache import cache +import time + +from redis import Redis from common.utils.lock import DistributedLock from common.utils import lazyproperty from common.utils import get_logger +from jumpserver.const import CONFIG logger = get_logger(__file__) +class ComputeLock(DistributedLock): + """ + 需要重建缓存的时候加上该锁，避免重复计算 + """ + def __init__(self, key): + name = f'compute:{key}' + super().__init__(name=name) + + class CacheFieldBase: field_type = str @@ -25,7 +36,7 @@ class IntegerField(CacheFieldBase): field_type = int -class CacheBase(type): +class CacheType(type): def __new__(cls, name, bases, attrs: dict): to_update = {} field_desc_mapper = {} @@ -41,12 +52,31 @@ class CacheBase(type): return type.__new__(cls, name, bases, attrs) -class Cache(metaclass=CacheBase): +class Cache(metaclass=CacheType): field_desc_mapper: dict timeout = None def __init__(self): self._data = None + self.redis = Redis(host=CONFIG.REDIS_HOST, port=CONFIG.REDIS_PORT, password=CONFIG.REDIS_PASSWORD) + + def __getitem__(self, item): + return self.field_desc_mapper[item] + + def __contains__(self, item): + return item in self.field_desc_mapper + + def get_field(self, name): + return self.field_desc_mapper[name] + + @property + def fields(self): + return self.field_desc_mapper.values() + + @property + def field_names(self): + names = self.field_desc_mapper.keys() + return names @lazyproperty def key_suffix(self): @@ -64,91 +94,75 @@ class Cache(metaclass=CacheBase): @property def data(self): if self._data is None: - data = self.get_data() - if data is None: - # 缓存中没有数据时，去数据库获取 - self.compute_and_set_all_data() + data = self.load_data_from_db() + if not data: + with ComputeLock(self.key): + data = self.load_data_from_db() + if not data: + # 缓存中没有数据时，去数据库获取 + self.init_all_values() return self._data - def get_data(self) -> dict: - data = cache.get(self.key) + def to_internal_value(self, data: dict): + internal_data = {} + for k, v in data.items(): + field = k.decode() + if field in self: + value = self[field].to_internal_value(v.decode()) + internal_data[field] = value + else: + logger.warn(f'Cache got invalid field: ' + f'key={self.key} ' + f'invalid_field={field} ' + f'valid_fields={self.field_names}') + return internal_data + + def load_data_from_db(self) -> dict: + data = self.redis.hgetall(self.key) logger.debug(f'Get data from cache: key={self.key} data={data}') - if data is not None: - data = json.loads(data) + if data: + data = self.to_internal_value(data) self._data = data return data - def set_data(self, data): - self._data = data - to_json = json.dumps(data) - logger.info(f'Set data to cache: key={self.key} data={to_json} timeout={self.timeout}') - cache.set(self.key, to_json, timeout=self.timeout) + def save_data_to_db(self, data): + logger.info(f'Set data to cache: key={self.key} data={data}') + self.redis.hset(self.key, mapping=data) + self.load_data_from_db() + + def compute_values(self, *fields): + field_objs = [] + for field in fields: + field_objs.append(self[field]) - def compute_data(self, *fields): - field_descs = [] - if not fields: - field_descs = self.field_desc_mapper.values() - else: - for field in fields: - assert field in self.field_desc_mapper, f'{field} is not a valid field' - field_descs.append(self.field_desc_mapper[field]) data = { - field_desc.field_name: field_desc.compute_value(self) - for field_desc in field_descs + field_obj.field_name: field_obj.compute_value(self) + for field_obj in field_objs } return data - def compute_and_set_all_data(self, computed_data: dict = None): - """ - TODO 怎样防止并发更新全部数据，浪费数据库资源 - """ - uncomputed_keys = () - if computed_data: - computed_keys = computed_data.keys() - all_keys = self.field_desc_mapper.keys() - uncomputed_keys = all_keys - computed_keys - else: - computed_data = {} - data = self.compute_data(*uncomputed_keys) - data.update(computed_data) - self.set_data(data) + def init_all_values(self): + t_start = time.time() + logger.info(f'Start init cache: key={self.key}') + data = self.compute_values(*self.field_names) + self.save_data_to_db(data) + logger.info(f'End init cache: cost={time.time()-t_start} key={self.key}') return data - def refresh_part_data_with_lock(self, refresh_data): - with DistributedLock(name=f'{self.key}.refresh'): - data = self.get_data() - if data is not None: - data.update(refresh_data) - self.set_data(data) - return data - - def expire_fields_with_lock(self, *fields): - with DistributedLock(name=f'{self.key}.refresh'): - data = self.get_data() - if data is not None: - logger.info(f'Expire cached fields: key={self.key} fields={fields}') - for f in fields: - data.pop(f, None) - self.set_data(data) - return data - def refresh(self, *fields): if not fields: # 没有指定 field 要刷新所有的值 - self.compute_and_set_all_data() + self.init_all_values() return - data = self.get_data() - if data is None: + data = self.load_data_from_db() + if not data: # 缓存中没有数据，设置所有的值 - self.compute_and_set_all_data() + self.init_all_values() return - refresh_data = self.compute_data(*fields) - if not self.refresh_part_data_with_lock(refresh_data): - # 刷新部分失败，缓存中没有数据，更新所有的值 - self.compute_and_set_all_data(refresh_data) - return + refresh_values = self.compute_values(*fields) + self.save_data_to_db(refresh_values) def get_key_suffix(self): raise NotImplementedError @@ -157,12 +171,13 @@ class Cache(metaclass=CacheBase): self._data = None def expire(self, *fields): + self._data = None if not fields: - self._data = None logger.info(f'Delete cached key: key={self.key}') - cache.delete(self.key) + self.redis.delete(self.key) else: - self.expire_fields_with_lock(*fields) + self.redis.hdel(self.key, *fields) + logger.info(f'Expire cached fields: key={self.key} fields={fields}') class CacheValueDesc: @@ -185,6 +200,8 @@ class CacheValueDesc: return value def compute_value(self, instance: Cache): + t_start = time.time() + logger.info(f'Start compute cache field: field={self.field_name} key={instance.key}') if self.field_type.queryset is not None: new_value = self.field_type.queryset.count() else: @@ -197,5 +214,8 @@ class CacheValueDesc: new_value = compute_func() new_value = self.field_type.field_type(new_value) - logger.info(f'Compute cache field value: key={instance.key} field={self.field_name} value={new_value}') + logger.info(f'End compute cache field: cost={time.time()-t_start} field={self.field_name} value={new_value} key={instance.key}') return new_value + + def to_internal_value(self, value): + return self.field_type.field_type(value) diff --git a/apps/orgs/cache.py b/apps/orgs/cache.py deleted file mode 100644 index f0a9cb83e..000000000 --- a/apps/orgs/cache.py +++ /dev/null @@ -1,39 +0,0 @@ -from django.db.transaction import on_commit - -from common.cache import * -from .utils import current_org, tmp_to_org -from .tasks import refresh_org_cache_task -from orgs.models import Organization - - -class OrgRelatedCache(Cache): - - def __init__(self): - super().__init__() - self.current_org = Organization.get_instance(current_org.id) - - def get_current_org(self): - """ - 暴露给子类控制组织的回调 - 1. 在交互式环境下能控制组织 - 2. 在 celery 任务下能控制组织 - """ - return self.current_org - - def compute_data(self, *fields): - with tmp_to_org(self.get_current_org()): - return super().compute_data(*fields) - - def refresh_async(self, *fields): - """ - 在事务提交之后再发送信号，防止因事务的隔离性导致未获得最新的数据 - """ - def func(): - logger.info(f'CACHE: Send refresh task {self}.{fields}') - refresh_org_cache_task.delay(self, *fields) - on_commit(func) - - def expire(self, *fields): - def func(): - super(OrgRelatedCache, self).expire(*fields) - on_commit(func) diff --git a/apps/orgs/caches.py b/apps/orgs/caches.py index b7e086d6a..9c29659e4 100644 --- a/apps/orgs/caches.py +++ b/apps/orgs/caches.py @@ -1,4 +1,10 @@ -from .cache import OrgRelatedCache, IntegerField +from django.db.transaction import on_commit +from orgs.models import Organization +from orgs.tasks import refresh_org_cache_task +from orgs.utils import current_org, tmp_to_org + +from common.cache import Cache, IntegerField +from common.utils import get_logger from users.models import UserGroup, User from assets.models import Node, AdminUser, SystemUser, Domain, Gateway from applications.models import Application @@ -6,6 +12,42 @@ from perms.models import AssetPermission, ApplicationPermission from .models import OrganizationMember +logger = get_logger(__file__) + + +class OrgRelatedCache(Cache): + + def __init__(self): + super().__init__() + self.current_org = Organization.get_instance(current_org.id) + + def get_current_org(self): + """ + 暴露给子类控制组织的回调 + 1. 在交互式环境下能控制组织 + 2. 在 celery 任务下能控制组织 + """ + return self.current_org + + def compute_values(self, *fields): + with tmp_to_org(self.get_current_org()): + return super().compute_values(*fields) + + def refresh_async(self, *fields): + """ + 在事务提交之后再发送信号，防止因事务的隔离性导致未获得最新的数据 + """ + def func(): + logger.info(f'CACHE: Send refresh task {self}.{fields}') + refresh_org_cache_task.delay(self, *fields) + on_commit(func) + + def expire(self, *fields): + def func(): + super(OrgRelatedCache, self).expire(*fields) + on_commit(func) + + class OrgResourceStatisticsCache(OrgRelatedCache): users_amount = IntegerField() groups_amount = IntegerField(queryset=UserGroup.objects) From 81170b4b7b316216a88487abeda7ac8201fea100 Mon Sep 17 00:00:00 2001 From: ibuler Date: Tue, 9 Mar 2021 12:18:04 +0800 Subject: [PATCH 59/71] =?UTF-8?q?perf:=20=E4=BC=98=E5=8C=96=E7=99=BB?= =?UTF-8?q?=E5=BD=95=E9=A1=B5=E9=9D=A2=EF=BC=8C=E9=9D=9E=E5=B8=B8=E7=BB=99?= =?UTF-8?q?=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit perf: 优化报错 perf: 优化忘记密码 perf: 添加注释 --- apps/authentication/forms.py | 26 +- apps/authentication/mixins.py | 11 +- .../templates/authentication/login.html | 256 ++--- apps/authentication/views/login.py | 9 +- apps/jumpserver/conf.py | 2 + apps/jumpserver/context_processor.py | 4 +- apps/jumpserver/settings/base.py | 5 +- apps/locale/zh/LC_MESSAGES/django.mo | Bin 69720 -> 70335 bytes apps/locale/zh/LC_MESSAGES/django.po | 912 ++++++++++-------- apps/static/css/login-style.css | 8 +- apps/users/api/profile.py | 6 - .../templates/users/forgot_password.html | 8 + 12 files changed, 732 insertions(+), 515 deletions(-) diff --git a/apps/authentication/forms.py b/apps/authentication/forms.py index fe28edb68..447d842bd 100644 --- a/apps/authentication/forms.py +++ b/apps/authentication/forms.py @@ -8,11 +8,26 @@ from captcha.fields import CaptchaField, CaptchaTextInput class UserLoginForm(forms.Form): - username = forms.CharField(label=_('Username'), max_length=100) + days_auto_login = int(settings.SESSION_COOKIE_AGE / 3600 / 24) + disable_days_auto_login = settings.SESSION_EXPIRE_AT_BROWSER_CLOSE_FORCE or days_auto_login < 1 + + username = forms.CharField( + label=_('Username'), max_length=100, + widget=forms.TextInput(attrs={ + 'placeholder': _("Username"), + 'autofocus': 'autofocus' + }) + ) password = forms.CharField( label=_('Password'), widget=forms.PasswordInput, max_length=1024, strip=False ) + auto_login = forms.BooleanField( + label=_("{} days auto login").format(days_auto_login or 1), + required=False, initial=False, widget=forms.CheckboxInput( + attrs={'disabled': disable_days_auto_login} + ) + ) def confirm_login_allowed(self, user): if not user.is_staff: @@ -35,8 +50,13 @@ class CaptchaMixin(forms.Form): class ChallengeMixin(forms.Form): - challenge = forms.CharField(label=_('MFA code'), max_length=6, - required=False) + challenge = forms.CharField( + label=_('MFA code'), max_length=6, required=False, + widget=forms.TextInput(attrs={ + 'placeholder': _("MFA code"), + 'style': 'width: 50%' + }) + ) def get_user_login_form_cls(*, captcha=False): diff --git a/apps/authentication/mixins.py b/apps/authentication/mixins.py index e4813ef3f..9702e6046 100644 --- a/apps/authentication/mixins.py +++ b/apps/authentication/mixins.py @@ -82,7 +82,7 @@ class AuthMixin: return raw_passwd def raise_credential_error(self, error): - raise self.partial_credential_error(error=errors.reason_password_decrypt_failed) + raise self.partial_credential_error(error=error) def get_auth_data(self, decrypt_passwd=False): request = self.request @@ -91,8 +91,8 @@ class AuthMixin: else: data = request.POST - items = ['username', 'password', 'challenge', 'public_key'] - username, password, challenge, public_key = bulk_get(data, *items, default='') + items = ['username', 'password', 'challenge', 'public_key', 'auto_login'] + username, password, challenge, public_key, auto_login = bulk_get(data, *items, default='') password = password + challenge.strip() ip = self.get_request_ip() self.partial_credential_error = partial(errors.CredentialError, username=username, ip=ip, request=request) @@ -101,7 +101,7 @@ class AuthMixin: password = self.decrypt_passwd(password) if not password: self.raise_credential_error(errors.reason_password_decrypt_failed) - return username, password, public_key, ip + return username, password, public_key, ip, auto_login def _check_only_allow_exists_user_auth(self, username): # 仅允许预先存在的用户认证 @@ -131,7 +131,7 @@ class AuthMixin: def check_user_auth(self, decrypt_passwd=False): self.check_is_block() request = self.request - username, password, public_key, ip = self.get_auth_data(decrypt_passwd=decrypt_passwd) + username, password, public_key, ip, auto_login = self.get_auth_data(decrypt_passwd=decrypt_passwd) self._check_only_allow_exists_user_auth(username) user = self._check_auth_user_is_valid(username, password, public_key) @@ -145,6 +145,7 @@ class AuthMixin: clean_failed_count(username, ip) request.session['auth_password'] = 1 request.session['user_id'] = str(user.id) + request.session['auto_login'] = auto_login request.session['auth_backend'] = auth_backend return user diff --git a/apps/authentication/templates/authentication/login.html b/apps/authentication/templates/authentication/login.html index e06567aa4..9866537ba 100644 --- a/apps/authentication/templates/authentication/login.html +++ b/apps/authentication/templates/authentication/login.html @@ -1,12 +1,11 @@ -{% load static %} {% load i18n %} +{% load bootstrap3 %} +{% load static %} - - - + {{ JMS_TITLE }} @@ -16,6 +15,8 @@ + + @@ -24,26 +25,54 @@ - -
-
-
- + +