From 540e8b377d204f532aac3e18e8e90961a07f7680 Mon Sep 17 00:00:00 2001 From: vapao Date: Mon, 5 Jul 2021 23:30:11 +0800 Subject: [PATCH] fix issues --- spug_api/apps/deploy/utils.py | 4 ++-- spug_api/apps/exec/management/commands/runworker.py | 3 ++- spug_api/apps/host/utils.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/spug_api/apps/deploy/utils.py b/spug_api/apps/deploy/utils.py index d72a182..c62fa0e 100644 --- a/spug_api/apps/deploy/utils.py +++ b/spug_api/apps/deploy/utils.py @@ -59,7 +59,7 @@ def _ext1_deploy(req, helper, env): extend = req.deploy.extend_obj env.update(SPUG_DST_DIR=extend.dst_dir) threads, latest_exception = [], None - max_workers = min(10, os.cpu_count() * 4) if req.deploy.is_parallel else 1 + max_workers = max(10, os.cpu_count() * 5) if req.deploy.is_parallel else 1 with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: for h_id in json.loads(req.host_ids): env = AttrDict(env.items()) @@ -124,7 +124,7 @@ def _ext2_deploy(req, helper, env): break if host_actions: threads, latest_exception = [], None - max_workers = min(10, os.cpu_count() * 4) if req.deploy.is_parallel else 1 + max_workers = max(10, os.cpu_count() * 5) if req.deploy.is_parallel else 1 with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: for h_id in json.loads(req.host_ids): env = AttrDict(env.items()) diff --git a/spug_api/apps/exec/management/commands/runworker.py b/spug_api/apps/exec/management/commands/runworker.py index 87591e3..70418ee 100644 --- a/spug_api/apps/exec/management/commands/runworker.py +++ b/spug_api/apps/exec/management/commands/runworker.py @@ -9,6 +9,7 @@ from apps.schedule.executors import schedule_worker_handler from apps.monitor.executors import monitor_worker_handler from apps.exec.executors import exec_worker_handler import logging +import os EXEC_WORKER_KEY = settings.EXEC_WORKER_KEY MONITOR_WORKER_KEY = settings.MONITOR_WORKER_KEY @@ -20,7 +21,7 @@ logging.basicConfig(level=logging.WARNING, format='%(asctime)s %(message)s') class Worker: def __init__(self): self.rds = get_redis_connection() - self._executor = ThreadPoolExecutor(max_workers=1) + self._executor = ThreadPoolExecutor(max_workers=max(50, os.cpu_count() * 20)) def run(self): logging.warning('Running worker') diff --git a/spug_api/apps/host/utils.py b/spug_api/apps/host/utils.py index bc10389..c8a18b8 100644 --- a/spug_api/apps/host/utils.py +++ b/spug_api/apps/host/utils.py @@ -214,7 +214,7 @@ def fetch_host_extend(ssh): def batch_sync_host(token, hosts, password, ): private_key, public_key = AppSetting.get_ssh_key() threads, latest_exception, rds = [], None, get_redis_connection() - max_workers = min(10, os.cpu_count() * 4) + max_workers = max(10, os.cpu_count() * 5) with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: for host in hosts: t = executor.submit(_sync_host_extend, host, private_key, public_key, password)