# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug # Copyright: (c) # Released under the AGPL-3.0 License. from django.conf import settings from libs.utils import AttrDict, render_str, human_seconds_time from apps.host.models import Host from apps.repository.models import Repository from apps.repository.utils import dispatch as build_repository from apps.deploy.helper import SpugError from concurrent import futures import json import time import os BUILD_DIR = settings.BUILD_DIR def ext1_deploy(req, helper, env): if not req.repository_id: rep = Repository( app_id=req.deploy.app_id, env_id=req.deploy.env_id, deploy_id=req.deploy_id, version=req.version, spug_version=req.spug_version, extra=req.extra, remarks='SPUG AUTO MAKE', created_by_id=req.created_by_id ) build_repository(rep, helper) req.repository = rep env.update(SPUG_BUILD_ID=str(req.repository_id)) env.update(helper.get_cross_env(req.spug_version)) extras = json.loads(req.extra) if extras[0] == 'repository': extras = extras[1:] if extras[0] == 'branch': env.update(SPUG_GIT_BRANCH=extras[1], SPUG_GIT_COMMIT_ID=extras[2]) else: env.update(SPUG_GIT_TAG=extras[1]) if req.deploy.is_parallel: threads, latest_exception = [], None max_workers = max(10, os.cpu_count() * 5) with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: for h_id in helper.deploy_host_ids: new_env = AttrDict(env.items()) t = executor.submit(_deploy_ext1_host, req, helper, h_id, new_env) t.h_id = h_id threads.append(t) for t in futures.as_completed(threads): exception = t.exception() if exception: helper.set_deploy_fail(t.h_id) latest_exception = exception if not isinstance(exception, SpugError): helper.send_error(t.h_id, f'Exception: {exception}', with_break=False) else: helper.set_deploy_success(t.h_id) if latest_exception: raise latest_exception else: host_ids = sorted(helper.deploy_host_ids, reverse=True) while host_ids: h_id = host_ids.pop() new_env = AttrDict(env.items()) try: _deploy_ext1_host(req, helper, h_id, new_env) helper.set_deploy_success(h_id) except Exception as e: helper.set_deploy_fail(h_id) helper.send_error(h_id, f'Exception: {e}', with_break=False) for h_id in host_ids: helper.set_deploy_fail(h_id) helper.send_error(h_id, '终止发布', with_break=False) raise e def _deploy_ext1_host(req, helper, h_id, env): flag = time.time() helper.set_deploy_process(h_id) helper.send_clear(h_id) helper.send_info(h_id, '数据准备... ', status='doing') host = Host.objects.filter(pk=h_id).first() if not host: helper.send_error(h_id, 'no such host') env.update({'SPUG_HOST_ID': h_id, 'SPUG_HOST_NAME': host.hostname}) extend = req.deploy.extend_obj extend.dst_dir = render_str(extend.dst_dir, env) extend.dst_repo = render_str(extend.dst_repo, env) env.update(SPUG_DST_DIR=extend.dst_dir) with host.get_ssh(default_env=env) as ssh: helper.save_pid(ssh.get_pid(), h_id) base_dst_dir = os.path.dirname(extend.dst_dir) code, _ = ssh.exec_command_raw( f'mkdir -p {extend.dst_repo} {base_dst_dir} && [ -e {extend.dst_dir} ] && [ ! -L {extend.dst_dir} ]') if code == 0: helper.send_error(host.id, f'\r\n检测到该主机的发布目录 {extend.dst_dir!r} 已存在,为了数据安全请自行备份后删除该目录,Spug 将会创建并接管该目录。') if req.type == '2': helper.send_warn(h_id, '跳过√\r\n') else: # clean clean_command = f'ls -d {extend.deploy_id}_* 2> /dev/null | sort -t _ -rnk2 | tail -n +{extend.versions + 1} | xargs rm -rf' helper.remote_raw(host.id, ssh, f'cd {extend.dst_repo} && {clean_command}') # transfer files tar_gz_file = f'{req.spug_version}.tar.gz' try: callback = helper.progress_callback(host.id) ssh.put_file( os.path.join(BUILD_DIR, tar_gz_file), os.path.join(extend.dst_repo, tar_gz_file), callback ) except Exception as e: helper.send_error(host.id, f'Exception: {e}') command = f'cd {extend.dst_repo} && rm -rf {req.spug_version} && tar xf {tar_gz_file} && rm -f {req.deploy_id}_*.tar.gz' helper.remote_raw(host.id, ssh, command) helper.send_success(h_id, '完成√\r\n') # pre host repo_dir = os.path.join(extend.dst_repo, req.spug_version) if extend.hook_pre_host: helper.send_info(h_id, '发布前任务... \r\n') command = f'cd {repo_dir} && {extend.hook_pre_host}' helper.remote(host.id, ssh, command) # do deploy helper.send_info(h_id, '执行发布... ') helper.remote_raw(host.id, ssh, f'rm -f {extend.dst_dir} && ln -sfn {repo_dir} {extend.dst_dir}') helper.send_success(h_id, '完成√\r\n') # post host if extend.hook_post_host: helper.send_info(h_id, '发布后任务... \r\n') command = f'cd {extend.dst_dir} && {extend.hook_post_host}' helper.remote(host.id, ssh, command) human_time = human_seconds_time(time.time() - flag) helper.send_success(h_id, f'\r\n** 发布成功,耗时:{human_time} **', status='success')