mirror of https://github.com/openspug/spug
141 lines
6.0 KiB
Python
141 lines
6.0 KiB
Python
# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug
|
||
# Copyright: (c) <spug.dev@gmail.com>
|
||
# Released under the AGPL-3.0 License.
|
||
from django.conf import settings
|
||
from libs.utils import AttrDict, render_str, human_seconds_time
|
||
from apps.host.models import Host
|
||
from apps.repository.models import Repository
|
||
from apps.repository.utils import dispatch as build_repository
|
||
from apps.deploy.helper import SpugError
|
||
from concurrent import futures
|
||
import json
|
||
import time
|
||
import os
|
||
|
||
BUILD_DIR = settings.BUILD_DIR
|
||
|
||
|
||
def ext1_deploy(req, helper, env):
|
||
if not req.repository_id:
|
||
rep = Repository(
|
||
app_id=req.deploy.app_id,
|
||
env_id=req.deploy.env_id,
|
||
deploy_id=req.deploy_id,
|
||
version=req.version,
|
||
spug_version=req.spug_version,
|
||
extra=req.extra,
|
||
remarks='SPUG AUTO MAKE',
|
||
created_by_id=req.created_by_id
|
||
)
|
||
build_repository(rep, helper)
|
||
req.repository = rep
|
||
env.update(SPUG_BUILD_ID=str(req.repository_id))
|
||
env.update(helper.get_cross_env(req.spug_version))
|
||
extras = json.loads(req.extra)
|
||
if extras[0] == 'repository':
|
||
extras = extras[1:]
|
||
if extras[0] == 'branch':
|
||
env.update(SPUG_GIT_BRANCH=extras[1], SPUG_GIT_COMMIT_ID=extras[2])
|
||
else:
|
||
env.update(SPUG_GIT_TAG=extras[1])
|
||
if req.deploy.is_parallel:
|
||
threads, latest_exception = [], None
|
||
max_workers = max(10, os.cpu_count() * 5)
|
||
with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||
for h_id in helper.deploy_host_ids:
|
||
new_env = AttrDict(env.items())
|
||
t = executor.submit(_deploy_ext1_host, req, helper, h_id, new_env)
|
||
t.h_id = h_id
|
||
threads.append(t)
|
||
for t in futures.as_completed(threads):
|
||
exception = t.exception()
|
||
if exception:
|
||
helper.set_deploy_fail(t.h_id)
|
||
latest_exception = exception
|
||
if not isinstance(exception, SpugError):
|
||
helper.send_error(t.h_id, f'Exception: {exception}', with_break=False)
|
||
else:
|
||
helper.set_deploy_success(t.h_id)
|
||
if latest_exception:
|
||
raise latest_exception
|
||
else:
|
||
host_ids = sorted(helper.deploy_host_ids, reverse=True)
|
||
while host_ids:
|
||
h_id = host_ids.pop()
|
||
new_env = AttrDict(env.items())
|
||
try:
|
||
_deploy_ext1_host(req, helper, h_id, new_env)
|
||
helper.set_deploy_success(h_id)
|
||
except Exception as e:
|
||
helper.set_deploy_fail(h_id)
|
||
helper.send_error(h_id, f'Exception: {e}', with_break=False)
|
||
for h_id in host_ids:
|
||
helper.set_deploy_fail(h_id)
|
||
helper.send_error(h_id, '终止发布', with_break=False)
|
||
raise e
|
||
|
||
|
||
def _deploy_ext1_host(req, helper, h_id, env):
|
||
flag = time.time()
|
||
helper.set_deploy_process(h_id)
|
||
helper.send_clear(h_id)
|
||
helper.send_info(h_id, '数据准备... ', status='doing')
|
||
host = Host.objects.filter(pk=h_id).first()
|
||
if not host:
|
||
helper.send_error(h_id, 'no such host')
|
||
env.update({'SPUG_HOST_ID': h_id, 'SPUG_HOST_NAME': host.hostname})
|
||
extend = req.deploy.extend_obj
|
||
extend.dst_dir = render_str(extend.dst_dir, env)
|
||
extend.dst_repo = render_str(extend.dst_repo, env)
|
||
env.update(SPUG_DST_DIR=extend.dst_dir)
|
||
with host.get_ssh(default_env=env) as ssh:
|
||
helper.save_pid(ssh.get_pid(), h_id)
|
||
base_dst_dir = os.path.dirname(extend.dst_dir)
|
||
code, _ = ssh.exec_command_raw(
|
||
f'mkdir -p {extend.dst_repo} {base_dst_dir} && [ -e {extend.dst_dir} ] && [ ! -L {extend.dst_dir} ]')
|
||
if code == 0:
|
||
helper.send_error(host.id,
|
||
f'\r\n检测到该主机的发布目录 {extend.dst_dir!r} 已存在，为了数据安全请自行备份后删除该目录，Spug 将会创建并接管该目录。')
|
||
if req.type == '2':
|
||
helper.send_warn(h_id, '跳过√\r\n')
|
||
else:
|
||
# clean
|
||
clean_command = f'ls -d {extend.deploy_id}_* 2> /dev/null | sort -t _ -rnk2 | tail -n +{extend.versions + 1} | xargs rm -rf'
|
||
helper.remote_raw(host.id, ssh, f'cd {extend.dst_repo} && {clean_command}')
|
||
# transfer files
|
||
tar_gz_file = f'{req.spug_version}.tar.gz'
|
||
try:
|
||
callback = helper.progress_callback(host.id)
|
||
ssh.put_file(
|
||
os.path.join(BUILD_DIR, tar_gz_file),
|
||
os.path.join(extend.dst_repo, tar_gz_file),
|
||
callback
|
||
)
|
||
except Exception as e:
|
||
helper.send_error(host.id, f'Exception: {e}')
|
||
|
||
command = f'cd {extend.dst_repo} && rm -rf {req.spug_version} && tar xf {tar_gz_file} && rm -f {req.deploy_id}_*.tar.gz'
|
||
helper.remote_raw(host.id, ssh, command)
|
||
helper.send_success(h_id, '完成√\r\n')
|
||
|
||
# pre host
|
||
repo_dir = os.path.join(extend.dst_repo, req.spug_version)
|
||
if extend.hook_pre_host:
|
||
helper.send_info(h_id, '发布前任务... \r\n')
|
||
command = f'cd {repo_dir} && {extend.hook_pre_host}'
|
||
helper.remote(host.id, ssh, command)
|
||
|
||
# do deploy
|
||
helper.send_info(h_id, '执行发布... ')
|
||
helper.remote_raw(host.id, ssh, f'rm -f {extend.dst_dir} && ln -sfn {repo_dir} {extend.dst_dir}')
|
||
helper.send_success(h_id, '完成√\r\n')
|
||
|
||
# post host
|
||
if extend.hook_post_host:
|
||
helper.send_info(h_id, '发布后任务... \r\n')
|
||
command = f'cd {extend.dst_dir} && {extend.hook_post_host}'
|
||
helper.remote(host.id, ssh, command)
|
||
|
||
human_time = human_seconds_time(time.time() - flag)
|
||
helper.send_success(h_id, f'\r\n** 发布成功，耗时：{human_time} **', status='success')
|