From 0cd3b99b8559656f912f95a8f14edd7984ed5f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9B=B7=E4=BA=8C=E7=8C=9B?= Date: Thu, 19 Dec 2019 23:44:56 +0800 Subject: [PATCH] A api update --- spug_api/apps/deploy/utils.py | 85 +++++++++++++++++++---------------- spug_api/apps/deploy/views.py | 6 ++- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/spug_api/apps/deploy/utils.py b/spug_api/apps/deploy/utils.py index 92f8ce5..a836abf 100644 --- a/spug_api/apps/deploy/utils.py +++ b/spug_api/apps/deploy/utils.py @@ -7,6 +7,7 @@ from threading import Thread import socket import subprocess import json +import time import os REPOS_DIR = settings.REPOS_DIR @@ -14,22 +15,28 @@ REPOS_DIR = settings.REPOS_DIR def deploy_dispatch(request, req, token): now = datetime.now() + rds = get_redis_connection() + helper = Helper(rds, token) + helper.send_step('local', 1, '发布准备... ') + rds.expire(token, 60 * 60) env = AttrDict( APP_NAME=req.app.name, APP_ID=str(req.app_id), TASK_NAME=req.name, TASK_ID=str(req.id), VERSION=f'{req.app_id}_{req.id}_{now.strftime("%Y%m%d%H%M%S")}', - TIME=str(now.strftime('%Y-%m-%d\ %H:%M:%S')) + TIME=str(now.strftime(r'%Y-%m-%d\ %H:%M:%S')) ) if req.app.extend == '1': env.update(json.loads(req.app.extend_obj.custom_envs)) - _ext1_deploy(request, req, token, env) + _ext1_deploy(request, req, helper, env) else: - _ext2_deploy(request, req, token, env) + _ext2_deploy(request, req, helper, env) + + print('!!!!!!!!!!!!!!!!!') -def _ext1_deploy(request, req, token, env): +def _ext1_deploy(request, req, helper, env): app = req.app extend = req.app.extend_obj extras = json.loads(req.extra) @@ -39,91 +46,91 @@ def _ext1_deploy(request, req, token, env): else: tree_ish = extras[1] env.update(TAG=extras[1]) - rds = get_redis_connection() - rds.rpush(token, json.dumps({'key': 'local', 'data': 'Starting...\n'})) - rds.expire(token, 60 * 60) - executor = Executor(rds, token) + helper.send_step('local', 2, '完成\r\n检出前任务... ') if extend.hook_pre_server: - executor.local(f'cd /tmp && {extend.hook_pre_server}', env) + helper.local(f'cd /tmp && {extend.hook_pre_server}', env) + helper.send_step('local', 3, '执行检出... ') git_dir = os.path.join(REPOS_DIR, str(app.id)) command = f'cd {git_dir} && git archive --prefix={env.VERSION}/ {tree_ish} | (cd .. && tar xf -)' - executor.local(command) + helper.local(command) + + time.sleep(3) + + helper.send_step('local', 4, '完成\r\n检出后任务... ') if extend.hook_post_server: - executor.local(f'cd {os.path.join(REPOS_DIR, env.VERSION)} && {extend.hook_post_server}', env) - - executor.local(f'cd {REPOS_DIR} && tar zcf {env.VERSION}.tar.gz {env.VERSION}') + helper.local(f'cd {os.path.join(REPOS_DIR, env.VERSION)} && {extend.hook_post_server}', env) + helper.send_step('local', 5) + helper.local(f'cd {REPOS_DIR} && tar zcf {env.VERSION}.tar.gz {env.VERSION}') for h_id in json.loads(req.host_ids): - Thread(target=_deploy_host, args=(executor, h_id, extend, env)).start() + Thread(target=_deploy_host, args=(helper, h_id, extend, env)).start() -def _ext2_deploy(request, req, token, env): +def _ext2_deploy(request, rds, req, token, env): pass -def _deploy_host(executor, h_id, extend, env): +def _deploy_host(helper, h_id, extend, env): + helper.send_step(h_id, 1) host = Host.objects.filter(pk=h_id).first() if not host: - executor.send_error(h_id, 'no such host') + helper.send_error(h_id, 'no such host') ssh = host.get_ssh() code, _ = ssh.exec_command(f'mkdir -p {extend.dst_repo} && [ -e {extend.dst_dir} ] && [ ! -L {extend.dst_dir} ]') if code == 0: - executor.send_error(host.id, f'please make sure the {extend.dst_dir!r} is not exists.') + helper.send_error(host.id, f'please make sure the {extend.dst_dir!r} is not exists.') # transfer files tar_gz_file = f'{env.VERSION}.tar.gz' - executor.send_info(host.id, 'Transferring files ...') try: ssh.put_file(os.path.join(REPOS_DIR, tar_gz_file), os.path.join(extend.dst_repo, tar_gz_file)) except Exception as e: - executor.send_error(host.id, f'exception: {e}') + helper.send_error(host.id, f'exception: {e}') command = f'cd {extend.dst_repo} && tar xf {tar_gz_file} && rm -f {env.APP_ID}_*.tar.gz' - executor.remote(host.id, ssh, command) - # exit_code, out = ssh.exec_command() - # if exit_code != 0: - # executor.send_error(host.id, f'{out}\r\nexit code: {exit_code}') + helper.remote(host.id, ssh, command) - repo_dir = os.path.join(extend.dst_repo, env.VERSION) # pre host + helper.send_step(h_id, 2) + repo_dir = os.path.join(extend.dst_repo, env.VERSION) if extend.hook_pre_host: command = f'cd {repo_dir} && {extend.hook_pre_host}' - executor.remote(host.id, ssh, command, env) + helper.remote(host.id, ssh, command, env) # do deploy + helper.send_step(h_id, 3) tmp_path = os.path.join(extend.dst_repo, f'tmp_{env.VERSION}') - executor.remote(host.id, ssh, f'ln -sfn {repo_dir} {tmp_path} && mv -fT {tmp_path} {extend.dst_dir}') - # exit_code, out = ssh.exec_command(f'') - # if exit_code != 0: - # executor.send_error(host.id, f'{out}\r\nexit code: {exit_code}') + helper.remote(host.id, ssh, f'ln -sfn {repo_dir} {tmp_path} && mv -fT {tmp_path} {extend.dst_dir}') # post host + helper.send_step(h_id, 4) if extend.hook_post_host: command = f'cd {extend.dst_dir} && {extend.hook_post_host}' - executor.remote(host.id, ssh, command, env) + helper.remote(host.id, ssh, command, env) - executor.send_done(host.id) + helper.send_step(h_id, 5) -class Executor: +class Helper: def __init__(self, rds, token): self.rds = rds self.token = token def send_info(self, key, message): - self.rds.lpush(self.token, json.dumps({'key': key, 'status': 'info', 'data': message})) + self.rds.rpush(self.token, json.dumps({'key': key, 'status': 'info', 'data': message})) def send_error(self, key, message): - self.rds.lpush(self.token, json.dumps({'key': key, 'status': 'error', 'data': message})) + self.rds.rpush(self.token, json.dumps({'key': key, 'status': 'error', 'data': message})) raise Exception(message) - def send_done(self, key): - self.rds.lpush(self.token, json.dumps({'key': key, 'status': 'done'})) + def send_step(self, key, step, data): + data = datetime.now().strftime('%H:%M:%S ') + data + self.rds.rpush(self.token, json.dumps({'key': key, 'step': step, 'data': data})) def local(self, command, env=None): - print(f'executor.local: {command!r}') + print(f'helper.local: {command!r}') task = subprocess.Popen(command, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) while True: message = task.stdout.readline() @@ -134,7 +141,7 @@ class Executor: self.send_error('local', f'exit code: {task.returncode}') def remote(self, key, ssh, command, env=None): - print(f'executor.remote: {command!r} env: {env!r}') + print(f'helper.remote: {command!r} env: {env!r}') code = -1 try: for code, out in ssh.exec_command_with_stream(command, environment=env): diff --git a/spug_api/apps/deploy/views.py b/spug_api/apps/deploy/views.py index df88255..04b0645 100644 --- a/spug_api/apps/deploy/views.py +++ b/spug_api/apps/deploy/views.py @@ -4,6 +4,7 @@ from libs import json_response, JsonParser, Argument from apps.deploy.models import DeployRequest from apps.deploy.utils import deploy_dispatch from apps.app.models import App +from apps.host.models import Host from threading import Thread import json import uuid @@ -63,6 +64,9 @@ def do_deploy(request, r_id): return json_response(error='未找到指定发布申请') if req.status != '2': return json_response(error='该申请单当前状态还不能执行发布') + hosts = Host.objects.filter(id__in=json.loads(req.host_ids)) token = uuid.uuid4().hex Thread(target=deploy_dispatch, args=(request, req, token)).start() - return json_response(token) + outputs = {str(x): {'data': ''} for x in json.loads(req.host_ids) + ['local']} + targets = [{'id': x.id, 'title': f'{x.name}({x.hostname}:{x.port})'} for x in hosts] + return json_response({'token': token, 'outputs': outputs, 'targets': targets})