diff --git a/spug_api/apps/deploy/ext1.py b/spug_api/apps/deploy/ext1.py index 41f02b7..810318b 100644 --- a/spug_api/apps/deploy/ext1.py +++ b/spug_api/apps/deploy/ext1.py @@ -29,6 +29,8 @@ def ext1_deploy(req, helper, env): ) build_repository(rep, helper) req.repository = rep + env.update(SPUG_BUILD_ID=str(req.repository_id)) + env.update(helper.get_cross_env(req.spug_version)) extras = json.loads(req.extra) if extras[0] == 'repository': extras = extras[1:] diff --git a/spug_api/apps/deploy/ext2.py b/spug_api/apps/deploy/ext2.py index 95f3b1b..74270e6 100644 --- a/spug_api/apps/deploy/ext2.py +++ b/spug_api/apps/deploy/ext2.py @@ -3,6 +3,7 @@ # Released under the AGPL-3.0 License. from django.conf import settings from libs.utils import AttrDict, render_str, human_seconds_time +from libs.executor import Executor from apps.host.models import Host from apps.deploy.helper import SpugError from concurrent import futures @@ -38,48 +39,52 @@ def ext2_deploy(req, helper, env, with_local): transfer_action = action break - if with_local: - helper.set_deploy_process('local') - helper.send_success('local', '', status='doing') - if server_actions or transfer_action: - helper.send_clear('local') - for action in server_actions: - helper.send_info('local', f'{action["title"]}...\r\n') - helper.local(f'cd /tmp && {action["data"]}', env) - step += 1 - if transfer_action: - action = transfer_action - helper.send_info('local', '检测到来源为本地路径的数据传输动作,执行打包... \r\n') - action['src'] = action['src'].rstrip('/ ') - action['dst'] = action['dst'].rstrip('/ ') - if not action['src'] or not action['dst']: - helper.send_error('local', f'Invalid path for transfer, src: {action["src"]} dst: {action["dst"]}') - if not os.path.exists(action['src']): - helper.send_error('local', f'No such file or directory: {action["src"]}') - is_dir, exclude = os.path.isdir(action['src']), '' - sp_dir, sd_dst = os.path.split(action['src']) - contain = sd_dst - if action['mode'] != '0' and is_dir: - files = helper.parse_filter_rule(action['rule'], ',', env) - if files: - if action['mode'] == '1': - contain = ' '.join(f'{sd_dst}/{x}' for x in files) - else: - excludes = [] - for x in files: - if x.startswith('/'): - excludes.append(f'--exclude={sd_dst}{x}') - else: - excludes.append(f'--exclude={x}') - exclude = ' '.join(excludes) - tar_gz_file = os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, f'{req.spug_version}.tar.gz') - helper.local(f'cd {sp_dir} && tar -zcf {tar_gz_file} {exclude} {contain}') - helper.send_info('local', '打包完成\r\n') - helper.set_deploy_success('local') - human_time = human_seconds_time(time.time() - flag) - helper.send_success('local', f'\r\n** 执行完成,耗时:{human_time} **', status='success') + if with_local or True: + with Executor(env) as et: + helper.save_pid(et.pid, 'local') + helper.set_deploy_process('local') + helper.send_success('local', '', status='doing') + if server_actions or transfer_action: + helper.send_clear('local') + for action in server_actions: + helper.send_info('local', f'{action["title"]}...\r\n') + helper.local(et, f'cd /tmp && {action["data"]}') + step += 1 + if transfer_action: + action = transfer_action + helper.send_info('local', '检测到来源为本地路径的数据传输动作,执行打包... \r\n') + action['src'] = action['src'].rstrip('/ ') + action['dst'] = action['dst'].rstrip('/ ') + if not action['src'] or not action['dst']: + helper.send_error('local', f'Invalid path for transfer, src: {action["src"]} dst: {action["dst"]}') + if not os.path.exists(action['src']): + helper.send_error('local', f'No such file or directory: {action["src"]}') + is_dir, exclude = os.path.isdir(action['src']), '' + sp_dir, sd_dst = os.path.split(action['src']) + contain = sd_dst + if action['mode'] != '0' and is_dir: + files = helper.parse_filter_rule(action['rule'], ',', env) + if files: + if action['mode'] == '1': + contain = ' '.join(f'{sd_dst}/{x}' for x in files) + else: + excludes = [] + for x in files: + if x.startswith('/'): + excludes.append(f'--exclude={sd_dst}{x}') + else: + excludes.append(f'--exclude={x}') + exclude = ' '.join(excludes) + tar_gz_file = os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, f'{req.spug_version}.tar.gz') + helper.local_raw(f'cd {sp_dir} && tar -zcf {tar_gz_file} {exclude} {contain}') + helper.send_info('local', '打包完成\r\n') + helper.set_deploy_success('local') + human_time = human_seconds_time(time.time() - flag) + helper.send_success('local', f'\r\n** 执行完成,耗时:{human_time} **', status='success') + helper.set_cross_env(req.spug_version, et.get_envs()) if host_actions: + env.update(helper.get_cross_env(req.spug_version)) if req.deploy.is_parallel: threads, latest_exception = [], None max_workers = max(10, os.cpu_count() * 5) diff --git a/spug_api/apps/deploy/helper.py b/spug_api/apps/deploy/helper.py index c911cac..f04fe2f 100644 --- a/spug_api/apps/deploy/helper.py +++ b/spug_api/apps/deploy/helper.py @@ -303,6 +303,18 @@ class Helper(NotifyMixin, KitMixin): self.files[key] = file return file + def get_cross_env(self, key): + file = os.path.join(settings.DEPLOY_DIR, key) + if os.path.exists(file): + with open(file, 'r') as f: + return json.loads(f.read()) + return {} + + def set_cross_env(self, key, envs): + file = os.path.join(settings.DEPLOY_DIR, key) + with open(file, 'w') as f: + f.write(json.dumps(envs)) + def add_callback(self, func): self.callback.append(func) @@ -389,7 +401,14 @@ class Helper(NotifyMixin, KitMixin): self._send(key, '\r\n') return partial(func, key) - def local(self, command, env=None): + def local(self, executor, command): + code = -1 + for code, out in executor.exec_command_with_stream(command): + self._send('local', out) + if code != 0: + self.send_error('local', f'exit code: {code}') + + def local_raw(self, command, env=None): if env: env = dict(env.items()) env.update(os.environ) @@ -398,9 +417,7 @@ class Helper(NotifyMixin, KitMixin): env=env, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - preexec_fn=os.setsid) - self.save_pid(task.pid, 'local') + stderr=subprocess.STDOUT) message = b'' while True: output = task.stdout.read(1) diff --git a/spug_api/apps/deploy/models.py b/spug_api/apps/deploy/models.py index 89528a5..27c56ce 100644 --- a/spug_api/apps/deploy/models.py +++ b/spug_api/apps/deploy/models.py @@ -72,6 +72,10 @@ class DeployRequest(models.Model, ModelMixin): p.unlink() except FileNotFoundError: pass + try: + Path(settings.DEPLOY_DIR, self.spug_version).unlink() + except FileNotFoundError: + pass super().delete(using, keep_parents) def __repr__(self): diff --git a/spug_api/apps/repository/utils.py b/spug_api/apps/repository/utils.py index 8ecade9..77e10c7 100644 --- a/spug_api/apps/repository/utils.py +++ b/spug_api/apps/repository/utils.py @@ -5,6 +5,7 @@ from django_redis import get_redis_connection from django.conf import settings from django.db import close_old_connections from libs.utils import AttrDict, human_datetime, render_str, human_seconds_time +from libs.executor import Executor from apps.repository.models import Repository from apps.config.utils import compose_configs from apps.deploy.helper import Helper @@ -69,7 +70,7 @@ def dispatch(rep: Repository, helper=None): helper.set_deploy_fail('local') raise e finally: - helper.local(f'cd {REPOS_DIR} && rm -rf {rep.spug_version}') + helper.local_raw(f'cd {REPOS_DIR} && rm -rf {rep.spug_version}') close_old_connections() if alone_build: helper.clear() @@ -88,35 +89,38 @@ def _build(rep: Repository, helper, env): env.update(SPUG_DST_DIR=render_str(extend.dst_dir, env)) helper.send_success('local', '完成√\r\n') - if extend.hook_pre_server: - helper.send_info('local', '检出前任务...\r\n') - helper.local(f'cd {git_dir} && {extend.hook_pre_server}', env) + with Executor(env) as et: + helper.save_pid(et.pid, 'local') + if extend.hook_pre_server: + helper.send_info('local', '检出前任务...\r\n') + helper.local(et, f'cd {git_dir} && {extend.hook_pre_server}') - helper.send_info('local', '执行检出... ') - tree_ish = env.get('SPUG_GIT_COMMIT_ID') or env.get('SPUG_GIT_TAG') - command = f'cd {git_dir} && git archive --prefix={rep.spug_version}/ {tree_ish} | (cd .. && tar xf -)' - helper.local(command) - helper.send_success('local', '完成√\r\n') + helper.send_info('local', '执行检出... ') + tree_ish = env.get('SPUG_GIT_COMMIT_ID') or env.get('SPUG_GIT_TAG') + command = f'cd {git_dir} && git archive --prefix={rep.spug_version}/ {tree_ish} | (cd .. && tar xf -)' + helper.local_raw(command) + helper.send_success('local', '完成√\r\n') - if extend.hook_post_server: - helper.send_info('local', '检出后任务...\r\n') - helper.local(f'cd {build_dir} && {extend.hook_post_server}', env) + if extend.hook_post_server: + helper.send_info('local', '检出后任务...\r\n') + helper.local(et, f'cd {build_dir} && {extend.hook_post_server}') - helper.send_info('local', '执行打包... ') - filter_rule, exclude, contain = json.loads(extend.filter_rule), '', rep.spug_version - files = helper.parse_filter_rule(filter_rule['data'], env=env) - if files: - if filter_rule['type'] == 'exclude': - excludes = [] - for x in files: - if x.startswith('/'): - excludes.append(f'--exclude={rep.spug_version}{x}') - else: - excludes.append(f'--exclude={x}') - exclude = ' '.join(excludes) - else: - contain = ' '.join(f'{rep.spug_version}/{x}' for x in files) - helper.local(f'mkdir -p {BUILD_DIR} && cd {REPOS_DIR} && tar zcf {tar_file} {exclude} {contain}') - helper.send_success('local', '完成√\r\n') - human_time = human_seconds_time(time.time() - flag) - helper.send_success('local', f'\r\n** 构建成功,耗时:{human_time} **\r\n', status='success') + helper.send_info('local', '执行打包... ') + filter_rule, exclude, contain = json.loads(extend.filter_rule), '', rep.spug_version + files = helper.parse_filter_rule(filter_rule['data'], env=env) + if files: + if filter_rule['type'] == 'exclude': + excludes = [] + for x in files: + if x.startswith('/'): + excludes.append(f'--exclude={rep.spug_version}{x}') + else: + excludes.append(f'--exclude={x}') + exclude = ' '.join(excludes) + else: + contain = ' '.join(f'{rep.spug_version}/{x}' for x in files) + helper.local_raw(f'mkdir -p {BUILD_DIR} && cd {REPOS_DIR} && tar zcf {tar_file} {exclude} {contain}') + helper.send_success('local', '完成√\r\n') + human_time = human_seconds_time(time.time() - flag) + helper.send_success('local', f'\r\n** 构建成功,耗时:{human_time} **\r\n', status='success') + helper.set_cross_env(rep.spug_version, et.get_envs()) diff --git a/spug_api/libs/executor.py b/spug_api/libs/executor.py new file mode 100644 index 0000000..b0feda2 --- /dev/null +++ b/spug_api/libs/executor.py @@ -0,0 +1,73 @@ +from libs.utils import str_decode +import subprocess +import re +import os + + +class Executor: + def __init__(self, default_env=None): + self.regex = re.compile(r'(? { - for (let key of Object.keys(store.outputs)) { - if (outputs[key].status === -2) { - outputs[key].status = -1 - } - outputs[key].data += '\r\n\x1b[31mWebsocket connection failed!\x1b[0m' - term.write('\r\n\x1b[31mWebsocket connection failed!\x1b[0m') + const data = '\r\n\x1b[31mWebsocket connection failed!\x1b[0m' + for (let key of Object.keys(outputs)) { + if (key === gCurrent) term.write(data) + outputs[key].data += data } } return socket