mirror of https://github.com/openspug/spug
A 新增可跨阶段的全局共享变量
parent
359989af4a
commit
48a0348427
|
@ -29,6 +29,8 @@ def ext1_deploy(req, helper, env):
|
|||
)
|
||||
build_repository(rep, helper)
|
||||
req.repository = rep
|
||||
env.update(SPUG_BUILD_ID=str(req.repository_id))
|
||||
env.update(helper.get_cross_env(req.spug_version))
|
||||
extras = json.loads(req.extra)
|
||||
if extras[0] == 'repository':
|
||||
extras = extras[1:]
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
# Released under the AGPL-3.0 License.
|
||||
from django.conf import settings
|
||||
from libs.utils import AttrDict, render_str, human_seconds_time
|
||||
from libs.executor import Executor
|
||||
from apps.host.models import Host
|
||||
from apps.deploy.helper import SpugError
|
||||
from concurrent import futures
|
||||
|
@ -38,48 +39,52 @@ def ext2_deploy(req, helper, env, with_local):
|
|||
transfer_action = action
|
||||
break
|
||||
|
||||
if with_local:
|
||||
helper.set_deploy_process('local')
|
||||
helper.send_success('local', '', status='doing')
|
||||
if server_actions or transfer_action:
|
||||
helper.send_clear('local')
|
||||
for action in server_actions:
|
||||
helper.send_info('local', f'{action["title"]}...\r\n')
|
||||
helper.local(f'cd /tmp && {action["data"]}', env)
|
||||
step += 1
|
||||
if transfer_action:
|
||||
action = transfer_action
|
||||
helper.send_info('local', '检测到来源为本地路径的数据传输动作,执行打包... \r\n')
|
||||
action['src'] = action['src'].rstrip('/ ')
|
||||
action['dst'] = action['dst'].rstrip('/ ')
|
||||
if not action['src'] or not action['dst']:
|
||||
helper.send_error('local', f'Invalid path for transfer, src: {action["src"]} dst: {action["dst"]}')
|
||||
if not os.path.exists(action['src']):
|
||||
helper.send_error('local', f'No such file or directory: {action["src"]}')
|
||||
is_dir, exclude = os.path.isdir(action['src']), ''
|
||||
sp_dir, sd_dst = os.path.split(action['src'])
|
||||
contain = sd_dst
|
||||
if action['mode'] != '0' and is_dir:
|
||||
files = helper.parse_filter_rule(action['rule'], ',', env)
|
||||
if files:
|
||||
if action['mode'] == '1':
|
||||
contain = ' '.join(f'{sd_dst}/{x}' for x in files)
|
||||
else:
|
||||
excludes = []
|
||||
for x in files:
|
||||
if x.startswith('/'):
|
||||
excludes.append(f'--exclude={sd_dst}{x}')
|
||||
else:
|
||||
excludes.append(f'--exclude={x}')
|
||||
exclude = ' '.join(excludes)
|
||||
tar_gz_file = os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, f'{req.spug_version}.tar.gz')
|
||||
helper.local(f'cd {sp_dir} && tar -zcf {tar_gz_file} {exclude} {contain}')
|
||||
helper.send_info('local', '打包完成\r\n')
|
||||
helper.set_deploy_success('local')
|
||||
human_time = human_seconds_time(time.time() - flag)
|
||||
helper.send_success('local', f'\r\n** 执行完成,耗时:{human_time} **', status='success')
|
||||
if with_local or True:
|
||||
with Executor(env) as et:
|
||||
helper.save_pid(et.pid, 'local')
|
||||
helper.set_deploy_process('local')
|
||||
helper.send_success('local', '', status='doing')
|
||||
if server_actions or transfer_action:
|
||||
helper.send_clear('local')
|
||||
for action in server_actions:
|
||||
helper.send_info('local', f'{action["title"]}...\r\n')
|
||||
helper.local(et, f'cd /tmp && {action["data"]}')
|
||||
step += 1
|
||||
if transfer_action:
|
||||
action = transfer_action
|
||||
helper.send_info('local', '检测到来源为本地路径的数据传输动作,执行打包... \r\n')
|
||||
action['src'] = action['src'].rstrip('/ ')
|
||||
action['dst'] = action['dst'].rstrip('/ ')
|
||||
if not action['src'] or not action['dst']:
|
||||
helper.send_error('local', f'Invalid path for transfer, src: {action["src"]} dst: {action["dst"]}')
|
||||
if not os.path.exists(action['src']):
|
||||
helper.send_error('local', f'No such file or directory: {action["src"]}')
|
||||
is_dir, exclude = os.path.isdir(action['src']), ''
|
||||
sp_dir, sd_dst = os.path.split(action['src'])
|
||||
contain = sd_dst
|
||||
if action['mode'] != '0' and is_dir:
|
||||
files = helper.parse_filter_rule(action['rule'], ',', env)
|
||||
if files:
|
||||
if action['mode'] == '1':
|
||||
contain = ' '.join(f'{sd_dst}/{x}' for x in files)
|
||||
else:
|
||||
excludes = []
|
||||
for x in files:
|
||||
if x.startswith('/'):
|
||||
excludes.append(f'--exclude={sd_dst}{x}')
|
||||
else:
|
||||
excludes.append(f'--exclude={x}')
|
||||
exclude = ' '.join(excludes)
|
||||
tar_gz_file = os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, f'{req.spug_version}.tar.gz')
|
||||
helper.local_raw(f'cd {sp_dir} && tar -zcf {tar_gz_file} {exclude} {contain}')
|
||||
helper.send_info('local', '打包完成\r\n')
|
||||
helper.set_deploy_success('local')
|
||||
human_time = human_seconds_time(time.time() - flag)
|
||||
helper.send_success('local', f'\r\n** 执行完成,耗时:{human_time} **', status='success')
|
||||
helper.set_cross_env(req.spug_version, et.get_envs())
|
||||
|
||||
if host_actions:
|
||||
env.update(helper.get_cross_env(req.spug_version))
|
||||
if req.deploy.is_parallel:
|
||||
threads, latest_exception = [], None
|
||||
max_workers = max(10, os.cpu_count() * 5)
|
||||
|
|
|
@ -303,6 +303,18 @@ class Helper(NotifyMixin, KitMixin):
|
|||
self.files[key] = file
|
||||
return file
|
||||
|
||||
def get_cross_env(self, key):
|
||||
file = os.path.join(settings.DEPLOY_DIR, key)
|
||||
if os.path.exists(file):
|
||||
with open(file, 'r') as f:
|
||||
return json.loads(f.read())
|
||||
return {}
|
||||
|
||||
def set_cross_env(self, key, envs):
|
||||
file = os.path.join(settings.DEPLOY_DIR, key)
|
||||
with open(file, 'w') as f:
|
||||
f.write(json.dumps(envs))
|
||||
|
||||
def add_callback(self, func):
|
||||
self.callback.append(func)
|
||||
|
||||
|
@ -389,7 +401,14 @@ class Helper(NotifyMixin, KitMixin):
|
|||
self._send(key, '\r\n')
|
||||
return partial(func, key)
|
||||
|
||||
def local(self, command, env=None):
|
||||
def local(self, executor, command):
|
||||
code = -1
|
||||
for code, out in executor.exec_command_with_stream(command):
|
||||
self._send('local', out)
|
||||
if code != 0:
|
||||
self.send_error('local', f'exit code: {code}')
|
||||
|
||||
def local_raw(self, command, env=None):
|
||||
if env:
|
||||
env = dict(env.items())
|
||||
env.update(os.environ)
|
||||
|
@ -398,9 +417,7 @@ class Helper(NotifyMixin, KitMixin):
|
|||
env=env,
|
||||
shell=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
preexec_fn=os.setsid)
|
||||
self.save_pid(task.pid, 'local')
|
||||
stderr=subprocess.STDOUT)
|
||||
message = b''
|
||||
while True:
|
||||
output = task.stdout.read(1)
|
||||
|
|
|
@ -72,6 +72,10 @@ class DeployRequest(models.Model, ModelMixin):
|
|||
p.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
try:
|
||||
Path(settings.DEPLOY_DIR, self.spug_version).unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
super().delete(using, keep_parents)
|
||||
|
||||
def __repr__(self):
|
||||
|
|
|
@ -5,6 +5,7 @@ from django_redis import get_redis_connection
|
|||
from django.conf import settings
|
||||
from django.db import close_old_connections
|
||||
from libs.utils import AttrDict, human_datetime, render_str, human_seconds_time
|
||||
from libs.executor import Executor
|
||||
from apps.repository.models import Repository
|
||||
from apps.config.utils import compose_configs
|
||||
from apps.deploy.helper import Helper
|
||||
|
@ -69,7 +70,7 @@ def dispatch(rep: Repository, helper=None):
|
|||
helper.set_deploy_fail('local')
|
||||
raise e
|
||||
finally:
|
||||
helper.local(f'cd {REPOS_DIR} && rm -rf {rep.spug_version}')
|
||||
helper.local_raw(f'cd {REPOS_DIR} && rm -rf {rep.spug_version}')
|
||||
close_old_connections()
|
||||
if alone_build:
|
||||
helper.clear()
|
||||
|
@ -88,35 +89,38 @@ def _build(rep: Repository, helper, env):
|
|||
env.update(SPUG_DST_DIR=render_str(extend.dst_dir, env))
|
||||
helper.send_success('local', '完成√\r\n')
|
||||
|
||||
if extend.hook_pre_server:
|
||||
helper.send_info('local', '检出前任务...\r\n')
|
||||
helper.local(f'cd {git_dir} && {extend.hook_pre_server}', env)
|
||||
with Executor(env) as et:
|
||||
helper.save_pid(et.pid, 'local')
|
||||
if extend.hook_pre_server:
|
||||
helper.send_info('local', '检出前任务...\r\n')
|
||||
helper.local(et, f'cd {git_dir} && {extend.hook_pre_server}')
|
||||
|
||||
helper.send_info('local', '执行检出... ')
|
||||
tree_ish = env.get('SPUG_GIT_COMMIT_ID') or env.get('SPUG_GIT_TAG')
|
||||
command = f'cd {git_dir} && git archive --prefix={rep.spug_version}/ {tree_ish} | (cd .. && tar xf -)'
|
||||
helper.local(command)
|
||||
helper.send_success('local', '完成√\r\n')
|
||||
helper.send_info('local', '执行检出... ')
|
||||
tree_ish = env.get('SPUG_GIT_COMMIT_ID') or env.get('SPUG_GIT_TAG')
|
||||
command = f'cd {git_dir} && git archive --prefix={rep.spug_version}/ {tree_ish} | (cd .. && tar xf -)'
|
||||
helper.local_raw(command)
|
||||
helper.send_success('local', '完成√\r\n')
|
||||
|
||||
if extend.hook_post_server:
|
||||
helper.send_info('local', '检出后任务...\r\n')
|
||||
helper.local(f'cd {build_dir} && {extend.hook_post_server}', env)
|
||||
if extend.hook_post_server:
|
||||
helper.send_info('local', '检出后任务...\r\n')
|
||||
helper.local(et, f'cd {build_dir} && {extend.hook_post_server}')
|
||||
|
||||
helper.send_info('local', '执行打包... ')
|
||||
filter_rule, exclude, contain = json.loads(extend.filter_rule), '', rep.spug_version
|
||||
files = helper.parse_filter_rule(filter_rule['data'], env=env)
|
||||
if files:
|
||||
if filter_rule['type'] == 'exclude':
|
||||
excludes = []
|
||||
for x in files:
|
||||
if x.startswith('/'):
|
||||
excludes.append(f'--exclude={rep.spug_version}{x}')
|
||||
else:
|
||||
excludes.append(f'--exclude={x}')
|
||||
exclude = ' '.join(excludes)
|
||||
else:
|
||||
contain = ' '.join(f'{rep.spug_version}/{x}' for x in files)
|
||||
helper.local(f'mkdir -p {BUILD_DIR} && cd {REPOS_DIR} && tar zcf {tar_file} {exclude} {contain}')
|
||||
helper.send_success('local', '完成√\r\n')
|
||||
human_time = human_seconds_time(time.time() - flag)
|
||||
helper.send_success('local', f'\r\n** 构建成功,耗时:{human_time} **\r\n', status='success')
|
||||
helper.send_info('local', '执行打包... ')
|
||||
filter_rule, exclude, contain = json.loads(extend.filter_rule), '', rep.spug_version
|
||||
files = helper.parse_filter_rule(filter_rule['data'], env=env)
|
||||
if files:
|
||||
if filter_rule['type'] == 'exclude':
|
||||
excludes = []
|
||||
for x in files:
|
||||
if x.startswith('/'):
|
||||
excludes.append(f'--exclude={rep.spug_version}{x}')
|
||||
else:
|
||||
excludes.append(f'--exclude={x}')
|
||||
exclude = ' '.join(excludes)
|
||||
else:
|
||||
contain = ' '.join(f'{rep.spug_version}/{x}' for x in files)
|
||||
helper.local_raw(f'mkdir -p {BUILD_DIR} && cd {REPOS_DIR} && tar zcf {tar_file} {exclude} {contain}')
|
||||
helper.send_success('local', '完成√\r\n')
|
||||
human_time = human_seconds_time(time.time() - flag)
|
||||
helper.send_success('local', f'\r\n** 构建成功,耗时:{human_time} **\r\n', status='success')
|
||||
helper.set_cross_env(rep.spug_version, et.get_envs())
|
||||
|
|
|
@ -0,0 +1,73 @@
|
|||
from libs.utils import str_decode
|
||||
import subprocess
|
||||
import re
|
||||
import os
|
||||
|
||||
|
||||
class Executor:
|
||||
def __init__(self, default_env=None):
|
||||
self.regex = re.compile(r'(?<!echo )Spug EOF 2108111926 (-?\d+)[\r\n]?')
|
||||
self.default_env = default_env
|
||||
self.task = None
|
||||
|
||||
@property
|
||||
def pid(self):
|
||||
if self.task:
|
||||
return self.task.pid
|
||||
return None
|
||||
|
||||
def get_envs(self):
|
||||
envs = {}
|
||||
_, output = self.exec_command('env | grep SPUG_GEV_')
|
||||
if output:
|
||||
for item in output.splitlines():
|
||||
if '=' in item:
|
||||
key, val = item.split('=', 1)
|
||||
envs[key] = val
|
||||
return envs
|
||||
|
||||
def exec_command_with_stream(self, command):
|
||||
command += '\necho Spug EOF 2108111926 $?\n'
|
||||
self.task.stdin.write(command.encode())
|
||||
self.task.stdin.flush()
|
||||
exit_code, message, = -1, b''
|
||||
while True:
|
||||
output = self.task.stdout.read(1)
|
||||
if not output:
|
||||
exit_code, message = self.task.wait(), ''
|
||||
break
|
||||
if output in (b'\r', b'\n'):
|
||||
message += b'\r\n' if output == b'\n' else b'\r'
|
||||
message = str_decode(message)
|
||||
match = self.regex.search(message)
|
||||
if match:
|
||||
exit_code = int(match.group(1))
|
||||
message = message[:match.start()]
|
||||
break
|
||||
yield exit_code, message
|
||||
message = b''
|
||||
else:
|
||||
message += output
|
||||
yield exit_code, message
|
||||
|
||||
def exec_command(self, command):
|
||||
exit_code, message = -1, ''
|
||||
for exit_code, line in self.exec_command_with_stream(command):
|
||||
message += line
|
||||
return exit_code, message
|
||||
|
||||
def __enter__(self):
|
||||
self.task = subprocess.Popen(
|
||||
'bash',
|
||||
env=self.default_env,
|
||||
shell=True,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
preexec_fn=os.setsid
|
||||
)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.task:
|
||||
self.task.kill()
|
|
@ -61,7 +61,7 @@ class SSH:
|
|||
self.pid = None
|
||||
self.eof = 'Spug EOF 2108111926'
|
||||
self.default_env = default_env
|
||||
self.regex = re.compile(r'Spug EOF 2108111926 (-?\d+)[\r\n]?')
|
||||
self.regex = re.compile(r'(?<!echo )Spug EOF 2108111926 (-?\d+)[\r\n]?')
|
||||
self.arguments = {
|
||||
'hostname': hostname,
|
||||
'port': port,
|
||||
|
|
|
@ -124,12 +124,10 @@ function Console(props) {
|
|||
}
|
||||
}
|
||||
socket.onerror = () => {
|
||||
for (let key of Object.keys(store.outputs)) {
|
||||
if (outputs[key].status === -2) {
|
||||
outputs[key].status = -1
|
||||
}
|
||||
outputs[key].data += '\r\n\x1b[31mWebsocket connection failed!\x1b[0m'
|
||||
term.write('\r\n\x1b[31mWebsocket connection failed!\x1b[0m')
|
||||
const data = '\r\n\x1b[31mWebsocket connection failed!\x1b[0m'
|
||||
for (let key of Object.keys(outputs)) {
|
||||
if (key === gCurrent) term.write(data)
|
||||
outputs[key].data += data
|
||||
}
|
||||
}
|
||||
return socket
|
||||
|
|
Loading…
Reference in New Issue