A 新增可跨阶段的全局共享变量

spug_api/apps/deploy/ext1.py

@@ -29,6 +29,8 @@ def ext1_deploy(req, helper, env):
         )
         build_repository(rep, helper)
         req.repository = rep
+    env.update(SPUG_BUILD_ID=str(req.repository_id))
+    env.update(helper.get_cross_env(req.spug_version))
     extras = json.loads(req.extra)
     if extras[0] == 'repository':
         extras = extras[1:]

spug_api/apps/deploy/ext2.py

@@ -3,6 +3,7 @@
 # Released under the AGPL-3.0 License.
 from django.conf import settings
 from libs.utils import AttrDict, render_str, human_seconds_time
+from libs.executor import Executor
 from apps.host.models import Host
 from apps.deploy.helper import SpugError
 from concurrent import futures
@@ -38,48 +39,52 @@ def ext2_deploy(req, helper, env, with_local):
                 transfer_action = action
             break
 
-    if with_local:
-        helper.set_deploy_process('local')
-        helper.send_success('local', '', status='doing')
-        if server_actions or transfer_action:
-            helper.send_clear('local')
-        for action in server_actions:
-            helper.send_info('local', f'{action["title"]}...\r\n')
-            helper.local(f'cd /tmp && {action["data"]}', env)
-            step += 1
-        if transfer_action:
-            action = transfer_action
-            helper.send_info('local', '检测到来源为本地路径的数据传输动作，执行打包...   \r\n')
-            action['src'] = action['src'].rstrip('/ ')
-            action['dst'] = action['dst'].rstrip('/ ')
-            if not action['src'] or not action['dst']:
-                helper.send_error('local', f'Invalid path for transfer, src: {action["src"]} dst: {action["dst"]}')
-            if not os.path.exists(action['src']):
-                helper.send_error('local', f'No such file or directory: {action["src"]}')
-            is_dir, exclude = os.path.isdir(action['src']), ''
-            sp_dir, sd_dst = os.path.split(action['src'])
-            contain = sd_dst
-            if action['mode'] != '0' and is_dir:
-                files = helper.parse_filter_rule(action['rule'], ',', env)
-                if files:
-                    if action['mode'] == '1':
-                        contain = ' '.join(f'{sd_dst}/{x}' for x in files)
-                    else:
-                        excludes = []
-                        for x in files:
-                            if x.startswith('/'):
-                                excludes.append(f'--exclude={sd_dst}{x}')
-                            else:
-                                excludes.append(f'--exclude={x}')
-                        exclude = ' '.join(excludes)
-            tar_gz_file = os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, f'{req.spug_version}.tar.gz')
-            helper.local(f'cd {sp_dir} && tar -zcf {tar_gz_file} {exclude} {contain}')
-            helper.send_info('local', '打包完成\r\n')
-        helper.set_deploy_success('local')
-        human_time = human_seconds_time(time.time() - flag)
-        helper.send_success('local', f'\r\n** 执行完成，耗时：{human_time} **', status='success')
+    if with_local or True:
+        with Executor(env) as et:
+            helper.save_pid(et.pid, 'local')
+            helper.set_deploy_process('local')
+            helper.send_success('local', '', status='doing')
+            if server_actions or transfer_action:
+                helper.send_clear('local')
+            for action in server_actions:
+                helper.send_info('local', f'{action["title"]}...\r\n')
+                helper.local(et, f'cd /tmp && {action["data"]}')
+                step += 1
+            if transfer_action:
+                action = transfer_action
+                helper.send_info('local', '检测到来源为本地路径的数据传输动作，执行打包...   \r\n')
+                action['src'] = action['src'].rstrip('/ ')
+                action['dst'] = action['dst'].rstrip('/ ')
+                if not action['src'] or not action['dst']:
+                    helper.send_error('local', f'Invalid path for transfer, src: {action["src"]} dst: {action["dst"]}')
+                if not os.path.exists(action['src']):
+                    helper.send_error('local', f'No such file or directory: {action["src"]}')
+                is_dir, exclude = os.path.isdir(action['src']), ''
+                sp_dir, sd_dst = os.path.split(action['src'])
+                contain = sd_dst
+                if action['mode'] != '0' and is_dir:
+                    files = helper.parse_filter_rule(action['rule'], ',', env)
+                    if files:
+                        if action['mode'] == '1':
+                            contain = ' '.join(f'{sd_dst}/{x}' for x in files)
+                        else:
+                            excludes = []
+                            for x in files:
+                                if x.startswith('/'):
+                                    excludes.append(f'--exclude={sd_dst}{x}')
+                                else:
+                                    excludes.append(f'--exclude={x}')
+                            exclude = ' '.join(excludes)
+                tar_gz_file = os.path.join(REPOS_DIR, env.SPUG_DEPLOY_ID, f'{req.spug_version}.tar.gz')
+                helper.local_raw(f'cd {sp_dir} && tar -zcf {tar_gz_file} {exclude} {contain}')
+                helper.send_info('local', '打包完成\r\n')
+            helper.set_deploy_success('local')
+            human_time = human_seconds_time(time.time() - flag)
+            helper.send_success('local', f'\r\n** 执行完成，耗时：{human_time} **', status='success')
+            helper.set_cross_env(req.spug_version, et.get_envs())
 
     if host_actions:
+        env.update(helper.get_cross_env(req.spug_version))
         if req.deploy.is_parallel:
             threads, latest_exception = [], None
             max_workers = max(10, os.cpu_count() * 5)

spug_api/apps/deploy/helper.py

@@ -303,6 +303,18 @@ class Helper(NotifyMixin, KitMixin):
         self.files[key] = file
         return file
 
+    def get_cross_env(self, key):
+        file = os.path.join(settings.DEPLOY_DIR, key)
+        if os.path.exists(file):
+            with open(file, 'r') as f:
+                return json.loads(f.read())
+        return {}
+
+    def set_cross_env(self, key, envs):
+        file = os.path.join(settings.DEPLOY_DIR, key)
+        with open(file, 'w') as f:
+            f.write(json.dumps(envs))
+
     def add_callback(self, func):
         self.callback.append(func)
 
@@ -389,7 +401,14 @@ class Helper(NotifyMixin, KitMixin):
         self._send(key, '\r\n')
         return partial(func, key)
 
-    def local(self, command, env=None):
+    def local(self, executor, command):
+        code = -1
+        for code, out in executor.exec_command_with_stream(command):
+            self._send('local', out)
+        if code != 0:
+            self.send_error('local', f'exit code: {code}')
+
+    def local_raw(self, command, env=None):
         if env:
             env = dict(env.items())
             env.update(os.environ)
@@ -398,9 +417,7 @@ class Helper(NotifyMixin, KitMixin):
             env=env,
             shell=True,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            preexec_fn=os.setsid)
-        self.save_pid(task.pid, 'local')
+            stderr=subprocess.STDOUT)
         message = b''
         while True:
             output = task.stdout.read(1)

spug_api/apps/deploy/models.py

@@ -72,6 +72,10 @@ class DeployRequest(models.Model, ModelMixin):
                 p.unlink()
             except FileNotFoundError:
                 pass
+        try:
+            Path(settings.DEPLOY_DIR, self.spug_version).unlink()
+        except FileNotFoundError:
+            pass
         super().delete(using, keep_parents)
 
     def __repr__(self):

spug_api/apps/repository/utils.py

@@ -5,6 +5,7 @@ from django_redis import get_redis_connection
 from django.conf import settings
 from django.db import close_old_connections
 from libs.utils import AttrDict, human_datetime, render_str, human_seconds_time
+from libs.executor import Executor
 from apps.repository.models import Repository
 from apps.config.utils import compose_configs
 from apps.deploy.helper import Helper
@@ -69,7 +70,7 @@ def dispatch(rep: Repository, helper=None):
         helper.set_deploy_fail('local')
         raise e
     finally:
-        helper.local(f'cd {REPOS_DIR} && rm -rf {rep.spug_version}')
+        helper.local_raw(f'cd {REPOS_DIR} && rm -rf {rep.spug_version}')
         close_old_connections()
         if alone_build:
             helper.clear()
@@ -88,35 +89,38 @@ def _build(rep: Repository, helper, env):
     env.update(SPUG_DST_DIR=render_str(extend.dst_dir, env))
     helper.send_success('local', '完成√\r\n')
 
-    if extend.hook_pre_server:
-        helper.send_info('local', '检出前任务...\r\n')
-        helper.local(f'cd {git_dir} && {extend.hook_pre_server}', env)
+    with Executor(env) as et:
+        helper.save_pid(et.pid, 'local')
+        if extend.hook_pre_server:
+            helper.send_info('local', '检出前任务...\r\n')
+            helper.local(et, f'cd {git_dir} && {extend.hook_pre_server}')
 
-    helper.send_info('local', '执行检出...        ')
-    tree_ish = env.get('SPUG_GIT_COMMIT_ID') or env.get('SPUG_GIT_TAG')
-    command = f'cd {git_dir} && git archive --prefix={rep.spug_version}/ {tree_ish} | (cd .. && tar xf -)'
-    helper.local(command)
-    helper.send_success('local', '完成√\r\n')
+        helper.send_info('local', '执行检出...        ')
+        tree_ish = env.get('SPUG_GIT_COMMIT_ID') or env.get('SPUG_GIT_TAG')
+        command = f'cd {git_dir} && git archive --prefix={rep.spug_version}/ {tree_ish} | (cd .. && tar xf -)'
+        helper.local_raw(command)
+        helper.send_success('local', '完成√\r\n')
 
-    if extend.hook_post_server:
-        helper.send_info('local', '检出后任务...\r\n')
-        helper.local(f'cd {build_dir} && {extend.hook_post_server}', env)
+        if extend.hook_post_server:
+            helper.send_info('local', '检出后任务...\r\n')
+            helper.local(et, f'cd {build_dir} && {extend.hook_post_server}')
 
-    helper.send_info('local', '执行打包...        ')
-    filter_rule, exclude, contain = json.loads(extend.filter_rule), '', rep.spug_version
-    files = helper.parse_filter_rule(filter_rule['data'], env=env)
-    if files:
-        if filter_rule['type'] == 'exclude':
-            excludes = []
-            for x in files:
-                if x.startswith('/'):
-                    excludes.append(f'--exclude={rep.spug_version}{x}')
-                else:
-                    excludes.append(f'--exclude={x}')
-            exclude = ' '.join(excludes)
-        else:
-            contain = ' '.join(f'{rep.spug_version}/{x}' for x in files)
-    helper.local(f'mkdir -p {BUILD_DIR} && cd {REPOS_DIR} && tar zcf {tar_file} {exclude} {contain}')
-    helper.send_success('local', '完成√\r\n')
-    human_time = human_seconds_time(time.time() - flag)
-    helper.send_success('local', f'\r\n** 构建成功，耗时：{human_time} **\r\n', status='success')
+        helper.send_info('local', '执行打包...        ')
+        filter_rule, exclude, contain = json.loads(extend.filter_rule), '', rep.spug_version
+        files = helper.parse_filter_rule(filter_rule['data'], env=env)
+        if files:
+            if filter_rule['type'] == 'exclude':
+                excludes = []
+                for x in files:
+                    if x.startswith('/'):
+                        excludes.append(f'--exclude={rep.spug_version}{x}')
+                    else:
+                        excludes.append(f'--exclude={x}')
+                exclude = ' '.join(excludes)
+            else:
+                contain = ' '.join(f'{rep.spug_version}/{x}' for x in files)
+        helper.local_raw(f'mkdir -p {BUILD_DIR} && cd {REPOS_DIR} && tar zcf {tar_file} {exclude} {contain}')
+        helper.send_success('local', '完成√\r\n')
+        human_time = human_seconds_time(time.time() - flag)
+        helper.send_success('local', f'\r\n** 构建成功，耗时：{human_time} **\r\n', status='success')
+        helper.set_cross_env(rep.spug_version, et.get_envs())

spug_api/libs/executor.py

@@ -0,0 +1,73 @@
+from libs.utils import str_decode
+import subprocess
+import re
+import os
+
+
+class Executor:
+    def __init__(self, default_env=None):
+        self.regex = re.compile(r'(?<!echo )Spug EOF 2108111926 (-?\d+)[\r\n]?')
+        self.default_env = default_env
+        self.task = None
+
+    @property
+    def pid(self):
+        if self.task:
+            return self.task.pid
+        return None
+
+    def get_envs(self):
+        envs = {}
+        _, output = self.exec_command('env | grep SPUG_GEV_')
+        if output:
+            for item in output.splitlines():
+                if '=' in item:
+                    key, val = item.split('=', 1)
+                    envs[key] = val
+        return envs
+
+    def exec_command_with_stream(self, command):
+        command += '\necho Spug EOF 2108111926 $?\n'
+        self.task.stdin.write(command.encode())
+        self.task.stdin.flush()
+        exit_code, message,  = -1, b''
+        while True:
+            output = self.task.stdout.read(1)
+            if not output:
+                exit_code, message = self.task.wait(), ''
+                break
+            if output in (b'\r', b'\n'):
+                message += b'\r\n' if output == b'\n' else b'\r'
+                message = str_decode(message)
+                match = self.regex.search(message)
+                if match:
+                    exit_code = int(match.group(1))
+                    message = message[:match.start()]
+                    break
+                yield exit_code, message
+                message = b''
+            else:
+                message += output
+        yield exit_code, message
+
+    def exec_command(self, command):
+        exit_code, message = -1, ''
+        for exit_code, line in self.exec_command_with_stream(command):
+            message += line
+        return exit_code, message
+
+    def __enter__(self):
+        self.task = subprocess.Popen(
+            'bash',
+            env=self.default_env,
+            shell=True,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            preexec_fn=os.setsid
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.task:
+            self.task.kill()

spug_api/libs/ssh.py

@@ -61,7 +61,7 @@ class SSH:
         self.pid = None
         self.eof = 'Spug EOF 2108111926'
         self.default_env = default_env
-        self.regex = re.compile(r'Spug EOF 2108111926 (-?\d+)[\r\n]?')
+        self.regex = re.compile(r'(?<!echo )Spug EOF 2108111926 (-?\d+)[\r\n]?')
         self.arguments = {
             'hostname': hostname,
             'port': port,

spug_web/src/pages/deploy/request/Console.js

@@ -124,12 +124,10 @@ function Console(props) {
       }
     }
     socket.onerror = () => {
-      for (let key of Object.keys(store.outputs)) {
-        if (outputs[key].status === -2) {
-          outputs[key].status = -1
-        }
-        outputs[key].data += '\r\n\x1b[31mWebsocket connection failed!\x1b[0m'
-        term.write('\r\n\x1b[31mWebsocket connection failed!\x1b[0m')
+      const data = '\r\n\x1b[31mWebsocket connection failed!\x1b[0m'
+      for (let key of Object.keys(outputs)) {
+        if (key === gCurrent) term.write(data)
+        outputs[key].data += data
       }
     }
     return socket