update pipeline module

spug_api/apps/pipeline/helper.py

@@ -0,0 +1,212 @@
+# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug
+# Copyright: (c) <spug.dev@gmail.com>
+# Released under the AGPL-3.0 License.
+from django.conf import settings
+from django.template.defaultfilters import filesizeformat
+from django_redis import get_redis_connection
+from libs.utils import SpugError, human_time, render_str, human_seconds_time
+from apps.config.utils import update_config_by_var
+from collections import defaultdict
+import time
+import json
+import os
+import re
+
+
+class KitMixin:
+    regex = re.compile(r'^((\r\n)*)(.*?)((\r\n)*)$', re.DOTALL)
+
+    @classmethod
+    def term_message(cls, message, color_mode='info', with_time=False):
+        prefix = f'{human_time()} ' if with_time else ''
+        if color_mode == 'info':
+            mode = '36m'
+        elif color_mode == 'warn':
+            mode = '33m'
+        elif color_mode == 'error':
+            mode = '31m'
+        elif color_mode == 'success':
+            mode = '32m'
+        else:
+            raise TypeError
+
+        return cls.regex.sub(fr'\1\033[{mode}{prefix}\3\033[0m\4', message)
+
+
+class Helper(KitMixin):
+    def __init__(self, rds, rds_key):
+        self.rds = rds
+        self.rds_key = rds_key
+        self.buffers = defaultdict(str)
+        self.flags = defaultdict(bool)
+        self.already_clear = False
+        self.files = {}
+
+    def __del__(self):
+        self.clear()
+
+    @classmethod
+    def make(cls, rds, rds_key):
+        rds.delete(rds_key)
+        return cls(rds, rds_key)
+
+    @classmethod
+    def fill_outputs(cls, outputs, deploy_key):
+        rds = get_redis_connection()
+        key_ttl = rds.ttl(deploy_key)
+        counter, hit_keys = 0, set()
+        if key_ttl > 30 or key_ttl == -1:
+            data = rds.lrange(deploy_key, counter, counter + 9)
+            while data:
+                for item in data:
+                    counter += 1
+                    item = json.loads(item.decode())
+                    key = item['key']
+                    if key in outputs:
+                        hit_keys.add(key)
+                        if 'data' in item:
+                            outputs[key]['data'] += item['data']
+                        if 'status' in item:
+                            outputs[key]['status'] = item['status']
+                data = rds.lrange(deploy_key, counter, counter + 9)
+
+        for key in outputs.keys():
+            if key in hit_keys:
+                continue
+            file_name = os.path.join(settings.DEPLOY_DIR, f'{deploy_key}:{key}')
+            if not os.path.exists(file_name):
+                continue
+            with open(file_name, newline='\r\n') as f:
+                line = f.readline()
+                while line:
+                    status, data = line.split(',', 1)
+                    if data:
+                        outputs[key]['data'] += data
+                    if status:
+                        outputs[key]['status'] = status
+                    line = f.readline()
+        return counter
+
+    def get_file(self, key):
+        if key in self.files:
+            return self.files[key]
+        file = open(os.path.join(settings.DEPLOY_DIR, f'{self.rds_key}:{key}'), 'w')
+        self.files[key] = file
+        return file
+
+    def get_cross_env(self, key):
+        file = os.path.join(settings.DEPLOY_DIR, key)
+        if os.path.exists(file):
+            with open(file, 'r') as f:
+                return json.loads(f.read())
+        return {}
+
+    def set_cross_env(self, key, envs):
+        file_envs = {}
+        for k, v in envs.items():
+            if k == 'SPUG_SET':
+                try:
+                    update_config_by_var(v)
+                except SpugError as e:
+                    self.send_error(key, f'{e}')
+            elif k.startswith('SPUG_GEV_'):
+                file_envs[k] = v
+
+        file = os.path.join(settings.DEPLOY_DIR, key)
+        with open(file, 'w') as f:
+            f.write(json.dumps(file_envs))
+
+    def save_pid(self, pid, key):
+        self.rds.set(f'PID:{self.rds_key}:{key}', pid, 3600)
+
+    def parse_filter_rule(self, data: str, sep='\n', env=None):
+        data, files = data.strip(), []
+        if data:
+            for line in data.split(sep):
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    files.append(render_str(line, env))
+        return files
+
+    def send(self, key, data, *, status=''):
+        message = {'key': key, 'data': data}
+        if status:
+            message['status'] = status
+        print(self.rds_key, message)
+        self.rds.rpush(self.rds_key, json.dumps(message))
+
+        file = self.get_file(key)
+        for idx, line in enumerate(data.split('\r\n')):
+            if idx != 0:
+                tmp = [status, self.buffers[key] + '\r\n']
+                file.write(','.join(tmp))
+                file.flush()
+                self.buffers[key] = ''
+                self.flags[key] = False
+            if line:
+                for idx2, item in enumerate(line.split('\r')):
+                    if idx2 != 0:
+                        self.flags[key] = True
+                    if item:
+                        if self.flags[key]:
+                            self.buffers[key] = item
+                            self.flags[key] = False
+                        else:
+                            self.buffers[key] += item
+
+    def send_clear(self, key):
+        self.send(key, '\033[2J\033[3J\033[1;1H')
+
+    def send_info(self, key, message, status='', with_time=True):
+        message = self.term_message(message, 'info', with_time)
+        self.send(key, message, status=status)
+
+    def send_warn(self, key, message, status=''):
+        message = self.term_message(message, 'warn')
+        self.send(key, message, status=status)
+
+    def send_success(self, key, message, with_time=True, start_time=None):
+        if start_time:
+            message += f', 耗时: {human_seconds_time(time.time() - start_time)}'
+        message = self.term_message(f'\r\n** {message} **', 'success', with_time)
+        self.send(key, message, status='success')
+
+    def send_error(self, key, message, with_break=False):
+        message = self.term_message(message, 'error')
+        if not message.endswith('\r\n'):
+            message += '\r\n'
+        self.send(key, message, status='error')
+        if with_break:
+            raise SpugError
+
+    def send_status(self, key, status):
+        self.send(key, '', status=status)
+
+    def clear(self):
+        if self.already_clear:
+            return
+        self.already_clear = True
+        for key, value in self.buffers.items():
+            if value:
+                file = self.get_file(key)
+                file.write(f',{value}')
+        for file in self.files.values():
+            file.close()
+        if self.rds.ttl(self.rds_key) == -1:
+            self.rds.expire(self.rds_key, 60)
+
+    def progress_callback(self, key):
+        def func(n, t):
+            message = f'\r         {filesizeformat(n):<8}/{filesizeformat(t):>8}  '
+            self.send(key, message)
+
+        self.send(key, '\r\n')
+        return func
+
+    def remote_exec(self, key, ssh, command, env=None):
+        code = -1
+        for code, out in ssh.exec_command_with_stream(command, environment=env):
+            self.send(key, out)
+        if code != 0:
+            self.send_error(key, f'exit code: {code}')
+        return code == 0

spug_api/apps/pipeline/models.py

@@ -2,6 +2,7 @@
 # Copyright: (c) <spug.dev@gmail.com>
 # Released under the AGPL-3.0 License.
 from django.db import models
+from django.conf import settings
 from libs.mixins import ModelMixin
 from apps.account.models import User
 import json
@@ -18,6 +19,25 @@ class Pipeline(models.Model, ModelMixin):
         tmp['nodes'] = json.loads(self.nodes)
         return tmp
 
+    def to_list(self):
+        tmp = self.to_dict(selects=('id', 'name', 'created_at'))
+        return tmp
+
     class Meta:
         db_table = 'pipelines'
         ordering = ('-id',)
+
+
+class PipeHistory(models.Model, ModelMixin):
+    pipeline = models.ForeignKey(Pipeline, on_delete=models.CASCADE)
+    ordinal = models.IntegerField()
+    created_by = models.ForeignKey(User, on_delete=models.PROTECT)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    @property
+    def deploy_key(self):
+        return f'{settings.PIPELINE_KEY}:{self.id}'
+
+    class Meta:
+        db_table = 'pipeline_histories'
+        ordering = ('-id',)

spug_api/apps/pipeline/urls.py

@@ -3,8 +3,9 @@
 # Released under the AGPL-3.0 License.
 from django.urls import path
 
-from apps.pipeline.views import PipeView
+from apps.pipeline.views import PipeView, DoView
 
 urlpatterns = [
     path('', PipeView.as_view()),
+    path('do/', DoView.as_view()),
 ]

spug_api/apps/pipeline/utils.py

@@ -0,0 +1,86 @@
+# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug
+# Copyright: (c) <spug.dev@gmail.com>
+# Released under the AGPL-3.0 License.
+from apps.credential.models import Credential
+from apps.host.models import Host
+from libs.utils import AttrDict, human_seconds_time
+from libs.gitlib import RemoteGit
+from apps.pipeline.helper import Helper
+from functools import partial, partialmethod
+from threading import Thread
+from concurrent import futures
+import time
+import os
+
+
+class NodeExecutor:
+    def __init__(self, rds, rds_key, nodes):
+        self.rds = rds
+        self.rds_key = rds_key
+        self.nodes = {x.id: x for x in map(AttrDict, nodes)}
+        self.node = AttrDict(nodes[0])
+        self.helper = Helper.make(self.rds, self.rds_key)
+
+    def run(self, node=None, state=None):
+        print(node, state)
+        if node:
+            downstream = getattr(node, 'downstream', [])
+            down_nodes = [self.nodes[x] for x in downstream]
+            available_nodes = [x for x in down_nodes if x.condition in (state, 'always')]
+            if len(available_nodes) >= 2:
+                for node in available_nodes[1:]:
+                    Thread(target=self._dispatch, args=(node,)).start()
+            if available_nodes:
+                self._dispatch(available_nodes[0])
+        else:
+            self._dispatch(self.node)
+
+    def _dispatch(self, node):
+        print('!!!!! _dispatch', node.name)
+        if node.module == 'build':
+            self._do_build(node)
+        elif node.module == 'ssh_exec':
+            self._do_ssh_exec(node)
+
+    def _do_build(self, node, marker=None):
+        # if node.mode == 'branch':
+        #     marker = node.branch
+        timestamp = time.time()
+        marker = 'origin/bugfix'
+        host = Host.objects.get(pk=node.target)
+        credential = None
+        if node.credential_id:
+            credential = Credential.objects.get(pk=node.credential_id)
+        with RemoteGit(host, node.git_url, node.workspace, credential) as git:
+            self.helper.send_info(node.id, '同步并检出Git仓库\r\n', 'processing')
+            git.set_remote_exec(partial(self.helper.remote_exec, node.id))
+            is_success = git.checkout(marker)
+            if is_success and node.command:
+                self.helper.send_info(node.id, '执行构建命令\r\n')
+                is_success = self.helper.remote_exec(node.id, git.ssh, node.command)
+            if is_success:
+                self.helper.send_success(node.id, '构建完成', start_time=timestamp)
+        self.run(node, 'success' if is_success else 'error')
+
+    def _do_ssh_exec(self, node):
+        threads = []
+        max_workers = max(10, os.cpu_count() * 5)
+        self.helper.send_status(node.id, 'processing')
+        with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            for host in Host.objects.filter(id__in=node.targets):
+                t = executor.submit(self._ssh_exec, host, node)
+                threads.append(t)
+            results = [x.result() for x in futures.as_completed(threads)]
+        state = 'success' if all(results) else 'error'
+        self.helper.send_status(node.id, state)
+        self.run(node, state)
+
+    def _ssh_exec(self, host, node):
+        timestamp = time.time()
+        key = f'{node.id}.{host.id}'
+        self.helper.send_info(key, '开始执行\r\n', 'processing')
+        with host.get_ssh() as ssh:
+            is_success = self.helper.remote_exec(key, ssh, node.command)
+        if is_success:
+            self.helper.send_success(key, '执行结束', start_time=timestamp)
+        return is_success

spug_api/apps/pipeline/views.py

@@ -2,8 +2,14 @@
 # Copyright: (c) <spug.dev@gmail.com>
 # Released under the AGPL-3.0 License.
 from django.views.generic import View
+from django_redis import get_redis_connection
 from libs import JsonParser, Argument, json_response, auth
-from apps.pipeline.models import Pipeline
+from libs.utils import AttrDict
+from apps.pipeline.models import Pipeline, PipeHistory
+from apps.pipeline.utils import NodeExecutor
+from apps.host.models import Host
+from threading import Thread
+from uuid import uuid4
 import json
 
 
@@ -57,3 +63,58 @@ class PipeView(View):
         if error is None:
             Pipeline.objects.filter(pk=form.id).delete()
         return json_response(error=error)
+
+
+class DoView(View):
+    @auth('exec.task.do')
+    def get(self, request):
+        pass
+
+    @auth('exec.task.do')
+    def post(self, request):
+        form, error = JsonParser(
+            Argument('id', type=int, help='参数错误'),
+        ).parse(request.body)
+        if error is None:
+            pipe = Pipeline.objects.get(pk=form.id)
+            latest_history = pipe.pipehistory_set.first()
+            ordinal = latest_history.ordinal + 1 if latest_history else 1
+            history = PipeHistory.objects.create(pipeline=pipe, ordinal=ordinal, created_by=request.user)
+            nodes, ids = json.loads(pipe.nodes), set()
+            for item in filter(lambda x: x['module'] == 'ssh_exec', nodes):
+                ids.update(item['targets'])
+
+            host_map = {x.id: f'{x.name}({x.hostname})' for x in Host.objects.filter(id__in=ids)}
+            for item in filter(lambda x: x['module'] == 'ssh_exec', nodes):
+                item['targets'] = [{'id': x, 'name': host_map[x]} for x in item['targets']]
+
+            rds = get_redis_connection()
+            executor = NodeExecutor(rds, history.deploy_key, json.loads(pipe.nodes))
+            Thread(target=executor.run).start()
+
+            response = AttrDict(token=history.id, nodes=nodes)
+            return json_response(response)
+        return json_response(error=error)
+
+    @auth('exec.task.do')
+    def patch(self, request):
+        form, error = JsonParser(
+            Argument('id', type=int, help='参数错误'),
+            Argument('cols', type=int, required=False),
+            Argument('rows', type=int, required=False)
+        ).parse(request.body)
+        if error is None:
+            term = None
+            if form.cols and form.rows:
+                term = {'width': form.cols, 'height': form.rows}
+            pipe = Pipeline.objects.get(pk=form.id)
+            latest_history = pipe.pipehistory_set.first()
+            ordinal = latest_history.ordinal + 1 if latest_history else 1
+            history = PipeHistory.objects.create(pipeline=pipe, ordinal=ordinal, created_by=request.user)
+            rds = get_redis_connection()
+            nodes = json.loads(pipe.nodes)
+            executor = NodeExecutor(rds, history.deploy_key, nodes)
+            Thread(target=executor.run).start()
+            response = AttrDict(token=history.id, nodes=nodes)
+            return json_response(response)
+        return json_response(error=error)

spug_api/consumer/consumers.py

@@ -22,6 +22,8 @@ class ComConsumer(BaseConsumer):
             self.key = f'{settings.BUILD_KEY}:{token}'
         elif module == 'request':
             self.key = f'{settings.REQUEST_KEY}:{token}'
+        elif module == 'pipeline':
+            self.key = f'{settings.PIPELINE_KEY}:{token}'
         elif module == 'host':
             self.key = token
         else:

spug_api/libs/gitlib.py

@@ -5,6 +5,7 @@ from git import Repo, RemoteReference, TagReference, InvalidGitRepositoryError,
 from tempfile import NamedTemporaryFile
 from datetime import datetime
 from io import StringIO
+from functools import partial
 import subprocess
 import shutil
 import os
@@ -114,6 +115,7 @@ class RemoteGit:
         self.url = url
         self.path = path
         self.credential = credential
+        self.remote_exec = self.ssh.exec_command
         self._ask_env = None
 
     def _make_ask_env(self):
@@ -123,7 +125,7 @@ class RemoteGit:
             return self._ask_env
         ask_file = f'{self.ssh.exec_file}.1'
         if self.credential.type == 'pw':
-            env = dict(GIT_ASKPASS=ask_file)
+            self._ask_env = dict(GIT_ASKPASS=ask_file)
             body = '#!/bin/bash\n'
             body += 'case "$1" in\n'
             body += '  Username*)\n'
@@ -134,20 +136,28 @@ class RemoteGit:
             body = body.format(self.credential)
             self.ssh.put_file_by_fl(StringIO(body), ask_file)
         else:
-            env = dict(GIT_SSH=ask_file)
+            self._ask_env = dict(GIT_SSH=ask_file)
             key_file = f'{self.ssh.exec_file}.2'
             self.ssh.put_file_by_fl(StringIO(self.credential.secret), key_file)
             self.ssh.sftp.chmod(key_file, 0o600)
             body = f'ssh -o StrictHostKeyChecking=no -i {key_file} $@'
             self.ssh.put_file_by_fl(StringIO(body), ask_file)
         self.ssh.sftp.chmod(ask_file, 0o755)
-        return env
+        return self._ask_env
 
     def _check_path(self):
         body = f'git rev-parse --resolve-git-dir {self.path}/.git'
         code, _ = self.ssh.exec_command(body)
         return code == 0
 
+    def _clone(self):
+        env = self._make_ask_env()
+        print(env)
+        return self.remote_exec(f'git clone {self.url} {self.path}', env)
+
+    def set_remote_exec(self, remote_exec):
+        self.remote_exec = partial(remote_exec, self.ssh)
+
     @classmethod
     def check_auth(cls, url, credential=None):
         env = dict()
@@ -185,16 +195,12 @@ class RemoteGit:
         res = subprocess.run(command, shell=True, capture_output=True, env=env)
         return res.returncode == 0, res.stderr.decode()
 
-    def clone(self):
-        env = self._make_ask_env()
-        code, out = self.ssh.exec_command(f'git clone {self.url} {self.path}', env)
-        if code != 0:
-            raise Exception(out)
-
     def fetch_branches_tags(self):
         body = f'set -e\ncd {self.path}\n'
         if not self._check_path():
-            self.clone()
+            code, out = self._clone()
+            if code != 0:
+                raise Exception(out)
         else:
             body += 'git fetch -q --tags --force\n'
 
@@ -224,15 +230,15 @@ class RemoteGit:
     def checkout(self, marker):
         body = f'set -e\ncd {self.path}\n'
         if not self._check_path():
-            self.clone()
+            is_success = self._clone()
+            if not is_success:
+                return False
         else:
             body += 'git fetch -q --tags --force\n'
 
         body += f'git checkout -f {marker}'
         env = self._make_ask_env()
-        code, out = self.ssh.exec_command(body, env)
+        return self.remote_exec(body, env)
-        if code != 0:
-            raise Exception(out)
 
     def __enter__(self):
         self.ssh.get_client()

spug_api/spug/settings.py

@@ -112,6 +112,7 @@ MONITOR_WORKER_KEY = 'spug:monitor:worker'
 EXEC_WORKER_KEY = 'spug:exec:worker'
 REQUEST_KEY = 'spug:request'
 BUILD_KEY = 'spug:build'
+PIPELINE_KEY = 'spug:pipeline'
 REPOS_DIR = os.path.join(os.path.dirname(os.path.dirname(BASE_DIR)), 'repos')
 BUILD_DIR = os.path.join(REPOS_DIR, 'build')
 TRANSFER_DIR = os.path.join(BASE_DIR, 'storage', 'transfer')

spug_web/package.json

@@ -18,8 +18,8 @@
     "react-dom": "^16.13.1",
     "react-router-dom": "^5.2.0",
     "react-scripts": "3.4.3",
-    "xterm": "^4.6.0",
+    "xterm": "^5.1.0",
-    "xterm-addon-fit": "^0.5.0"
+    "xterm-addon-fit": "^0.7.0"
   },
   "scripts": {
     "start": "react-app-rewired start",

spug_web/src/pages/deploy/request/Console.js

@@ -41,11 +41,11 @@ function Console(props) {
 
   useEffect(() => {
     gCurrent = current
-    term.setOption('disableStdin', true)
+    term.options.disableStdin = true
-    term.setOption('fontSize', 14)
+    term.options.fontSize = 14
-    term.setOption('lineHeight', 1.2)
+    term.options.lineHeight = 1.2
-    term.setOption('fontFamily', gStore.terminal.fontFamily)
+    term.options.fontFamily = gStore.terminal.fontFamily
-    term.setOption('theme', {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'})
+    term.options.theme = {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'}
     term.attachCustomKeyEventHandler((arg) => {
       if (arg.ctrlKey && arg.code === 'KeyC' && arg.type === 'keydown') {
         document.execCommand('copy')

spug_web/src/pages/pipeline/Editor.js

@@ -12,6 +12,7 @@ import NodeConfig from './NodeConfig';
 import PipeForm from './Form';
 import Node from './Node';
 import { transfer } from './utils';
+import { history } from 'libs';
 import S from './store';
 import lds from 'lodash';
 import css from './editor.module.less';
@@ -32,7 +33,7 @@ function Editor(props) {
   }, [])
 
   useEffect(() => {
-    if (S.record.nodes.length) {
+    if ((S.record?.nodes ?? []).length) {
       const data = transfer(S.record.nodes)
       setNodes(data)
     }
@@ -119,6 +120,7 @@ function Editor(props) {
     }
     S.record.nodes.splice(index, 1)
     S.record = {...S.record}
+    S.updateRecord()
   }
 
   function handleRefresh(node) {
@@ -133,7 +135,8 @@ function Editor(props) {
         <div className={css.title}>{S.record.name}</div>
         <EditOutlined className={css.edit} onClick={() => setVisible(true)}/>
         <div style={{flex: 1}}/>
-        <Button className={css.back} type="link" icon={<RollbackOutlined/>}>返回列表</Button>
+        <Button className={css.back} type="link" icon={<RollbackOutlined/>}
+                onClick={() => history.goBack()}>返回列表</Button>
       </div>
       <div className={css.body}>
         <div className={css.nodes}>

spug_web/src/pages/pipeline/Icon.js

@@ -1,6 +1,11 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
 import React from 'react';
 import { Avatar } from 'antd';
-import iconRemoteExec from './assets/icon_remote_exec.png';
+import iconSSHExec from './assets/icon_ssh_exec.png';
 import iconBuild from './assets/icon_build.png';
 import iconParameter from './assets/icon_parameter.png';
 import iconDataTransfer from './assets/icon_data_transfer.png';
@@ -11,8 +16,8 @@ import iconSelect from './assets/icon_select.png';
 
 function Icon(props) {
   switch (props.module) {
-    case 'remote_exec':
+    case 'ssh_exec':
-      return <Avatar size={props.size || 42} src={iconRemoteExec}/>
+      return <Avatar size={props.size || 42} src={iconSSHExec}/>
     case 'build':
       return <Avatar size={props.size || 42} src={iconBuild}/>
     case 'parameter':

spug_web/src/pages/pipeline/Node.js

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
 import React from 'react'
 import { observer } from 'mobx-react';
 import { Dropdown } from 'antd';
@@ -40,6 +45,14 @@ function Node(props) {
     }
   ]
 
+  function dropdownRender(menus) {
+    return (
+      <div onMouseDown={e => e.stopPropagation()}>
+        {menus}
+      </div>
+    )
+  }
+
   const node = props.node
   switch (node) {
     case '  ':
@@ -69,7 +82,8 @@ function Node(props) {
             ) : (
               <div className={css.title} style={{color: '#595959'}}>请选择节点</div>
             )}
-            <Dropdown className={css.action} trigger="click" menu={{items: menus}} onMouseDown={handleActionClick}>
+            <Dropdown dropdownRender={dropdownRender} className={css.action}
+                      trigger="click" menu={{items: menus}} onMouseDown={handleActionClick}>
               <MoreOutlined/>
             </Dropdown>
           </div>

spug_web/src/pages/pipeline/NodeConfig.js

@@ -12,7 +12,8 @@ import Icon from './Icon';
 import { clsNames } from 'libs';
 import S from './store';
 import css from './nodeConfig.module.less';
-import { NODES } from './data'
+import { NODES } from './data';
+import lds from 'lodash';
 
 function NodeConfig(props) {
   const [tab, setTab] = useState('node')
@@ -35,7 +36,8 @@ function NodeConfig(props) {
     const data = handler()
     if (typeof data === 'object') {
       setLoading(true)
-      Object.assign(S.node, data)
+      const basic = lds.pick(S.node, ['id', 'module', 'downstream'])
+      S.node = Object.assign(data, basic)
       props.doRefresh(S.node)
         .finally(() => setLoading(false))
     }

spug_web/src/pages/pipeline/Table.js

@@ -0,0 +1,73 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
+import React, { useEffect } from 'react';
+import { observer } from 'mobx-react';
+import { Table, Modal, message } from 'antd';
+import { PlusOutlined } from '@ant-design/icons';
+import { Action, TableCard, AuthButton } from 'components';
+import { http, hasPermission, history } from 'libs';
+import S from './store';
+
+function ComTable() {
+  useEffect(() => {
+    S.fetchRecords()
+  }, [])
+
+  function handleDelete(text) {
+    Modal.confirm({
+      title: '删除确认',
+      content: `确定要删除【${text['name']}】?`,
+      onOk: () => {
+        return http.delete('/api/pipeline/', {params: {id: text.id}})
+          .then(() => {
+            message.success('删除成功');
+            S.fetchRecords()
+          })
+      }
+    })
+  }
+
+  function toDetail(info) {
+    history.push(`/pipeline/${info ? info.id : 'new'}`)
+  }
+
+  return (
+    <TableCard
+      tKey="pipe"
+      rowKey="id"
+      title="流程列表"
+      loading={S.isFetching}
+      dataSource={S.dataSource}
+      onReload={S.fetchRecords}
+      actions={[
+        <AuthButton
+          auth="pipeline.pipeline.add"
+          type="primary"
+          icon={<PlusOutlined/>}
+          onClick={() => toDetail()}>新建</AuthButton>
+      ]}
+      pagination={{
+        showSizeChanger: true,
+        showLessItems: true,
+        showTotal: total => `共 ${total} 条`,
+        pageSizeOptions: ['10', '20', '50', '100']
+      }}>
+      <Table.Column title="流程名称" dataIndex="name"/>
+      <Table.Column ellipsis title="备注信息" dataIndex="desc"/>
+      {hasPermission('pipeline.pipeline.edit|pipeline.pipeline.del') && (
+        <Table.Column width={210} title="操作" render={info => (
+          <Action>
+            <Action.Button auth="config.app.edit" onClick={() => toDetail(info)}>编辑</Action.Button>
+            <Action.Button auth="config.app.edit" onClick={() => S.showConsole(info)}>执行</Action.Button>
+            <Action.Button danger auth="config.app.del" onClick={() => handleDelete(info)}>删除</Action.Button>
+          </Action>
+        )}/>
+      )}
+    </TableCard>
+  )
+}
+
+export default observer(ComTable)

spug_web/src/pages/pipeline/console/Body.js

@@ -0,0 +1,143 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
+import React, { useEffect, useRef, useState } from 'react';
+import { observer } from 'mobx-react';
+import { Tooltip, Tabs } from 'antd';
+import { CodeOutlined, StopOutlined } from '@ant-design/icons';
+import { Terminal } from 'xterm';
+import { FitAddon } from 'xterm-addon-fit';
+import { http, X_TOKEN } from 'libs';
+import css from './body.module.less';
+import S from './store';
+import gStore from 'gStore';
+
+function Body() {
+  const el = useRef()
+  const [term] = useState(new Terminal());
+  const [fitPlugin] = useState(new FitAddon());
+  const [wsState, setWSState] = useState();
+
+  useEffect(() => {
+    let socket;
+    http.post('/api/pipeline/do/', {id: 1})
+      .then(res => {
+        socket = _makeSocket(res.token)
+        S.nodes = res.nodes
+        S.node = res.nodes[0]
+      })
+    return () => socket && socket.close()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
+  useEffect(() => {
+    term.options.disableStdin = true
+    term.options.fontSize = 14
+    term.options.lineHeight = 1.2
+    term.options.fontFamily = gStore.terminal.fontFamily
+    term.options.theme = {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'}
+    term.attachCustomKeyEventHandler((arg) => {
+      if (arg.ctrlKey && arg.code === 'KeyC' && arg.type === 'keydown') {
+        document.execCommand('copy')
+        return false
+      }
+      return true
+    })
+    const resize = () => fitPlugin.fit();
+    term.loadAddon(fitPlugin)
+    term.open(el.current)
+    fitPlugin.fit()
+    term.write('\x1b[36m### WebSocket connecting ...\x1b[0m')
+    window.addEventListener('resize', resize)
+
+    return () => window.removeEventListener('resize', resize);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
+
+  function _makeSocket(token, index = 0) {
+    const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
+    const socket = new WebSocket(`${protocol}//${window.location.host}/api/ws/pipeline/${token}/?x-token=${X_TOKEN}`);
+    socket.onopen = () => socket.send(String(index));
+    socket.onmessage = e => {
+      if (e.data === 'pong') {
+        socket.send(String(index))
+      } else {
+        index += 1;
+        const {key, data, status} = JSON.parse(e.data);
+        if (S.outputs[key]) {
+          if (status) {
+            S.outputs[key].status = status
+            if (key === S.nodeID) {
+              S.node.status = status
+            }
+          }
+          if (data) {
+            S.outputs[key].data += data
+            if (key === S.nodeID) {
+              term.write(data)
+            }
+          }
+        } else {
+          S.outputs[key] = {data, status}
+          S.node.state = ''
+        }
+      }
+    }
+    socket.onerror = () => {
+      setWSState('Websocket connection failed')
+    }
+    return socket
+  }
+
+  useEffect(() => {
+    if (S.node.id) {
+      fitPlugin.fit()
+      term.reset()
+      if (S.outputs[S.nodeID]) {
+        term.write(S.outputs[S.nodeID].data)
+      } else {
+        S.outputs[S.nodeID] = {data: ''}
+      }
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [S.node])
+
+  function handleTerminate() {
+
+  }
+
+  function openTerminal() {
+
+  }
+
+  return (
+    <div className={css.container}>
+      <div className={css.header}>
+        <div className={css.title}>{S.node?.name}</div>
+        <div className={css.tips}>{wsState}</div>
+        <Tooltip title="终止执行">
+          {S.outputs[S.nodeID]?.status === 'processing' ? (
+            <StopOutlined className={css.icon} style={{color: '#faad14'}} onClick={handleTerminate}/>
+          ) : (
+            <StopOutlined className={css.icon} style={{color: '#dfdfdf'}}/>
+          )}
+        </Tooltip>
+        <Tooltip title="打开web终端">
+          <CodeOutlined className={css.icon} onClick={() => openTerminal()}/>
+        </Tooltip>
+      </div>
+      {S.node?.module === 'ssh_exec' && (
+        <Tabs items={(S.node?.targets ?? []).map(x => ({label: x.name, key: `${S.node.id}.${x.id}`}))}
+              tabBarStyle={{fontSize: 13}} onChange={v => S.node = Object.assign({}, S.node, {_id: v})}/>
+      )}
+      <div className={css.termContainer}>
+        <div ref={el} className={css.term}/>
+      </div>
+    </div>
+  )
+}
+
+export default observer(Body)

spug_web/src/pages/pipeline/console/Node.js

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
+import React from 'react'
+import { observer } from 'mobx-react';
+import { LoadingOutlined } from '@ant-design/icons';
+import Icon from '../Icon';
+import { clsNames } from 'libs';
+import S from './store';
+import css from './node.module.less';
+
+function Node(props) {
+  const node = props.node
+  switch (node) {
+    case '  ':
+      return <div className={css.box}/>
+    case ' -':
+      return <div className={clsNames(css.box, css.line)}/>
+    case '--':
+      return <div className={clsNames(css.box, css.line, css.line2)}/>
+    case ' 7':
+      return <div className={clsNames(css.box, css.angle)}/>
+    case '-7':
+      return <div className={clsNames(css.box, css.angle, css.angle2)}/>
+    case ' |':
+      return (
+        <div className={clsNames(css.box, css.arrow)}>
+          <div className={css.triangle}/>
+        </div>
+      )
+    default:
+      return (
+        <div className={clsNames(css.box, css.node, S.node?.id === node.id && css.active)} onClick={props.onClick}>
+          {S.outputs[node.id]?.status === 'processing' ? <LoadingOutlined className={css.loading}/> : null}
+          <Icon size={36} module={node.module}/>
+        </div>
+      )
+  }
+}
+
+export default observer(Node)

spug_web/src/pages/pipeline/console/Sider.js

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
+import React from 'react';
+import { observer } from 'mobx-react';
+import Node from './Node';
+import S from './store';
+import css from './sider.module.less';
+
+
+function Sider() {
+  function handleClick(node) {
+    if (node.module === 'ssh_exec') {
+      node._id = `${node.id}.${node.targets[0].id}`
+    }
+    S.node = node
+  }
+
+  return (
+    <div className={css.sider}>
+      {S.matrixNodes.map((row, idx) => (
+        <div key={idx} style={{display: 'flex', flexDirection: 'row'}}>
+          {row.map((item, idx) => (
+            <Node key={idx} node={item} onClick={() => handleClick(item)}/>
+          ))}
+        </div>
+      ))}
+    </div>
+  )
+}
+
+export default observer(Sider)

spug_web/src/pages/pipeline/console/body.module.less

@@ -0,0 +1,73 @@
+.container {
+  height: calc(100vh - 300px);
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  padding-left: 24px;
+  overflow: hidden;
+}
+
+.header {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 8px;
+
+  .tips {
+    color: #ff4d4f;
+  }
+
+  .icon {
+    font-size: 18px;
+    color: #1890ff;
+    cursor: pointer;
+    margin-left: 12px;
+  }
+
+  .title {
+    flex: 1;
+    font-weight: 500;
+  }
+}
+
+
+.termContainer {
+  flex: 1;
+  background-color: #2b2b2b;
+  padding: 8px 0 4px 12px;
+  border-radius: 4px;
+  overflow: hidden;
+
+  .term {
+    width: 100%;
+    height: 100%;
+  }
+}
+
+.execContainer {
+
+  .header {
+    color: #ffffff;
+    background: #36435c;
+    border-radius: 4px;
+    padding: 6px 12px;
+    margin-right: 12px;
+    font-size: 14px;
+    display: flex;
+    flex-direction: row;
+    align-items: center;
+
+    .title {
+      flex: 1;
+      margin-left: 12px;
+    }
+  }
+
+  .item {
+
+  }
+}
+
+:global(.ant-tabs-tab) {
+  font-size: 13px;
+}

spug_web/src/pages/pipeline/console/index.js

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
+import React from 'react';
+import { observer } from 'mobx-react';
+import { Modal, Row } from 'antd';
+import Sider from './Sider';
+import Body from './Body';
+import pS from '../store';
+import S from './store';
+
+function Index() {
+  return (
+    <Modal
+      open={pS.consoleVisible}
+      width="80%"
+      title="运行控制台"
+      footer={null}
+      destroyOnClose
+      afterClose={S.initial}
+      onCancel={() => pS.consoleVisible = false}>
+      <Row>
+        <Sider/>
+        <Body/>
+      </Row>
+    </Modal>
+  )
+}
+
+export default observer(Index)

spug_web/src/pages/pipeline/console/node.module.less

@@ -0,0 +1,108 @@
+.box {
+  position: relative;
+  width: 38px;
+  height: 38px;
+  margin-right: 16px;
+  flex-shrink: 0;
+
+  &:last-child {
+    margin: 0;
+  }
+}
+
+.triangle {
+  position: absolute;
+  bottom: -2px;
+  left: 15px;
+  width: 0;
+  height: 0;
+  border: 4px solid transparent;
+  border-top-color: #999999;
+}
+
+.node {
+  position: relative;
+  box-shadow: 0 0 6px #9999994c;
+  border: 1px solid transparent;
+  border-radius: 50%;
+  cursor: pointer;
+
+  &:hover {
+    box-shadow: 0 0 6px #2563fcbb;
+    .action {
+      display: block;
+    }
+  }
+}
+
+.loading {
+  position: absolute;
+  top: 4px;
+  left: 4px;
+  color: #ffffff;
+  font-size: 28px;
+  z-index: 999;
+}
+
+.active {
+  box-shadow: 0 0 6px #2563fc;
+}
+
+.line {
+  &:after {
+    content: ' ';
+    height: 2px;
+    background: #999999;
+    position: absolute;
+    top: 18px;
+    left: -14px;
+    right: -4px;
+  }
+}
+
+.line2 {
+  &:after {
+    left: -34px;
+  }
+}
+
+.angle {
+  &:before {
+    content: ' ';
+    position: absolute;
+    background: #999999;
+    top: 18px;
+    bottom: 18px;
+    left: -14px;
+    right: 18px;
+  }
+
+  &:after {
+    content: ' ';
+    position: absolute;
+    background: #999999;
+    top: 18px;
+    left: 18px;
+    right: 18px;
+    bottom: -4px;
+  }
+}
+
+.angle2 {
+  &:before {
+    left: -34px;
+  }
+}
+
+.arrow {
+  &:after {
+    content: ' ';
+    background: #999999;
+    position: absolute;
+    top: 2px;
+    left: 18px;
+    right: 18px;
+    bottom: 4px;
+  }
+}
+

spug_web/src/pages/pipeline/console/sider.module.less

@@ -0,0 +1,6 @@
+.sider {
+  max-width: 200px;
+  min-width: 104px;
+  padding: 6px;
+  overflow: auto;
+}

spug_web/src/pages/pipeline/console/store.js

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
+import { observable, computed } from 'mobx';
+import { transfer } from '../utils';
+
+class Store {
+  @observable node = {};
+  @observable nodes = [];
+  @observable outputs = {};
+
+  @computed get nodeID() {
+    return this.node._id ?? this.node.id
+  }
+
+  @computed get matrixNodes() {
+    return transfer(this.nodes)
+  }
+
+  initial = () => {
+    this.node = {}
+    this.nodes = []
+    this.outputs = {}
+  }
+}
+
+export default new Store()

spug_web/src/pages/pipeline/data.js

@@ -1,5 +1,5 @@
 export const NODES = [
-  {module: 'remote_exec', name: '执行命令'},
+  {module: 'ssh_exec', name: '执行命令'},
   {module: 'build', name: '构建'},
   {module: 'parameter', name: '参数化'},
   {module: 'data_transfer', name: '数据传输'},
@@ -60,7 +60,7 @@ export const DATAS = {
       'command': 'date && sleep 3',
     },
     {
-      'module': 'remote_exec',
+      'module': 'ssh_exec',
       'name': '执行命令',
       'id': 5,
       'targets': [2, 3],

spug_web/src/pages/pipeline/index.js

@@ -6,12 +6,16 @@
 import React from 'react';
 import { observer } from 'mobx-react';
 import { AuthDiv } from 'components';
-import Editor from './Editor';
+import Table from './Table';
+import Console from './console';
 
-export default observer(function () {
+function Index() {
   return (
-    <AuthDiv auth="system.account.view">
+    <AuthDiv auth="pipeline.pipeline.view">
-      <Editor/>
+      <Table/>
+      <Console/>
     </AuthDiv>
   )
-})
+}
+
+export default observer(Index)

spug_web/src/pages/pipeline/modules/Build.js

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
 import React, { useEffect, useState } from 'react';
 import { observer } from 'mobx-react';
 import { Form, Input, Select, Radio, message } from 'antd';
@@ -47,7 +52,7 @@ function Build(props) {
       <Form.Item required name="condition" label="执行条件" tooltip="当该节点为流程的起始节点时（无上游节点），该条件将会被忽略。">
         <Radio.Group>
           <Radio.Button value="success">上游执行成功时</Radio.Button>
-          <Radio.Button value="failure">上游执行失败时</Radio.Button>
+          <Radio.Button value="error">上游执行失败时</Radio.Button>
           <Radio.Button value="always">总是执行</Radio.Button>
         </Radio.Group>
       </Form.Item>

spug_web/src/pages/pipeline/modules/DataTransfer.js

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
 import React, { useEffect } from 'react';
 import { Form, Input, message, Card, Radio } from 'antd';
 import HostSelector from 'pages/host/Selector';
@@ -29,7 +34,7 @@ function DataTransfer(props) {
       <Form.Item required name="condition" label="执行条件" tooltip="当该节点为流程的起始节点时（无上游节点），该条件将会被忽略。">
         <Radio.Group>
           <Radio.Button value="success">上游执行成功时</Radio.Button>
-          <Radio.Button value="failure">上游执行失败时</Radio.Button>
+          <Radio.Button value="error">上游执行失败时</Radio.Button>
           <Radio.Button value="always">总是执行</Radio.Button>
         </Radio.Group>
       </Form.Item>

spug_web/src/pages/pipeline/modules/SSHExec.js

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
 import React, { useEffect } from 'react';
 import { Form, Input, Radio, message } from 'antd';
 import { ACEditor } from 'components';
@@ -11,12 +16,16 @@ function SSHExec(props) {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
+  useEffect(() => {
+    form.resetFields()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [props.node])
+
   function handleSave() {
     const data = form.getFieldsValue()
     if (!data.name) return message.error('请输入节点名称')
     if (!data.condition) return message.error('请选择节点的执行条件')
     if (!data.targets || data.targets.length === 0) return message.error('请选择执行主机')
-    if (!data.interpreter) return  message.error('请选择执行解释器')
     if (!data.command) return message.error('请输入执行内容')
     return data
   }
@@ -29,19 +38,13 @@ function SSHExec(props) {
       <Form.Item required name="condition" label="执行条件" tooltip="当该节点为流程的起始节点时（无上游节点），该条件将会被忽略。">
         <Radio.Group>
           <Radio.Button value="success">上游执行成功时</Radio.Button>
-          <Radio.Button value="failure">上游执行失败时</Radio.Button>
+          <Radio.Button value="error">上游执行失败时</Radio.Button>
           <Radio.Button value="always">总是执行</Radio.Button>
         </Radio.Group>
       </Form.Item>
       <Form.Item required name="targets" label="选择主机">
         <HostSelector type="button"/>
       </Form.Item>
-      <Form.Item required name="interpreter" label="执行解释器">
-        <Radio.Group buttonStyle="solid">
-          <Radio.Button value="sh">Shell</Radio.Button>
-          <Radio.Button value="python">Python</Radio.Button>
-        </Radio.Group>
-      </Form.Item>
       <Form.Item required label="执行内容" shouldUpdate={(p, c) => p.interpreter !== c.interpreter}>
         {({getFieldValue}) => (
           <Form.Item name="command" noStyle>

spug_web/src/pages/pipeline/modules/index.js

@@ -1,3 +1,8 @@
+/**
+ * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
+ * Copyright (c) <spug.dev@gmail.com>
+ * Released under the AGPL-3.0 License.
+ */
 import React from 'react';
 import SSHExec from './SSHExec';
 import Build from './Build';
@@ -5,7 +10,7 @@ import DataTransfer from './DataTransfer';
 
 function ModuleConfig(props) {
   switch (props.node.module) {
-    case 'remote_exec':
+    case 'ssh_exec':
       return <SSHExec {...props}/>
     case 'build':
       return <Build {...props}/>

spug_web/src/pages/pipeline/store.js

@@ -3,20 +3,28 @@
  * Copyright (c) <spug.dev@gmail.com>
  * Released under the AGPL-3.0 License.
  */
-import { observable } from 'mobx';
+import { computed, observable } from 'mobx';
-import { http } from 'libs';
+import { http, includes } from 'libs';
 import { message } from 'antd';
 
 class Store {
+  @observable records = [];
   @observable record = {nodes: []};
   @observable nodes = [];
   @observable node = {};
   @observable actionNode = {};
   @observable isFetching = true;
+  @observable consoleVisible = false;
 
-  fetchRecords = (id, isFetching) => {
+  @computed get dataSource() {
+    let records = this.records;
+    if (this.f_name) records = records.filter(x => includes(x.name, this.f_name));
+    return records
+  }
+
+  fetchRecords = () => {
     this.isFetching = true;
-    return http.get('/api/pipline/')
+    return http.get('/api/pipeline/')
       .then(res => this.records = res)
       .finally(() => this.isFetching = false)
   }
@@ -35,6 +43,11 @@ class Store {
         message.success('保存成功')
       })
   }
+
+  showConsole = (record) => {
+    this.record = record
+    this.consoleVisible = true
+  }
 }
 
 export default new Store()

spug_web/src/routes.js

@@ -43,7 +43,7 @@ import SystemCredential from './pages/system/credential';
 import WelcomeIndex from './pages/welcome/index';
 import WelcomeInfo from './pages/welcome/info';
 import PipelineIndex from './pages/pipeline';
-import PipelineEditor from './pages/pipeline';
+import PipelineEditor from './pages/pipeline/Editor';
 
 export default [
   {icon: <DesktopOutlined/>, title: '工作台', path: '/home', component: HomeIndex},
@@ -69,8 +69,8 @@ export default [
       {title: '发布申请', auth: 'deploy.request.view', path: '/deploy/request', component: DeployRequest},
     ]
   },
-  {icon: <FlagOutlined/>, title: '流水线', path: '/pipeline', component: PipelineIndex},
   {path: '/pipeline/:id', component: PipelineEditor},
+  {icon: <FlagOutlined/>, title: '流水线', path: '/pipeline', component: PipelineIndex},
   {
     icon: <ScheduleOutlined/>,
     title: '任务计划',