From 6cda7a9d2a9d8f8fce20c178f83505d55ec49bf8 Mon Sep 17 00:00:00 2001 From: vapao Date: Thu, 2 Mar 2023 00:00:51 +0800 Subject: [PATCH] update pipeline module --- spug_api/apps/pipeline/helper.py | 212 ++++++++++++++++++ spug_api/apps/pipeline/models.py | 20 ++ spug_api/apps/pipeline/urls.py | 3 +- spug_api/apps/pipeline/utils.py | 86 +++++++ spug_api/apps/pipeline/views.py | 63 +++++- spug_api/consumer/consumers.py | 2 + spug_api/libs/gitlib.py | 34 +-- spug_api/spug/settings.py | 1 + spug_web/package.json | 4 +- spug_web/src/pages/deploy/request/Console.js | 10 +- spug_web/src/pages/pipeline/Editor.js | 7 +- spug_web/src/pages/pipeline/Icon.js | 11 +- spug_web/src/pages/pipeline/Node.js | 16 +- spug_web/src/pages/pipeline/NodeConfig.js | 6 +- spug_web/src/pages/pipeline/Table.js | 73 ++++++ ...icon_remote_exec.png => icon_ssh_exec.png} | Bin spug_web/src/pages/pipeline/console/Body.js | 143 ++++++++++++ spug_web/src/pages/pipeline/console/Node.js | 43 ++++ spug_web/src/pages/pipeline/console/Sider.js | 34 +++ .../pages/pipeline/console/body.module.less | 73 ++++++ spug_web/src/pages/pipeline/console/index.js | 32 +++ .../pages/pipeline/console/node.module.less | 108 +++++++++ .../pages/pipeline/console/sider.module.less | 6 + spug_web/src/pages/pipeline/console/store.js | 29 +++ spug_web/src/pages/pipeline/data.js | 4 +- spug_web/src/pages/pipeline/index.js | 14 +- spug_web/src/pages/pipeline/modules/Build.js | 7 +- .../pages/pipeline/modules/DataTransfer.js | 7 +- .../src/pages/pipeline/modules/SSHExec.js | 19 +- spug_web/src/pages/pipeline/modules/index.js | 7 +- spug_web/src/pages/pipeline/store.js | 21 +- spug_web/src/routes.js | 4 +- 32 files changed, 1044 insertions(+), 55 deletions(-) create mode 100644 spug_api/apps/pipeline/helper.py create mode 100644 spug_api/apps/pipeline/utils.py create mode 100644 spug_web/src/pages/pipeline/Table.js rename spug_web/src/pages/pipeline/assets/{icon_remote_exec.png => icon_ssh_exec.png} (100%) create mode 100644 spug_web/src/pages/pipeline/console/Body.js create mode 100644 spug_web/src/pages/pipeline/console/Node.js create mode 100644 spug_web/src/pages/pipeline/console/Sider.js create mode 100644 spug_web/src/pages/pipeline/console/body.module.less create mode 100644 spug_web/src/pages/pipeline/console/index.js create mode 100644 spug_web/src/pages/pipeline/console/node.module.less create mode 100644 spug_web/src/pages/pipeline/console/sider.module.less create mode 100644 spug_web/src/pages/pipeline/console/store.js diff --git a/spug_api/apps/pipeline/helper.py b/spug_api/apps/pipeline/helper.py new file mode 100644 index 0000000..b40088e --- /dev/null +++ b/spug_api/apps/pipeline/helper.py @@ -0,0 +1,212 @@ +# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug +# Copyright: (c) +# Released under the AGPL-3.0 License. +from django.conf import settings +from django.template.defaultfilters import filesizeformat +from django_redis import get_redis_connection +from libs.utils import SpugError, human_time, render_str, human_seconds_time +from apps.config.utils import update_config_by_var +from collections import defaultdict +import time +import json +import os +import re + + +class KitMixin: + regex = re.compile(r'^((\r\n)*)(.*?)((\r\n)*)$', re.DOTALL) + + @classmethod + def term_message(cls, message, color_mode='info', with_time=False): + prefix = f'{human_time()} ' if with_time else '' + if color_mode == 'info': + mode = '36m' + elif color_mode == 'warn': + mode = '33m' + elif color_mode == 'error': + mode = '31m' + elif color_mode == 'success': + mode = '32m' + else: + raise TypeError + + return cls.regex.sub(fr'\1\033[{mode}{prefix}\3\033[0m\4', message) + + +class Helper(KitMixin): + def __init__(self, rds, rds_key): + self.rds = rds + self.rds_key = rds_key + self.buffers = defaultdict(str) + self.flags = defaultdict(bool) + self.already_clear = False + self.files = {} + + def __del__(self): + self.clear() + + @classmethod + def make(cls, rds, rds_key): + rds.delete(rds_key) + return cls(rds, rds_key) + + @classmethod + def fill_outputs(cls, outputs, deploy_key): + rds = get_redis_connection() + key_ttl = rds.ttl(deploy_key) + counter, hit_keys = 0, set() + if key_ttl > 30 or key_ttl == -1: + data = rds.lrange(deploy_key, counter, counter + 9) + while data: + for item in data: + counter += 1 + item = json.loads(item.decode()) + key = item['key'] + if key in outputs: + hit_keys.add(key) + if 'data' in item: + outputs[key]['data'] += item['data'] + if 'status' in item: + outputs[key]['status'] = item['status'] + data = rds.lrange(deploy_key, counter, counter + 9) + + for key in outputs.keys(): + if key in hit_keys: + continue + file_name = os.path.join(settings.DEPLOY_DIR, f'{deploy_key}:{key}') + if not os.path.exists(file_name): + continue + with open(file_name, newline='\r\n') as f: + line = f.readline() + while line: + status, data = line.split(',', 1) + if data: + outputs[key]['data'] += data + if status: + outputs[key]['status'] = status + line = f.readline() + return counter + + def get_file(self, key): + if key in self.files: + return self.files[key] + file = open(os.path.join(settings.DEPLOY_DIR, f'{self.rds_key}:{key}'), 'w') + self.files[key] = file + return file + + def get_cross_env(self, key): + file = os.path.join(settings.DEPLOY_DIR, key) + if os.path.exists(file): + with open(file, 'r') as f: + return json.loads(f.read()) + return {} + + def set_cross_env(self, key, envs): + file_envs = {} + for k, v in envs.items(): + if k == 'SPUG_SET': + try: + update_config_by_var(v) + except SpugError as e: + self.send_error(key, f'{e}') + elif k.startswith('SPUG_GEV_'): + file_envs[k] = v + + file = os.path.join(settings.DEPLOY_DIR, key) + with open(file, 'w') as f: + f.write(json.dumps(file_envs)) + + def save_pid(self, pid, key): + self.rds.set(f'PID:{self.rds_key}:{key}', pid, 3600) + + def parse_filter_rule(self, data: str, sep='\n', env=None): + data, files = data.strip(), [] + if data: + for line in data.split(sep): + line = line.strip() + if line and not line.startswith('#'): + files.append(render_str(line, env)) + return files + + def send(self, key, data, *, status=''): + message = {'key': key, 'data': data} + if status: + message['status'] = status + print(self.rds_key, message) + self.rds.rpush(self.rds_key, json.dumps(message)) + + file = self.get_file(key) + for idx, line in enumerate(data.split('\r\n')): + if idx != 0: + tmp = [status, self.buffers[key] + '\r\n'] + file.write(','.join(tmp)) + file.flush() + self.buffers[key] = '' + self.flags[key] = False + if line: + for idx2, item in enumerate(line.split('\r')): + if idx2 != 0: + self.flags[key] = True + if item: + if self.flags[key]: + self.buffers[key] = item + self.flags[key] = False + else: + self.buffers[key] += item + + def send_clear(self, key): + self.send(key, '\033[2J\033[3J\033[1;1H') + + def send_info(self, key, message, status='', with_time=True): + message = self.term_message(message, 'info', with_time) + self.send(key, message, status=status) + + def send_warn(self, key, message, status=''): + message = self.term_message(message, 'warn') + self.send(key, message, status=status) + + def send_success(self, key, message, with_time=True, start_time=None): + if start_time: + message += f', 耗时: {human_seconds_time(time.time() - start_time)}' + message = self.term_message(f'\r\n** {message} **', 'success', with_time) + self.send(key, message, status='success') + + def send_error(self, key, message, with_break=False): + message = self.term_message(message, 'error') + if not message.endswith('\r\n'): + message += '\r\n' + self.send(key, message, status='error') + if with_break: + raise SpugError + + def send_status(self, key, status): + self.send(key, '', status=status) + + def clear(self): + if self.already_clear: + return + self.already_clear = True + for key, value in self.buffers.items(): + if value: + file = self.get_file(key) + file.write(f',{value}') + for file in self.files.values(): + file.close() + if self.rds.ttl(self.rds_key) == -1: + self.rds.expire(self.rds_key, 60) + + def progress_callback(self, key): + def func(n, t): + message = f'\r {filesizeformat(n):<8}/{filesizeformat(t):>8} ' + self.send(key, message) + + self.send(key, '\r\n') + return func + + def remote_exec(self, key, ssh, command, env=None): + code = -1 + for code, out in ssh.exec_command_with_stream(command, environment=env): + self.send(key, out) + if code != 0: + self.send_error(key, f'exit code: {code}') + return code == 0 diff --git a/spug_api/apps/pipeline/models.py b/spug_api/apps/pipeline/models.py index eb25777..3ed74c6 100644 --- a/spug_api/apps/pipeline/models.py +++ b/spug_api/apps/pipeline/models.py @@ -2,6 +2,7 @@ # Copyright: (c) # Released under the AGPL-3.0 License. from django.db import models +from django.conf import settings from libs.mixins import ModelMixin from apps.account.models import User import json @@ -18,6 +19,25 @@ class Pipeline(models.Model, ModelMixin): tmp['nodes'] = json.loads(self.nodes) return tmp + def to_list(self): + tmp = self.to_dict(selects=('id', 'name', 'created_at')) + return tmp + class Meta: db_table = 'pipelines' ordering = ('-id',) + + +class PipeHistory(models.Model, ModelMixin): + pipeline = models.ForeignKey(Pipeline, on_delete=models.CASCADE) + ordinal = models.IntegerField() + created_by = models.ForeignKey(User, on_delete=models.PROTECT) + created_at = models.DateTimeField(auto_now_add=True) + + @property + def deploy_key(self): + return f'{settings.PIPELINE_KEY}:{self.id}' + + class Meta: + db_table = 'pipeline_histories' + ordering = ('-id',) diff --git a/spug_api/apps/pipeline/urls.py b/spug_api/apps/pipeline/urls.py index 93a6be7..48ab43b 100644 --- a/spug_api/apps/pipeline/urls.py +++ b/spug_api/apps/pipeline/urls.py @@ -3,8 +3,9 @@ # Released under the AGPL-3.0 License. from django.urls import path -from apps.pipeline.views import PipeView +from apps.pipeline.views import PipeView, DoView urlpatterns = [ path('', PipeView.as_view()), + path('do/', DoView.as_view()), ] diff --git a/spug_api/apps/pipeline/utils.py b/spug_api/apps/pipeline/utils.py new file mode 100644 index 0000000..3663b50 --- /dev/null +++ b/spug_api/apps/pipeline/utils.py @@ -0,0 +1,86 @@ +# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug +# Copyright: (c) +# Released under the AGPL-3.0 License. +from apps.credential.models import Credential +from apps.host.models import Host +from libs.utils import AttrDict, human_seconds_time +from libs.gitlib import RemoteGit +from apps.pipeline.helper import Helper +from functools import partial, partialmethod +from threading import Thread +from concurrent import futures +import time +import os + + +class NodeExecutor: + def __init__(self, rds, rds_key, nodes): + self.rds = rds + self.rds_key = rds_key + self.nodes = {x.id: x for x in map(AttrDict, nodes)} + self.node = AttrDict(nodes[0]) + self.helper = Helper.make(self.rds, self.rds_key) + + def run(self, node=None, state=None): + print(node, state) + if node: + downstream = getattr(node, 'downstream', []) + down_nodes = [self.nodes[x] for x in downstream] + available_nodes = [x for x in down_nodes if x.condition in (state, 'always')] + if len(available_nodes) >= 2: + for node in available_nodes[1:]: + Thread(target=self._dispatch, args=(node,)).start() + if available_nodes: + self._dispatch(available_nodes[0]) + else: + self._dispatch(self.node) + + def _dispatch(self, node): + print('!!!!! _dispatch', node.name) + if node.module == 'build': + self._do_build(node) + elif node.module == 'ssh_exec': + self._do_ssh_exec(node) + + def _do_build(self, node, marker=None): + # if node.mode == 'branch': + # marker = node.branch + timestamp = time.time() + marker = 'origin/bugfix' + host = Host.objects.get(pk=node.target) + credential = None + if node.credential_id: + credential = Credential.objects.get(pk=node.credential_id) + with RemoteGit(host, node.git_url, node.workspace, credential) as git: + self.helper.send_info(node.id, '同步并检出Git仓库\r\n', 'processing') + git.set_remote_exec(partial(self.helper.remote_exec, node.id)) + is_success = git.checkout(marker) + if is_success and node.command: + self.helper.send_info(node.id, '执行构建命令\r\n') + is_success = self.helper.remote_exec(node.id, git.ssh, node.command) + if is_success: + self.helper.send_success(node.id, '构建完成', start_time=timestamp) + self.run(node, 'success' if is_success else 'error') + + def _do_ssh_exec(self, node): + threads = [] + max_workers = max(10, os.cpu_count() * 5) + self.helper.send_status(node.id, 'processing') + with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + for host in Host.objects.filter(id__in=node.targets): + t = executor.submit(self._ssh_exec, host, node) + threads.append(t) + results = [x.result() for x in futures.as_completed(threads)] + state = 'success' if all(results) else 'error' + self.helper.send_status(node.id, state) + self.run(node, state) + + def _ssh_exec(self, host, node): + timestamp = time.time() + key = f'{node.id}.{host.id}' + self.helper.send_info(key, '开始执行\r\n', 'processing') + with host.get_ssh() as ssh: + is_success = self.helper.remote_exec(key, ssh, node.command) + if is_success: + self.helper.send_success(key, '执行结束', start_time=timestamp) + return is_success diff --git a/spug_api/apps/pipeline/views.py b/spug_api/apps/pipeline/views.py index 29cd61c..8099aa6 100644 --- a/spug_api/apps/pipeline/views.py +++ b/spug_api/apps/pipeline/views.py @@ -2,8 +2,14 @@ # Copyright: (c) # Released under the AGPL-3.0 License. from django.views.generic import View +from django_redis import get_redis_connection from libs import JsonParser, Argument, json_response, auth -from apps.pipeline.models import Pipeline +from libs.utils import AttrDict +from apps.pipeline.models import Pipeline, PipeHistory +from apps.pipeline.utils import NodeExecutor +from apps.host.models import Host +from threading import Thread +from uuid import uuid4 import json @@ -57,3 +63,58 @@ class PipeView(View): if error is None: Pipeline.objects.filter(pk=form.id).delete() return json_response(error=error) + + +class DoView(View): + @auth('exec.task.do') + def get(self, request): + pass + + @auth('exec.task.do') + def post(self, request): + form, error = JsonParser( + Argument('id', type=int, help='参数错误'), + ).parse(request.body) + if error is None: + pipe = Pipeline.objects.get(pk=form.id) + latest_history = pipe.pipehistory_set.first() + ordinal = latest_history.ordinal + 1 if latest_history else 1 + history = PipeHistory.objects.create(pipeline=pipe, ordinal=ordinal, created_by=request.user) + nodes, ids = json.loads(pipe.nodes), set() + for item in filter(lambda x: x['module'] == 'ssh_exec', nodes): + ids.update(item['targets']) + + host_map = {x.id: f'{x.name}({x.hostname})' for x in Host.objects.filter(id__in=ids)} + for item in filter(lambda x: x['module'] == 'ssh_exec', nodes): + item['targets'] = [{'id': x, 'name': host_map[x]} for x in item['targets']] + + rds = get_redis_connection() + executor = NodeExecutor(rds, history.deploy_key, json.loads(pipe.nodes)) + Thread(target=executor.run).start() + + response = AttrDict(token=history.id, nodes=nodes) + return json_response(response) + return json_response(error=error) + + @auth('exec.task.do') + def patch(self, request): + form, error = JsonParser( + Argument('id', type=int, help='参数错误'), + Argument('cols', type=int, required=False), + Argument('rows', type=int, required=False) + ).parse(request.body) + if error is None: + term = None + if form.cols and form.rows: + term = {'width': form.cols, 'height': form.rows} + pipe = Pipeline.objects.get(pk=form.id) + latest_history = pipe.pipehistory_set.first() + ordinal = latest_history.ordinal + 1 if latest_history else 1 + history = PipeHistory.objects.create(pipeline=pipe, ordinal=ordinal, created_by=request.user) + rds = get_redis_connection() + nodes = json.loads(pipe.nodes) + executor = NodeExecutor(rds, history.deploy_key, nodes) + Thread(target=executor.run).start() + response = AttrDict(token=history.id, nodes=nodes) + return json_response(response) + return json_response(error=error) diff --git a/spug_api/consumer/consumers.py b/spug_api/consumer/consumers.py index 2ab9908..8b238f9 100644 --- a/spug_api/consumer/consumers.py +++ b/spug_api/consumer/consumers.py @@ -22,6 +22,8 @@ class ComConsumer(BaseConsumer): self.key = f'{settings.BUILD_KEY}:{token}' elif module == 'request': self.key = f'{settings.REQUEST_KEY}:{token}' + elif module == 'pipeline': + self.key = f'{settings.PIPELINE_KEY}:{token}' elif module == 'host': self.key = token else: diff --git a/spug_api/libs/gitlib.py b/spug_api/libs/gitlib.py index 265d84e..80d79a0 100644 --- a/spug_api/libs/gitlib.py +++ b/spug_api/libs/gitlib.py @@ -5,6 +5,7 @@ from git import Repo, RemoteReference, TagReference, InvalidGitRepositoryError, from tempfile import NamedTemporaryFile from datetime import datetime from io import StringIO +from functools import partial import subprocess import shutil import os @@ -114,6 +115,7 @@ class RemoteGit: self.url = url self.path = path self.credential = credential + self.remote_exec = self.ssh.exec_command self._ask_env = None def _make_ask_env(self): @@ -123,7 +125,7 @@ class RemoteGit: return self._ask_env ask_file = f'{self.ssh.exec_file}.1' if self.credential.type == 'pw': - env = dict(GIT_ASKPASS=ask_file) + self._ask_env = dict(GIT_ASKPASS=ask_file) body = '#!/bin/bash\n' body += 'case "$1" in\n' body += ' Username*)\n' @@ -134,20 +136,28 @@ class RemoteGit: body = body.format(self.credential) self.ssh.put_file_by_fl(StringIO(body), ask_file) else: - env = dict(GIT_SSH=ask_file) + self._ask_env = dict(GIT_SSH=ask_file) key_file = f'{self.ssh.exec_file}.2' self.ssh.put_file_by_fl(StringIO(self.credential.secret), key_file) self.ssh.sftp.chmod(key_file, 0o600) body = f'ssh -o StrictHostKeyChecking=no -i {key_file} $@' self.ssh.put_file_by_fl(StringIO(body), ask_file) self.ssh.sftp.chmod(ask_file, 0o755) - return env + return self._ask_env def _check_path(self): body = f'git rev-parse --resolve-git-dir {self.path}/.git' code, _ = self.ssh.exec_command(body) return code == 0 + def _clone(self): + env = self._make_ask_env() + print(env) + return self.remote_exec(f'git clone {self.url} {self.path}', env) + + def set_remote_exec(self, remote_exec): + self.remote_exec = partial(remote_exec, self.ssh) + @classmethod def check_auth(cls, url, credential=None): env = dict() @@ -185,16 +195,12 @@ class RemoteGit: res = subprocess.run(command, shell=True, capture_output=True, env=env) return res.returncode == 0, res.stderr.decode() - def clone(self): - env = self._make_ask_env() - code, out = self.ssh.exec_command(f'git clone {self.url} {self.path}', env) - if code != 0: - raise Exception(out) - def fetch_branches_tags(self): body = f'set -e\ncd {self.path}\n' if not self._check_path(): - self.clone() + code, out = self._clone() + if code != 0: + raise Exception(out) else: body += 'git fetch -q --tags --force\n' @@ -224,15 +230,15 @@ class RemoteGit: def checkout(self, marker): body = f'set -e\ncd {self.path}\n' if not self._check_path(): - self.clone() + is_success = self._clone() + if not is_success: + return False else: body += 'git fetch -q --tags --force\n' body += f'git checkout -f {marker}' env = self._make_ask_env() - code, out = self.ssh.exec_command(body, env) - if code != 0: - raise Exception(out) + return self.remote_exec(body, env) def __enter__(self): self.ssh.get_client() diff --git a/spug_api/spug/settings.py b/spug_api/spug/settings.py index c4b3df5..978e686 100644 --- a/spug_api/spug/settings.py +++ b/spug_api/spug/settings.py @@ -112,6 +112,7 @@ MONITOR_WORKER_KEY = 'spug:monitor:worker' EXEC_WORKER_KEY = 'spug:exec:worker' REQUEST_KEY = 'spug:request' BUILD_KEY = 'spug:build' +PIPELINE_KEY = 'spug:pipeline' REPOS_DIR = os.path.join(os.path.dirname(os.path.dirname(BASE_DIR)), 'repos') BUILD_DIR = os.path.join(REPOS_DIR, 'build') TRANSFER_DIR = os.path.join(BASE_DIR, 'storage', 'transfer') diff --git a/spug_web/package.json b/spug_web/package.json index 7ea1719..8206bc0 100644 --- a/spug_web/package.json +++ b/spug_web/package.json @@ -18,8 +18,8 @@ "react-dom": "^16.13.1", "react-router-dom": "^5.2.0", "react-scripts": "3.4.3", - "xterm": "^4.6.0", - "xterm-addon-fit": "^0.5.0" + "xterm": "^5.1.0", + "xterm-addon-fit": "^0.7.0" }, "scripts": { "start": "react-app-rewired start", diff --git a/spug_web/src/pages/deploy/request/Console.js b/spug_web/src/pages/deploy/request/Console.js index b83a93a..5925a9e 100644 --- a/spug_web/src/pages/deploy/request/Console.js +++ b/spug_web/src/pages/deploy/request/Console.js @@ -41,11 +41,11 @@ function Console(props) { useEffect(() => { gCurrent = current - term.setOption('disableStdin', true) - term.setOption('fontSize', 14) - term.setOption('lineHeight', 1.2) - term.setOption('fontFamily', gStore.terminal.fontFamily) - term.setOption('theme', {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'}) + term.options.disableStdin = true + term.options.fontSize = 14 + term.options.lineHeight = 1.2 + term.options.fontFamily = gStore.terminal.fontFamily + term.options.theme = {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'} term.attachCustomKeyEventHandler((arg) => { if (arg.ctrlKey && arg.code === 'KeyC' && arg.type === 'keydown') { document.execCommand('copy') diff --git a/spug_web/src/pages/pipeline/Editor.js b/spug_web/src/pages/pipeline/Editor.js index 17f451e..39b98ff 100644 --- a/spug_web/src/pages/pipeline/Editor.js +++ b/spug_web/src/pages/pipeline/Editor.js @@ -12,6 +12,7 @@ import NodeConfig from './NodeConfig'; import PipeForm from './Form'; import Node from './Node'; import { transfer } from './utils'; +import { history } from 'libs'; import S from './store'; import lds from 'lodash'; import css from './editor.module.less'; @@ -32,7 +33,7 @@ function Editor(props) { }, []) useEffect(() => { - if (S.record.nodes.length) { + if ((S.record?.nodes ?? []).length) { const data = transfer(S.record.nodes) setNodes(data) } @@ -119,6 +120,7 @@ function Editor(props) { } S.record.nodes.splice(index, 1) S.record = {...S.record} + S.updateRecord() } function handleRefresh(node) { @@ -133,7 +135,8 @@ function Editor(props) {
{S.record.name}
setVisible(true)}/>
- +
diff --git a/spug_web/src/pages/pipeline/Icon.js b/spug_web/src/pages/pipeline/Icon.js index dbe9364..af0c1a7 100644 --- a/spug_web/src/pages/pipeline/Icon.js +++ b/spug_web/src/pages/pipeline/Icon.js @@ -1,6 +1,11 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ import React from 'react'; import { Avatar } from 'antd'; -import iconRemoteExec from './assets/icon_remote_exec.png'; +import iconSSHExec from './assets/icon_ssh_exec.png'; import iconBuild from './assets/icon_build.png'; import iconParameter from './assets/icon_parameter.png'; import iconDataTransfer from './assets/icon_data_transfer.png'; @@ -11,8 +16,8 @@ import iconSelect from './assets/icon_select.png'; function Icon(props) { switch (props.module) { - case 'remote_exec': - return + case 'ssh_exec': + return case 'build': return case 'parameter': diff --git a/spug_web/src/pages/pipeline/Node.js b/spug_web/src/pages/pipeline/Node.js index bfbaad6..ee65c37 100644 --- a/spug_web/src/pages/pipeline/Node.js +++ b/spug_web/src/pages/pipeline/Node.js @@ -1,3 +1,8 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ import React from 'react' import { observer } from 'mobx-react'; import { Dropdown } from 'antd'; @@ -40,6 +45,14 @@ function Node(props) { } ] + function dropdownRender(menus) { + return ( +
e.stopPropagation()}> + {menus} +
+ ) + } + const node = props.node switch (node) { case ' ': @@ -69,7 +82,8 @@ function Node(props) { ) : (
请选择节点
)} - +
diff --git a/spug_web/src/pages/pipeline/NodeConfig.js b/spug_web/src/pages/pipeline/NodeConfig.js index a2ebc84..d2cbbbb 100644 --- a/spug_web/src/pages/pipeline/NodeConfig.js +++ b/spug_web/src/pages/pipeline/NodeConfig.js @@ -12,7 +12,8 @@ import Icon from './Icon'; import { clsNames } from 'libs'; import S from './store'; import css from './nodeConfig.module.less'; -import { NODES } from './data' +import { NODES } from './data'; +import lds from 'lodash'; function NodeConfig(props) { const [tab, setTab] = useState('node') @@ -35,7 +36,8 @@ function NodeConfig(props) { const data = handler() if (typeof data === 'object') { setLoading(true) - Object.assign(S.node, data) + const basic = lds.pick(S.node, ['id', 'module', 'downstream']) + S.node = Object.assign(data, basic) props.doRefresh(S.node) .finally(() => setLoading(false)) } diff --git a/spug_web/src/pages/pipeline/Table.js b/spug_web/src/pages/pipeline/Table.js new file mode 100644 index 0000000..c7b157d --- /dev/null +++ b/spug_web/src/pages/pipeline/Table.js @@ -0,0 +1,73 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ +import React, { useEffect } from 'react'; +import { observer } from 'mobx-react'; +import { Table, Modal, message } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; +import { Action, TableCard, AuthButton } from 'components'; +import { http, hasPermission, history } from 'libs'; +import S from './store'; + +function ComTable() { + useEffect(() => { + S.fetchRecords() + }, []) + + function handleDelete(text) { + Modal.confirm({ + title: '删除确认', + content: `确定要删除【${text['name']}】?`, + onOk: () => { + return http.delete('/api/pipeline/', {params: {id: text.id}}) + .then(() => { + message.success('删除成功'); + S.fetchRecords() + }) + } + }) + } + + function toDetail(info) { + history.push(`/pipeline/${info ? info.id : 'new'}`) + } + + return ( + } + onClick={() => toDetail()}>新建 + ]} + pagination={{ + showSizeChanger: true, + showLessItems: true, + showTotal: total => `共 ${total} 条`, + pageSizeOptions: ['10', '20', '50', '100'] + }}> + + + {hasPermission('pipeline.pipeline.edit|pipeline.pipeline.del') && ( + ( + + toDetail(info)}>编辑 + S.showConsole(info)}>执行 + handleDelete(info)}>删除 + + )}/> + )} + + ) +} + +export default observer(ComTable) diff --git a/spug_web/src/pages/pipeline/assets/icon_remote_exec.png b/spug_web/src/pages/pipeline/assets/icon_ssh_exec.png similarity index 100% rename from spug_web/src/pages/pipeline/assets/icon_remote_exec.png rename to spug_web/src/pages/pipeline/assets/icon_ssh_exec.png diff --git a/spug_web/src/pages/pipeline/console/Body.js b/spug_web/src/pages/pipeline/console/Body.js new file mode 100644 index 0000000..152c78f --- /dev/null +++ b/spug_web/src/pages/pipeline/console/Body.js @@ -0,0 +1,143 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ +import React, { useEffect, useRef, useState } from 'react'; +import { observer } from 'mobx-react'; +import { Tooltip, Tabs } from 'antd'; +import { CodeOutlined, StopOutlined } from '@ant-design/icons'; +import { Terminal } from 'xterm'; +import { FitAddon } from 'xterm-addon-fit'; +import { http, X_TOKEN } from 'libs'; +import css from './body.module.less'; +import S from './store'; +import gStore from 'gStore'; + +function Body() { + const el = useRef() + const [term] = useState(new Terminal()); + const [fitPlugin] = useState(new FitAddon()); + const [wsState, setWSState] = useState(); + + useEffect(() => { + let socket; + http.post('/api/pipeline/do/', {id: 1}) + .then(res => { + socket = _makeSocket(res.token) + S.nodes = res.nodes + S.node = res.nodes[0] + }) + return () => socket && socket.close() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + useEffect(() => { + term.options.disableStdin = true + term.options.fontSize = 14 + term.options.lineHeight = 1.2 + term.options.fontFamily = gStore.terminal.fontFamily + term.options.theme = {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'} + term.attachCustomKeyEventHandler((arg) => { + if (arg.ctrlKey && arg.code === 'KeyC' && arg.type === 'keydown') { + document.execCommand('copy') + return false + } + return true + }) + const resize = () => fitPlugin.fit(); + term.loadAddon(fitPlugin) + term.open(el.current) + fitPlugin.fit() + term.write('\x1b[36m### WebSocket connecting ...\x1b[0m') + window.addEventListener('resize', resize) + + return () => window.removeEventListener('resize', resize); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + + function _makeSocket(token, index = 0) { + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const socket = new WebSocket(`${protocol}//${window.location.host}/api/ws/pipeline/${token}/?x-token=${X_TOKEN}`); + socket.onopen = () => socket.send(String(index)); + socket.onmessage = e => { + if (e.data === 'pong') { + socket.send(String(index)) + } else { + index += 1; + const {key, data, status} = JSON.parse(e.data); + if (S.outputs[key]) { + if (status) { + S.outputs[key].status = status + if (key === S.nodeID) { + S.node.status = status + } + } + if (data) { + S.outputs[key].data += data + if (key === S.nodeID) { + term.write(data) + } + } + } else { + S.outputs[key] = {data, status} + S.node.state = '' + } + } + } + socket.onerror = () => { + setWSState('Websocket connection failed') + } + return socket + } + + useEffect(() => { + if (S.node.id) { + fitPlugin.fit() + term.reset() + if (S.outputs[S.nodeID]) { + term.write(S.outputs[S.nodeID].data) + } else { + S.outputs[S.nodeID] = {data: ''} + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [S.node]) + + function handleTerminate() { + + } + + function openTerminal() { + + } + + return ( +
+
+
{S.node?.name}
+
{wsState}
+ + {S.outputs[S.nodeID]?.status === 'processing' ? ( + + ) : ( + + )} + + + openTerminal()}/> + +
+ {S.node?.module === 'ssh_exec' && ( + ({label: x.name, key: `${S.node.id}.${x.id}`}))} + tabBarStyle={{fontSize: 13}} onChange={v => S.node = Object.assign({}, S.node, {_id: v})}/> + )} +
+
+
+
+ ) +} + +export default observer(Body) \ No newline at end of file diff --git a/spug_web/src/pages/pipeline/console/Node.js b/spug_web/src/pages/pipeline/console/Node.js new file mode 100644 index 0000000..bef1756 --- /dev/null +++ b/spug_web/src/pages/pipeline/console/Node.js @@ -0,0 +1,43 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ +import React from 'react' +import { observer } from 'mobx-react'; +import { LoadingOutlined } from '@ant-design/icons'; +import Icon from '../Icon'; +import { clsNames } from 'libs'; +import S from './store'; +import css from './node.module.less'; + +function Node(props) { + const node = props.node + switch (node) { + case ' ': + return
+ case ' -': + return
+ case '--': + return
+ case ' 7': + return
+ case '-7': + return
+ case ' |': + return ( +
+
+
+ ) + default: + return ( +
+ {S.outputs[node.id]?.status === 'processing' ? : null} + +
+ ) + } +} + +export default observer(Node) \ No newline at end of file diff --git a/spug_web/src/pages/pipeline/console/Sider.js b/spug_web/src/pages/pipeline/console/Sider.js new file mode 100644 index 0000000..29b9ad2 --- /dev/null +++ b/spug_web/src/pages/pipeline/console/Sider.js @@ -0,0 +1,34 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ +import React from 'react'; +import { observer } from 'mobx-react'; +import Node from './Node'; +import S from './store'; +import css from './sider.module.less'; + + +function Sider() { + function handleClick(node) { + if (node.module === 'ssh_exec') { + node._id = `${node.id}.${node.targets[0].id}` + } + S.node = node + } + + return ( +
+ {S.matrixNodes.map((row, idx) => ( +
+ {row.map((item, idx) => ( + handleClick(item)}/> + ))} +
+ ))} +
+ ) +} + +export default observer(Sider) \ No newline at end of file diff --git a/spug_web/src/pages/pipeline/console/body.module.less b/spug_web/src/pages/pipeline/console/body.module.less new file mode 100644 index 0000000..e6e2246 --- /dev/null +++ b/spug_web/src/pages/pipeline/console/body.module.less @@ -0,0 +1,73 @@ +.container { + height: calc(100vh - 300px); + flex: 1; + display: flex; + flex-direction: column; + padding-left: 24px; + overflow: hidden; +} + +.header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; + + .tips { + color: #ff4d4f; + } + + .icon { + font-size: 18px; + color: #1890ff; + cursor: pointer; + margin-left: 12px; + } + + .title { + flex: 1; + font-weight: 500; + } +} + + +.termContainer { + flex: 1; + background-color: #2b2b2b; + padding: 8px 0 4px 12px; + border-radius: 4px; + overflow: hidden; + + .term { + width: 100%; + height: 100%; + } +} + +.execContainer { + + .header { + color: #ffffff; + background: #36435c; + border-radius: 4px; + padding: 6px 12px; + margin-right: 12px; + font-size: 14px; + display: flex; + flex-direction: row; + align-items: center; + + .title { + flex: 1; + margin-left: 12px; + } + } + + .item { + + } +} + +:global(.ant-tabs-tab) { + font-size: 13px; +} diff --git a/spug_web/src/pages/pipeline/console/index.js b/spug_web/src/pages/pipeline/console/index.js new file mode 100644 index 0000000..ee4045e --- /dev/null +++ b/spug_web/src/pages/pipeline/console/index.js @@ -0,0 +1,32 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ +import React from 'react'; +import { observer } from 'mobx-react'; +import { Modal, Row } from 'antd'; +import Sider from './Sider'; +import Body from './Body'; +import pS from '../store'; +import S from './store'; + +function Index() { + return ( + pS.consoleVisible = false}> + + + + + + ) +} + +export default observer(Index) diff --git a/spug_web/src/pages/pipeline/console/node.module.less b/spug_web/src/pages/pipeline/console/node.module.less new file mode 100644 index 0000000..9dc6d8a --- /dev/null +++ b/spug_web/src/pages/pipeline/console/node.module.less @@ -0,0 +1,108 @@ +.box { + position: relative; + width: 38px; + height: 38px; + margin-right: 16px; + flex-shrink: 0; + + &:last-child { + margin: 0; + } +} + +.triangle { + position: absolute; + bottom: -2px; + left: 15px; + width: 0; + height: 0; + border: 4px solid transparent; + border-top-color: #999999; +} + +.node { + position: relative; + box-shadow: 0 0 6px #9999994c; + border: 1px solid transparent; + border-radius: 50%; + cursor: pointer; + + &:hover { + box-shadow: 0 0 6px #2563fcbb; + .action { + display: block; + } + } +} + +.loading { + position: absolute; + top: 4px; + left: 4px; + color: #ffffff; + font-size: 28px; + z-index: 999; +} + +.active { + box-shadow: 0 0 6px #2563fc; +} + +.line { + &:after { + content: ' '; + height: 2px; + background: #999999; + position: absolute; + top: 18px; + left: -14px; + right: -4px; + } +} + +.line2 { + &:after { + left: -34px; + } +} + +.angle { + &:before { + content: ' '; + position: absolute; + background: #999999; + top: 18px; + bottom: 18px; + left: -14px; + right: 18px; + } + + &:after { + content: ' '; + position: absolute; + background: #999999; + top: 18px; + left: 18px; + right: 18px; + bottom: -4px; + } +} + +.angle2 { + &:before { + left: -34px; + } +} + +.arrow { + &:after { + content: ' '; + background: #999999; + position: absolute; + top: 2px; + left: 18px; + right: 18px; + bottom: 4px; + } +} + diff --git a/spug_web/src/pages/pipeline/console/sider.module.less b/spug_web/src/pages/pipeline/console/sider.module.less new file mode 100644 index 0000000..bb0dde2 --- /dev/null +++ b/spug_web/src/pages/pipeline/console/sider.module.less @@ -0,0 +1,6 @@ +.sider { + max-width: 200px; + min-width: 104px; + padding: 6px; + overflow: auto; +} \ No newline at end of file diff --git a/spug_web/src/pages/pipeline/console/store.js b/spug_web/src/pages/pipeline/console/store.js new file mode 100644 index 0000000..6124a77 --- /dev/null +++ b/spug_web/src/pages/pipeline/console/store.js @@ -0,0 +1,29 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ +import { observable, computed } from 'mobx'; +import { transfer } from '../utils'; + +class Store { + @observable node = {}; + @observable nodes = []; + @observable outputs = {}; + + @computed get nodeID() { + return this.node._id ?? this.node.id + } + + @computed get matrixNodes() { + return transfer(this.nodes) + } + + initial = () => { + this.node = {} + this.nodes = [] + this.outputs = {} + } +} + +export default new Store() diff --git a/spug_web/src/pages/pipeline/data.js b/spug_web/src/pages/pipeline/data.js index 0d392d0..0c980c2 100644 --- a/spug_web/src/pages/pipeline/data.js +++ b/spug_web/src/pages/pipeline/data.js @@ -1,5 +1,5 @@ export const NODES = [ - {module: 'remote_exec', name: '执行命令'}, + {module: 'ssh_exec', name: '执行命令'}, {module: 'build', name: '构建'}, {module: 'parameter', name: '参数化'}, {module: 'data_transfer', name: '数据传输'}, @@ -60,7 +60,7 @@ export const DATAS = { 'command': 'date && sleep 3', }, { - 'module': 'remote_exec', + 'module': 'ssh_exec', 'name': '执行命令', 'id': 5, 'targets': [2, 3], diff --git a/spug_web/src/pages/pipeline/index.js b/spug_web/src/pages/pipeline/index.js index 040e562..574b601 100644 --- a/spug_web/src/pages/pipeline/index.js +++ b/spug_web/src/pages/pipeline/index.js @@ -6,12 +6,16 @@ import React from 'react'; import { observer } from 'mobx-react'; import { AuthDiv } from 'components'; -import Editor from './Editor'; +import Table from './Table'; +import Console from './console'; -export default observer(function () { +function Index() { return ( - - + + + ) -}) +} + +export default observer(Index) diff --git a/spug_web/src/pages/pipeline/modules/Build.js b/spug_web/src/pages/pipeline/modules/Build.js index c83a7b7..e47a975 100644 --- a/spug_web/src/pages/pipeline/modules/Build.js +++ b/spug_web/src/pages/pipeline/modules/Build.js @@ -1,3 +1,8 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ import React, { useEffect, useState } from 'react'; import { observer } from 'mobx-react'; import { Form, Input, Select, Radio, message } from 'antd'; @@ -47,7 +52,7 @@ function Build(props) { 上游执行成功时 - 上游执行失败时 + 上游执行失败时 总是执行 diff --git a/spug_web/src/pages/pipeline/modules/DataTransfer.js b/spug_web/src/pages/pipeline/modules/DataTransfer.js index a3f78a8..1b2739a 100644 --- a/spug_web/src/pages/pipeline/modules/DataTransfer.js +++ b/spug_web/src/pages/pipeline/modules/DataTransfer.js @@ -1,3 +1,8 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ import React, { useEffect } from 'react'; import { Form, Input, message, Card, Radio } from 'antd'; import HostSelector from 'pages/host/Selector'; @@ -29,7 +34,7 @@ function DataTransfer(props) { 上游执行成功时 - 上游执行失败时 + 上游执行失败时 总是执行 diff --git a/spug_web/src/pages/pipeline/modules/SSHExec.js b/spug_web/src/pages/pipeline/modules/SSHExec.js index f8537dc..bc43246 100644 --- a/spug_web/src/pages/pipeline/modules/SSHExec.js +++ b/spug_web/src/pages/pipeline/modules/SSHExec.js @@ -1,3 +1,8 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ import React, { useEffect } from 'react'; import { Form, Input, Radio, message } from 'antd'; import { ACEditor } from 'components'; @@ -11,12 +16,16 @@ function SSHExec(props) { // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + useEffect(() => { + form.resetFields() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [props.node]) + function handleSave() { const data = form.getFieldsValue() if (!data.name) return message.error('请输入节点名称') if (!data.condition) return message.error('请选择节点的执行条件') if (!data.targets || data.targets.length === 0) return message.error('请选择执行主机') - if (!data.interpreter) return message.error('请选择执行解释器') if (!data.command) return message.error('请输入执行内容') return data } @@ -29,19 +38,13 @@ function SSHExec(props) { 上游执行成功时 - 上游执行失败时 + 上游执行失败时 总是执行 - - - Shell - Python - - p.interpreter !== c.interpreter}> {({getFieldValue}) => ( diff --git a/spug_web/src/pages/pipeline/modules/index.js b/spug_web/src/pages/pipeline/modules/index.js index 97ee454..bbc22d8 100644 --- a/spug_web/src/pages/pipeline/modules/index.js +++ b/spug_web/src/pages/pipeline/modules/index.js @@ -1,3 +1,8 @@ +/** + * Copyright (c) OpenSpug Organization. https://github.com/openspug/spug + * Copyright (c) + * Released under the AGPL-3.0 License. + */ import React from 'react'; import SSHExec from './SSHExec'; import Build from './Build'; @@ -5,7 +10,7 @@ import DataTransfer from './DataTransfer'; function ModuleConfig(props) { switch (props.node.module) { - case 'remote_exec': + case 'ssh_exec': return case 'build': return diff --git a/spug_web/src/pages/pipeline/store.js b/spug_web/src/pages/pipeline/store.js index 00bcba1..40bd475 100644 --- a/spug_web/src/pages/pipeline/store.js +++ b/spug_web/src/pages/pipeline/store.js @@ -3,20 +3,28 @@ * Copyright (c) * Released under the AGPL-3.0 License. */ -import { observable } from 'mobx'; -import { http } from 'libs'; +import { computed, observable } from 'mobx'; +import { http, includes } from 'libs'; import { message } from 'antd'; class Store { + @observable records = []; @observable record = {nodes: []}; @observable nodes = []; @observable node = {}; @observable actionNode = {}; @observable isFetching = true; + @observable consoleVisible = false; - fetchRecords = (id, isFetching) => { + @computed get dataSource() { + let records = this.records; + if (this.f_name) records = records.filter(x => includes(x.name, this.f_name)); + return records + } + + fetchRecords = () => { this.isFetching = true; - return http.get('/api/pipline/') + return http.get('/api/pipeline/') .then(res => this.records = res) .finally(() => this.isFetching = false) } @@ -35,6 +43,11 @@ class Store { message.success('保存成功') }) } + + showConsole = (record) => { + this.record = record + this.consoleVisible = true + } } export default new Store() diff --git a/spug_web/src/routes.js b/spug_web/src/routes.js index abe5b9c..097e283 100644 --- a/spug_web/src/routes.js +++ b/spug_web/src/routes.js @@ -43,7 +43,7 @@ import SystemCredential from './pages/system/credential'; import WelcomeIndex from './pages/welcome/index'; import WelcomeInfo from './pages/welcome/info'; import PipelineIndex from './pages/pipeline'; -import PipelineEditor from './pages/pipeline'; +import PipelineEditor from './pages/pipeline/Editor'; export default [ {icon: , title: '工作台', path: '/home', component: HomeIndex}, @@ -69,8 +69,8 @@ export default [ {title: '发布申请', auth: 'deploy.request.view', path: '/deploy/request', component: DeployRequest}, ] }, - {icon: , title: '流水线', path: '/pipeline', component: PipelineIndex}, {path: '/pipeline/:id', component: PipelineEditor}, + {icon: , title: '流水线', path: '/pipeline', component: PipelineIndex}, { icon: , title: '任务计划',