github
/
spug
mirror of https://github.com/openspug/spug
1
0
Fork 0
Code Issues Releases Wiki Activity

update pipeline module

4.0
vapao 2023-03-02 00:00:51 +08:00
parent 50030d544a
commit 6cda7a9d2a
32 changed files with 1044 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
spug_api/apps/pipeline/helper.py Normal file
View File

@ -0,0 +1,212 @@
# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug
# Copyright: (c) <spug.dev@gmail.com>
# Released under the AGPL-3.0 License.
from django.conf import settings
from django.template.defaultfilters import filesizeformat
from django_redis import get_redis_connection
from libs.utils import SpugError, human_time, render_str, human_seconds_time
from apps.config.utils import update_config_by_var
from collections import defaultdict
import time
import json
import os
import re
class KitMixin:
regex = re.compile(r'^((\r\n)*)(.*?)((\r\n)*)$', re.DOTALL)
@classmethod
def term_message(cls, message, color_mode='info', with_time=False):
prefix = f'{human_time()} ' if with_time else ''
if color_mode == 'info':
mode = '36m'
elif color_mode == 'warn':
mode = '33m'
elif color_mode == 'error':
mode = '31m'
elif color_mode == 'success':
mode = '32m'
else:
raise TypeError
return cls.regex.sub(fr'\1\033[{mode}{prefix}\3\033[0m\4', message)
class Helper(KitMixin):
def __init__(self, rds, rds_key):
self.rds = rds
self.rds_key = rds_key
self.buffers = defaultdict(str)
self.flags = defaultdict(bool)
self.already_clear = False
self.files = {}
def __del__(self):
self.clear()
@classmethod
def make(cls, rds, rds_key):
rds.delete(rds_key)
return cls(rds, rds_key)
@classmethod
def fill_outputs(cls, outputs, deploy_key):
rds = get_redis_connection()
key_ttl = rds.ttl(deploy_key)
counter, hit_keys = 0, set()
if key_ttl > 30 or key_ttl == -1:
data = rds.lrange(deploy_key, counter, counter + 9)
while data:
for item in data:
counter += 1
item = json.loads(item.decode())
key = item['key']
if key in outputs:
hit_keys.add(key)
if 'data' in item:
outputs[key]['data'] += item['data']
if 'status' in item:
outputs[key]['status'] = item['status']
data = rds.lrange(deploy_key, counter, counter + 9)
for key in outputs.keys():
if key in hit_keys:
continue
file_name = os.path.join(settings.DEPLOY_DIR, f'{deploy_key}:{key}')
if not os.path.exists(file_name):
continue
with open(file_name, newline='\r\n') as f:
line = f.readline()
while line:
status, data = line.split(',', 1)
if data:
outputs[key]['data'] += data
if status:
outputs[key]['status'] = status
line = f.readline()
return counter
def get_file(self, key):
if key in self.files:
return self.files[key]
file = open(os.path.join(settings.DEPLOY_DIR, f'{self.rds_key}:{key}'), 'w')
self.files[key] = file
return file
def get_cross_env(self, key):
file = os.path.join(settings.DEPLOY_DIR, key)
if os.path.exists(file):
with open(file, 'r') as f:
return json.loads(f.read())
return {}
def set_cross_env(self, key, envs):
file_envs = {}
for k, v in envs.items():
if k == 'SPUG_SET':
try:
update_config_by_var(v)
except SpugError as e:
self.send_error(key, f'{e}')
elif k.startswith('SPUG_GEV_'):
file_envs[k] = v
file = os.path.join(settings.DEPLOY_DIR, key)
with open(file, 'w') as f:
f.write(json.dumps(file_envs))
def save_pid(self, pid, key):
self.rds.set(f'PID:{self.rds_key}:{key}', pid, 3600)
def parse_filter_rule(self, data: str, sep='\n', env=None):
data, files = data.strip(), []
if data:
for line in data.split(sep):
line = line.strip()
if line and not line.startswith('#'):
files.append(render_str(line, env))
return files
def send(self, key, data, *, status=''):
message = {'key': key, 'data': data}
if status:
message['status'] = status
print(self.rds_key, message)
self.rds.rpush(self.rds_key, json.dumps(message))
file = self.get_file(key)
for idx, line in enumerate(data.split('\r\n')):
if idx != 0:
tmp = [status, self.buffers[key] + '\r\n']
file.write(','.join(tmp))
file.flush()
self.buffers[key] = ''
self.flags[key] = False
if line:
for idx2, item in enumerate(line.split('\r')):
if idx2 != 0:
self.flags[key] = True
if item:
if self.flags[key]:
self.buffers[key] = item
self.flags[key] = False
else:
self.buffers[key] += item
def send_clear(self, key):
self.send(key, '\033[2J\033[3J\033[1;1H')
def send_info(self, key, message, status='', with_time=True):
message = self.term_message(message, 'info', with_time)
self.send(key, message, status=status)
def send_warn(self, key, message, status=''):
message = self.term_message(message, 'warn')
self.send(key, message, status=status)
def send_success(self, key, message, with_time=True, start_time=None):
if start_time:
message += f', 耗时: {human_seconds_time(time.time() - start_time)}'
message = self.term_message(f'\r\n** {message} **', 'success', with_time)
self.send(key, message, status='success')
def send_error(self, key, message, with_break=False):
message = self.term_message(message, 'error')
if not message.endswith('\r\n'):
message += '\r\n'
self.send(key, message, status='error')
if with_break:
raise SpugError
def send_status(self, key, status):
self.send(key, '', status=status)
def clear(self):
if self.already_clear:
return
self.already_clear = True
for key, value in self.buffers.items():
if value:
file = self.get_file(key)
file.write(f',{value}')
for file in self.files.values():
file.close()
if self.rds.ttl(self.rds_key) == -1:
self.rds.expire(self.rds_key, 60)
def progress_callback(self, key):
def func(n, t):
message = f'\r {filesizeformat(n):<8}/{filesizeformat(t):>8} '
self.send(key, message)
self.send(key, '\r\n')
return func
def remote_exec(self, key, ssh, command, env=None):
code = -1
for code, out in ssh.exec_command_with_stream(command, environment=env):
self.send(key, out)
if code != 0:
self.send_error(key, f'exit code: {code}')
return code == 0

20
spug_api/apps/pipeline/models.py
View File

@ -2,6 +2,7 @@
# Copyright: (c) <spug.dev@gmail.com>
# Released under the AGPL-3.0 License.
from django.db import models
from django.conf import settings
from libs.mixins import ModelMixin
from apps.account.models import User
import json
@ -18,6 +19,25 @@ class Pipeline(models.Model, ModelMixin):
tmp['nodes'] = json.loads(self.nodes)
return tmp
def to_list(self):
tmp = self.to_dict(selects=('id', 'name', 'created_at'))
return tmp
class Meta:
db_table = 'pipelines'
ordering = ('-id',)
class PipeHistory(models.Model, ModelMixin):
pipeline = models.ForeignKey(Pipeline, on_delete=models.CASCADE)
ordinal = models.IntegerField()
created_by = models.ForeignKey(User, on_delete=models.PROTECT)
created_at = models.DateTimeField(auto_now_add=True)
@property
def deploy_key(self):
return f'{settings.PIPELINE_KEY}:{self.id}'
class Meta:
db_table = 'pipeline_histories'
ordering = ('-id',)

3
spug_api/apps/pipeline/urls.py
View File

@ -3,8 +3,9 @@
# Released under the AGPL-3.0 License.
from django.urls import path
from apps.pipeline.views import PipeView
from apps.pipeline.views import PipeView, DoView
urlpatterns = [
path('', PipeView.as_view()),
path('do/', DoView.as_view()),
]

86
spug_api/apps/pipeline/utils.py Normal file
View File

@ -0,0 +1,86 @@
# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug
# Copyright: (c) <spug.dev@gmail.com>
# Released under the AGPL-3.0 License.
from apps.credential.models import Credential
from apps.host.models import Host
from libs.utils import AttrDict, human_seconds_time
from libs.gitlib import RemoteGit
from apps.pipeline.helper import Helper
from functools import partial, partialmethod
from threading import Thread
from concurrent import futures
import time
import os
class NodeExecutor:
def __init__(self, rds, rds_key, nodes):
self.rds = rds
self.rds_key = rds_key
self.nodes = {x.id: x for x in map(AttrDict, nodes)}
self.node = AttrDict(nodes[0])
self.helper = Helper.make(self.rds, self.rds_key)
def run(self, node=None, state=None):
print(node, state)
if node:
downstream = getattr(node, 'downstream', [])
down_nodes = [self.nodes[x] for x in downstream]
available_nodes = [x for x in down_nodes if x.condition in (state, 'always')]
if len(available_nodes) >= 2:
for node in available_nodes[1:]:
Thread(target=self._dispatch, args=(node,)).start()
if available_nodes:
self._dispatch(available_nodes[0])
else:
self._dispatch(self.node)
def _dispatch(self, node):
print('!!!!! _dispatch', node.name)
if node.module == 'build':
self._do_build(node)
elif node.module == 'ssh_exec':
self._do_ssh_exec(node)
def _do_build(self, node, marker=None):
# if node.mode == 'branch':
# marker = node.branch
timestamp = time.time()
marker = 'origin/bugfix'
host = Host.objects.get(pk=node.target)
credential = None
if node.credential_id:
credential = Credential.objects.get(pk=node.credential_id)
with RemoteGit(host, node.git_url, node.workspace, credential) as git:
self.helper.send_info(node.id, '同步并检出Git仓库\r\n', 'processing')
git.set_remote_exec(partial(self.helper.remote_exec, node.id))
is_success = git.checkout(marker)
if is_success and node.command:
self.helper.send_info(node.id, '执行构建命令\r\n')
is_success = self.helper.remote_exec(node.id, git.ssh, node.command)
if is_success:
self.helper.send_success(node.id, '构建完成', start_time=timestamp)
self.run(node, 'success' if is_success else 'error')
def _do_ssh_exec(self, node):
threads = []
max_workers = max(10, os.cpu_count() * 5)
self.helper.send_status(node.id, 'processing')
with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
for host in Host.objects.filter(id__in=node.targets):
t = executor.submit(self._ssh_exec, host, node)
threads.append(t)
results = [x.result() for x in futures.as_completed(threads)]
state = 'success' if all(results) else 'error'
self.helper.send_status(node.id, state)
self.run(node, state)
def _ssh_exec(self, host, node):
timestamp = time.time()
key = f'{node.id}.{host.id}'
self.helper.send_info(key, '开始执行\r\n', 'processing')
with host.get_ssh() as ssh:
is_success = self.helper.remote_exec(key, ssh, node.command)
if is_success:
self.helper.send_success(key, '执行结束', start_time=timestamp)
return is_success

63
spug_api/apps/pipeline/views.py
View File

@ -2,8 +2,14 @@
# Copyright: (c) <spug.dev@gmail.com>
# Released under the AGPL-3.0 License.
from django.views.generic import View
from django_redis import get_redis_connection
from libs import JsonParser, Argument, json_response, auth
from apps.pipeline.models import Pipeline
from libs.utils import AttrDict
from apps.pipeline.models import Pipeline, PipeHistory
from apps.pipeline.utils import NodeExecutor
from apps.host.models import Host
from threading import Thread
from uuid import uuid4
import json
@ -57,3 +63,58 @@ class PipeView(View):
if error is None:
Pipeline.objects.filter(pk=form.id).delete()
return json_response(error=error)
class DoView(View):
@auth('exec.task.do')
def get(self, request):
pass
@auth('exec.task.do')
def post(self, request):
form, error = JsonParser(
Argument('id', type=int, help='参数错误'),
).parse(request.body)
if error is None:
pipe = Pipeline.objects.get(pk=form.id)
latest_history = pipe.pipehistory_set.first()
ordinal = latest_history.ordinal + 1 if latest_history else 1
history = PipeHistory.objects.create(pipeline=pipe, ordinal=ordinal, created_by=request.user)
nodes, ids = json.loads(pipe.nodes), set()
for item in filter(lambda x: x['module'] == 'ssh_exec', nodes):
ids.update(item['targets'])
host_map = {x.id: f'{x.name}({x.hostname})' for x in Host.objects.filter(id__in=ids)}
for item in filter(lambda x: x['module'] == 'ssh_exec', nodes):
item['targets'] = [{'id': x, 'name': host_map[x]} for x in item['targets']]
rds = get_redis_connection()
executor = NodeExecutor(rds, history.deploy_key, json.loads(pipe.nodes))
Thread(target=executor.run).start()
response = AttrDict(token=history.id, nodes=nodes)
return json_response(response)
return json_response(error=error)
@auth('exec.task.do')
def patch(self, request):
form, error = JsonParser(
Argument('id', type=int, help='参数错误'),
Argument('cols', type=int, required=False),
Argument('rows', type=int, required=False)
).parse(request.body)
if error is None:
term = None
if form.cols and form.rows:
term = {'width': form.cols, 'height': form.rows}
pipe = Pipeline.objects.get(pk=form.id)
latest_history = pipe.pipehistory_set.first()
ordinal = latest_history.ordinal + 1 if latest_history else 1
history = PipeHistory.objects.create(pipeline=pipe, ordinal=ordinal, created_by=request.user)
rds = get_redis_connection()
nodes = json.loads(pipe.nodes)
executor = NodeExecutor(rds, history.deploy_key, nodes)
Thread(target=executor.run).start()
response = AttrDict(token=history.id, nodes=nodes)
return json_response(response)
return json_response(error=error)

2
spug_api/consumer/consumers.py
View File

@ -22,6 +22,8 @@ class ComConsumer(BaseConsumer):
self.key = f'{settings.BUILD_KEY}:{token}'
elif module == 'request':
self.key = f'{settings.REQUEST_KEY}:{token}'
elif module == 'pipeline':
self.key = f'{settings.PIPELINE_KEY}:{token}'
elif module == 'host':
self.key = token
else:

34
spug_api/libs/gitlib.py
View File

@ -5,6 +5,7 @@ from git import Repo, RemoteReference, TagReference, InvalidGitRepositoryError,
from tempfile import NamedTemporaryFile
from datetime import datetime
from io import StringIO
from functools import partial
import subprocess
import shutil
import os
@ -114,6 +115,7 @@ class RemoteGit:
self.url = url
self.path = path
self.credential = credential
self.remote_exec = self.ssh.exec_command
self._ask_env = None
def _make_ask_env(self):
@ -123,7 +125,7 @@ class RemoteGit:
return self._ask_env
ask_file = f'{self.ssh.exec_file}.1'
if self.credential.type == 'pw':
env = dict(GIT_ASKPASS=ask_file)
self._ask_env = dict(GIT_ASKPASS=ask_file)
body = '#!/bin/bash\n'
body += 'case "$1" in\n'
body += ' Username*)\n'
@ -134,20 +136,28 @@ class RemoteGit:
body = body.format(self.credential)
self.ssh.put_file_by_fl(StringIO(body), ask_file)
else:
env = dict(GIT_SSH=ask_file)
self._ask_env = dict(GIT_SSH=ask_file)
key_file = f'{self.ssh.exec_file}.2'
self.ssh.put_file_by_fl(StringIO(self.credential.secret), key_file)
self.ssh.sftp.chmod(key_file, 0o600)
body = f'ssh -o StrictHostKeyChecking=no -i {key_file} $@'
self.ssh.put_file_by_fl(StringIO(body), ask_file)
self.ssh.sftp.chmod(ask_file, 0o755)
return env
return self._ask_env
def _check_path(self):
body = f'git rev-parse --resolve-git-dir {self.path}/.git'
code, _ = self.ssh.exec_command(body)
return code == 0
def _clone(self):
env = self._make_ask_env()
print(env)
return self.remote_exec(f'git clone {self.url} {self.path}', env)
def set_remote_exec(self, remote_exec):
self.remote_exec = partial(remote_exec, self.ssh)
@classmethod
def check_auth(cls, url, credential=None):
env = dict()
@ -185,16 +195,12 @@ class RemoteGit:
res = subprocess.run(command, shell=True, capture_output=True, env=env)
return res.returncode == 0, res.stderr.decode()
def clone(self):
env = self._make_ask_env()
code, out = self.ssh.exec_command(f'git clone {self.url} {self.path}', env)
if code != 0:
raise Exception(out)
def fetch_branches_tags(self):
body = f'set -e\ncd {self.path}\n'
if not self._check_path():
self.clone()
code, out = self._clone()
if code != 0:
raise Exception(out)
else:
body += 'git fetch -q --tags --force\n'
@ -224,15 +230,15 @@ class RemoteGit:
def checkout(self, marker):
body = f'set -e\ncd {self.path}\n'
if not self._check_path():
self.clone()
is_success = self._clone()
if not is_success:
return False
else:
body += 'git fetch -q --tags --force\n'
body += f'git checkout -f {marker}'
env = self._make_ask_env()
code, out = self.ssh.exec_command(body, env)
if code != 0:
raise Exception(out)
return self.remote_exec(body, env)
def __enter__(self):
self.ssh.get_client()

1
spug_api/spug/settings.py
View File

@ -112,6 +112,7 @@ MONITOR_WORKER_KEY = 'spug:monitor:worker'
EXEC_WORKER_KEY = 'spug:exec:worker'
REQUEST_KEY = 'spug:request'
BUILD_KEY = 'spug:build'
PIPELINE_KEY = 'spug:pipeline'
REPOS_DIR = os.path.join(os.path.dirname(os.path.dirname(BASE_DIR)), 'repos')
BUILD_DIR = os.path.join(REPOS_DIR, 'build')
TRANSFER_DIR = os.path.join(BASE_DIR, 'storage', 'transfer')

4
spug_web/package.json
View File

@ -18,8 +18,8 @@
"react-dom": "^16.13.1",
"react-router-dom": "^5.2.0",
"react-scripts": "3.4.3",
"xterm": "^4.6.0",
"xterm-addon-fit": "^0.5.0"
"xterm": "^5.1.0",
"xterm-addon-fit": "^0.7.0"
},
"scripts": {
"start": "react-app-rewired start",

10
spug_web/src/pages/deploy/request/Console.js
View File

@ -41,11 +41,11 @@ function Console(props) {
useEffect(() => {
gCurrent = current
term.setOption('disableStdin', true)
term.setOption('fontSize', 14)
term.setOption('lineHeight', 1.2)
term.setOption('fontFamily', gStore.terminal.fontFamily)
term.setOption('theme', {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'})
term.options.disableStdin = true
term.options.fontSize = 14
term.options.lineHeight = 1.2
term.options.fontFamily = gStore.terminal.fontFamily
term.options.theme = {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'}
term.attachCustomKeyEventHandler((arg) => {
if (arg.ctrlKey && arg.code === 'KeyC' && arg.type === 'keydown') {
document.execCommand('copy')

7
spug_web/src/pages/pipeline/Editor.js
View File

@ -12,6 +12,7 @@ import NodeConfig from './NodeConfig';
import PipeForm from './Form';
import Node from './Node';
import { transfer } from './utils';
import { history } from 'libs';
import S from './store';
import lds from 'lodash';
import css from './editor.module.less';
@ -32,7 +33,7 @@ function Editor(props) {
}, [])
useEffect(() => {
if (S.record.nodes.length) {
if ((S.record?.nodes ?? []).length) {
const data = transfer(S.record.nodes)
setNodes(data)
}
@ -119,6 +120,7 @@ function Editor(props) {
}
S.record.nodes.splice(index, 1)
S.record = {...S.record}
S.updateRecord()
}
function handleRefresh(node) {
@ -133,7 +135,8 @@ function Editor(props) {
<div className={css.title}>{S.record.name}</div>
<EditOutlined className={css.edit} onClick={() => setVisible(true)}/>
<div style={{flex: 1}}/>
<Button className={css.back} type="link" icon={<RollbackOutlined/>}>返回列表</Button>
<Button className={css.back} type="link" icon={<RollbackOutlined/>}
onClick={() => history.goBack()}>返回列表</Button>
</div>
<div className={css.body}>
<div className={css.nodes}>

11
spug_web/src/pages/pipeline/Icon.js
View File

@ -1,6 +1,11 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React from 'react';
import { Avatar } from 'antd';
import iconRemoteExec from './assets/icon_remote_exec.png';
import iconSSHExec from './assets/icon_ssh_exec.png';
import iconBuild from './assets/icon_build.png';
import iconParameter from './assets/icon_parameter.png';
import iconDataTransfer from './assets/icon_data_transfer.png';
@ -11,8 +16,8 @@ import iconSelect from './assets/icon_select.png';
function Icon(props) {
switch (props.module) {
case 'remote_exec':
return <Avatar size={props.size || 42} src={iconRemoteExec}/>
case 'ssh_exec':
return <Avatar size={props.size || 42} src={iconSSHExec}/>
case 'build':
return <Avatar size={props.size || 42} src={iconBuild}/>
case 'parameter':

16
spug_web/src/pages/pipeline/Node.js
View File

@ -1,3 +1,8 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React from 'react'
import { observer } from 'mobx-react';
import { Dropdown } from 'antd';
@ -40,6 +45,14 @@ function Node(props) {
}
]
function dropdownRender(menus) {
return (
<div onMouseDown={e => e.stopPropagation()}>
{menus}
</div>
)
}
const node = props.node
switch (node) {
case ' ':
@ -69,7 +82,8 @@ function Node(props) {
) : (
<div className={css.title} style={{color: '#595959'}}>请选择节点</div>
)}
<Dropdown className={css.action} trigger="click" menu={{items: menus}} onMouseDown={handleActionClick}>
<Dropdown dropdownRender={dropdownRender} className={css.action}
trigger="click" menu={{items: menus}} onMouseDown={handleActionClick}>
<MoreOutlined/>
</Dropdown>
</div>

6
spug_web/src/pages/pipeline/NodeConfig.js
View File

@ -12,7 +12,8 @@ import Icon from './Icon';
import { clsNames } from 'libs';
import S from './store';
import css from './nodeConfig.module.less';
import { NODES } from './data'
import { NODES } from './data';
import lds from 'lodash';
function NodeConfig(props) {
const [tab, setTab] = useState('node')
@ -35,7 +36,8 @@ function NodeConfig(props) {
const data = handler()
if (typeof data === 'object') {
setLoading(true)
Object.assign(S.node, data)
const basic = lds.pick(S.node, ['id', 'module', 'downstream'])
S.node = Object.assign(data, basic)
props.doRefresh(S.node)
.finally(() => setLoading(false))
}

73
spug_web/src/pages/pipeline/Table.js Normal file
View File

@ -0,0 +1,73 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React, { useEffect } from 'react';
import { observer } from 'mobx-react';
import { Table, Modal, message } from 'antd';
import { PlusOutlined } from '@ant-design/icons';
import { Action, TableCard, AuthButton } from 'components';
import { http, hasPermission, history } from 'libs';
import S from './store';
function ComTable() {
useEffect(() => {
S.fetchRecords()
}, [])
function handleDelete(text) {
Modal.confirm({
title: '删除确认',
content: `确定要删除【${text['name']}】?`,
onOk: () => {
return http.delete('/api/pipeline/', {params: {id: text.id}})
.then(() => {
message.success('删除成功');
S.fetchRecords()
})
}
})
}
function toDetail(info) {
history.push(`/pipeline/${info ? info.id : 'new'}`)
}
return (
<TableCard
tKey="pipe"
rowKey="id"
title="流程列表"
loading={S.isFetching}
dataSource={S.dataSource}
onReload={S.fetchRecords}
actions={[
<AuthButton
auth="pipeline.pipeline.add"
type="primary"
icon={<PlusOutlined/>}
onClick={() => toDetail()}>新建</AuthButton>
]}
pagination={{
showSizeChanger: true,
showLessItems: true,
showTotal: total => `共 ${total} 条`,
pageSizeOptions: ['10', '20', '50', '100']
}}>
<Table.Column title="流程名称" dataIndex="name"/>
<Table.Column ellipsis title="备注信息" dataIndex="desc"/>
{hasPermission('pipeline.pipeline.edit|pipeline.pipeline.del') && (
<Table.Column width={210} title="操作" render={info => (
<Action>
<Action.Button auth="config.app.edit" onClick={() => toDetail(info)}>编辑</Action.Button>
<Action.Button auth="config.app.edit" onClick={() => S.showConsole(info)}>执行</Action.Button>
<Action.Button danger auth="config.app.del" onClick={() => handleDelete(info)}>删除</Action.Button>
</Action>
)}/>
)}
</TableCard>
)
}
export default observer(ComTable)

0
spug_web/src/pages/pipeline/assets/icon_remote_exec.png → spug_web/src/pages/pipeline/assets/icon_ssh_exec.png
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

143
spug_web/src/pages/pipeline/console/Body.js Normal file
View File

@ -0,0 +1,143 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React, { useEffect, useRef, useState } from 'react';
import { observer } from 'mobx-react';
import { Tooltip, Tabs } from 'antd';
import { CodeOutlined, StopOutlined } from '@ant-design/icons';
import { Terminal } from 'xterm';
import { FitAddon } from 'xterm-addon-fit';
import { http, X_TOKEN } from 'libs';
import css from './body.module.less';
import S from './store';
import gStore from 'gStore';
function Body() {
const el = useRef()
const [term] = useState(new Terminal());
const [fitPlugin] = useState(new FitAddon());
const [wsState, setWSState] = useState();
useEffect(() => {
let socket;
http.post('/api/pipeline/do/', {id: 1})
.then(res => {
socket = _makeSocket(res.token)
S.nodes = res.nodes
S.node = res.nodes[0]
})
return () => socket && socket.close()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
useEffect(() => {
term.options.disableStdin = true
term.options.fontSize = 14
term.options.lineHeight = 1.2
term.options.fontFamily = gStore.terminal.fontFamily
term.options.theme = {background: '#2b2b2b', foreground: '#A9B7C6', cursor: '#2b2b2b'}
term.attachCustomKeyEventHandler((arg) => {
if (arg.ctrlKey && arg.code === 'KeyC' && arg.type === 'keydown') {
document.execCommand('copy')
return false
}
return true
})
const resize = () => fitPlugin.fit();
term.loadAddon(fitPlugin)
term.open(el.current)
fitPlugin.fit()
term.write('\x1b[36m### WebSocket connecting ...\x1b[0m')
window.addEventListener('resize', resize)
return () => window.removeEventListener('resize', resize);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
function _makeSocket(token, index = 0) {
const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
const socket = new WebSocket(`${protocol}//${window.location.host}/api/ws/pipeline/${token}/?x-token=${X_TOKEN}`);
socket.onopen = () => socket.send(String(index));
socket.onmessage = e => {
if (e.data === 'pong') {
socket.send(String(index))
} else {
index += 1;
const {key, data, status} = JSON.parse(e.data);
if (S.outputs[key]) {
if (status) {
S.outputs[key].status = status
if (key === S.nodeID) {
S.node.status = status
}
}
if (data) {
S.outputs[key].data += data
if (key === S.nodeID) {
term.write(data)
}
}
} else {
S.outputs[key] = {data, status}
S.node.state = ''
}
}
}
socket.onerror = () => {
setWSState('Websocket connection failed')
}
return socket
}
useEffect(() => {
if (S.node.id) {
fitPlugin.fit()
term.reset()
if (S.outputs[S.nodeID]) {
term.write(S.outputs[S.nodeID].data)
} else {
S.outputs[S.nodeID] = {data: ''}
}
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [S.node])
function handleTerminate() {
}
function openTerminal() {
}
return (
<div className={css.container}>
<div className={css.header}>
<div className={css.title}>{S.node?.name}</div>
<div className={css.tips}>{wsState}</div>
<Tooltip title="终止执行">
{S.outputs[S.nodeID]?.status === 'processing' ? (
<StopOutlined className={css.icon} style={{color: '#faad14'}} onClick={handleTerminate}/>
) : (
<StopOutlined className={css.icon} style={{color: '#dfdfdf'}}/>
)}
</Tooltip>
<Tooltip title="打开web终端">
<CodeOutlined className={css.icon} onClick={() => openTerminal()}/>
</Tooltip>
</div>
{S.node?.module === 'ssh_exec' && (
<Tabs items={(S.node?.targets ?? []).map(x => ({label: x.name, key: `${S.node.id}.${x.id}`}))}
tabBarStyle={{fontSize: 13}} onChange={v => S.node = Object.assign({}, S.node, {_id: v})}/>
)}
<div className={css.termContainer}>
<div ref={el} className={css.term}/>
</div>
</div>
)
}
export default observer(Body)

43
spug_web/src/pages/pipeline/console/Node.js Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React from 'react'
import { observer } from 'mobx-react';
import { LoadingOutlined } from '@ant-design/icons';
import Icon from '../Icon';
import { clsNames } from 'libs';
import S from './store';
import css from './node.module.less';
function Node(props) {
const node = props.node
switch (node) {
case ' ':
return <div className={css.box}/>
case ' -':
return <div className={clsNames(css.box, css.line)}/>
case '--':
return <div className={clsNames(css.box, css.line, css.line2)}/>
case ' 7':
return <div className={clsNames(css.box, css.angle)}/>
case '-7':
return <div className={clsNames(css.box, css.angle, css.angle2)}/>
case ' |':
return (
<div className={clsNames(css.box, css.arrow)}>
<div className={css.triangle}/>
</div>
)
default:
return (
<div className={clsNames(css.box, css.node, S.node?.id === node.id && css.active)} onClick={props.onClick}>
{S.outputs[node.id]?.status === 'processing' ? <LoadingOutlined className={css.loading}/> : null}
<Icon size={36} module={node.module}/>
</div>
)
}
}
export default observer(Node)

34
spug_web/src/pages/pipeline/console/Sider.js Normal file
View File

@ -0,0 +1,34 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React from 'react';
import { observer } from 'mobx-react';
import Node from './Node';
import S from './store';
import css from './sider.module.less';
function Sider() {
function handleClick(node) {
if (node.module === 'ssh_exec') {
node._id = `${node.id}.${node.targets[0].id}`
}
S.node = node
}
return (
<div className={css.sider}>
{S.matrixNodes.map((row, idx) => (
<div key={idx} style={{display: 'flex', flexDirection: 'row'}}>
{row.map((item, idx) => (
<Node key={idx} node={item} onClick={() => handleClick(item)}/>
))}
</div>
))}
</div>
)
}
export default observer(Sider)

73
spug_web/src/pages/pipeline/console/body.module.less Normal file
View File

@ -0,0 +1,73 @@
.container {
height: calc(100vh - 300px);
flex: 1;
display: flex;
flex-direction: column;
padding-left: 24px;
overflow: hidden;
}
.header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
.tips {
color: #ff4d4f;
}
.icon {
font-size: 18px;
color: #1890ff;
cursor: pointer;
margin-left: 12px;
}
.title {
flex: 1;
font-weight: 500;
}
}
.termContainer {
flex: 1;
background-color: #2b2b2b;
padding: 8px 0 4px 12px;
border-radius: 4px;
overflow: hidden;
.term {
width: 100%;
height: 100%;
}
}
.execContainer {
.header {
color: #ffffff;
background: #36435c;
border-radius: 4px;
padding: 6px 12px;
margin-right: 12px;
font-size: 14px;
display: flex;
flex-direction: row;
align-items: center;
.title {
flex: 1;
margin-left: 12px;
}
}
.item {
}
}
:global(.ant-tabs-tab) {
font-size: 13px;
}

32
spug_web/src/pages/pipeline/console/index.js Normal file
View File

@ -0,0 +1,32 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React from 'react';
import { observer } from 'mobx-react';
import { Modal, Row } from 'antd';
import Sider from './Sider';
import Body from './Body';
import pS from '../store';
import S from './store';
function Index() {
return (
<Modal
open={pS.consoleVisible}
width="80%"
title="运行控制台"
footer={null}
destroyOnClose
afterClose={S.initial}
onCancel={() => pS.consoleVisible = false}>
<Row>
<Sider/>
<Body/>
</Row>
</Modal>
)
}
export default observer(Index)

108
spug_web/src/pages/pipeline/console/node.module.less Normal file
View File

@ -0,0 +1,108 @@
.box {
position: relative;
width: 38px;
height: 38px;
margin-right: 16px;
flex-shrink: 0;
&:last-child {
margin: 0;
}
}
.triangle {
position: absolute;
bottom: -2px;
left: 15px;
width: 0;
height: 0;
border: 4px solid transparent;
border-top-color: #999999;
}
.node {
position: relative;
box-shadow: 0 0 6px #9999994c;
border: 1px solid transparent;
border-radius: 50%;
cursor: pointer;
&:hover {
box-shadow: 0 0 6px #2563fcbb;
.action {
display: block;
}
}
}
.loading {
position: absolute;
top: 4px;
left: 4px;
color: #ffffff;
font-size: 28px;
z-index: 999;
}
.active {
box-shadow: 0 0 6px #2563fc;
}
.line {
&:after {
content: ' ';
height: 2px;
background: #999999;
position: absolute;
top: 18px;
left: -14px;
right: -4px;
}
}
.line2 {
&:after {
left: -34px;
}
}
.angle {
&:before {
content: ' ';
position: absolute;
background: #999999;
top: 18px;
bottom: 18px;
left: -14px;
right: 18px;
}
&:after {
content: ' ';
position: absolute;
background: #999999;
top: 18px;
left: 18px;
right: 18px;
bottom: -4px;
}
}
.angle2 {
&:before {
left: -34px;
}
}
.arrow {
&:after {
content: ' ';
background: #999999;
position: absolute;
top: 2px;
left: 18px;
right: 18px;
bottom: 4px;
}
}

6
spug_web/src/pages/pipeline/console/sider.module.less Normal file
View File

@ -0,0 +1,6 @@
.sider {
max-width: 200px;
min-width: 104px;
padding: 6px;
overflow: auto;
}

29
spug_web/src/pages/pipeline/console/store.js Normal file
View File

@ -0,0 +1,29 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import { observable, computed } from 'mobx';
import { transfer } from '../utils';
class Store {
@observable node = {};
@observable nodes = [];
@observable outputs = {};
@computed get nodeID() {
return this.node._id ?? this.node.id
}
@computed get matrixNodes() {
return transfer(this.nodes)
}
initial = () => {
this.node = {}
this.nodes = []
this.outputs = {}
}
}
export default new Store()

4
spug_web/src/pages/pipeline/data.js
View File

@ -1,5 +1,5 @@
export const NODES = [
{module: 'remote_exec', name: '执行命令'},
{module: 'ssh_exec', name: '执行命令'},
{module: 'build', name: '构建'},
{module: 'parameter', name: '参数化'},
{module: 'data_transfer', name: '数据传输'},
@ -60,7 +60,7 @@ export const DATAS = {
'command': 'date && sleep 3',
},
{
'module': 'remote_exec',
'module': 'ssh_exec',
'name': '执行命令',
'id': 5,
'targets': [2, 3],

14
spug_web/src/pages/pipeline/index.js
View File

@ -6,12 +6,16 @@
import React from 'react';
import { observer } from 'mobx-react';
import { AuthDiv } from 'components';
import Editor from './Editor';
import Table from './Table';
import Console from './console';
export default observer(function () {
function Index() {
return (
<AuthDiv auth="system.account.view">
<Editor/>
<AuthDiv auth="pipeline.pipeline.view">
<Table/>
<Console/>
</AuthDiv>
)
})
}
export default observer(Index)

7
spug_web/src/pages/pipeline/modules/Build.js
View File

@ -1,3 +1,8 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React, { useEffect, useState } from 'react';
import { observer } from 'mobx-react';
import { Form, Input, Select, Radio, message } from 'antd';
@ -47,7 +52,7 @@ function Build(props) {
<Form.Item required name="condition" label="执行条件" tooltip="当该节点为流程的起始节点时（无上游节点），该条件将会被忽略。">
<Radio.Group>
<Radio.Button value="success">上游执行成功时</Radio.Button>
<Radio.Button value="failure">上游执行失败时</Radio.Button>
<Radio.Button value="error">上游执行失败时</Radio.Button>
<Radio.Button value="always">总是执行</Radio.Button>
</Radio.Group>
</Form.Item>

7
spug_web/src/pages/pipeline/modules/DataTransfer.js
View File

@ -1,3 +1,8 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React, { useEffect } from 'react';
import { Form, Input, message, Card, Radio } from 'antd';
import HostSelector from 'pages/host/Selector';
@ -29,7 +34,7 @@ function DataTransfer(props) {
<Form.Item required name="condition" label="执行条件" tooltip="当该节点为流程的起始节点时（无上游节点），该条件将会被忽略。">
<Radio.Group>
<Radio.Button value="success">上游执行成功时</Radio.Button>
<Radio.Button value="failure">上游执行失败时</Radio.Button>
<Radio.Button value="error">上游执行失败时</Radio.Button>
<Radio.Button value="always">总是执行</Radio.Button>
</Radio.Group>
</Form.Item>

19
spug_web/src/pages/pipeline/modules/SSHExec.js
View File

@ -1,3 +1,8 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React, { useEffect } from 'react';
import { Form, Input, Radio, message } from 'antd';
import { ACEditor } from 'components';
@ -11,12 +16,16 @@ function SSHExec(props) {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
useEffect(() => {
form.resetFields()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [props.node])
function handleSave() {
const data = form.getFieldsValue()
if (!data.name) return message.error('请输入节点名称')
if (!data.condition) return message.error('请选择节点的执行条件')
if (!data.targets || data.targets.length === 0) return message.error('请选择执行主机')
if (!data.interpreter) return message.error('请选择执行解释器')
if (!data.command) return message.error('请输入执行内容')
return data
}
@ -29,19 +38,13 @@ function SSHExec(props) {
<Form.Item required name="condition" label="执行条件" tooltip="当该节点为流程的起始节点时（无上游节点），该条件将会被忽略。">
<Radio.Group>
<Radio.Button value="success">上游执行成功时</Radio.Button>
<Radio.Button value="failure">上游执行失败时</Radio.Button>
<Radio.Button value="error">上游执行失败时</Radio.Button>
<Radio.Button value="always">总是执行</Radio.Button>
</Radio.Group>
</Form.Item>
<Form.Item required name="targets" label="选择主机">
<HostSelector type="button"/>
</Form.Item>
<Form.Item required name="interpreter" label="执行解释器">
<Radio.Group buttonStyle="solid">
<Radio.Button value="sh">Shell</Radio.Button>
<Radio.Button value="python">Python</Radio.Button>
</Radio.Group>
</Form.Item>
<Form.Item required label="执行内容" shouldUpdate={(p, c) => p.interpreter !== c.interpreter}>
{({getFieldValue}) => (
<Form.Item name="command" noStyle>

7
spug_web/src/pages/pipeline/modules/index.js
View File

@ -1,3 +1,8 @@
/**
* Copyright (c) OpenSpug Organization. https://github.com/openspug/spug
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import React from 'react';
import SSHExec from './SSHExec';
import Build from './Build';
@ -5,7 +10,7 @@ import DataTransfer from './DataTransfer';
function ModuleConfig(props) {
switch (props.node.module) {
case 'remote_exec':
case 'ssh_exec':
return <SSHExec {...props}/>
case 'build':
return <Build {...props}/>

21
spug_web/src/pages/pipeline/store.js
View File

@ -3,20 +3,28 @@
* Copyright (c) <spug.dev@gmail.com>
* Released under the AGPL-3.0 License.
*/
import { observable } from 'mobx';
import { http } from 'libs';
import { computed, observable } from 'mobx';
import { http, includes } from 'libs';
import { message } from 'antd';
class Store {
@observable records = [];
@observable record = {nodes: []};
@observable nodes = [];
@observable node = {};
@observable actionNode = {};
@observable isFetching = true;
@observable consoleVisible = false;
fetchRecords = (id, isFetching) => {
@computed get dataSource() {
let records = this.records;
if (this.f_name) records = records.filter(x => includes(x.name, this.f_name));
return records
}
fetchRecords = () => {
this.isFetching = true;
return http.get('/api/pipline/')
return http.get('/api/pipeline/')
.then(res => this.records = res)
.finally(() => this.isFetching = false)
}
@ -35,6 +43,11 @@ class Store {
message.success('保存成功')
})
}
showConsole = (record) => {
this.record = record
this.consoleVisible = true
}
}
export default new Store()

4
spug_web/src/routes.js
View File

@ -43,7 +43,7 @@ import SystemCredential from './pages/system/credential';
import WelcomeIndex from './pages/welcome/index';
import WelcomeInfo from './pages/welcome/info';
import PipelineIndex from './pages/pipeline';
import PipelineEditor from './pages/pipeline';
import PipelineEditor from './pages/pipeline/Editor';
export default [
{icon: <DesktopOutlined/>, title: '工作台', path: '/home', component: HomeIndex},
@ -69,8 +69,8 @@ export default [
{title: '发布申请', auth: 'deploy.request.view', path: '/deploy/request', component: DeployRequest},
]
},
{icon: <FlagOutlined/>, title: '流水线', path: '/pipeline', component: PipelineIndex},
{path: '/pipeline/:id', component: PipelineEditor},
{icon: <FlagOutlined/>, title: '流水线', path: '/pipeline', component: PipelineIndex},
{
icon: <ScheduleOutlined/>,
title: '任务计划',