From 50030d544a6716a46df77fc434f109606b194216 Mon Sep 17 00:00:00 2001 From: vapao Date: Wed, 22 Feb 2023 16:52:53 +0800 Subject: [PATCH] add pipeline module --- spug_api/apps/pipeline/__init__.py | 3 + spug_api/apps/pipeline/models.py | 23 +++ spug_api/apps/pipeline/urls.py | 10 ++ spug_api/apps/pipeline/views.py | 59 ++++++++ spug_api/consumer/consumers.py | 13 +- spug_api/libs/gitlib.py | 137 ++++++++++++++++++ spug_api/libs/ssh.py | 95 +++--------- spug_api/spug/settings.py | 2 + spug_api/spug/urls.py | 2 + spug_web/package.json | 2 +- spug_web/src/layout/index.js | 2 - spug_web/src/pages/host/Selector.js | 34 +++-- spug_web/src/pages/pipeline/Editor.js | 113 +++++++++------ spug_web/src/pages/pipeline/Form.js | 48 ++++++ spug_web/src/pages/pipeline/Node.js | 32 ++-- spug_web/src/pages/pipeline/NodeConfig.js | 58 +++----- spug_web/src/pages/pipeline/data.js | 36 ++++- .../src/pages/pipeline/editor.module.less | 87 +++++------ spug_web/src/pages/pipeline/index.js | 7 +- spug_web/src/pages/pipeline/modules/Build.js | 88 +++++++++++ .../pages/pipeline/modules/DataTransfer.js | 56 +++++++ .../src/pages/pipeline/modules/SSHExec.js | 59 ++++++++ spug_web/src/pages/pipeline/modules/index.js | 19 +++ .../pages/pipeline/modules/index.module.less | 10 ++ spug_web/src/pages/pipeline/node.module.less | 19 ++- .../src/pages/pipeline/nodeConfig.module.less | 7 +- spug_web/src/pages/pipeline/store.js | 25 ++++ spug_web/src/pages/pipeline/utils.js | 9 +- spug_web/src/routes.js | 6 + 29 files changed, 798 insertions(+), 263 deletions(-) create mode 100644 spug_api/apps/pipeline/__init__.py create mode 100644 spug_api/apps/pipeline/models.py create mode 100644 spug_api/apps/pipeline/urls.py create mode 100644 spug_api/apps/pipeline/views.py create mode 100644 spug_web/src/pages/pipeline/Form.js create mode 100644 spug_web/src/pages/pipeline/modules/Build.js create mode 100644 spug_web/src/pages/pipeline/modules/DataTransfer.js create mode 100644 spug_web/src/pages/pipeline/modules/SSHExec.js create mode 100644 spug_web/src/pages/pipeline/modules/index.js create mode 100644 spug_web/src/pages/pipeline/modules/index.module.less diff --git a/spug_api/apps/pipeline/__init__.py b/spug_api/apps/pipeline/__init__.py new file mode 100644 index 0000000..89f622a --- /dev/null +++ b/spug_api/apps/pipeline/__init__.py @@ -0,0 +1,3 @@ +# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug +# Copyright: (c) +# Released under the AGPL-3.0 License. diff --git a/spug_api/apps/pipeline/models.py b/spug_api/apps/pipeline/models.py new file mode 100644 index 0000000..eb25777 --- /dev/null +++ b/spug_api/apps/pipeline/models.py @@ -0,0 +1,23 @@ +# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug +# Copyright: (c) +# Released under the AGPL-3.0 License. +from django.db import models +from libs.mixins import ModelMixin +from apps.account.models import User +import json + + +class Pipeline(models.Model, ModelMixin): + name = models.CharField(max_length=64) + nodes = models.TextField(default='[]') + created_by = models.ForeignKey(User, on_delete=models.PROTECT) + created_at = models.DateTimeField(auto_now_add=True) + + def to_view(self): + tmp = self.to_dict() + tmp['nodes'] = json.loads(self.nodes) + return tmp + + class Meta: + db_table = 'pipelines' + ordering = ('-id',) diff --git a/spug_api/apps/pipeline/urls.py b/spug_api/apps/pipeline/urls.py new file mode 100644 index 0000000..93a6be7 --- /dev/null +++ b/spug_api/apps/pipeline/urls.py @@ -0,0 +1,10 @@ +# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug +# Copyright: (c) +# Released under the AGPL-3.0 License. +from django.urls import path + +from apps.pipeline.views import PipeView + +urlpatterns = [ + path('', PipeView.as_view()), +] diff --git a/spug_api/apps/pipeline/views.py b/spug_api/apps/pipeline/views.py new file mode 100644 index 0000000..29cd61c --- /dev/null +++ b/spug_api/apps/pipeline/views.py @@ -0,0 +1,59 @@ +# Copyright: (c) OpenSpug Organization. https://github.com/openspug/spug +# Copyright: (c) +# Released under the AGPL-3.0 License. +from django.views.generic import View +from libs import JsonParser, Argument, json_response, auth +from apps.pipeline.models import Pipeline +import json + + +class PipeView(View): + def get(self, request): + form, error = JsonParser( + Argument('id', type=int, required=False) + ).parse(request.GET) + if error is None: + if form.id: + pipe = Pipeline.objects.filter(pk=form.id).first() + if not pipe: + return json_response(error='未找到指定流程') + response = pipe.to_view() + else: + pipes = Pipeline.objects.all() + response = [x.to_list() for x in pipes] + return json_response(response) + + @auth('deploy.app.add|deploy.app.edit|config.app.add|config.app.edit') + def post(self, request): + form, error = JsonParser( + Argument('id', type=int, required=False), + Argument('name', help='请输入流程名称'), + Argument('nodes', type=list, handler=json.dumps, default='[]') + ).parse(request.body) + if error is None: + if form.id: + Pipeline.objects.filter(pk=form.id).update(**form) + pipe = Pipeline.objects.get(pk=form.id) + else: + pipe = Pipeline.objects.create(created_by=request.user, **form) + return json_response(pipe.to_view()) + return json_response(error=error) + + def patch(self, request): + form, error = JsonParser( + Argument('id', type=int, help='请指定操作对象'), + Argument('name', required=False), + Argument('nodes', type=list, handler=json.dumps, required=False), + ).parse(request.body, True) + if error is None: + Pipeline.objects.filter(pk=form.id).update(**form) + return json_response(error=error) + + @auth('deploy.app.del|config.app.del') + def delete(self, request): + form, error = JsonParser( + Argument('id', type=int, help='请指定操作对象') + ).parse(request.GET) + if error is None: + Pipeline.objects.filter(pk=form.id).delete() + return json_response(error=error) diff --git a/spug_api/consumer/consumers.py b/spug_api/consumer/consumers.py index f150bb9..2ab9908 100644 --- a/spug_api/consumer/consumers.py +++ b/spug_api/consumer/consumers.py @@ -59,29 +59,26 @@ class SSHConsumer(BaseConsumer): self.ssh = None def loop_read(self): - is_ready, data = False, b'' + is_ready, buf_size = False, 4096 while True: - out = self.chan.recv(32 * 1024) - if not out: + data = self.chan.recv(buf_size) + if not data: self.close(3333) break - data += out + while self.chan.recv_ready(): + data += self.chan.recv(buf_size) try: text = data.decode() except UnicodeDecodeError: try: text = data.decode(encoding='GBK') except UnicodeDecodeError: - time.sleep(0.01) - if self.chan.recv_ready(): - continue text = data.decode(errors='ignore') if not is_ready: self.send(text_data='\033[2J\033[3J\033[1;1H') is_ready = True self.send(text_data=text) - data = b'' def receive(self, text_data=None, bytes_data=None): data = text_data or bytes_data diff --git a/spug_api/libs/gitlib.py b/spug_api/libs/gitlib.py index 6569a0d..265d84e 100644 --- a/spug_api/libs/gitlib.py +++ b/spug_api/libs/gitlib.py @@ -4,6 +4,8 @@ from git import Repo, RemoteReference, TagReference, InvalidGitRepositoryError, GitCommandError from tempfile import NamedTemporaryFile from datetime import datetime +from io import StringIO +import subprocess import shutil import os @@ -104,3 +106,138 @@ class Git: def __exit__(self, exc_type, exc_val, exc_tb): if self.fd: self.fd.close() + + +class RemoteGit: + def __init__(self, host, url, path, credential=None): + self.ssh = host.get_ssh() + self.url = url + self.path = path + self.credential = credential + self._ask_env = None + + def _make_ask_env(self): + if not self.credential: + return None + if self._ask_env: + return self._ask_env + ask_file = f'{self.ssh.exec_file}.1' + if self.credential.type == 'pw': + env = dict(GIT_ASKPASS=ask_file) + body = '#!/bin/bash\n' + body += 'case "$1" in\n' + body += ' Username*)\n' + body += ' echo "{0.username}";;\n' + body += ' Password*)\n' + body += ' echo "{0.secret}";;\n' + body += 'esac' + body = body.format(self.credential) + self.ssh.put_file_by_fl(StringIO(body), ask_file) + else: + env = dict(GIT_SSH=ask_file) + key_file = f'{self.ssh.exec_file}.2' + self.ssh.put_file_by_fl(StringIO(self.credential.secret), key_file) + self.ssh.sftp.chmod(key_file, 0o600) + body = f'ssh -o StrictHostKeyChecking=no -i {key_file} $@' + self.ssh.put_file_by_fl(StringIO(body), ask_file) + self.ssh.sftp.chmod(ask_file, 0o755) + return env + + def _check_path(self): + body = f'git rev-parse --resolve-git-dir {self.path}/.git' + code, _ = self.ssh.exec_command(body) + return code == 0 + + @classmethod + def check_auth(cls, url, credential=None): + env = dict() + if credential: + if credential.type == 'pw': + ask_command = '#!/bin/bash\n' + ask_command += 'case "$1" in\n' + ask_command += ' Username*)\n' + ask_command += ' echo "{0.username}";;\n' + ask_command += ' Password*)\n' + ask_command += ' echo "{0.secret}";;\n' + ask_command += 'esac' + ask_command = ask_command.format(credential) + ask_file = NamedTemporaryFile() + ask_file.write(ask_command.encode()) + ask_file.flush() + os.chmod(ask_file.name, 0o755) + env.update(GIT_ASKPASS=ask_file.name) + print(ask_file.name) + else: + key_file = NamedTemporaryFile() + key_file.write(credential.secret.encode()) + key_file.flush() + os.chmod(key_file.name, 0o600) + ask_command = f'ssh -o StrictHostKeyChecking=no -i {key_file.name} $@' + ask_file = NamedTemporaryFile() + ask_file.write(ask_command.encode()) + ask_file.flush() + os.chmod(ask_file.name, 0o755) + env.update(GIT_SSH=ask_file.name) + print(ask_file.name) + print(key_file.name) + + command = f'git ls-remote -h {url} HEAD' + res = subprocess.run(command, shell=True, capture_output=True, env=env) + return res.returncode == 0, res.stderr.decode() + + def clone(self): + env = self._make_ask_env() + code, out = self.ssh.exec_command(f'git clone {self.url} {self.path}', env) + if code != 0: + raise Exception(out) + + def fetch_branches_tags(self): + body = f'set -e\ncd {self.path}\n' + if not self._check_path(): + self.clone() + else: + body += 'git fetch -q --tags --force\n' + + body += 'git --no-pager branch -r --format="%(refname:short)" | grep -v /HEAD | while read branch; do\n' + body += ' echo "Branch: $branch"\n' + body += ' git --no-pager log -20 --date="format-local:%Y-%m-%d %H:%M" --format="%H %cd %cn %s" $branch\n' + body += 'done\n' + body += 'echo "Tags:"\n' + body += 'git --no-pager for-each-ref --format="%(refname:short) %(if)%(taggername)%(then)%(taggername)' + body += '%(else)%(authorname)%(end) %(creatordate:format-local:%Y-%m-%d %H:%M) %(subject)" ' + body += '--sort=-creatordate refs/tags\n' + env = self._make_ask_env() + code, out = self.ssh.exec_command(body, env) + if code != 0: + raise Exception(out) + branches, tags, each = {}, [], [] + for line in out.splitlines(): + if line.startswith('Branch:'): + branch = line.split()[-1] + branches[branch] = each = [] + elif line.startswith('Tags:'): + tags = each = [] + else: + each.append(line) + return branches, tags + + def checkout(self, marker): + body = f'set -e\ncd {self.path}\n' + if not self._check_path(): + self.clone() + else: + body += 'git fetch -q --tags --force\n' + + body += f'git checkout -f {marker}' + env = self._make_ask_env() + code, out = self.ssh.exec_command(body, env) + if code != 0: + raise Exception(out) + + def __enter__(self): + self.ssh.get_client() + return self + + def __exit__(self, *args, **kwargs): + self.ssh.client.close() + self.ssh.client = None diff --git a/spug_api/libs/ssh.py b/spug_api/libs/ssh.py index 472977a..8ec4bb7 100644 --- a/spug_api/libs/ssh.py +++ b/spug_api/libs/ssh.py @@ -3,52 +3,13 @@ # Released under the AGPL-3.0 License. from paramiko.client import SSHClient, AutoAddPolicy from paramiko.rsakey import RSAKey -from paramiko.auth_handler import AuthHandler from paramiko.ssh_exception import AuthenticationException, SSHException -from paramiko.py3compat import b, u from io import StringIO from uuid import uuid4 import time import re -def _finalize_pubkey_algorithm(self, key_type): - if "rsa" not in key_type: - return key_type - if re.search(r"-OpenSSH_(?:[1-6]|7\.[0-7])", self.transport.remote_version): - pubkey_algo = "ssh-rsa" - if key_type.endswith("-cert-v01@openssh.com"): - pubkey_algo += "-cert-v01@openssh.com" - - self.transport._agreed_pubkey_algorithm = pubkey_algo - return pubkey_algo - my_algos = [x for x in self.transport.preferred_pubkeys if "rsa" in x] - if not my_algos: - raise SSHException( - "An RSA key was specified, but no RSA pubkey algorithms are configured!" # noqa - ) - server_algo_str = u( - self.transport.server_extensions.get("server-sig-algs", b("")) - ) - if server_algo_str: - server_algos = server_algo_str.split(",") - agreement = list(filter(server_algos.__contains__, my_algos)) - if agreement: - pubkey_algo = agreement[0] - else: - err = "Unable to agree on a pubkey algorithm for signing a {!r} key!" # noqa - raise AuthenticationException(err.format(key_type)) - else: - pubkey_algo = "ssh-rsa" - if key_type.endswith("-cert-v01@openssh.com"): - pubkey_algo += "-cert-v01@openssh.com" - self.transport._agreed_pubkey_algorithm = pubkey_algo - return pubkey_algo - - -AuthHandler._finalize_pubkey_algorithm = _finalize_pubkey_algorithm - - class SSH: def __init__(self, hostname, port=22, username='root', pkey=None, password=None, default_env=None, connect_timeout=10, term=None): @@ -56,7 +17,7 @@ class SSH: self.client = None self.channel = None self.sftp = None - self.exec_file = None + self.exec_file = f'/tmp/spug.{uuid4().hex}' self.term = term or {} self.pid = None self.eof = 'Spug EOF 2108111926' @@ -124,29 +85,18 @@ class SSH: out += line return exit_code, out - def _win_exec_command_with_stream(self, command, environment=None): - channel = self.client.get_transport().open_session() - if environment: - channel.update_environment(environment) - channel.set_combine_stderr(True) - channel.get_pty(width=102) - channel.exec_command(command) - stdout = channel.makefile("rb", -1) - out = stdout.readline() - while out: - yield channel.exit_status, self._decode(out) - out = stdout.readline() - yield channel.recv_exit_status(), self._decode(out) - def exec_command_with_stream(self, command, environment=None): channel = self._get_channel() command = self._handle_command(command, environment) channel.sendall(command) - exit_code, line = -1, '' + buf_size, exit_code, line = 4096, -1, '' while True: - line = self._decode(channel.recv(8196)) - if not line: + out = channel.recv(buf_size) + if not out: break + while channel.recv_ready(): + out += channel.recv(buf_size) + line = self._decode(out) match = self.regex.search(line) if match: exit_code = int(match.group(1)) @@ -185,18 +135,23 @@ class SSH: if self.channel: return self.channel - counter = 0 - self.channel = self.client.invoke_shell(**self.term) + self.channel = self.client.invoke_shell(term='xterm', **self.term) + self.channel.settimeout(3600) command = '[ -n "$BASH_VERSION" ] && set +o history\n' - command += '[ -n "$ZSH_VERSION" ] && set +o zle && set -o no_nomatch\n' + command += '[ -n "$ZSH_VERSION" ] && set +o zle && set -o no_nomatch && HISTFILE=""\n' command += 'export PS1= && stty -echo\n' + command += f'trap \'rm -f {self.exec_file}*\' EXIT\n' command += self._make_env_command(self.default_env) command += f'echo {self.eof} $$\n' + time.sleep(0.2) # compatibility self.channel.sendall(command) - out = '' + counter, buf_size = 0, 4096 while True: if self.channel.recv_ready(): - out += self._decode(self.channel.recv(8196)) + out = self.channel.recv(buf_size) + if self.channel.recv_ready(): + out += self.channel.recv(buf_size) + out = self._decode(out) match = self.regex.search(out) if match: self.pid = int(match.group(1)) @@ -232,17 +187,11 @@ class SSH: return f'export {str_envs}\n' def _handle_command(self, command, environment): - new_command = commands = '' - if not self.exec_file: - self.exec_file = f'/tmp/spug.{uuid4().hex}' - commands += f'trap \'rm -f {self.exec_file}\' EXIT\n' - - new_command += self._make_env_command(environment) + new_command = self._make_env_command(environment) new_command += command new_command += f'\necho {self.eof} $?\n' self.put_file_by_fl(StringIO(new_command), self.exec_file) - commands += f'. {self.exec_file}\n' - return commands + return f'. {self.exec_file}\n' def _decode(self, content): try: @@ -253,12 +202,8 @@ class SSH: def __enter__(self): self.get_client() - transport = self.client.get_transport() - if 'windows' in transport.remote_version.lower(): - self.exec_command = self.exec_command_raw - self.exec_command_with_stream = self._win_exec_command_with_stream return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, *args, **kwargs): self.client.close() self.client = None diff --git a/spug_api/spug/settings.py b/spug_api/spug/settings.py index 4c3a04d..c4b3df5 100644 --- a/spug_api/spug/settings.py +++ b/spug_api/spug/settings.py @@ -47,6 +47,8 @@ INSTALLED_APPS = [ 'apps.notify', 'apps.repository', 'apps.home', + 'apps.credential', + 'apps.pipeline', 'channels', ] diff --git a/spug_api/spug/urls.py b/spug_api/spug/urls.py index 0635e8f..ad98bf6 100644 --- a/spug_api/spug/urls.py +++ b/spug_api/spug/urls.py @@ -33,5 +33,7 @@ urlpatterns = [ path('home/', include('apps.home.urls')), path('notify/', include('apps.notify.urls')), path('file/', include('apps.file.urls')), + path('credential/', include('apps.credential.urls')), + path('pipeline/', include('apps.pipeline.urls')), path('apis/', include('apps.apis.urls')), ] diff --git a/spug_web/package.json b/spug_web/package.json index 08c4401..7ea1719 100644 --- a/spug_web/package.json +++ b/spug_web/package.json @@ -5,7 +5,7 @@ "dependencies": { "@ant-design/icons": "^4.3.0", "ace-builds": "^1.4.13", - "antd": "4.21.5", + "antd": "^4.24.5", "axios": "^0.21.0", "bizcharts": "^3.5.9", "history": "^4.10.1", diff --git a/spug_web/src/layout/index.js b/spug_web/src/layout/index.js index ad8249a..efa4f06 100644 --- a/spug_web/src/layout/index.js +++ b/spug_web/src/layout/index.js @@ -9,7 +9,6 @@ import { Layout, message } from 'antd'; import { NotFound } from 'components'; import Sider from './Sider'; import Header from './Header'; -import Footer from './Footer' import routes from '../routes'; import { hasPermission, isMobile } from 'libs'; import styles from './layout.module.less'; @@ -50,7 +49,6 @@ export default function () { {Routes} -