From ca2775c9bf0416921253e59ec445f610bcea36bf Mon Sep 17 00:00:00 2001 From: vapao Date: Wed, 13 Apr 2022 11:14:14 +0800 Subject: [PATCH] =?UTF-8?q?A=20=E5=A2=9E=E5=8A=A0=E8=A1=A5=E5=81=BF?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spug_api/apps/deploy/helper.py | 24 +++++++++- spug_api/apps/deploy/models.py | 1 + spug_api/apps/deploy/utils.py | 44 ++++++++++++++----- spug_api/apps/deploy/views.py | 9 +++- spug_api/apps/repository/utils.py | 2 +- .../src/pages/deploy/request/Ext1Console.js | 3 +- .../src/pages/deploy/request/Ext2Console.js | 3 +- spug_web/src/pages/deploy/request/Table.js | 38 ++++++++++++---- .../pages/deploy/request/index.module.less | 1 + spug_web/src/pages/deploy/request/store.js | 2 +- 10 files changed, 99 insertions(+), 28 deletions(-) diff --git a/spug_api/apps/deploy/helper.py b/spug_api/apps/deploy/helper.py index 5945ed1..2f7a4ea 100644 --- a/spug_api/apps/deploy/helper.py +++ b/spug_api/apps/deploy/helper.py @@ -17,9 +17,28 @@ class Helper: def __init__(self, rds, key): self.rds = rds self.key = key - self.rds.delete(self.key) self.callback = [] + @classmethod + def make(cls, rds, key, host_ids=None): + if host_ids: + counter, tmp_key = 0, f'{key}_tmp' + data = rds.lrange(key, counter, counter + 9) + while data: + for item in data: + counter += 1 + print(item) + tmp = json.loads(item.decode()) + if tmp['key'] not in host_ids: + rds.rpush(tmp_key, item) + data = rds.lrange(key, counter, counter + 9) + rds.delete(key) + if rds.exists(tmp_key): + rds.rename(tmp_key, key) + else: + rds.delete(key) + return cls(rds, key) + @classmethod def _make_dd_notify(cls, url, action, req, version, host_str): texts = [ @@ -169,7 +188,7 @@ class Helper: @classmethod def send_deploy_notify(cls, req, action=None): rst_notify = json.loads(req.deploy.rst_notify) - host_ids = json.loads(req.host_ids) + host_ids = req.host_ids if rst_notify['mode'] != '0' and rst_notify.get('value'): url = rst_notify['value'] version = req.version @@ -232,6 +251,7 @@ class Helper: self._send({'key': key, 'step': step, 'data': data}) def clear(self): + self.rds.delete(f'{self.key}_tmp') # save logs for two weeks self.rds.expire(self.key, 14 * 24 * 60 * 60) self.rds.close() diff --git a/spug_api/apps/deploy/models.py b/spug_api/apps/deploy/models.py index ef70609..f813253 100644 --- a/spug_api/apps/deploy/models.py +++ b/spug_api/apps/deploy/models.py @@ -37,6 +37,7 @@ class DeployRequest(models.Model, ModelMixin): version = models.CharField(max_length=50, null=True) spug_version = models.CharField(max_length=50, null=True) plan = models.DateTimeField(null=True) + fail_host_ids = models.TextField(default='[]') created_at = models.CharField(max_length=20, default=human_datetime) created_by = models.ForeignKey(User, models.PROTECT, related_name='+') diff --git a/spug_api/apps/deploy/utils.py b/spug_api/apps/deploy/utils.py index e2149a0..a516174 100644 --- a/spug_api/apps/deploy/utils.py +++ b/spug_api/apps/deploy/utils.py @@ -9,6 +9,7 @@ from apps.host.models import Host from apps.config.utils import compose_configs from apps.repository.models import Repository from apps.repository.utils import dispatch as build_repository +from apps.deploy.models import DeployRequest from apps.deploy.helper import Helper, SpugError from concurrent import futures from functools import partial @@ -19,10 +20,16 @@ import os REPOS_DIR = settings.REPOS_DIR -def dispatch(req): +def dispatch(req, fail_mode=False): rds = get_redis_connection() rds_key = f'{settings.REQUEST_KEY}:{req.id}' - helper = Helper(rds, rds_key) + if fail_mode: + req.host_ids = req.fail_host_ids + req.fail_mode = fail_mode + req.host_ids = json.loads(req.host_ids) + req.fail_host_ids = req.host_ids[:] + helper = Helper.make(rds, rds_key, req.host_ids if fail_mode else None) + try: api_token = uuid.uuid4().hex rds.setex(api_token, 60 * 60, f'{req.deploy.app_id},{req.deploy.env_id}') @@ -56,7 +63,11 @@ def dispatch(req): raise e finally: close_old_connections() - req.save() + DeployRequest.objects.filter(pk=req.id).update( + status=req.status, + repository=req.repository, + fail_host_ids=json.dumps(req.fail_host_ids), + ) helper.clear() Helper.send_deploy_notify(req) @@ -86,7 +97,7 @@ def _ext1_deploy(req, helper, env): threads, latest_exception = [], None max_workers = max(10, os.cpu_count() * 5) with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - for h_id in json.loads(req.host_ids): + for h_id in req.host_ids: new_env = AttrDict(env.items()) t = executor.submit(_deploy_ext1_host, req, helper, h_id, new_env) t.h_id = h_id @@ -97,15 +108,18 @@ def _ext1_deploy(req, helper, env): latest_exception = exception if not isinstance(exception, SpugError): helper.send_error(t.h_id, f'Exception: {exception}', False) + else: + req.fail_host_ids.remove(t.h_id) if latest_exception: raise latest_exception else: - host_ids = sorted(json.loads(req.host_ids), reverse=True) + host_ids = sorted(req.host_ids, reverse=True) while host_ids: h_id = host_ids.pop() new_env = AttrDict(env.items()) try: _deploy_ext1_host(req, helper, h_id, new_env) + req.fail_host_ids.remove(h_id) except Exception as e: helper.send_error(h_id, f'Exception: {e}', False) for h_id in host_ids: @@ -114,7 +128,6 @@ def _ext1_deploy(req, helper, env): def _ext2_deploy(req, helper, env): - helper.send_info('local', f'\033[32m完成√\033[0m\r\n') extend, step = req.deploy.extend_obj, 1 host_actions = json.loads(extend.host_actions) server_actions = json.loads(extend.server_actions) @@ -122,10 +135,13 @@ def _ext2_deploy(req, helper, env): if req.version: for index, value in enumerate(req.version.split()): env.update({f'SPUG_RELEASE_{index + 1}': value}) - for action in server_actions: - helper.send_step('local', step, f'{human_time()} {action["title"]}...\r\n') - helper.local(f'cd /tmp && {action["data"]}', env) - step += 1 + + if not req.fail_mode: + helper.send_info('local', f'\033[32m完成√\033[0m\r\n') + for action in server_actions: + helper.send_step('local', step, f'{human_time()} {action["title"]}...\r\n') + helper.local(f'cd /tmp && {action["data"]}', env) + step += 1 for action in host_actions: if action.get('type') == 'transfer': @@ -171,7 +187,7 @@ def _ext2_deploy(req, helper, env): threads, latest_exception = [], None max_workers = max(10, os.cpu_count() * 5) with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - for h_id in json.loads(req.host_ids): + for h_id in req.host_ids: new_env = AttrDict(env.items()) t = executor.submit(_deploy_ext2_host, helper, h_id, host_actions, new_env, req.spug_version) t.h_id = h_id @@ -182,21 +198,25 @@ def _ext2_deploy(req, helper, env): latest_exception = exception if not isinstance(exception, SpugError): helper.send_error(t.h_id, f'Exception: {exception}', False) + else: + req.fail_host_ids.remove(t.h_id) if latest_exception: raise latest_exception else: - host_ids = sorted(json.loads(req.host_ids), reverse=True) + host_ids = sorted(req.host_ids, reverse=True) while host_ids: h_id = host_ids.pop() new_env = AttrDict(env.items()) try: _deploy_ext2_host(helper, h_id, host_actions, new_env, req.spug_version) + req.fail_host_ids.remove(h_id) except Exception as e: helper.send_error(h_id, f'Exception: {e}', False) for h_id in host_ids: helper.send_error(h_id, '终止发布', False) raise e else: + req.fail_host_ids = [] helper.send_step('local', 100, f'\r\n{human_time()} ** 发布成功 **') diff --git a/spug_api/apps/deploy/views.py b/spug_api/apps/deploy/views.py index 4298c91..bcf8681 100644 --- a/spug_api/apps/deploy/views.py +++ b/spug_api/apps/deploy/views.py @@ -46,6 +46,7 @@ class RequestView(View): tmp['app_name'] = item.app_name tmp['app_extend'] = item.app_extend tmp['host_ids'] = json.loads(item.host_ids) + tmp['fail_host_ids'] = json.loads(item.fail_host_ids) tmp['extra'] = json.loads(item.extra) if item.extra else None tmp['rep_extra'] = json.loads(item.rep_extra) if item.rep_extra else None tmp['app_host_ids'] = json.loads(item.app_host_ids) @@ -138,6 +139,7 @@ class RequestDetailView(View): @auth('deploy.request.do') def post(self, request, r_id): + form, _ = JsonParser(Argument('mode', default='all')).parse(request.body) query = {'pk': r_id} if not request.user.is_supper: perms = request.user.deploy_perms @@ -148,14 +150,16 @@ class RequestDetailView(View): return json_response(error='未找到指定发布申请') if req.status not in ('1', '-3'): return json_response(error='该申请单当前状态还不能执行发布') - hosts = Host.objects.filter(id__in=json.loads(req.host_ids)) + + host_ids = req.fail_host_ids if form.mode == 'fail' else req.host_ids + hosts = Host.objects.filter(id__in=json.loads(host_ids)) message = f'{human_time()} 等待调度... ' outputs = {x.id: {'id': x.id, 'title': x.name, 'step': 0, 'data': message} for x in hosts} req.status = '2' req.do_at = human_datetime() req.do_by = request.user req.save() - Thread(target=dispatch, args=(req,)).start() + Thread(target=dispatch, args=(req, form.mode == 'fail')).start() if req.is_quick_deploy: if req.repository_id: outputs['local'] = {'id': 'local', 'step': 100, 'data': f'{human_time()} 已构建完成忽略执行。'} @@ -325,6 +329,7 @@ def get_request_info(request): if error is None: req = DeployRequest.objects.get(pk=form.id) response = req.to_dict(selects=('status', 'reason')) + response['fail_host_ids'] = json.loads(req.fail_host_ids) response['status_alias'] = req.get_status_display() return json_response(response) return json_response(error=error) diff --git a/spug_api/apps/repository/utils.py b/spug_api/apps/repository/utils.py index 37651cc..5c44377 100644 --- a/spug_api/apps/repository/utils.py +++ b/spug_api/apps/repository/utils.py @@ -23,7 +23,7 @@ def dispatch(rep: Repository, helper=None): if not helper: rds = get_redis_connection() rds_key = f'{settings.BUILD_KEY}:{rep.spug_version}' - helper = Helper(rds, rds_key) + helper = Helper.make(rds, rds_key) rep.save() try: api_token = uuid.uuid4().hex diff --git a/spug_web/src/pages/deploy/request/Ext1Console.js b/spug_web/src/pages/deploy/request/Ext1Console.js index 9f75356..b1f3840 100644 --- a/spug_web/src/pages/deploy/request/Ext1Console.js +++ b/spug_web/src/pages/deploy/request/Ext1Console.js @@ -36,7 +36,7 @@ function Ext1Console(props) { function doDeploy() { let socket; - http.post(`/api/deploy/request/${props.request.id}/`) + http.post(`/api/deploy/request/${props.request.id}/`, {mode: props.request.mode}) .then(res => { Object.assign(outputs, res.outputs) setTimeout(() => setFetching(false), 100) @@ -57,6 +57,7 @@ function Ext1Console(props) { } else { index += 1; const {key, data, step, status} = JSON.parse(e.data); + if (!outputs[key]) return if (data !== undefined) { outputs[key].data += data if (terms[key]) terms[key].write(data) diff --git a/spug_web/src/pages/deploy/request/Ext2Console.js b/spug_web/src/pages/deploy/request/Ext2Console.js index e548562..a3da332 100644 --- a/spug_web/src/pages/deploy/request/Ext2Console.js +++ b/spug_web/src/pages/deploy/request/Ext2Console.js @@ -40,7 +40,7 @@ function Ext2Console(props) { function doDeploy() { let socket; - http.post(`/api/deploy/request/${props.request.id}/`) + http.post(`/api/deploy/request/${props.request.id}/`, {mode: props.request.mode}) .then(res => { setSActions(res.s_actions); setHActions(res.h_actions); @@ -63,6 +63,7 @@ function Ext2Console(props) { } else { index += 1; const {key, data, step, status} = JSON.parse(e.data); + if (!outputs[key]) return if (data !== undefined) { outputs[key].data += data if (terms[key]) terms[key].write(data) diff --git a/spug_web/src/pages/deploy/request/Table.js b/spug_web/src/pages/deploy/request/Table.js index 007d36e..2e58b71 100644 --- a/spug_web/src/pages/deploy/request/Table.js +++ b/spug_web/src/pages/deploy/request/Table.js @@ -12,6 +12,16 @@ import { Action, AuthButton, TableCard } from 'components'; import S from './index.module.less'; import store from './store'; +function DeployConfirm() { + return ( +
+
确认发布方式
+
补偿：仅发布上次发布失败的主机。
+
全量：再次发布所有主机。
+
+ ) +} + function ComTable() { const columns = [{ title: '申请标题', @@ -121,9 +131,7 @@ function ComTable() { case '-3': return store.readConsole(info)}>查看 - handleDeploy(e, info)}> - 发布 - + {info.visible_rollback && ( store.rollback(info)}>回滚 )} @@ -148,9 +156,7 @@ function ComTable() { ; case '1': return - handleDeploy(e, info)}> - 发布 - + handleDelete(info)}>删除 ; case '2': @@ -163,6 +169,21 @@ function ComTable() { } }]; + function DoAction(props) { + const {host_ids, fail_host_ids} = props.info; + return ( + } + okText="全量" + cancelText="补偿" + cancelButtonProps={{disabled: [0, host_ids.length].includes(fail_host_ids.length)}} + onConfirm={e => handleDeploy(e, props.info, 'all')} + onCancel={e => handleDeploy(e, props.info, 'fail')}> + 发布 + + ) + } + function handleDelete(info) { Modal.confirm({ title: '删除确认', @@ -177,14 +198,15 @@ function ComTable() { }) } - function handleDeploy(e, info) { + function handleDeploy(e, info, mode) { + info.mode = mode store.showConsole(info); } return ( row.key || row.id} title="申请列表" columns={columns} scroll={{x: 1500}} diff --git a/spug_web/src/pages/deploy/request/index.module.less b/spug_web/src/pages/deploy/request/index.module.less index af4f8c8..3ff3d9f 100644 --- a/spug_web/src/pages/deploy/request/index.module.less +++ b/spug_web/src/pages/deploy/request/index.module.less @@ -13,6 +13,7 @@ bottom: 12px; right: 24px; align-items: flex-end; + z-index: 999; .item { width: 180px; diff --git a/spug_web/src/pages/deploy/request/store.js b/spug_web/src/pages/deploy/request/store.js index abca671..254460e 100644 --- a/spug_web/src/pages/deploy/request/store.js +++ b/spug_web/src/pages/deploy/request/store.js @@ -58,7 +58,7 @@ class Store { .then(res => { for (let item of this.records) { if (item.id === id) { - Object.assign(item, res) + Object.assign(item, res, {key: Date.now()}) break } }