A 增加补偿发布功能

spug_api/apps/deploy/helper.py

@@ -17,9 +17,28 @@ class Helper:
     def __init__(self, rds, key):
         self.rds = rds
         self.key = key
-        self.rds.delete(self.key)
         self.callback = []
 
+    @classmethod
+    def make(cls, rds, key, host_ids=None):
+        if host_ids:
+            counter, tmp_key = 0, f'{key}_tmp'
+            data = rds.lrange(key, counter, counter + 9)
+            while data:
+                for item in data:
+                    counter += 1
+                    print(item)
+                    tmp = json.loads(item.decode())
+                    if tmp['key'] not in host_ids:
+                        rds.rpush(tmp_key, item)
+                data = rds.lrange(key, counter, counter + 9)
+            rds.delete(key)
+            if rds.exists(tmp_key):
+                rds.rename(tmp_key, key)
+        else:
+            rds.delete(key)
+        return cls(rds, key)
+
     @classmethod
     def _make_dd_notify(cls, url, action, req, version, host_str):
         texts = [
@@ -169,7 +188,7 @@ class Helper:
     @classmethod
     def send_deploy_notify(cls, req, action=None):
         rst_notify = json.loads(req.deploy.rst_notify)
-        host_ids = json.loads(req.host_ids)
+        host_ids = req.host_ids
         if rst_notify['mode'] != '0' and rst_notify.get('value'):
             url = rst_notify['value']
             version = req.version
@@ -232,6 +251,7 @@ class Helper:
         self._send({'key': key, 'step': step, 'data': data})
 
     def clear(self):
+        self.rds.delete(f'{self.key}_tmp')
         # save logs for two weeks
         self.rds.expire(self.key, 14 * 24 * 60 * 60)
         self.rds.close()

spug_api/apps/deploy/models.py

@@ -37,6 +37,7 @@ class DeployRequest(models.Model, ModelMixin):
     version = models.CharField(max_length=50, null=True)
     spug_version = models.CharField(max_length=50, null=True)
     plan = models.DateTimeField(null=True)
+    fail_host_ids = models.TextField(default='[]')
 
     created_at = models.CharField(max_length=20, default=human_datetime)
     created_by = models.ForeignKey(User, models.PROTECT, related_name='+')

spug_api/apps/deploy/utils.py

@@ -9,6 +9,7 @@ from apps.host.models import Host
 from apps.config.utils import compose_configs
 from apps.repository.models import Repository
 from apps.repository.utils import dispatch as build_repository
+from apps.deploy.models import DeployRequest
 from apps.deploy.helper import Helper, SpugError
 from concurrent import futures
 from functools import partial
@@ -19,10 +20,16 @@ import os
 REPOS_DIR = settings.REPOS_DIR
 
 
-def dispatch(req):
+def dispatch(req, fail_mode=False):
     rds = get_redis_connection()
     rds_key = f'{settings.REQUEST_KEY}:{req.id}'
-    helper = Helper(rds, rds_key)
+    if fail_mode:
+        req.host_ids = req.fail_host_ids
+    req.fail_mode = fail_mode
+    req.host_ids = json.loads(req.host_ids)
+    req.fail_host_ids = req.host_ids[:]
+    helper = Helper.make(rds, rds_key, req.host_ids if fail_mode else None)
+
     try:
         api_token = uuid.uuid4().hex
         rds.setex(api_token, 60 * 60, f'{req.deploy.app_id},{req.deploy.env_id}')
@@ -56,7 +63,11 @@ def dispatch(req):
         raise e
     finally:
         close_old_connections()
-        req.save()
+        DeployRequest.objects.filter(pk=req.id).update(
+            status=req.status,
+            repository=req.repository,
+            fail_host_ids=json.dumps(req.fail_host_ids),
+        )
         helper.clear()
         Helper.send_deploy_notify(req)
 
@@ -86,7 +97,7 @@ def _ext1_deploy(req, helper, env):
         threads, latest_exception = [], None
         max_workers = max(10, os.cpu_count() * 5)
         with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-            for h_id in json.loads(req.host_ids):
+            for h_id in req.host_ids:
                 new_env = AttrDict(env.items())
                 t = executor.submit(_deploy_ext1_host, req, helper, h_id, new_env)
                 t.h_id = h_id
@@ -97,15 +108,18 @@ def _ext1_deploy(req, helper, env):
                     latest_exception = exception
                     if not isinstance(exception, SpugError):
                         helper.send_error(t.h_id, f'Exception: {exception}', False)
+                else:
+                    req.fail_host_ids.remove(t.h_id)
         if latest_exception:
             raise latest_exception
     else:
-        host_ids = sorted(json.loads(req.host_ids), reverse=True)
+        host_ids = sorted(req.host_ids, reverse=True)
         while host_ids:
             h_id = host_ids.pop()
             new_env = AttrDict(env.items())
             try:
                 _deploy_ext1_host(req, helper, h_id, new_env)
+                req.fail_host_ids.remove(h_id)
             except Exception as e:
                 helper.send_error(h_id,  f'Exception: {e}', False)
                 for h_id in host_ids:
@@ -114,7 +128,6 @@ def _ext1_deploy(req, helper, env):
 
 
 def _ext2_deploy(req, helper, env):
-    helper.send_info('local', f'\033[32m完成√\033[0m\r\n')
     extend, step = req.deploy.extend_obj, 1
     host_actions = json.loads(extend.host_actions)
     server_actions = json.loads(extend.server_actions)
@@ -122,10 +135,13 @@ def _ext2_deploy(req, helper, env):
     if req.version:
         for index, value in enumerate(req.version.split()):
             env.update({f'SPUG_RELEASE_{index + 1}': value})
-    for action in server_actions:
+
-        helper.send_step('local', step, f'{human_time()} {action["title"]}...\r\n')
+    if not req.fail_mode:
-        helper.local(f'cd /tmp && {action["data"]}', env)
+        helper.send_info('local', f'\033[32m完成√\033[0m\r\n')
-        step += 1
+        for action in server_actions:
+            helper.send_step('local', step, f'{human_time()} {action["title"]}...\r\n')
+            helper.local(f'cd /tmp && {action["data"]}', env)
+            step += 1
 
     for action in host_actions:
         if action.get('type') == 'transfer':
@@ -171,7 +187,7 @@ def _ext2_deploy(req, helper, env):
             threads, latest_exception = [], None
             max_workers = max(10, os.cpu_count() * 5)
             with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-                for h_id in json.loads(req.host_ids):
+                for h_id in req.host_ids:
                     new_env = AttrDict(env.items())
                     t = executor.submit(_deploy_ext2_host, helper, h_id, host_actions, new_env, req.spug_version)
                     t.h_id = h_id
@@ -182,21 +198,25 @@ def _ext2_deploy(req, helper, env):
                         latest_exception = exception
                         if not isinstance(exception, SpugError):
                             helper.send_error(t.h_id, f'Exception: {exception}', False)
+                    else:
+                        req.fail_host_ids.remove(t.h_id)
             if latest_exception:
                 raise latest_exception
         else:
-            host_ids = sorted(json.loads(req.host_ids), reverse=True)
+            host_ids = sorted(req.host_ids, reverse=True)
             while host_ids:
                 h_id = host_ids.pop()
                 new_env = AttrDict(env.items())
                 try:
                     _deploy_ext2_host(helper, h_id, host_actions, new_env, req.spug_version)
+                    req.fail_host_ids.remove(h_id)
                 except Exception as e:
                     helper.send_error(h_id, f'Exception: {e}', False)
                     for h_id in host_ids:
                         helper.send_error(h_id, '终止发布', False)
                     raise e
     else:
+        req.fail_host_ids = []
         helper.send_step('local', 100, f'\r\n{human_time()} ** 发布成功 **')
 
 

spug_api/apps/deploy/views.py

@@ -46,6 +46,7 @@ class RequestView(View):
             tmp['app_name'] = item.app_name
             tmp['app_extend'] = item.app_extend
             tmp['host_ids'] = json.loads(item.host_ids)
+            tmp['fail_host_ids'] = json.loads(item.fail_host_ids)
             tmp['extra'] = json.loads(item.extra) if item.extra else None
             tmp['rep_extra'] = json.loads(item.rep_extra) if item.rep_extra else None
             tmp['app_host_ids'] = json.loads(item.app_host_ids)
@@ -138,6 +139,7 @@ class RequestDetailView(View):
 
     @auth('deploy.request.do')
     def post(self, request, r_id):
+        form, _ = JsonParser(Argument('mode', default='all')).parse(request.body)
         query = {'pk': r_id}
         if not request.user.is_supper:
             perms = request.user.deploy_perms
@@ -148,14 +150,16 @@ class RequestDetailView(View):
             return json_response(error='未找到指定发布申请')
         if req.status not in ('1', '-3'):
             return json_response(error='该申请单当前状态还不能执行发布')
-        hosts = Host.objects.filter(id__in=json.loads(req.host_ids))
+
+        host_ids = req.fail_host_ids if form.mode == 'fail' else req.host_ids
+        hosts = Host.objects.filter(id__in=json.loads(host_ids))
         message = f'{human_time()} 等待调度...        '
         outputs = {x.id: {'id': x.id, 'title': x.name, 'step': 0, 'data': message} for x in hosts}
         req.status = '2'
         req.do_at = human_datetime()
         req.do_by = request.user
         req.save()
-        Thread(target=dispatch, args=(req,)).start()
+        Thread(target=dispatch, args=(req, form.mode == 'fail')).start()
         if req.is_quick_deploy:
             if req.repository_id:
                 outputs['local'] = {'id': 'local', 'step': 100, 'data': f'{human_time()} 已构建完成忽略执行。'}
@@ -325,6 +329,7 @@ def get_request_info(request):
     if error is None:
         req = DeployRequest.objects.get(pk=form.id)
         response = req.to_dict(selects=('status', 'reason'))
+        response['fail_host_ids'] = json.loads(req.fail_host_ids)
         response['status_alias'] = req.get_status_display()
         return json_response(response)
     return json_response(error=error)

spug_api/apps/repository/utils.py

@@ -23,7 +23,7 @@ def dispatch(rep: Repository, helper=None):
     if not helper:
         rds = get_redis_connection()
         rds_key = f'{settings.BUILD_KEY}:{rep.spug_version}'
-        helper = Helper(rds, rds_key)
+        helper = Helper.make(rds, rds_key)
         rep.save()
     try:
         api_token = uuid.uuid4().hex

spug_web/src/pages/deploy/request/Ext1Console.js

@@ -36,7 +36,7 @@ function Ext1Console(props) {
 
   function doDeploy() {
     let socket;
-    http.post(`/api/deploy/request/${props.request.id}/`)
+    http.post(`/api/deploy/request/${props.request.id}/`, {mode: props.request.mode})
       .then(res => {
         Object.assign(outputs, res.outputs)
         setTimeout(() => setFetching(false), 100)
@@ -57,6 +57,7 @@ function Ext1Console(props) {
       } else {
         index += 1;
         const {key, data, step, status} = JSON.parse(e.data);
+        if (!outputs[key]) return
         if (data !== undefined) {
           outputs[key].data += data
           if (terms[key]) terms[key].write(data)

spug_web/src/pages/deploy/request/Ext2Console.js

@@ -40,7 +40,7 @@ function Ext2Console(props) {
 
   function doDeploy() {
     let socket;
-    http.post(`/api/deploy/request/${props.request.id}/`)
+    http.post(`/api/deploy/request/${props.request.id}/`, {mode: props.request.mode})
       .then(res => {
         setSActions(res.s_actions);
         setHActions(res.h_actions);
@@ -63,6 +63,7 @@ function Ext2Console(props) {
       } else {
         index += 1;
         const {key, data, step, status} = JSON.parse(e.data);
+        if (!outputs[key]) return
         if (data !== undefined) {
           outputs[key].data += data
           if (terms[key]) terms[key].write(data)

spug_web/src/pages/deploy/request/Table.js

@@ -12,6 +12,16 @@ import { Action, AuthButton, TableCard } from 'components';
 import S from './index.module.less';
 import store from './store';
 
+function DeployConfirm() {
+  return (
+    <div>
+      <div>确认发布方式</div>
+      <div style={{color: '#999', fontSize: 12}}>补偿：仅发布上次发布失败的主机。</div>
+      <div style={{color: '#999', fontSize: 12}}>全量：再次发布所有主机。</div>
+    </div>
+  )
+}
+
 function ComTable() {
   const columns = [{
     title: '申请标题',
@@ -121,9 +131,7 @@ function ComTable() {
         case '-3':
           return <Action>
             <Action.Button auth="deploy.request.do" onClick={() => store.readConsole(info)}>查看</Action.Button>
-            <Popconfirm title="确认要执行该发布申请？" onConfirm={e => handleDeploy(e, info)}>
+            <DoAction info={info}/>
-              <Action.Button auth="deploy.request.do">发布</Action.Button>
-            </Popconfirm>
             {info.visible_rollback && (
               <Action.Button auth="deploy.request.do" onClick={() => store.rollback(info)}>回滚</Action.Button>
             )}
@@ -148,9 +156,7 @@ function ComTable() {
           </Action>;
         case '1':
           return <Action>
-            <Popconfirm title="确认要执行该发布申请？" onConfirm={e => handleDeploy(e, info)}>
+            <DoAction info={info}/>
-              <Action.Button auth="deploy.request.do">发布</Action.Button>
-            </Popconfirm>
             <Action.Button auth="deploy.request.del" onClick={() => handleDelete(info)}>删除</Action.Button>
           </Action>;
         case '2':
@@ -163,6 +169,21 @@ function ComTable() {
     }
   }];
 
+  function DoAction(props) {
+    const {host_ids, fail_host_ids} = props.info;
+    return (
+      <Popconfirm
+        title={<DeployConfirm/>}
+        okText="全量"
+        cancelText="补偿"
+        cancelButtonProps={{disabled: [0, host_ids.length].includes(fail_host_ids.length)}}
+        onConfirm={e => handleDeploy(e, props.info, 'all')}
+        onCancel={e => handleDeploy(e, props.info, 'fail')}>
+        <Action.Button auth="deploy.request.do">发布</Action.Button>
+      </Popconfirm>
+    )
+  }
+
   function handleDelete(info) {
     Modal.confirm({
       title: '删除确认',
@@ -177,14 +198,15 @@ function ComTable() {
     })
   }
 
-  function handleDeploy(e, info) {
+  function handleDeploy(e, info, mode) {
+    info.mode = mode
     store.showConsole(info);
   }
 
   return (
     <TableCard
       tKey="dr"
-      rowKey="id"
+      rowKey={row => row.key || row.id}
       title="申请列表"
       columns={columns}
       scroll={{x: 1500}}

spug_web/src/pages/deploy/request/index.module.less

@@ -13,6 +13,7 @@
   bottom: 12px;
   right: 24px;
   align-items: flex-end;
+  z-index: 999;
 
   .item {
     width: 180px;

spug_web/src/pages/deploy/request/store.js

@@ -58,7 +58,7 @@ class Store {
       .then(res => {
         for (let item of this.records) {
           if (item.id === id) {
-            Object.assign(item, res)
+            Object.assign(item, res, {key: Date.now()})
             break
           }
         }