Enable the ability to cordon/uncordon/drain nodes (#4723)

* feat(node): Enable the ability to cordon/uncordon/drain nodes

* feat(cluster): check if there is a drain operation somewhere

* feat(kubernetes): allow to cordon, uncordon, drain nodes

* refacto(kubernetes): set a constant for drain label name

* fix(node): Relocate the warning message next to the dropdown and change the information message

app/kubernetes/node/converter.js

@@ -1,6 +1,6 @@
 import _ from 'lodash-es';
 
-import { KubernetesNode, KubernetesNodeDetails, KubernetesNodeTaint } from 'Kubernetes/node/models';
+import { KubernetesNode, KubernetesNodeDetails, KubernetesNodeTaint, KubernetesNodeAvailabilities, KubernetesPortainerNodeDrainLabel } from 'Kubernetes/node/models';
 import KubernetesResourceReservationHelper from 'Kubernetes/helpers/resourceReservationHelper';
 import { KubernetesNodeFormValues, KubernetesNodeTaintFormValues, KubernetesNodeLabelFormValues } from 'Kubernetes/node/formValues';
 import { KubernetesNodeCreatePayload, KubernetesNodeTaintPayload } from 'Kubernetes/node/payload';
@@ -30,6 +30,11 @@ class KubernetesNodeConverter {
       NetworkUnavailable: networkUnavailable && networkUnavailable.status === 'True',
     };
 
+    res.Availability = KubernetesNodeAvailabilities.ACTIVE;
+    if (data.spec.unschedulable === true) {
+      res.Availability = _.has(data.metadata.labels, KubernetesPortainerNodeDrainLabel) ? KubernetesNodeAvailabilities.DRAIN : KubernetesNodeAvailabilities.PAUSE;
+    }
+
     if (ready.status === 'False') {
       res.Status = 'Unhealthy';
     } else if (ready.status === 'Unknown' || res.Conditions.MemoryPressure || res.Conditions.PIDPressure || res.Conditions.DiskPressure || res.Conditions.NetworkUnavailable) {
@@ -67,6 +72,8 @@ class KubernetesNodeConverter {
   static nodeToFormValues(node) {
     const res = new KubernetesNodeFormValues();
 
+    res.Availability = node.Availability;
+
     res.Taints = _.map(node.Taints, (taint) => {
       const res = new KubernetesNodeTaintFormValues();
       res.Key = taint.Key;
@@ -92,6 +99,8 @@ class KubernetesNodeConverter {
   static formValuesToNode(node, formValues) {
     const res = angular.copy(node);
 
+    res.Availability = formValues.Availability;
+
     const filteredTaints = _.filter(formValues.Taints, (taint) => !taint.NeedsDeletion);
     res.Taints = _.map(filteredTaints, (item) => {
       const taint = new KubernetesNodeTaint();
@@ -130,6 +139,15 @@ class KubernetesNodeConverter {
 
     payload.metadata.labels = node.Labels;
 
+    if (node.Availability !== KubernetesNodeAvailabilities.ACTIVE) {
+      payload.spec.unschedulable = true;
+      if (node.Availability === KubernetesNodeAvailabilities.DRAIN) {
+        payload.metadata.labels[KubernetesPortainerNodeDrainLabel] = '';
+      } else {
+        delete payload.metadata.labels[KubernetesPortainerNodeDrainLabel];
+      }
+    }
+
     return payload;
   }
 

app/kubernetes/node/formValues.js

@@ -1,6 +1,7 @@
 const _KubernetesNodeFormValues = Object.freeze({
   Taints: [],
   Labels: [],
+  Availability: '',
 });
 
 export class KubernetesNodeFormValues {

app/kubernetes/node/models.js

@@ -1,3 +1,5 @@
+export const KubernetesPortainerNodeDrainLabel = 'io.portainer/node-status-drain';
+
 /**
  * KubernetesNode Model
  */
@@ -14,6 +16,7 @@ const _KubernetesNode = Object.freeze({
   Api: false,
   Taints: [],
   Port: 0,
+  Availability: '',
 });
 
 export class KubernetesNode {
@@ -58,6 +61,12 @@ export class KubernetesNodeTaint {
   }
 }
 
+export const KubernetesNodeAvailabilities = Object.freeze({
+  ACTIVE: 'Active',
+  PAUSE: 'Pause',
+  DRAIN: 'Drain',
+});
+
 export const KubernetesNodeTaintEffects = Object.freeze({
   NOSCHEDULE: 'NoSchedule',
   PREFERNOSCHEDULE: 'PreferNoSchedule',

app/kubernetes/node/service.js

@@ -57,7 +57,8 @@ class KubernetesNodeService {
       const newNode = KubernetesNodeConverter.formValuesToNode(node, nodeFormValues);
       const payload = KubernetesNodeConverter.patchPayload(node, newNode);
       const data = await this.KubernetesNodes().patch(params, payload).$promise;
-      return data;
+      const patchedNode = KubernetesNodeConverter.apiToNodeDetails(data);
+      return patchedNode;
     } catch (err) {
       throw { msg: 'Unable to patch node', err: err };
     }

app/kubernetes/pod/converter.js

@@ -10,7 +10,7 @@ import {
 } from 'Kubernetes/models/application/models';
 
 import { createPayloadFactory } from './payloads/create';
-import { KubernetesPod, KubernetesPodToleration, KubernetesPodAffinity, KubernetesPodContainer, KubernetesPodContainerTypes } from './models';
+import { KubernetesPod, KubernetesPodToleration, KubernetesPodAffinity, KubernetesPodContainer, KubernetesPodContainerTypes, KubernetesPodEviction } from 'Kubernetes/pod/models';
 
 function computeStatus(statuses) {
   const containerStatuses = _.map(statuses, 'state');
@@ -117,6 +117,13 @@ export default class KubernetesPodConverter {
     return res;
   }
 
+  static evictionPayload(pod) {
+    const res = new KubernetesPodEviction();
+    res.metadata.name = pod.Name;
+    res.metadata.namespace = pod.Namespace;
+    return res;
+  }
+
   static patchPayload(oldPod, newPod) {
     const oldPayload = createPayload(oldPod);
     const newPayload = createPayload(newPod);

app/kubernetes/pod/models/index.js

@@ -65,6 +65,21 @@ export class KubernetesPodContainer {
   }
 }
 
+const _KubernetesPodEviction = Object.freeze({
+  apiVersion: 'policy/v1beta1',
+  kind: 'Eviction',
+  metadata: {
+    name: '',
+    namespace: '',
+  },
+});
+
+export class KubernetesPodEviction {
+  constructor() {
+    Object.assign(this, JSON.parse(JSON.stringify(_KubernetesPodEviction)));
+  }
+}
+
 export const KubernetesPodContainerTypes = {
   INIT: 1,
   APP: 2,

app/kubernetes/pod/service.js

@@ -2,7 +2,7 @@ import angular from 'angular';
 import PortainerError from 'Portainer/error';
 
 import { KubernetesCommonParams } from 'Kubernetes/models/common/params';
-import KubernetesPodConverter from './converter';
+import KubernetesPodConverter from 'Kubernetes/pod/converter';
 
 class KubernetesPodService {
   /* @ngInject */
@@ -15,6 +15,7 @@ class KubernetesPodService {
     this.logsAsync = this.logsAsync.bind(this);
     this.deleteAsync = this.deleteAsync.bind(this);
     this.patchAsync = this.patchAsync.bind(this);
+    this.evictionAsync = this.evictionAsync.bind(this);
   }
 
   async getAsync(namespace, name) {
@@ -116,6 +117,26 @@ class KubernetesPodService {
   delete(pod) {
     return this.$async(this.deleteAsync, pod);
   }
+
+  /**
+   * EVICT
+   */
+  async evictionAsync(pod) {
+    try {
+      const params = new KubernetesCommonParams();
+      params.id = pod.Name;
+      params.action = 'eviction';
+      const namespace = pod.Namespace;
+      const podEvictionPayload = KubernetesPodConverter.evictionPayload(pod);
+      await this.KubernetesPods(namespace).evict(params, podEvictionPayload).$promise;
+    } catch (err) {
+      throw new PortainerError('Unable to evict pod', err);
+    }
+  }
+
+  eviction(pod) {
+    return this.$async(this.evictionAsync, pod);
+  }
 }
 
 export default KubernetesPodService;

app/kubernetes/rest/pod.js

@@ -42,6 +42,7 @@ angular.module('portainer.kubernetes').factory('KubernetesPods', [
             params: { action: 'log' },
             transformResponse: logsHandler,
           },
+          evict: { method: 'POST' },
         }
       );
     };

app/kubernetes/views/cluster/node/node.html

@@ -52,6 +52,26 @@
                         </span>
                       </td>
                     </tr>
+                    <tr>
+                      <td class="col-xs-3">
+                        Availability
+                      </td>
+                      <td class="col-xs-9">
+                        <select class="form-control" name="availability" style="display: inline-block; width: 16rem;" ng-model="ctrl.formValues.Availability">
+                          <option>{{ ctrl.availabilities.ACTIVE }}</option>
+                          <option>{{ ctrl.availabilities.PAUSE }}</option>
+                          <option>{{ ctrl.availabilities.DRAIN }}</option>
+                        </select>
+                        <span class="small text-warning" ng-if="ctrl.state.isDrainOperation && ctrl.formValues.Availability === ctrl.availabilities.DRAIN">
+                          <i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
+                          Cannot use this action while another node is currently being drained.
+                        </span>
+                        <span class="small text-warning" ng-if="ctrl.state.isContainPortainer && ctrl.formValues.Availability === ctrl.availabilities.DRAIN">
+                          <i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
+                          Cannot drain a node where this Portainer instance is running.
+                        </span>
+                      </td>
+                    </tr>
                   </tbody>
                 </table>
 

app/kubernetes/views/cluster/node/nodeController.js

@@ -5,7 +5,7 @@ import { KubernetesResourceReservation } from 'Kubernetes/models/resource-reserv
 import KubernetesEventHelper from 'Kubernetes/helpers/eventHelper';
 import KubernetesNodeConverter from 'Kubernetes/node/converter';
 import { KubernetesNodeLabelFormValues, KubernetesNodeTaintFormValues } from 'Kubernetes/node/formValues';
-import { KubernetesNodeTaintEffects } from 'Kubernetes/node/models';
+import { KubernetesNodeTaintEffects, KubernetesNodeAvailabilities } from 'Kubernetes/node/models';
 import KubernetesFormValidationHelper from 'Kubernetes/helpers/formValidationHelper';
 import { KubernetesNodeHelper } from 'Kubernetes/node/helper';
 
@@ -35,12 +35,13 @@ class KubernetesNodeController {
     this.KubernetesEndpointService = KubernetesEndpointService;
 
     this.onInit = this.onInit.bind(this);
-    this.getNodeAsync = this.getNodeAsync.bind(this);
+    this.getNodesAsync = this.getNodesAsync.bind(this);
     this.getEvents = this.getEvents.bind(this);
     this.getEventsAsync = this.getEventsAsync.bind(this);
     this.getApplicationsAsync = this.getApplicationsAsync.bind(this);
     this.getEndpointsAsync = this.getEndpointsAsync.bind(this);
     this.updateNodeAsync = this.updateNodeAsync.bind(this);
+    this.drainNodeAsync = this.drainNodeAsync.bind(this);
   }
 
   selectTab(index) {
@@ -152,6 +153,47 @@ class KubernetesNodeController {
 
   /* #endregion */
 
+  /* #region cordon */
+
+  computeCordonWarning() {
+    return this.formValues.Availability === this.availabilities.PAUSE;
+  }
+
+  /* #endregion */
+
+  /* #region drain */
+
+  computeDrainWarning() {
+    return this.formValues.Availability === this.availabilities.DRAIN;
+  }
+
+  async drainNodeAsync() {
+    const pods = _.flatten(_.map(this.applications, (app) => app.Pods));
+    let actionCount = pods.length;
+    for (const pod of pods) {
+      try {
+        await this.KubernetesPodService.eviction(pod);
+        this.Notifications.success('Pod successfully evicted', pod.Name);
+      } catch (err) {
+        this.Notifications.error('Failure', err, 'Unable to evict pod');
+        this.formValues.Availability = this.availabilities.PAUSE;
+        await this.KubernetesNodeService.patch(this.node, this.formValues);
+      } finally {
+        --actionCount;
+        if (actionCount === 0) {
+          this.formValues.Availability = this.availabilities.PAUSE;
+          await this.KubernetesNodeService.patch(this.node, this.formValues);
+        }
+      }
+    }
+  }
+
+  drainNode() {
+    return this.$async(this.drainNodeAsync);
+  }
+
+  /* #endregion */
+
   /* #region actions */
 
   isNoChangesMade() {
@@ -160,8 +202,12 @@ class KubernetesNodeController {
     return !payload.length;
   }
 
+  isDrainError() {
+    return (this.state.isDrainOperation || this.state.isContainPortainer) && this.formValues.Availability === this.availabilities.DRAIN;
+  }
+
   isFormValid() {
-    return !this.state.hasDuplicateTaintKeys && !this.state.hasDuplicateLabelKeys && !this.isNoChangesMade();
+    return !this.state.hasDuplicateTaintKeys && !this.state.hasDuplicateLabelKeys && !this.isNoChangesMade() && !this.isDrainError();
   }
 
   resetFormValues() {
@@ -196,7 +242,10 @@ class KubernetesNodeController {
 
   async updateNodeAsync() {
     try {
-      await this.KubernetesNodeService.patch(this.node, this.formValues);
+      this.node = await this.KubernetesNodeService.patch(this.node, this.formValues);
+      if (this.formValues.Availability === 'Drain') {
+        await this.drainNode();
+      }
       this.Notifications.success('Node updated successfully');
       this.$state.reload();
     } catch (err) {
@@ -207,6 +256,8 @@ class KubernetesNodeController {
   updateNode() {
     const taintsWarning = this.computeTaintsWarning();
     const labelsWarning = this.computeLabelsWarning();
+    const cordonWarning = this.computeCordonWarning();
+    const drainWarning = this.computeDrainWarning();
 
     if (taintsWarning && !labelsWarning) {
       this.ModalService.confirmUpdate(
@@ -235,16 +286,36 @@ class KubernetesNodeController {
           }
         }
       );
+    } else if (cordonWarning) {
+      this.ModalService.confirmUpdate(
+        'Marking this node as unschedulable will effectively cordon the node and prevent any new workload from being scheduled on that node. Are you sure?',
+        (confirmed) => {
+          if (confirmed) {
+            return this.$async(this.updateNodeAsync);
+          }
+        }
+      );
+    } else if (drainWarning) {
+      this.ModalService.confirmUpdate(
+        'Draining this node will cause all workloads to be evicted from that node. This might lead to some service interruption. Are you sure?',
+        (confirmed) => {
+          if (confirmed) {
+            return this.$async(this.updateNodeAsync);
+          }
+        }
+      );
     } else {
       return this.$async(this.updateNodeAsync);
     }
   }
 
-  async getNodeAsync() {
+  async getNodesAsync() {
     try {
       this.state.dataLoading = true;
       const nodeName = this.$transition$.params().name;
-      this.node = await this.KubernetesNodeService.get(nodeName);
+      this.nodes = await this.KubernetesNodeService.get();
+      this.node = _.find(this.nodes, { Name: nodeName });
+      this.state.isDrainOperation = _.find(this.nodes, { Availability: this.availabilities.DRAIN });
     } catch (err) {
       this.Notifications.error('Failure', err, 'Unable to retrieve node');
     } finally {
@@ -252,8 +323,8 @@ class KubernetesNodeController {
     }
   }
 
-  getNode() {
-    return this.$async(this.getNodeAsync);
+  getNodes() {
+    return this.$async(this.getNodesAsync);
   }
 
   hasEventWarnings() {
@@ -303,6 +374,7 @@ class KubernetesNodeController {
       });
       this.resourceReservation.Memory = KubernetesResourceReservationHelper.megaBytesValue(this.resourceReservation.Memory);
       this.memoryLimit = KubernetesResourceReservationHelper.megaBytesValue(this.node.Memory);
+      this.state.isContainPortainer = _.find(this.applications, { ApplicationName: 'portainer' });
     } catch (err) {
       this.Notifications.error('Failure', err, 'Unable to retrieve applications');
     } finally {
@@ -328,11 +400,15 @@ class KubernetesNodeController {
       hasDuplicateTaintKeys: false,
       duplicateLabelKeys: [],
       hasDuplicateLabelKeys: false,
+      isDrainOperation: false,
+      isContainPortainer: false,
     };
 
+    this.availabilities = KubernetesNodeAvailabilities;
+
     this.state.activeTab = this.LocalStorage.getActiveTab('node');
 
-    await this.getNode();
+    await this.getNodes();
     await this.getEvents();
     await this.getApplications();
     await this.getEndpoints();