github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

Enable the ability to cordon/uncordon/drain nodes (#4723) 

* feat(node): Enable the ability to cordon/uncordon/drain nodes

* feat(cluster): check if there is a drain operation somewhere

* feat(kubernetes): allow to cordon, uncordon, drain nodes

* refacto(kubernetes): set a constant for drain label name

* fix(node): Relocate the warning message next to the dropdown and change the information message
pull/4666/head
Maxime Bajeux 2021-03-15 22:36:14 +01:00 committed by GitHub
parent 660bc2dadf
commit 32a9a2e46b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 182 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/kubernetes/node/converter.js
View File

@ -1,6 +1,6 @@
import _ from 'lodash-es';
import { KubernetesNode, KubernetesNodeDetails, KubernetesNodeTaint } from 'Kubernetes/node/models';
import { KubernetesNode, KubernetesNodeDetails, KubernetesNodeTaint, KubernetesNodeAvailabilities, KubernetesPortainerNodeDrainLabel } from 'Kubernetes/node/models';
import KubernetesResourceReservationHelper from 'Kubernetes/helpers/resourceReservationHelper';
import { KubernetesNodeFormValues, KubernetesNodeTaintFormValues, KubernetesNodeLabelFormValues } from 'Kubernetes/node/formValues';
import { KubernetesNodeCreatePayload, KubernetesNodeTaintPayload } from 'Kubernetes/node/payload';
@ -30,6 +30,11 @@ class KubernetesNodeConverter {
NetworkUnavailable: networkUnavailable && networkUnavailable.status === 'True',
};
res.Availability = KubernetesNodeAvailabilities.ACTIVE;
if (data.spec.unschedulable === true) {
res.Availability = _.has(data.metadata.labels, KubernetesPortainerNodeDrainLabel) ? KubernetesNodeAvailabilities.DRAIN : KubernetesNodeAvailabilities.PAUSE;
}
if (ready.status === 'False') {
res.Status = 'Unhealthy';
} else if (ready.status === 'Unknown' || res.Conditions.MemoryPressure || res.Conditions.PIDPressure || res.Conditions.DiskPressure || res.Conditions.NetworkUnavailable) {
@ -67,6 +72,8 @@ class KubernetesNodeConverter {
static nodeToFormValues(node) {
const res = new KubernetesNodeFormValues();
res.Availability = node.Availability;
res.Taints = _.map(node.Taints, (taint) => {
const res = new KubernetesNodeTaintFormValues();
res.Key = taint.Key;
@ -92,6 +99,8 @@ class KubernetesNodeConverter {
static formValuesToNode(node, formValues) {
const res = angular.copy(node);
res.Availability = formValues.Availability;
const filteredTaints = _.filter(formValues.Taints, (taint) => !taint.NeedsDeletion);
res.Taints = _.map(filteredTaints, (item) => {
const taint = new KubernetesNodeTaint();
@ -130,6 +139,15 @@ class KubernetesNodeConverter {
payload.metadata.labels = node.Labels;
if (node.Availability !== KubernetesNodeAvailabilities.ACTIVE) {
payload.spec.unschedulable = true;
if (node.Availability === KubernetesNodeAvailabilities.DRAIN) {
payload.metadata.labels[KubernetesPortainerNodeDrainLabel] = '';
} else {
delete payload.metadata.labels[KubernetesPortainerNodeDrainLabel];
}
}
return payload;
}

1
app/kubernetes/node/formValues.js
View File

@ -1,6 +1,7 @@
const _KubernetesNodeFormValues = Object.freeze({
Taints: [],
Labels: [],
Availability: '',
});
export class KubernetesNodeFormValues {

9
app/kubernetes/node/models.js
View File

@ -1,3 +1,5 @@
export const KubernetesPortainerNodeDrainLabel = 'io.portainer/node-status-drain';
/**
* KubernetesNode Model
*/
@ -14,6 +16,7 @@ const _KubernetesNode = Object.freeze({
Api: false,
Taints: [],
Port: 0,
Availability: '',
});
export class KubernetesNode {
@ -58,6 +61,12 @@ export class KubernetesNodeTaint {
}
}
export const KubernetesNodeAvailabilities = Object.freeze({
ACTIVE: 'Active',
PAUSE: 'Pause',
DRAIN: 'Drain',
});
export const KubernetesNodeTaintEffects = Object.freeze({
NOSCHEDULE: 'NoSchedule',
PREFERNOSCHEDULE: 'PreferNoSchedule',

3
app/kubernetes/node/service.js
View File

@ -57,7 +57,8 @@ class KubernetesNodeService {
const newNode = KubernetesNodeConverter.formValuesToNode(node, nodeFormValues);
const payload = KubernetesNodeConverter.patchPayload(node, newNode);
const data = await this.KubernetesNodes().patch(params, payload).$promise;
return data;
const patchedNode = KubernetesNodeConverter.apiToNodeDetails(data);
return patchedNode;
} catch (err) {
throw { msg: 'Unable to patch node', err: err };
}

9
app/kubernetes/pod/converter.js
View File

@ -10,7 +10,7 @@ import {
} from 'Kubernetes/models/application/models';
import { createPayloadFactory } from './payloads/create';
import { KubernetesPod, KubernetesPodToleration, KubernetesPodAffinity, KubernetesPodContainer, KubernetesPodContainerTypes } from './models';
import { KubernetesPod, KubernetesPodToleration, KubernetesPodAffinity, KubernetesPodContainer, KubernetesPodContainerTypes, KubernetesPodEviction } from 'Kubernetes/pod/models';
function computeStatus(statuses) {
const containerStatuses = _.map(statuses, 'state');
@ -117,6 +117,13 @@ export default class KubernetesPodConverter {
return res;
}
static evictionPayload(pod) {
const res = new KubernetesPodEviction();
res.metadata.name = pod.Name;
res.metadata.namespace = pod.Namespace;
return res;
}
static patchPayload(oldPod, newPod) {
const oldPayload = createPayload(oldPod);
const newPayload = createPayload(newPod);

15
app/kubernetes/pod/models/index.js
View File

@ -65,6 +65,21 @@ export class KubernetesPodContainer {
}
}
const _KubernetesPodEviction = Object.freeze({
apiVersion: 'policy/v1beta1',
kind: 'Eviction',
metadata: {
name: '',
namespace: '',
},
});
export class KubernetesPodEviction {
constructor() {
Object.assign(this, JSON.parse(JSON.stringify(_KubernetesPodEviction)));
}
}
export const KubernetesPodContainerTypes = {
INIT: 1,
APP: 2,

23
app/kubernetes/pod/service.js
View File

@ -2,7 +2,7 @@ import angular from 'angular';
import PortainerError from 'Portainer/error';
import { KubernetesCommonParams } from 'Kubernetes/models/common/params';
import KubernetesPodConverter from './converter';
import KubernetesPodConverter from 'Kubernetes/pod/converter';
class KubernetesPodService {
/* @ngInject */
@ -15,6 +15,7 @@ class KubernetesPodService {
this.logsAsync = this.logsAsync.bind(this);
this.deleteAsync = this.deleteAsync.bind(this);
this.patchAsync = this.patchAsync.bind(this);
this.evictionAsync = this.evictionAsync.bind(this);
}
async getAsync(namespace, name) {
@ -116,6 +117,26 @@ class KubernetesPodService {
delete(pod) {
return this.$async(this.deleteAsync, pod);
}
/**
* EVICT
*/
async evictionAsync(pod) {
try {
const params = new KubernetesCommonParams();
params.id = pod.Name;
params.action = 'eviction';
const namespace = pod.Namespace;
const podEvictionPayload = KubernetesPodConverter.evictionPayload(pod);
await this.KubernetesPods(namespace).evict(params, podEvictionPayload).$promise;
} catch (err) {
throw new PortainerError('Unable to evict pod', err);
}
}
eviction(pod) {
return this.$async(this.evictionAsync, pod);
}
}
export default KubernetesPodService;

1
app/kubernetes/rest/pod.js
View File

@ -42,6 +42,7 @@ angular.module('portainer.kubernetes').factory('KubernetesPods', [
params: { action: 'log' },
transformResponse: logsHandler,
},
evict: { method: 'POST' },
}
);
};

20
app/kubernetes/views/cluster/node/node.html
View File

@ -52,6 +52,26 @@
</span>
</td>
</tr>
<tr>
<td class="col-xs-3">
Availability
</td>
<td class="col-xs-9">
<select class="form-control" name="availability" style="display: inline-block; width: 16rem;" ng-model="ctrl.formValues.Availability">
<option>{{ ctrl.availabilities.ACTIVE }}</option>
<option>{{ ctrl.availabilities.PAUSE }}</option>
<option>{{ ctrl.availabilities.DRAIN }}</option>
</select>
<span class="small text-warning" ng-if="ctrl.state.isDrainOperation && ctrl.formValues.Availability === ctrl.availabilities.DRAIN">
<i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
Cannot use this action while another node is currently being drained.
</span>
<span class="small text-warning" ng-if="ctrl.state.isContainPortainer && ctrl.formValues.Availability === ctrl.availabilities.DRAIN">
<i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
Cannot drain a node where this Portainer instance is running.
</span>
</td>
</tr>
</tbody>
</table>

94
app/kubernetes/views/cluster/node/nodeController.js
View File

@ -5,7 +5,7 @@ import { KubernetesResourceReservation } from 'Kubernetes/models/resource-reserv
import KubernetesEventHelper from 'Kubernetes/helpers/eventHelper';
import KubernetesNodeConverter from 'Kubernetes/node/converter';
import { KubernetesNodeLabelFormValues, KubernetesNodeTaintFormValues } from 'Kubernetes/node/formValues';
import { KubernetesNodeTaintEffects } from 'Kubernetes/node/models';
import { KubernetesNodeTaintEffects, KubernetesNodeAvailabilities } from 'Kubernetes/node/models';
import KubernetesFormValidationHelper from 'Kubernetes/helpers/formValidationHelper';
import { KubernetesNodeHelper } from 'Kubernetes/node/helper';
@ -35,12 +35,13 @@ class KubernetesNodeController {
this.KubernetesEndpointService = KubernetesEndpointService;
this.onInit = this.onInit.bind(this);
this.getNodeAsync = this.getNodeAsync.bind(this);
this.getNodesAsync = this.getNodesAsync.bind(this);
this.getEvents = this.getEvents.bind(this);
this.getEventsAsync = this.getEventsAsync.bind(this);
this.getApplicationsAsync = this.getApplicationsAsync.bind(this);
this.getEndpointsAsync = this.getEndpointsAsync.bind(this);
this.updateNodeAsync = this.updateNodeAsync.bind(this);
this.drainNodeAsync = this.drainNodeAsync.bind(this);
}
selectTab(index) {
@ -152,6 +153,47 @@ class KubernetesNodeController {
/* #endregion */
/* #region cordon */
computeCordonWarning() {
return this.formValues.Availability === this.availabilities.PAUSE;
}
/* #endregion */
/* #region drain */
computeDrainWarning() {
return this.formValues.Availability === this.availabilities.DRAIN;
}
async drainNodeAsync() {
const pods = _.flatten(_.map(this.applications, (app) => app.Pods));
let actionCount = pods.length;
for (const pod of pods) {
try {
await this.KubernetesPodService.eviction(pod);
this.Notifications.success('Pod successfully evicted', pod.Name);
} catch (err) {
this.Notifications.error('Failure', err, 'Unable to evict pod');
this.formValues.Availability = this.availabilities.PAUSE;
await this.KubernetesNodeService.patch(this.node, this.formValues);
} finally {
--actionCount;
if (actionCount === 0) {
this.formValues.Availability = this.availabilities.PAUSE;
await this.KubernetesNodeService.patch(this.node, this.formValues);
}
}
}
}
drainNode() {
return this.$async(this.drainNodeAsync);
}
/* #endregion */
/* #region actions */
isNoChangesMade() {
@ -160,8 +202,12 @@ class KubernetesNodeController {
return !payload.length;
}
isDrainError() {
return (this.state.isDrainOperation || this.state.isContainPortainer) && this.formValues.Availability === this.availabilities.DRAIN;
}
isFormValid() {
return !this.state.hasDuplicateTaintKeys && !this.state.hasDuplicateLabelKeys && !this.isNoChangesMade();
return !this.state.hasDuplicateTaintKeys && !this.state.hasDuplicateLabelKeys && !this.isNoChangesMade() && !this.isDrainError();
}
resetFormValues() {
@ -196,7 +242,10 @@ class KubernetesNodeController {
async updateNodeAsync() {
try {
await this.KubernetesNodeService.patch(this.node, this.formValues);
this.node = await this.KubernetesNodeService.patch(this.node, this.formValues);
if (this.formValues.Availability === 'Drain') {
await this.drainNode();
}
this.Notifications.success('Node updated successfully');
this.$state.reload();
} catch (err) {
@ -207,6 +256,8 @@ class KubernetesNodeController {
updateNode() {
const taintsWarning = this.computeTaintsWarning();
const labelsWarning = this.computeLabelsWarning();
const cordonWarning = this.computeCordonWarning();
const drainWarning = this.computeDrainWarning();
if (taintsWarning && !labelsWarning) {
this.ModalService.confirmUpdate(
@ -235,16 +286,36 @@ class KubernetesNodeController {
}
}
);
} else if (cordonWarning) {
this.ModalService.confirmUpdate(
'Marking this node as unschedulable will effectively cordon the node and prevent any new workload from being scheduled on that node. Are you sure?',
(confirmed) => {
if (confirmed) {
return this.$async(this.updateNodeAsync);
}
}
);
} else if (drainWarning) {
this.ModalService.confirmUpdate(
'Draining this node will cause all workloads to be evicted from that node. This might lead to some service interruption. Are you sure?',
(confirmed) => {
if (confirmed) {
return this.$async(this.updateNodeAsync);
}
}
);
} else {
return this.$async(this.updateNodeAsync);
}
}
async getNodeAsync() {
async getNodesAsync() {
try {
this.state.dataLoading = true;
const nodeName = this.$transition$.params().name;
this.node = await this.KubernetesNodeService.get(nodeName);
this.nodes = await this.KubernetesNodeService.get();
this.node = _.find(this.nodes, { Name: nodeName });
this.state.isDrainOperation = _.find(this.nodes, { Availability: this.availabilities.DRAIN });
} catch (err) {
this.Notifications.error('Failure', err, 'Unable to retrieve node');
} finally {
@ -252,8 +323,8 @@ class KubernetesNodeController {
}
}
getNode() {
return this.$async(this.getNodeAsync);
getNodes() {
return this.$async(this.getNodesAsync);
}
hasEventWarnings() {
@ -303,6 +374,7 @@ class KubernetesNodeController {
});
this.resourceReservation.Memory = KubernetesResourceReservationHelper.megaBytesValue(this.resourceReservation.Memory);
this.memoryLimit = KubernetesResourceReservationHelper.megaBytesValue(this.node.Memory);
this.state.isContainPortainer = _.find(this.applications, { ApplicationName: 'portainer' });
} catch (err) {
this.Notifications.error('Failure', err, 'Unable to retrieve applications');
} finally {
@ -328,11 +400,15 @@ class KubernetesNodeController {
hasDuplicateTaintKeys: false,
duplicateLabelKeys: [],
hasDuplicateLabelKeys: false,
isDrainOperation: false,
isContainPortainer: false,
};
this.availabilities = KubernetesNodeAvailabilities;
this.state.activeTab = this.LocalStorage.getActiveTab('node');
await this.getNode();
await this.getNodes();
await this.getEvents();
await this.getApplications();
await this.getEndpoints();