Allow swapping NotReady and Unschedulable Taints

pkg/controller/node/controller_utils.go

@@ -36,8 +36,9 @@ import (
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	extensionslisters "k8s.io/kubernetes/pkg/client/listers/extensions/v1beta1"
 	"k8s.io/kubernetes/pkg/cloudprovider"
+	"k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
-	"k8s.io/kubernetes/pkg/util/node"
+	nodepkg "k8s.io/kubernetes/pkg/util/node"
 	utilversion "k8s.io/kubernetes/pkg/util/version"
 
 	"github.com/golang/glog"
@@ -102,12 +103,12 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n
 // setPodTerminationReason attempts to set a reason and message in the pod status, updates it in the apiserver,
 // and returns an error if it encounters one.
 func setPodTerminationReason(kubeClient clientset.Interface, pod *v1.Pod, nodeName string) (*v1.Pod, error) {
-	if pod.Status.Reason == node.NodeUnreachablePodReason {
+	if pod.Status.Reason == nodepkg.NodeUnreachablePodReason {
 		return pod, nil
 	}
 
-	pod.Status.Reason = node.NodeUnreachablePodReason
+	pod.Status.Reason = nodepkg.NodeUnreachablePodReason
-	pod.Status.Message = fmt.Sprintf(node.NodeUnreachablePodMessage, nodeName, pod.Name)
+	pod.Status.Message = fmt.Sprintf(nodepkg.NodeUnreachablePodMessage, nodeName, pod.Name)
 
 	var updatedPod *v1.Pod
 	var err error
@@ -286,3 +287,32 @@ func recordNodeStatusChange(recorder record.EventRecorder, node *v1.Node, new_st
 	// and event is recorded or neither should happen, see issue #6055.
 	recorder.Eventf(ref, v1.EventTypeNormal, new_status, "Node %s status is now: %s", node.Name, new_status)
 }
+
+// Returns true in case of success and false otherwise
+func swapNodeControllerTaint(kubeClient clientset.Interface, taintToAdd, taintToRemove *v1.Taint, node *v1.Node) bool {
+	taintToAdd.TimeAdded = metav1.Now()
+	err := controller.AddOrUpdateTaintOnNode(kubeClient, node.Name, taintToAdd)
+	if err != nil {
+		utilruntime.HandleError(
+			fmt.Errorf(
+				"unable to taint %v unresponsive Node %q: %v",
+				taintToAdd.Key,
+				node.Name,
+				err))
+		return false
+	}
+	glog.V(4).Infof("Added %v Taint to Node %v", taintToAdd, node.Name)
+
+	err = controller.RemoveTaintOffNode(kubeClient, node.Name, taintToRemove, node)
+	if err != nil {
+		utilruntime.HandleError(
+			fmt.Errorf(
+				"unable to remove %v unneeded taint from unresponsive Node %q: %v",
+				taintToRemove.Key,
+				node.Name,
+				err))
+		return false
+	}
+	glog.V(4).Infof("Made sure that Node %v has no %v Taint", node.Name, taintToRemove)
+	return true
+}

pkg/controller/node/nodecontroller.go

@@ -478,6 +478,74 @@ func NewNodeController(
 	return nc, nil
 }
 
+func (nc *NodeController) doEvictionPass() {
+	nc.evictorLock.Lock()
+	defer nc.evictorLock.Unlock()
+	for k := range nc.zonePodEvictor {
+		// Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded).
+		nc.zonePodEvictor[k].Try(func(value TimedValue) (bool, time.Duration) {
+			node, err := nc.nodeLister.Get(value.Value)
+			if apierrors.IsNotFound(err) {
+				glog.Warningf("Node %v no longer present in nodeLister!", value.Value)
+			} else if err != nil {
+				glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err)
+			} else {
+				zone := utilnode.GetZoneKey(node)
+				EvictionsNumber.WithLabelValues(zone).Inc()
+			}
+			nodeUid, _ := value.UID.(string)
+			remaining, err := deletePods(nc.kubeClient, nc.recorder, value.Value, nodeUid, nc.daemonSetStore)
+			if err != nil {
+				utilruntime.HandleError(fmt.Errorf("unable to evict node %q: %v", value.Value, err))
+				return false, 0
+			}
+			if remaining {
+				glog.Infof("Pods awaiting deletion due to NodeController eviction")
+			}
+			return true, 0
+		})
+	}
+}
+
+func (nc *NodeController) doTaintingPass() {
+	nc.evictorLock.Lock()
+	defer nc.evictorLock.Unlock()
+	for k := range nc.zoneNotReadyOrUnreachableTainer {
+		// Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded).
+		nc.zoneNotReadyOrUnreachableTainer[k].Try(func(value TimedValue) (bool, time.Duration) {
+			node, err := nc.nodeLister.Get(value.Value)
+			if apierrors.IsNotFound(err) {
+				glog.Warningf("Node %v no longer present in nodeLister!", value.Value)
+				return true, 0
+			} else if err != nil {
+				glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err)
+				// retry in 50 millisecond
+				return false, 50 * time.Millisecond
+			} else {
+				zone := utilnode.GetZoneKey(node)
+				EvictionsNumber.WithLabelValues(zone).Inc()
+			}
+			_, condition := v1.GetNodeCondition(&node.Status, v1.NodeReady)
+			// Because we want to mimic NodeStatus.Condition["Ready"] we make "unreachable" and "not ready" taints mutually exclusive.
+			taintToAdd := v1.Taint{}
+			oppositeTaint := v1.Taint{}
+			if condition.Status == v1.ConditionFalse {
+				taintToAdd = *NotReadyTaintTemplate
+				oppositeTaint = *UnreachableTaintTemplate
+			} else if condition.Status == v1.ConditionUnknown {
+				taintToAdd = *UnreachableTaintTemplate
+				oppositeTaint = *NotReadyTaintTemplate
+			} else {
+				// It seems that the Node is ready again, so there's no need to taint it.
+				glog.V(4).Infof("Node %v was in a taint queue, but it's ready now. Ignoring taint request.", value.Value)
+				return true, 0
+			}
+
+			return swapNodeControllerTaint(nc.kubeClient, &taintToAdd, &oppositeTaint, node), 0
+		})
+	}
+}
+
 // Run starts an asynchronous loop that monitors the status of cluster nodes.
 func (nc *NodeController) Run() {
 	go func() {
@@ -502,101 +570,12 @@ func (nc *NodeController) Run() {
 		if nc.useTaintBasedEvictions {
 			// Handling taint based evictions. Because we don't want a dedicated logic in TaintManager for NC-originated
 			// taints and we normally don't rate limit evictions caused by taints, we need to rate limit adding taints.
-			go wait.Until(func() {
+			go wait.Until(nc.doTaintingPass, nodeEvictionPeriod, wait.NeverStop)
-				nc.evictorLock.Lock()
-				defer nc.evictorLock.Unlock()
-				for k := range nc.zoneNotReadyOrUnreachableTainer {
-					// Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded).
-					nc.zoneNotReadyOrUnreachableTainer[k].Try(func(value TimedValue) (bool, time.Duration) {
-						node, err := nc.nodeLister.Get(value.Value)
-						if apierrors.IsNotFound(err) {
-							glog.Warningf("Node %v no longer present in nodeLister!", value.Value)
-							return true, 0
-						} else if err != nil {
-							glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err)
-							// retry in 50 millisecond
-							return false, 50 * time.Millisecond
-						} else {
-							zone := utilnode.GetZoneKey(node)
-							EvictionsNumber.WithLabelValues(zone).Inc()
-						}
-						_, condition := v1.GetNodeCondition(&node.Status, v1.NodeReady)
-						// Because we want to mimic NodeStatus.Condition["Ready"] we make "unreachable" and "not ready" taints mutually exclusive.
-						taintToAdd := v1.Taint{}
-						oppositeTaint := v1.Taint{}
-						if condition.Status == v1.ConditionFalse {
-							taintToAdd = *NotReadyTaintTemplate
-							oppositeTaint = *UnreachableTaintTemplate
-						} else if condition.Status == v1.ConditionUnknown {
-							taintToAdd = *UnreachableTaintTemplate
-							oppositeTaint = *NotReadyTaintTemplate
-						} else {
-							// It seems that the Node is ready again, so there's no need to taint it.
-							glog.V(4).Infof("Node %v was in a taint queue, but it's ready now. Ignoring taint request.", value.Value)
-							return true, 0
-						}
-
-						taintToAdd.TimeAdded = metav1.Now()
-						err = controller.AddOrUpdateTaintOnNode(nc.kubeClient, value.Value, &taintToAdd)
-						if err != nil {
-							utilruntime.HandleError(
-								fmt.Errorf(
-									"unable to taint %v unresponsive Node %q: %v",
-									taintToAdd.Key,
-									value.Value,
-									err))
-							return false, 0
-						} else {
-							glog.V(4).Info("Added %v Taint to Node %v", taintToAdd, value.Value)
-						}
-						err = controller.RemoveTaintOffNode(nc.kubeClient, value.Value, &oppositeTaint, node)
-						if err != nil {
-							utilruntime.HandleError(
-								fmt.Errorf(
-									"unable to remove %v unneeded taint from unresponsive Node %q: %v",
-									oppositeTaint.Key,
-									value.Value,
-									err))
-							return false, 0
-						} else {
-							glog.V(4).Info("Made sure that Node %v has no %v Taint", value.Value, oppositeTaint)
-						}
-						return true, 0
-					})
-				}
-			}, nodeEvictionPeriod, wait.NeverStop)
 		} else {
 			// Managing eviction of nodes:
 			// When we delete pods off a node, if the node was not empty at the time we then
 			// queue an eviction watcher. If we hit an error, retry deletion.
-			go wait.Until(func() {
+			go wait.Until(nc.doEvictionPass, nodeEvictionPeriod, wait.NeverStop)
-				nc.evictorLock.Lock()
-				defer nc.evictorLock.Unlock()
-				for k := range nc.zonePodEvictor {
-					// Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded).
-					nc.zonePodEvictor[k].Try(func(value TimedValue) (bool, time.Duration) {
-						node, err := nc.nodeLister.Get(value.Value)
-						if apierrors.IsNotFound(err) {
-							glog.Warningf("Node %v no longer present in nodeLister!", value.Value)
-						} else if err != nil {
-							glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err)
-						} else {
-							zone := utilnode.GetZoneKey(node)
-							EvictionsNumber.WithLabelValues(zone).Inc()
-						}
-						nodeUid, _ := value.UID.(string)
-						remaining, err := deletePods(nc.kubeClient, nc.recorder, value.Value, nodeUid, nc.daemonSetStore)
-						if err != nil {
-							utilruntime.HandleError(fmt.Errorf("unable to evict node %q: %v", value.Value, err))
-							return false, 0
-						}
-						if remaining {
-							glog.Infof("Pods awaiting deletion due to NodeController eviction")
-						}
-						return true, 0
-					})
-				}
-			}, nodeEvictionPeriod, wait.NeverStop)
 		}
 	}()
 }
@@ -685,7 +664,13 @@ func (nc *NodeController) monitorNodeStatus() error {
 			// Check eviction timeout against decisionTimestamp
 			if observedReadyCondition.Status == v1.ConditionFalse {
 				if nc.useTaintBasedEvictions {
-					if nc.markNodeForTainting(node) {
+					// We want to update the taint straight away if Node is already tainted with the UnreachableTaint
+					if v1.TaintExists(node.Spec.Taints, UnreachableTaintTemplate) {
+						taintToAdd := *NotReadyTaintTemplate
+						if !swapNodeControllerTaint(nc.kubeClient, &taintToAdd, UnreachableTaintTemplate, node) {
+							glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.")
+						}
+					} else if nc.markNodeForTainting(node) {
 						glog.V(2).Infof("Node %v is NotReady as of %v. Adding it to the Taint queue.",
 							node.Name,
 							decisionTimestamp,
@@ -706,7 +691,13 @@ func (nc *NodeController) monitorNodeStatus() error {
 			}
 			if observedReadyCondition.Status == v1.ConditionUnknown {
 				if nc.useTaintBasedEvictions {
-					if nc.markNodeForTainting(node) {
+					// We want to update the taint straight away if Node is already tainted with the UnreachableTaint
+					if v1.TaintExists(node.Spec.Taints, NotReadyTaintTemplate) {
+						taintToAdd := *UnreachableTaintTemplate
+						if !swapNodeControllerTaint(nc.kubeClient, &taintToAdd, NotReadyTaintTemplate, node) {
+							glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.")
+						}
+					} else if nc.markNodeForTainting(node) {
 						glog.V(2).Infof("Node %v is unresponsive as of %v. Adding it to the Taint queue.",
 							node.Name,
 							decisionTimestamp,

pkg/controller/node/nodecontroller_test.go

@@ -74,7 +74,9 @@ func NewNodeControllerFromClient(
 	clusterCIDR *net.IPNet,
 	serviceCIDR *net.IPNet,
 	nodeCIDRMaskSize int,
-	allocateNodeCIDRs bool) (*nodeController, error) {
+	allocateNodeCIDRs bool,
+	useTaints bool,
+) (*nodeController, error) {
 
 	factory := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
 
@@ -99,8 +101,8 @@ func NewNodeControllerFromClient(
 		serviceCIDR,
 		nodeCIDRMaskSize,
 		allocateNodeCIDRs,
-		false,
+		useTaints,
-		false,
+		useTaints,
 	)
 	if err != nil {
 		return nil, err
@@ -549,7 +551,7 @@ func TestMonitorNodeStatusEvictPods(t *testing.T) {
 	for _, item := range table {
 		nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler,
 			evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod,
-			testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false)
+			testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false)
 		nodeController.now = func() metav1.Time { return fakeNow }
 		nodeController.recorder = testutil.NewFakeRecorder()
 		for _, ds := range item.daemonSets {
@@ -698,7 +700,7 @@ func TestPodStatusChange(t *testing.T) {
 	for _, item := range table {
 		nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler,
 			evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod,
-			testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false)
+			testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false)
 		nodeController.now = func() metav1.Time { return fakeNow }
 		nodeController.recorder = testutil.NewFakeRecorder()
 		if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil {
@@ -1215,7 +1217,7 @@ func TestMonitorNodeStatusEvictPodsWithDisruption(t *testing.T) {
 		}
 		nodeController, _ := NewNodeControllerFromClient(nil, fakeNodeHandler,
 			evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod,
-			testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false)
+			testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false)
 		nodeController.now = func() metav1.Time { return fakeNow }
 		nodeController.enterPartialDisruptionFunc = func(nodeNum int) float32 {
 			return testRateLimiterQPS
@@ -1310,7 +1312,7 @@ func TestCloudProviderNoRateLimit(t *testing.T) {
 	nodeController, _ := NewNodeControllerFromClient(nil, fnh, 10*time.Minute,
 		testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold,
 		testNodeMonitorGracePeriod, testNodeStartupGracePeriod,
-		testNodeMonitorPeriod, nil, nil, 0, false)
+		testNodeMonitorPeriod, nil, nil, 0, false, false)
 	nodeController.cloud = &fakecloud.FakeCloud{}
 	nodeController.now = func() metav1.Time { return metav1.Date(2016, 1, 1, 12, 0, 0, 0, time.UTC) }
 	nodeController.recorder = testutil.NewFakeRecorder()
@@ -1579,7 +1581,7 @@ func TestMonitorNodeStatusUpdateStatus(t *testing.T) {
 	for i, item := range table {
 		nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler, 5*time.Minute,
 			testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold,
-			testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false)
+			testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false)
 		nodeController.now = func() metav1.Time { return fakeNow }
 		nodeController.recorder = testutil.NewFakeRecorder()
 		if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil {
@@ -1813,7 +1815,7 @@ func TestMonitorNodeStatusMarkPodsNotReady(t *testing.T) {
 	for i, item := range table {
 		nodeController, _ := NewNodeControllerFromClient(nil, item.fakeNodeHandler, 5*time.Minute,
 			testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold,
-			testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false)
+			testNodeMonitorGracePeriod, testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, false)
 		nodeController.now = func() metav1.Time { return fakeNow }
 		nodeController.recorder = testutil.NewFakeRecorder()
 		if err := syncNodeStore(nodeController, item.fakeNodeHandler); err != nil {
@@ -1845,6 +1847,146 @@ func TestMonitorNodeStatusMarkPodsNotReady(t *testing.T) {
 	}
 }
 
+func TestSwapUnreachableNotReadyTaints(t *testing.T) {
+	fakeNow := metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC)
+	evictionTimeout := 10 * time.Minute
+
+	fakeNodeHandler := &testutil.FakeNodeHandler{
+		Existing: []*v1.Node{
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node0",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					Labels: map[string]string{
+						metav1.LabelZoneRegion:        "region1",
+						metav1.LabelZoneFailureDomain: "zone1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Conditions: []v1.NodeCondition{
+						{
+							Type:               v1.NodeReady,
+							Status:             v1.ConditionUnknown,
+							LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC),
+							LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+			// Because of the logic that prevents NC from evicting anything when all Nodes are NotReady
+			// we need second healthy node in tests. Because of how the tests are written we need to update
+			// the status of this Node.
+			{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "node1",
+					CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					Labels: map[string]string{
+						metav1.LabelZoneRegion:        "region1",
+						metav1.LabelZoneFailureDomain: "zone1",
+					},
+				},
+				Status: v1.NodeStatus{
+					Conditions: []v1.NodeCondition{
+						{
+							Type:               v1.NodeReady,
+							Status:             v1.ConditionTrue,
+							LastHeartbeatTime:  metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC),
+							LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC),
+						},
+					},
+				},
+			},
+		},
+		Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*testutil.NewPod("pod0", "node0")}}),
+	}
+	timeToPass := evictionTimeout
+	newNodeStatus := v1.NodeStatus{
+		Conditions: []v1.NodeCondition{
+			{
+				Type:   v1.NodeReady,
+				Status: v1.ConditionFalse,
+				// Node status has just been updated, and is NotReady for 10min.
+				LastHeartbeatTime:  metav1.Date(2017, 1, 1, 12, 9, 0, 0, time.UTC),
+				LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC),
+			},
+		},
+	}
+	healthyNodeNewStatus := v1.NodeStatus{
+		Conditions: []v1.NodeCondition{
+			{
+				Type:               v1.NodeReady,
+				Status:             v1.ConditionTrue,
+				LastHeartbeatTime:  metav1.Date(2017, 1, 1, 12, 10, 0, 0, time.UTC),
+				LastTransitionTime: metav1.Date(2017, 1, 1, 12, 0, 0, 0, time.UTC),
+			},
+		},
+	}
+	originalTaint := UnreachableTaintTemplate
+	updatedTaint := NotReadyTaintTemplate
+
+	nodeController, _ := NewNodeControllerFromClient(nil, fakeNodeHandler,
+		evictionTimeout, testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold, testNodeMonitorGracePeriod,
+		testNodeStartupGracePeriod, testNodeMonitorPeriod, nil, nil, 0, false, true)
+	nodeController.now = func() metav1.Time { return fakeNow }
+	nodeController.recorder = testutil.NewFakeRecorder()
+	if err := syncNodeStore(nodeController, fakeNodeHandler); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if err := nodeController.monitorNodeStatus(); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	nodeController.doTaintingPass()
+
+	node0, err := fakeNodeHandler.Get("node0", metav1.GetOptions{})
+	if err != nil {
+		t.Errorf("Can't get current node0...")
+		return
+	}
+	node1, err := fakeNodeHandler.Get("node1", metav1.GetOptions{})
+	if err != nil {
+		t.Errorf("Can't get current node1...")
+		return
+	}
+
+	if originalTaint != nil && !v1.TaintExists(node0.Spec.Taints, originalTaint) {
+		t.Errorf("Can't find taint %v in %v", originalTaint, node0.Spec.Taints)
+	}
+
+	nodeController.now = func() metav1.Time { return metav1.Time{Time: fakeNow.Add(timeToPass)} }
+
+	node0.Status = newNodeStatus
+	node1.Status = healthyNodeNewStatus
+	_, err = fakeNodeHandler.UpdateStatus(node0)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+	_, err = fakeNodeHandler.UpdateStatus(node1)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+
+	if err := syncNodeStore(nodeController, fakeNodeHandler); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if err := nodeController.monitorNodeStatus(); err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	nodeController.doTaintingPass()
+
+	node0, err = fakeNodeHandler.Get("node0", metav1.GetOptions{})
+	if err != nil {
+		t.Errorf("Can't get current node0...")
+		return
+	}
+	if updatedTaint != nil {
+		if !v1.TaintExists(node0.Spec.Taints, updatedTaint) {
+			t.Errorf("Can't find taint %v in %v", updatedTaint, node0.Spec.Taints)
+		}
+	}
+}
+
 func TestNodeEventGeneration(t *testing.T) {
 	fakeNow := metav1.Date(2016, 9, 10, 12, 0, 0, 0, time.UTC)
 	fakeNodeHandler := &testutil.FakeNodeHandler{
@@ -1876,7 +2018,7 @@ func TestNodeEventGeneration(t *testing.T) {
 	nodeController, _ := NewNodeControllerFromClient(nil, fakeNodeHandler, 5*time.Minute,
 		testRateLimiterQPS, testRateLimiterQPS, testLargeClusterThreshold, testUnhealtyThreshold,
 		testNodeMonitorGracePeriod, testNodeStartupGracePeriod,
-		testNodeMonitorPeriod, nil, nil, 0, false)
+		testNodeMonitorPeriod, nil, nil, 0, false, false)
 	nodeController.cloud = &fakecloud.FakeCloud{}
 	nodeController.nodeExistsInCloudProvider = func(nodeName types.NodeName) (bool, error) {
 		return false, nil
@@ -1987,7 +2129,7 @@ func TestCheckPod(t *testing.T) {
 		},
 	}
 
-	nc, _ := NewNodeControllerFromClient(nil, fake.NewSimpleClientset(), 0, 0, 0, 0, 0, 0, 0, 0, nil, nil, 0, false)
+	nc, _ := NewNodeControllerFromClient(nil, fake.NewSimpleClientset(), 0, 0, 0, 0, 0, 0, 0, 0, nil, nil, 0, false, false)
 	nc.nodeInformer.Informer().GetStore().Add(&v1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "new",

pkg/controller/node/testutil/BUILD

@@ -17,6 +17,7 @@ go_library(
         "//pkg/client/clientset_generated/clientset/fake:go_default_library",
         "//pkg/client/clientset_generated/clientset/typed/core/v1:go_default_library",
         "//pkg/util/node:go_default_library",
+        "//vendor:github.com/evanphx/json-patch",
         "//vendor:github.com/golang/glog",
         "//vendor:k8s.io/apimachinery/pkg/api/errors",
         "//vendor:k8s.io/apimachinery/pkg/api/resource",
@@ -24,6 +25,7 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/runtime",
         "//vendor:k8s.io/apimachinery/pkg/types",
         "//vendor:k8s.io/apimachinery/pkg/util/sets",
+        "//vendor:k8s.io/apimachinery/pkg/util/strategicpatch",
         "//vendor:k8s.io/apimachinery/pkg/watch",
         "//vendor:k8s.io/client-go/pkg/api/v1",
         "//vendor:k8s.io/client-go/util/clock",

pkg/controller/node/testutil/test_utils.go

@@ -17,6 +17,7 @@ limitations under the License.
 package testutil
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"sync"
@@ -28,16 +29,19 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/watch"
 
 	clientv1 "k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/util/clock"
+
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset/fake"
 	v1core "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
 	utilnode "k8s.io/kubernetes/pkg/util/node"
 
+	"github.com/evanphx/json-patch"
 	"github.com/golang/glog"
 )
 
@@ -189,6 +193,7 @@ func (m *FakeNodeHandler) Update(node *v1.Node) (*v1.Node, error) {
 		m.RequestCount++
 		m.lock.Unlock()
 	}()
+
 	nodeCopy := *node
 	for i, updateNode := range m.UpdatedNodes {
 		if updateNode.Name == nodeCopy.Name {
@@ -207,6 +212,35 @@ func (m *FakeNodeHandler) UpdateStatus(node *v1.Node) (*v1.Node, error) {
 		m.RequestCount++
 		m.lock.Unlock()
 	}()
+
+	var origNodeCopy v1.Node
+	found := false
+	for i := range m.Existing {
+		if m.Existing[i].Name == node.Name {
+			origNodeCopy = *m.Existing[i]
+			found = true
+		}
+	}
+	updatedNodeIndex := -1
+	for i := range m.UpdatedNodes {
+		if m.UpdatedNodes[i].Name == node.Name {
+			origNodeCopy = *m.UpdatedNodes[i]
+			updatedNodeIndex = i
+			found = true
+		}
+	}
+
+	if !found {
+		return nil, fmt.Errorf("Not found node %v", node)
+	}
+
+	origNodeCopy.Status = node.Status
+	if updatedNodeIndex < 0 {
+		m.UpdatedNodes = append(m.UpdatedNodes, &origNodeCopy)
+	} else {
+		m.UpdatedNodes[updatedNodeIndex] = &origNodeCopy
+	}
+
 	nodeCopy := *node
 	m.UpdatedNodeStatuses = append(m.UpdatedNodeStatuses, &nodeCopy)
 	return node, nil
@@ -225,7 +259,76 @@ func (m *FakeNodeHandler) Watch(opts metav1.ListOptions) (watch.Interface, error
 
 // Patch patches a Node in the fake store.
 func (m *FakeNodeHandler) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*v1.Node, error) {
-	return nil, nil
+	m.lock.Lock()
+	defer func() {
+		m.RequestCount++
+		m.lock.Unlock()
+	}()
+	var nodeCopy v1.Node
+	for i := range m.Existing {
+		if m.Existing[i].Name == name {
+			nodeCopy = *m.Existing[i]
+		}
+	}
+	updatedNodeIndex := -1
+	for i := range m.UpdatedNodes {
+		if m.UpdatedNodes[i].Name == name {
+			nodeCopy = *m.UpdatedNodes[i]
+			updatedNodeIndex = i
+		}
+	}
+
+	originalObjJS, err := json.Marshal(nodeCopy)
+	if err != nil {
+		glog.Errorf("Failed to marshal %v", nodeCopy)
+		return nil, nil
+	}
+	var originalNode v1.Node
+	if err = json.Unmarshal(originalObjJS, &originalNode); err != nil {
+		glog.Errorf("Failed to unmarshall original object: %v", err)
+		return nil, nil
+	}
+
+	var patchedObjJS []byte
+	switch pt {
+	case types.JSONPatchType:
+		patchObj, err := jsonpatch.DecodePatch(data)
+		if err != nil {
+			glog.Error(err.Error())
+			return nil, nil
+		}
+		if patchedObjJS, err = patchObj.Apply(originalObjJS); err != nil {
+			glog.Error(err.Error())
+			return nil, nil
+		}
+	case types.MergePatchType:
+		if patchedObjJS, err = jsonpatch.MergePatch(originalObjJS, data); err != nil {
+			glog.Error(err.Error())
+			return nil, nil
+		}
+	case types.StrategicMergePatchType:
+		if patchedObjJS, err = strategicpatch.StrategicMergePatch(originalObjJS, data, originalNode); err != nil {
+			glog.Error(err.Error())
+			return nil, nil
+		}
+	default:
+		glog.Errorf("unknown Content-Type header for patch: %v", pt)
+		return nil, nil
+	}
+
+	var updatedNode v1.Node
+	if err = json.Unmarshal(patchedObjJS, &updatedNode); err != nil {
+		glog.Errorf("Failed to unmarshall patched object: %v", err)
+		return nil, nil
+	}
+
+	if updatedNodeIndex < 0 {
+		m.UpdatedNodes = append(m.UpdatedNodes, &updatedNode)
+	} else {
+		m.UpdatedNodes[updatedNodeIndex] = &updatedNode
+	}
+
+	return &updatedNode, nil
 }
 
 // FakeRecorder is used as a fake during testing.