Merge pull request #41864 from marun/kubectl-drain-orphans

Automatic merge from submit-queue (batch tested with PRs 41857, 41864, 40522, 41835, 41991)

kubectl: Allow 'drain --force' to remove orphaned pods

If the managing resource of a given pod (e.g. DaemonSet/ReplicaSet/etc) is deleted (effectively orphaning the pod), and ``kubectl drain --force`` is invoked on the node hosting the pod, the command would fail with an error indicating that the managing resource was not found.  This PR reduces the error to a warning if ``--force`` is specified, allowing nodes with orphaned pods to be drained.   

Reference: https://bugzilla.redhat.com/show_bug.cgi?id=1424678

cc: @derekwaynecarr 

```release-note
Allow drain --force to remove pods whose managing resource is deleted.
```

pkg/kubectl/cmd/drain.go

@@ -151,7 +151,8 @@ var (
 		DaemonSet controller, which ignores unschedulable markings.  If there are any
 		pods that are neither mirror pods nor managed by ReplicationController,
 		ReplicaSet, DaemonSet, StatefulSet or Job, then drain will not delete any pods unless you
-		use --force.
+		use --force.  --force will also allow deletion to proceed if the managing resource of one
+		or more pods is missing.
 
 		'drain' waits for graceful termination. You should not operate on the machine until
 		the command completes.
@@ -309,6 +310,10 @@ func (o *DrainOptions) unreplicatedFilter(pod api.Pod) (bool, *warning, *fatal)
 
 	sr, err := o.getPodCreator(pod)
 	if err != nil {
+		// if we're forcing, remove orphaned pods with a warning
+		if apierrors.IsNotFound(err) && o.Force {
+			return true, &warning{err.Error()}, nil
+		}
 		return false, nil, &fatal{err.Error()}
 	}
 	if sr != nil {
@@ -321,11 +326,19 @@ func (o *DrainOptions) unreplicatedFilter(pod api.Pod) (bool, *warning, *fatal)
 }
 
 func (o *DrainOptions) daemonsetFilter(pod api.Pod) (bool, *warning, *fatal) {
-	// Note that we return false in all cases where the pod is DaemonSet managed,
+	// Note that we return false in cases where the pod is DaemonSet managed,
 	// regardless of flags.  We never delete them, the only question is whether
 	// their presence constitutes an error.
+	//
+	// The exception is for pods that are orphaned (the referencing
+	// management resource - including DaemonSet - is not found).
+	// Such pods will be deleted if --force is used.
 	sr, err := o.getPodCreator(pod)
 	if err != nil {
+		// if we're forcing, remove orphaned pods with a warning
+		if apierrors.IsNotFound(err) && o.Force {
+			return true, &warning{err.Error()}, nil
+		}
 		return false, nil, &fatal{err.Error()}
 	}
 	if sr == nil || sr.Reference.Kind != "DaemonSet" {

pkg/kubectl/cmd/drain_test.go

@@ -278,6 +278,34 @@ func TestDrain(t *testing.T) {
 		},
 	}
 
+	missing_ds := extensions.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              "missing-ds",
+			Namespace:         "default",
+			CreationTimestamp: metav1.Time{Time: time.Now()},
+			SelfLink:          "/apis/extensions/v1beta1/namespaces/default/daemonsets/missing-ds",
+		},
+		Spec: extensions.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{MatchLabels: labels},
+		},
+	}
+
+	missing_ds_anno := make(map[string]string)
+	missing_ds_anno[api.CreatedByAnnotation] = refJson(t, &missing_ds)
+
+	orphaned_ds_pod := api.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              "bar",
+			Namespace:         "default",
+			CreationTimestamp: metav1.Time{Time: time.Now()},
+			Labels:            labels,
+			Annotations:       missing_ds_anno,
+		},
+		Spec: api.PodSpec{
+			NodeName: "node",
+		},
+	}
+
 	job := batch.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:              "job",
@@ -390,6 +418,26 @@ func TestDrain(t *testing.T) {
 			expectFatal:  true,
 			expectDelete: false,
 		},
+		{
+			description:  "orphaned DS-managed pod",
+			node:         node,
+			expected:     cordoned_node,
+			pods:         []api.Pod{orphaned_ds_pod},
+			rcs:          []api.ReplicationController{},
+			args:         []string{"node"},
+			expectFatal:  true,
+			expectDelete: false,
+		},
+		{
+			description:  "orphaned DS-managed pod with --force",
+			node:         node,
+			expected:     cordoned_node,
+			pods:         []api.Pod{orphaned_ds_pod},
+			rcs:          []api.ReplicationController{},
+			args:         []string{"node", "--force"},
+			expectFatal:  false,
+			expectDelete: true,
+		},
 		{
 			description:  "DS-managed pod with --ignore-daemonsets",
 			node:         node,
@@ -526,6 +574,8 @@ func TestDrain(t *testing.T) {
 						return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, &test.rcs[0])}, nil
 					case m.isFor("GET", "/namespaces/default/daemonsets/ds"):
 						return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(testapi.Extensions.Codec(), &ds)}, nil
+					case m.isFor("GET", "/namespaces/default/daemonsets/missing-ds"):
+						return &http.Response{StatusCode: 404, Header: defaultHeader(), Body: objBody(testapi.Extensions.Codec(), &extensions.DaemonSet{})}, nil
 					case m.isFor("GET", "/namespaces/default/jobs/job"):
 						return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(testapi.Batch.Codec(), &job)}, nil
 					case m.isFor("GET", "/namespaces/default/replicasets/rs"):