From 92d739bebc10692034779e33e1e3e57280d3b315 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Tue, 21 Feb 2017 17:07:42 -0800 Subject: [PATCH] kubectl: Allow 'drain --force' to remove orphaned pods --- pkg/kubectl/cmd/drain.go | 17 ++++++++++-- pkg/kubectl/cmd/drain_test.go | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/pkg/kubectl/cmd/drain.go b/pkg/kubectl/cmd/drain.go index 05c3d38c09..bf9020e35d 100644 --- a/pkg/kubectl/cmd/drain.go +++ b/pkg/kubectl/cmd/drain.go @@ -151,7 +151,8 @@ var ( DaemonSet controller, which ignores unschedulable markings. If there are any pods that are neither mirror pods nor managed by ReplicationController, ReplicaSet, DaemonSet, StatefulSet or Job, then drain will not delete any pods unless you - use --force. + use --force. --force will also allow deletion to proceed if the managing resource of one + or more pods is missing. 'drain' waits for graceful termination. You should not operate on the machine until the command completes. @@ -309,6 +310,10 @@ func (o *DrainOptions) unreplicatedFilter(pod api.Pod) (bool, *warning, *fatal) sr, err := o.getPodCreator(pod) if err != nil { + // if we're forcing, remove orphaned pods with a warning + if apierrors.IsNotFound(err) && o.Force { + return true, &warning{err.Error()}, nil + } return false, nil, &fatal{err.Error()} } if sr != nil { @@ -321,11 +326,19 @@ func (o *DrainOptions) unreplicatedFilter(pod api.Pod) (bool, *warning, *fatal) } func (o *DrainOptions) daemonsetFilter(pod api.Pod) (bool, *warning, *fatal) { - // Note that we return false in all cases where the pod is DaemonSet managed, + // Note that we return false in cases where the pod is DaemonSet managed, // regardless of flags. We never delete them, the only question is whether // their presence constitutes an error. + // + // The exception is for pods that are orphaned (the referencing + // management resource - including DaemonSet - is not found). + // Such pods will be deleted if --force is used. sr, err := o.getPodCreator(pod) if err != nil { + // if we're forcing, remove orphaned pods with a warning + if apierrors.IsNotFound(err) && o.Force { + return true, &warning{err.Error()}, nil + } return false, nil, &fatal{err.Error()} } if sr == nil || sr.Reference.Kind != "DaemonSet" { diff --git a/pkg/kubectl/cmd/drain_test.go b/pkg/kubectl/cmd/drain_test.go index 8bbc503aab..14a4c6d90d 100644 --- a/pkg/kubectl/cmd/drain_test.go +++ b/pkg/kubectl/cmd/drain_test.go @@ -278,6 +278,34 @@ func TestDrain(t *testing.T) { }, } + missing_ds := extensions.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "missing-ds", + Namespace: "default", + CreationTimestamp: metav1.Time{Time: time.Now()}, + SelfLink: "/apis/extensions/v1beta1/namespaces/default/daemonsets/missing-ds", + }, + Spec: extensions.DaemonSetSpec{ + Selector: &metav1.LabelSelector{MatchLabels: labels}, + }, + } + + missing_ds_anno := make(map[string]string) + missing_ds_anno[api.CreatedByAnnotation] = refJson(t, &missing_ds) + + orphaned_ds_pod := api.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "default", + CreationTimestamp: metav1.Time{Time: time.Now()}, + Labels: labels, + Annotations: missing_ds_anno, + }, + Spec: api.PodSpec{ + NodeName: "node", + }, + } + job := batch.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job", @@ -390,6 +418,26 @@ func TestDrain(t *testing.T) { expectFatal: true, expectDelete: false, }, + { + description: "orphaned DS-managed pod", + node: node, + expected: cordoned_node, + pods: []api.Pod{orphaned_ds_pod}, + rcs: []api.ReplicationController{}, + args: []string{"node"}, + expectFatal: true, + expectDelete: false, + }, + { + description: "orphaned DS-managed pod with --force", + node: node, + expected: cordoned_node, + pods: []api.Pod{orphaned_ds_pod}, + rcs: []api.ReplicationController{}, + args: []string{"node", "--force"}, + expectFatal: false, + expectDelete: true, + }, { description: "DS-managed pod with --ignore-daemonsets", node: node, @@ -526,6 +574,8 @@ func TestDrain(t *testing.T) { return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(codec, &test.rcs[0])}, nil case m.isFor("GET", "/namespaces/default/daemonsets/ds"): return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(testapi.Extensions.Codec(), &ds)}, nil + case m.isFor("GET", "/namespaces/default/daemonsets/missing-ds"): + return &http.Response{StatusCode: 404, Header: defaultHeader(), Body: objBody(testapi.Extensions.Codec(), &extensions.DaemonSet{})}, nil case m.isFor("GET", "/namespaces/default/jobs/job"): return &http.Response{StatusCode: 200, Header: defaultHeader(), Body: objBody(testapi.Batch.Codec(), &job)}, nil case m.isFor("GET", "/namespaces/default/replicasets/rs"):