From c192ba4c1a14def33853968abf0c1ec24d699e6e Mon Sep 17 00:00:00 2001 From: Antoine Pelisse Date: Tue, 23 Apr 2019 15:16:32 -0700 Subject: [PATCH] Fix eviction dry-run --- pkg/registry/core/pod/storage/BUILD | 13 +- pkg/registry/core/pod/storage/eviction.go | 46 +++++-- .../core/pod/storage/eviction_test.go | 125 ++++++++++++++++++ 3 files changed, 168 insertions(+), 16 deletions(-) diff --git a/pkg/registry/core/pod/storage/BUILD b/pkg/registry/core/pod/storage/BUILD index b02aa36550..8837d4cc1f 100644 --- a/pkg/registry/core/pod/storage/BUILD +++ b/pkg/registry/core/pod/storage/BUILD @@ -1,10 +1,4 @@ -package(default_visibility = ["//visibility:public"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", - "go_test", -) +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_test( name = "go_default_test", @@ -20,6 +14,7 @@ go_test( "//pkg/registry/registrytest:go_default_library", "//pkg/securitycontext:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/api/apitesting:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -27,7 +22,9 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", + "//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", @@ -46,6 +43,7 @@ go_library( "storage.go", ], importpath = "k8s.io/kubernetes/pkg/registry/core/pod/storage", + visibility = ["//visibility:public"], deps = [ "//pkg/api/pod:go_default_library", "//pkg/apis/core:go_default_library", @@ -85,4 +83,5 @@ filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], + visibility = ["//visibility:public"], ) diff --git a/pkg/registry/core/pod/storage/eviction.go b/pkg/registry/core/pod/storage/eviction.go index fdcf2d8dbd..b9d131b1f4 100644 --- a/pkg/registry/core/pod/storage/eviction.go +++ b/pkg/registry/core/pod/storage/eviction.go @@ -19,6 +19,7 @@ package storage import ( "context" "fmt" + "reflect" "time" "k8s.io/apimachinery/pkg/api/errors" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/util/dryrun" "k8s.io/client-go/util/retry" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" @@ -77,11 +79,36 @@ func (r *EvictionREST) New() runtime.Object { return &policy.Eviction{} } +// Propagate dry-run takes the dry-run option from the request and pushes it into the eviction object. +// It returns an error if they have non-matching dry-run options. +func propagateDryRun(eviction *policy.Eviction, options *metav1.CreateOptions) (*metav1.DeleteOptions, error) { + if eviction.DeleteOptions == nil { + return &metav1.DeleteOptions{DryRun: options.DryRun}, nil + } + if len(eviction.DeleteOptions.DryRun) == 0 { + eviction.DeleteOptions.DryRun = options.DryRun + return eviction.DeleteOptions, nil + } + if len(options.DryRun) == 0 { + return eviction.DeleteOptions, nil + } + + if !reflect.DeepEqual(options.DryRun, eviction.DeleteOptions.DryRun) { + return nil, fmt.Errorf("Non-matching dry-run options in request and content: %v and %v", options.DryRun, eviction.DeleteOptions.DryRun) + } + return eviction.DeleteOptions, nil +} + // Create attempts to create a new eviction. That is, it tries to evict a pod. func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { eviction := obj.(*policy.Eviction) - obj, err := r.store.Get(ctx, eviction.Name, &metav1.GetOptions{}) + deletionOptions, err := propagateDryRun(eviction, options) + if err != nil { + return nil, err + } + + obj, err = r.store.Get(ctx, eviction.Name, &metav1.GetOptions{}) if err != nil { return nil, err } @@ -89,7 +116,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal // Evicting a terminal pod should result in direct deletion of pod as it already caused disruption by the time we are evicting. // There is no need to check for pdb. if pod.Status.Phase == api.PodSucceeded || pod.Status.Phase == api.PodFailed { - _, _, err = r.store.Delete(ctx, eviction.Name, eviction.DeleteOptions) + _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions) if err != nil { return nil, err } @@ -118,7 +145,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal // If it was false already, or if it becomes false during the course of our retries, // raise an error marked as a 429. - if err := r.checkAndDecrement(pod.Namespace, pod.Name, pdb); err != nil { + if err := r.checkAndDecrement(pod.Namespace, pod.Name, pdb, dryrun.IsDryRun(deletionOptions.DryRun)); err != nil { return err } } @@ -138,11 +165,6 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal // At this point there was either no PDB or we succeeded in decrementing // Try the delete - deletionOptions := eviction.DeleteOptions - if deletionOptions == nil { - // default to non-nil to trigger graceful deletion - deletionOptions = &metav1.DeleteOptions{} - } _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions) if err != nil { return nil, err @@ -153,7 +175,7 @@ func (r *EvictionREST) Create(ctx context.Context, obj runtime.Object, createVal } // checkAndDecrement checks if the provided PodDisruptionBudget allows any disruption. -func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb policy.PodDisruptionBudget) error { +func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb policy.PodDisruptionBudget, dryRun bool) error { if pdb.Status.ObservedGeneration < pdb.Generation { // TODO(mml): Add a Retry-After header. Once there are time-based // budgets, we can sometimes compute a sensible suggested value. But @@ -179,6 +201,12 @@ func (r *EvictionREST) checkAndDecrement(namespace string, podName string, pdb p if pdb.Status.DisruptedPods == nil { pdb.Status.DisruptedPods = make(map[string]metav1.Time) } + + // If this is a dry-run, we don't need to go any further than that. + if dryRun == true { + return nil + } + // Eviction handler needs to inform the PDB controller that it is about to delete a pod // so it should not consider it as available in calculations when updating PodDisruptions allowed. // If the pod is not deleted within a reasonable time limit PDB controller will assume that it won't diff --git a/pkg/registry/core/pod/storage/eviction_test.go b/pkg/registry/core/pod/storage/eviction_test.go index a5cc1ac762..49d334404e 100644 --- a/pkg/registry/core/pod/storage/eviction_test.go +++ b/pkg/registry/core/pod/storage/eviction_test.go @@ -17,15 +17,24 @@ limitations under the License. package storage import ( + "context" "testing" + "k8s.io/apimachinery/pkg/api/apitesting" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + examplev1 "k8s.io/apiserver/pkg/apis/example/v1" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/registry/generic" + genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" + "k8s.io/apiserver/pkg/storage" + etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" + "k8s.io/kubernetes/pkg/registry/registrytest" ) func TestEviction(t *testing.T) { @@ -163,3 +172,119 @@ func TestEviction(t *testing.T) { }) } } + +type FailDeleteUpdateStorage struct { + storage.Interface +} + +func (f FailDeleteUpdateStorage) Delete(ctx context.Context, key string, out runtime.Object, precondition *storage.Preconditions) error { + return storage.NewKeyNotFoundError(key, 0) +} + +func (f FailDeleteUpdateStorage) GuaranteedUpdate(ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool, + preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error { + return storage.NewKeyNotFoundError(key, 0) +} + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +func newFailDeleteUpdateStorage(t *testing.T) (*REST, *etcdtesting.EtcdTestServer) { + etcdStorage, server := registrytest.NewEtcdStorage(t, "") + restOptions := generic.RESTOptions{ + StorageConfig: etcdStorage, + Decorator: generic.UndecoratedStorage, + DeleteCollectionWorkers: 3, + ResourcePrefix: "pods", + } + storage := NewStorage(restOptions, nil, nil, nil) + storage.Pod.Store.Storage = genericregistry.DryRunnableStorage{ + Storage: FailDeleteUpdateStorage{storage.Pod.Store.Storage.Storage}, + Codec: apitesting.TestStorageCodec(codecs, examplev1.SchemeGroupVersion), + } + return storage.Pod, server +} + +func TestEvictionDryRun(t *testing.T) { + testcases := []struct { + name string + evictionOptions *metav1.DeleteOptions + requestOptions *metav1.CreateOptions + pod *api.Pod + pdbs []runtime.Object + }{ + { + name: "just request-options", + requestOptions: &metav1.CreateOptions{DryRun: []string{"All"}}, + evictionOptions: &metav1.DeleteOptions{}, + pod: func() *api.Pod { + pod := validNewPod() + pod.Labels = map[string]string{"a": "true"} + pod.Spec.NodeName = "foo" + return pod + }(), + }, + { + name: "just eviction-options", + requestOptions: &metav1.CreateOptions{}, + evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}}, + pod: func() *api.Pod { + pod := validNewPod() + pod.Labels = map[string]string{"a": "true"} + pod.Spec.NodeName = "foo" + return pod + }(), + }, + { + name: "both options", + evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}}, + requestOptions: &metav1.CreateOptions{DryRun: []string{"All"}}, + pod: func() *api.Pod { + pod := validNewPod() + pod.Labels = map[string]string{"a": "true"} + pod.Spec.NodeName = "foo" + return pod + }(), + }, + { + name: "with pdbs", + evictionOptions: &metav1.DeleteOptions{DryRun: []string{"All"}}, + requestOptions: &metav1.CreateOptions{DryRun: []string{"All"}}, + pod: func() *api.Pod { + pod := validNewPod() + pod.Labels = map[string]string{"a": "true"} + pod.Spec.NodeName = "foo" + return pod + }(), + pdbs: []runtime.Object{&policy.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, + Spec: policy.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}}, + Status: policy.PodDisruptionBudgetStatus{PodDisruptionsAllowed: 1}, + }}, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault) + storage, server := newFailDeleteUpdateStorage(t) + defer server.Terminate(t) + defer storage.Store.DestroyFunc() + + pod := validNewPod() + pod.Labels = map[string]string{"a": "true"} + pod.Spec.NodeName = "foo" + if _, err := storage.Create(testContext, pod, nil, &metav1.CreateOptions{}); err != nil { + t.Error(err) + } + + client := fake.NewSimpleClientset(tc.pdbs...) + evictionRest := newEvictionStorage(storage.Store, client.Policy()) + eviction := &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: tc.evictionOptions} + _, err := evictionRest.Create(testContext, eviction, nil, tc.requestOptions) + if err != nil { + t.Fatalf("Failed to run eviction: %v", err) + } + }) + } +}