From 3c4c85f2125144bf8a20677afee1e14ed528b26c Mon Sep 17 00:00:00 2001 From: Rohit Agarwal Date: Wed, 15 Nov 2017 19:02:48 -0800 Subject: [PATCH] Add ExtendedResourceToleration admission controller. --- cmd/kube-apiserver/app/options/BUILD | 1 + cmd/kube-apiserver/app/options/plugins.go | 2 + plugin/BUILD | 1 + .../extendedresourcetoleration/BUILD | 42 ++ .../extendedresourcetoleration/admission.go | 94 +++++ .../admission_test.go | 382 ++++++++++++++++++ 6 files changed, 522 insertions(+) create mode 100644 plugin/pkg/admission/extendedresourcetoleration/BUILD create mode 100644 plugin/pkg/admission/extendedresourcetoleration/admission.go create mode 100644 plugin/pkg/admission/extendedresourcetoleration/admission_test.go diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD index 6d7c32772c..2ad00991da 100644 --- a/cmd/kube-apiserver/app/options/BUILD +++ b/cmd/kube-apiserver/app/options/BUILD @@ -30,6 +30,7 @@ go_library( "//plugin/pkg/admission/deny:go_default_library", "//plugin/pkg/admission/eventratelimit:go_default_library", "//plugin/pkg/admission/exec:go_default_library", + "//plugin/pkg/admission/extendedresourcetoleration:go_default_library", "//plugin/pkg/admission/gc:go_default_library", "//plugin/pkg/admission/imagepolicy:go_default_library", "//plugin/pkg/admission/initialresources:go_default_library", diff --git a/cmd/kube-apiserver/app/options/plugins.go b/cmd/kube-apiserver/app/options/plugins.go index 6d899d3b07..30ed306485 100644 --- a/cmd/kube-apiserver/app/options/plugins.go +++ b/cmd/kube-apiserver/app/options/plugins.go @@ -32,6 +32,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/deny" "k8s.io/kubernetes/plugin/pkg/admission/eventratelimit" "k8s.io/kubernetes/plugin/pkg/admission/exec" + "k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration" "k8s.io/kubernetes/plugin/pkg/admission/gc" "k8s.io/kubernetes/plugin/pkg/admission/imagepolicy" "k8s.io/kubernetes/plugin/pkg/admission/initialresources" @@ -61,6 +62,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { deny.Register(plugins) eventratelimit.Register(plugins) exec.Register(plugins) + extendedresourcetoleration.Register(plugins) gc.Register(plugins) imagepolicy.Register(plugins) initialresources.Register(plugins) diff --git a/plugin/BUILD b/plugin/BUILD index 843efb9c3e..764a8e3283 100644 --- a/plugin/BUILD +++ b/plugin/BUILD @@ -19,6 +19,7 @@ filegroup( "//plugin/pkg/admission/deny:all-srcs", "//plugin/pkg/admission/eventratelimit:all-srcs", "//plugin/pkg/admission/exec:all-srcs", + "//plugin/pkg/admission/extendedresourcetoleration:all-srcs", "//plugin/pkg/admission/gc:all-srcs", "//plugin/pkg/admission/imagepolicy:all-srcs", "//plugin/pkg/admission/initialresources:all-srcs", diff --git a/plugin/pkg/admission/extendedresourcetoleration/BUILD b/plugin/pkg/admission/extendedresourcetoleration/BUILD new file mode 100644 index 0000000000..882b966cd7 --- /dev/null +++ b/plugin/pkg/admission/extendedresourcetoleration/BUILD @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["admission.go"], + importpath = "k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/apis/core/helper:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["admission_test.go"], + importpath = "k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration", + library = ":go_default_library", + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/apis/core/helper:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/plugin/pkg/admission/extendedresourcetoleration/admission.go b/plugin/pkg/admission/extendedresourcetoleration/admission.go new file mode 100644 index 0000000000..410c18160b --- /dev/null +++ b/plugin/pkg/admission/extendedresourcetoleration/admission.go @@ -0,0 +1,94 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extendedresourcetoleration + +import ( + "fmt" + "io" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/admission" + "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/apis/core/helper" +) + +// Register is called by the apiserver to register the plugin factory. +func Register(plugins *admission.Plugins) { + plugins.Register("ExtendedResourceToleration", func(config io.Reader) (admission.Interface, error) { + return newExtendedResourceToleration(), nil + }) +} + +// newExtendedResourceToleration creates a new instance of the ExtendedResourceToleration admission controller. +func newExtendedResourceToleration() *plugin { + return &plugin{ + Handler: admission.NewHandler(admission.Create, admission.Update), + } +} + +// Make sure we are implementing the interface. +var _ admission.MutationInterface = &plugin{} + +type plugin struct { + *admission.Handler +} + +// Admit updates the toleration of a pod based on the resources requested by it. +// If an extended resource of name "example.com/device" is requested, it adds +// a toleration with key "example.com/device", operator "Exists" and effect "NoSchedule". +// The rationale for this is described in: +// https://github.com/kubernetes/kubernetes/issues/55080 +func (p *plugin) Admit(attributes admission.Attributes) error { + // Ignore all calls to subresources or resources other than pods. + if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != core.Resource("pods") { + return nil + } + + pod, ok := attributes.GetObject().(*core.Pod) + if !ok { + return errors.NewBadRequest(fmt.Sprintf("expected *core.Pod but got %T", attributes.GetObject())) + } + + resources := sets.String{} + for _, container := range pod.Spec.Containers { + for resourceName := range container.Resources.Requests { + if helper.IsExtendedResourceName(resourceName) { + resources.Insert(string(resourceName)) + } + } + } + for _, container := range pod.Spec.InitContainers { + for resourceName := range container.Resources.Requests { + if helper.IsExtendedResourceName(resourceName) { + resources.Insert(string(resourceName)) + } + } + } + + // Doing .List() so that we get a stable sorted list. + // This allows us to test adding tolerations for multiple extended resources. + for _, resource := range resources.List() { + helper.AddOrUpdateTolerationInPod(pod, &core.Toleration{ + Key: resource, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }) + } + + return nil +} diff --git a/plugin/pkg/admission/extendedresourcetoleration/admission_test.go b/plugin/pkg/admission/extendedresourcetoleration/admission_test.go new file mode 100644 index 0000000000..646ae007e1 --- /dev/null +++ b/plugin/pkg/admission/extendedresourcetoleration/admission_test.go @@ -0,0 +1,382 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package extendedresourcetoleration + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apiserver/pkg/admission" + "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/apis/core/helper" +) + +func TestAdmit(t *testing.T) { + + plugin := newExtendedResourceToleration() + + containerRequestingCPU := core.Container{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + } + + containerRequestingMemory := core.Container{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceMemory: *resource.NewQuantity(2048, resource.DecimalSI), + }, + }, + } + + extendedResource1 := "example.com/device-ek" + extendedResource2 := "example.com/device-do" + + containerRequestingExtendedResource1 := core.Container{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceName(extendedResource1): *resource.NewQuantity(1, resource.DecimalSI), + }, + }, + } + containerRequestingExtendedResource2 := core.Container{ + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceName(extendedResource2): *resource.NewQuantity(2, resource.DecimalSI), + }, + }, + } + + tests := []struct { + description string + requestedPod core.Pod + expectedPod core.Pod + }{ + { + description: "empty pod without any extended resources, expect no change in tolerations", + requestedPod: core.Pod{ + Spec: core.PodSpec{}, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{}, + }, + }, + { + description: "pod with container without any extended resources, expect no change in tolerations", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + }, + }, + }, + }, + { + description: "pod with init container without any extended resources, expect no change in tolerations", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + InitContainers: []core.Container{ + containerRequestingMemory, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + InitContainers: []core.Container{ + containerRequestingMemory, + }, + }, + }, + }, + { + description: "pod with container with extended resource, expect toleration to be added", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingExtendedResource1, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + { + description: "pod with init container with extended resource, expect toleration to be added", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + InitContainers: []core.Container{ + containerRequestingExtendedResource2, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + InitContainers: []core.Container{ + containerRequestingExtendedResource2, + }, + Tolerations: []core.Toleration{ + { + Key: extendedResource2, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + { + description: "pod with existing tolerations and container with extended resource, expect existing tolerations to be preserved and new toleration to be added", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: "foo", + Operator: core.TolerationOpEqual, + Value: "bar", + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: "foo", + Operator: core.TolerationOpEqual, + Value: "bar", + Effect: core.TaintEffectNoSchedule, + }, + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + { + description: "pod with multiple extended resources, expect multiple tolerations to be added", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + InitContainers: []core.Container{ + containerRequestingCPU, + containerRequestingExtendedResource2, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + InitContainers: []core.Container{ + containerRequestingCPU, + containerRequestingExtendedResource2, + }, + Tolerations: []core.Toleration{ + // Note the order, it's sorted by the Key + { + Key: extendedResource2, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + { + description: "pod with container requesting extended resource and existing correct toleration, expect no change in tolerations", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + { + description: "pod with container requesting extended resource and existing toleration with the same key but different effect and value, expect existing tolerations to be preserved and new toleration to be added", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: extendedResource1, + Operator: core.TolerationOpEqual, + Value: "foo", + Effect: core.TaintEffectNoExecute, + }, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Key: extendedResource1, + Operator: core.TolerationOpEqual, + Value: "foo", + Effect: core.TaintEffectNoExecute, + }, + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + { + description: "pod with wildcard toleration and container requesting extended resource, expect existing tolerations to be preserved and new toleration to be added", + requestedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Operator: core.TolerationOpExists, + }, + }, + }, + }, + expectedPod: core.Pod{ + Spec: core.PodSpec{ + Containers: []core.Container{ + containerRequestingCPU, + containerRequestingMemory, + containerRequestingExtendedResource1, + }, + Tolerations: []core.Toleration{ + { + Operator: core.TolerationOpExists, + }, + { + Key: extendedResource1, + Operator: core.TolerationOpExists, + Effect: core.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + } + for i, test := range tests { + err := plugin.Admit(admission.NewAttributesRecord(&test.requestedPod, nil, core.Kind("Pod").WithVersion("version"), "foo", "name", core.Resource("pods").WithVersion("version"), "", "ignored", nil)) + if err != nil { + t.Errorf("[%d: %s] unexpected error %v for pod %+v", i, test.description, err, test.requestedPod) + } + + if !helper.Semantic.DeepEqual(test.expectedPod.Spec.Tolerations, test.requestedPod.Spec.Tolerations) { + t.Errorf("[%d: %s] expected %#v got %#v", i, test.description, test.expectedPod.Spec.Tolerations, test.requestedPod.Spec.Tolerations) + } + } +} + +func TestHandles(t *testing.T) { + plugin := newExtendedResourceToleration() + tests := map[admission.Operation]bool{ + admission.Create: true, + admission.Update: true, + admission.Delete: false, + admission.Connect: false, + } + for op, expected := range tests { + result := plugin.Handles(op) + if result != expected { + t.Errorf("Unexpected result for operation %s: %v\n", op, result) + } + } +}