mirror of https://github.com/k3s-io/k3s
Add ExtendedResourceToleration admission controller.
Rohit Agarwal
parent
c3ed0f2663
commit
3c4c85f212
|
|
@ -30,6 +30,7 @@ go_library(
|
|||
"//plugin/pkg/admission/deny:go_default_library",
|
||||
"//plugin/pkg/admission/eventratelimit:go_default_library",
|
||||
"//plugin/pkg/admission/exec:go_default_library",
|
||||
"//plugin/pkg/admission/extendedresourcetoleration:go_default_library",
|
||||
"//plugin/pkg/admission/gc:go_default_library",
|
||||
"//plugin/pkg/admission/imagepolicy:go_default_library",
|
||||
"//plugin/pkg/admission/initialresources:go_default_library",
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import (
|
|||
"k8s.io/kubernetes/plugin/pkg/admission/deny"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/eventratelimit"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/exec"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/gc"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/imagepolicy"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/initialresources"
|
||||
|
|
@ -61,6 +62,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
|
|||
deny.Register(plugins)
|
||||
eventratelimit.Register(plugins)
|
||||
exec.Register(plugins)
|
||||
extendedresourcetoleration.Register(plugins)
|
||||
gc.Register(plugins)
|
||||
imagepolicy.Register(plugins)
|
||||
initialresources.Register(plugins)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ filegroup(
|
|||
"//plugin/pkg/admission/deny:all-srcs",
|
||||
"//plugin/pkg/admission/eventratelimit:all-srcs",
|
||||
"//plugin/pkg/admission/exec:all-srcs",
|
||||
"//plugin/pkg/admission/extendedresourcetoleration:all-srcs",
|
||||
"//plugin/pkg/admission/gc:all-srcs",
|
||||
"//plugin/pkg/admission/imagepolicy:all-srcs",
|
||||
"//plugin/pkg/admission/initialresources:all-srcs",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["admission.go"],
|
||||
importpath = "k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//pkg/apis/core:go_default_library",
|
||||
"//pkg/apis/core/helper:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["admission_test.go"],
|
||||
importpath = "k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration",
|
||||
library = ":go_default_library",
|
||||
deps = [
|
||||
"//pkg/apis/core:go_default_library",
|
||||
"//pkg/apis/core/helper:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package extendedresourcetoleration
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/apis/core/helper"
|
||||
)
|
||||
|
||||
// Register is called by the apiserver to register the plugin factory.
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("ExtendedResourceToleration", func(config io.Reader) (admission.Interface, error) {
|
||||
return newExtendedResourceToleration(), nil
|
||||
})
|
||||
}
|
||||
|
||||
// newExtendedResourceToleration creates a new instance of the ExtendedResourceToleration admission controller.
|
||||
func newExtendedResourceToleration() *plugin {
|
||||
return &plugin{
|
||||
Handler: admission.NewHandler(admission.Create, admission.Update),
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure we are implementing the interface.
|
||||
var _ admission.MutationInterface = &plugin{}
|
||||
|
||||
type plugin struct {
|
||||
*admission.Handler
|
||||
}
|
||||
|
||||
// Admit updates the toleration of a pod based on the resources requested by it.
|
||||
// If an extended resource of name "example.com/device" is requested, it adds
|
||||
// a toleration with key "example.com/device", operator "Exists" and effect "NoSchedule".
|
||||
// The rationale for this is described in:
|
||||
// https://github.com/kubernetes/kubernetes/issues/55080
|
||||
func (p *plugin) Admit(attributes admission.Attributes) error {
|
||||
// Ignore all calls to subresources or resources other than pods.
|
||||
if len(attributes.GetSubresource()) != 0 || attributes.GetResource().GroupResource() != core.Resource("pods") {
|
||||
return nil
|
||||
}
|
||||
|
||||
pod, ok := attributes.GetObject().(*core.Pod)
|
||||
if !ok {
|
||||
return errors.NewBadRequest(fmt.Sprintf("expected *core.Pod but got %T", attributes.GetObject()))
|
||||
}
|
||||
|
||||
resources := sets.String{}
|
||||
for _, container := range pod.Spec.Containers {
|
||||
for resourceName := range container.Resources.Requests {
|
||||
if helper.IsExtendedResourceName(resourceName) {
|
||||
resources.Insert(string(resourceName))
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, container := range pod.Spec.InitContainers {
|
||||
for resourceName := range container.Resources.Requests {
|
||||
if helper.IsExtendedResourceName(resourceName) {
|
||||
resources.Insert(string(resourceName))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Doing .List() so that we get a stable sorted list.
|
||||
// This allows us to test adding tolerations for multiple extended resources.
|
||||
for _, resource := range resources.List() {
|
||||
helper.AddOrUpdateTolerationInPod(pod, &core.Toleration{
|
||||
Key: resource,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,382 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package extendedresourcetoleration
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/apis/core/helper"
|
||||
)
|
||||
|
||||
func TestAdmit(t *testing.T) {
|
||||
|
||||
plugin := newExtendedResourceToleration()
|
||||
|
||||
containerRequestingCPU := core.Container{
|
||||
Resources: core.ResourceRequirements{
|
||||
Requests: core.ResourceList{
|
||||
core.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
containerRequestingMemory := core.Container{
|
||||
Resources: core.ResourceRequirements{
|
||||
Requests: core.ResourceList{
|
||||
core.ResourceMemory: *resource.NewQuantity(2048, resource.DecimalSI),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
extendedResource1 := "example.com/device-ek"
|
||||
extendedResource2 := "example.com/device-do"
|
||||
|
||||
containerRequestingExtendedResource1 := core.Container{
|
||||
Resources: core.ResourceRequirements{
|
||||
Requests: core.ResourceList{
|
||||
core.ResourceName(extendedResource1): *resource.NewQuantity(1, resource.DecimalSI),
|
||||
},
|
||||
},
|
||||
}
|
||||
containerRequestingExtendedResource2 := core.Container{
|
||||
Resources: core.ResourceRequirements{
|
||||
Requests: core.ResourceList{
|
||||
core.ResourceName(extendedResource2): *resource.NewQuantity(2, resource.DecimalSI),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
description string
|
||||
requestedPod core.Pod
|
||||
expectedPod core.Pod
|
||||
}{
|
||||
{
|
||||
description: "empty pod without any extended resources, expect no change in tolerations",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with container without any extended resources, expect no change in tolerations",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with init container without any extended resources, expect no change in tolerations",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
InitContainers: []core.Container{
|
||||
containerRequestingMemory,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
InitContainers: []core.Container{
|
||||
containerRequestingMemory,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with container with extended resource, expect toleration to be added",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with init container with extended resource, expect toleration to be added",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
InitContainers: []core.Container{
|
||||
containerRequestingExtendedResource2,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
InitContainers: []core.Container{
|
||||
containerRequestingExtendedResource2,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: extendedResource2,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with existing tolerations and container with extended resource, expect existing tolerations to be preserved and new toleration to be added",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: core.TolerationOpEqual,
|
||||
Value: "bar",
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: "foo",
|
||||
Operator: core.TolerationOpEqual,
|
||||
Value: "bar",
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with multiple extended resources, expect multiple tolerations to be added",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
InitContainers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingExtendedResource2,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
InitContainers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingExtendedResource2,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
// Note the order, it's sorted by the Key
|
||||
{
|
||||
Key: extendedResource2,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with container requesting extended resource and existing correct toleration, expect no change in tolerations",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with container requesting extended resource and existing toleration with the same key but different effect and value, expect existing tolerations to be preserved and new toleration to be added",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpEqual,
|
||||
Value: "foo",
|
||||
Effect: core.TaintEffectNoExecute,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpEqual,
|
||||
Value: "foo",
|
||||
Effect: core.TaintEffectNoExecute,
|
||||
},
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "pod with wildcard toleration and container requesting extended resource, expect existing tolerations to be preserved and new toleration to be added",
|
||||
requestedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Operator: core.TolerationOpExists,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedPod: core.Pod{
|
||||
Spec: core.PodSpec{
|
||||
Containers: []core.Container{
|
||||
containerRequestingCPU,
|
||||
containerRequestingMemory,
|
||||
containerRequestingExtendedResource1,
|
||||
},
|
||||
Tolerations: []core.Toleration{
|
||||
{
|
||||
Operator: core.TolerationOpExists,
|
||||
},
|
||||
{
|
||||
Key: extendedResource1,
|
||||
Operator: core.TolerationOpExists,
|
||||
Effect: core.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for i, test := range tests {
|
||||
err := plugin.Admit(admission.NewAttributesRecord(&test.requestedPod, nil, core.Kind("Pod").WithVersion("version"), "foo", "name", core.Resource("pods").WithVersion("version"), "", "ignored", nil))
|
||||
if err != nil {
|
||||
t.Errorf("[%d: %s] unexpected error %v for pod %+v", i, test.description, err, test.requestedPod)
|
||||
}
|
||||
|
||||
if !helper.Semantic.DeepEqual(test.expectedPod.Spec.Tolerations, test.requestedPod.Spec.Tolerations) {
|
||||
t.Errorf("[%d: %s] expected %#v got %#v", i, test.description, test.expectedPod.Spec.Tolerations, test.requestedPod.Spec.Tolerations)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandles(t *testing.T) {
|
||||
plugin := newExtendedResourceToleration()
|
||||
tests := map[admission.Operation]bool{
|
||||
admission.Create: true,
|
||||
admission.Update: true,
|
||||
admission.Delete: false,
|
||||
admission.Connect: false,
|
||||
}
|
||||
for op, expected := range tests {
|
||||
result := plugin.Handles(op)
|
||||
if result != expected {
|
||||
t.Errorf("Unexpected result for operation %s: %v\n", op, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue