diff --git a/hack/.linted_packages b/hack/.linted_packages index 11847a3178..44d641363b 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -254,6 +254,7 @@ plugin/cmd/kube-scheduler plugin/cmd/kube-scheduler/app/options plugin/pkg/admission/admit plugin/pkg/admission/alwayspullimages +plugin/pkg/admission/defaulttolerationseconds plugin/pkg/admission/deny plugin/pkg/admission/exec plugin/pkg/admission/gc diff --git a/pkg/api/v1/helpers.go b/pkg/api/v1/helpers.go index 110ab3b585..3b1190e8a1 100644 --- a/pkg/api/v1/helpers.go +++ b/pkg/api/v1/helpers.go @@ -296,6 +296,45 @@ func GetPodTolerations(pod *Pod) ([]Toleration, error) { return GetTolerationsFromPodAnnotations(pod.Annotations) } +// Tries to add a toleration to annotations list. Returns true if something was updated +// false otherwise. +func AddOrUpdateTolerationInPod(pod *Pod, toleration *Toleration) (bool, error) { + podTolerations, err := GetPodTolerations(pod) + if err != nil { + return false, err + } + + var newTolerations []*Toleration + updated := false + for i := range podTolerations { + if toleration.MatchToleration(&podTolerations[i]) { + if api.Semantic.DeepEqual(toleration, podTolerations[i]) { + return false, nil + } + newTolerations = append(newTolerations, toleration) + updated = true + continue + } + + newTolerations = append(newTolerations, &podTolerations[i]) + } + + if !updated { + newTolerations = append(newTolerations, toleration) + } + + tolerationsData, err := json.Marshal(newTolerations) + if err != nil { + return false, err + } + + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + pod.Annotations[TolerationsAnnotationKey] = string(tolerationsData) + return true, nil +} + // GetTaintsFromNodeAnnotations gets the json serialized taints data from Pod.Annotations // and converts it to the []Taint type in api. func GetTaintsFromNodeAnnotations(annotations map[string]string) ([]Taint, error) { @@ -313,6 +352,16 @@ func GetNodeTaints(node *Node) ([]Taint, error) { return GetTaintsFromNodeAnnotations(node.Annotations) } +// MatchToleration checks if the toleration matches tolerationToMatch. Tolerations are unique by , +// if the two tolerations have same combination, regard as they match. +// TODO: uniqueness check for tolerations in api validations. +func (t *Toleration) MatchToleration(tolerationToMatch *Toleration) bool { + return t.Key == tolerationToMatch.Key && + t.Effect == tolerationToMatch.Effect && + t.Operator == tolerationToMatch.Operator && + t.Value == tolerationToMatch.Value +} + // ToleratesTaint checks if the toleration tolerates the taint. // The matching follows the rules below: // (1) Empty toleration.effect means to match all taint effects, diff --git a/plugin/BUILD b/plugin/BUILD index 9225d06f72..a7604f7d1f 100644 --- a/plugin/BUILD +++ b/plugin/BUILD @@ -17,6 +17,7 @@ filegroup( "//plugin/pkg/admission/admit:all-srcs", "//plugin/pkg/admission/alwayspullimages:all-srcs", "//plugin/pkg/admission/antiaffinity:all-srcs", + "//plugin/pkg/admission/defaulttolerationseconds:all-srcs", "//plugin/pkg/admission/deny:all-srcs", "//plugin/pkg/admission/exec:all-srcs", "//plugin/pkg/admission/gc:all-srcs", diff --git a/plugin/pkg/admission/defaulttolerationseconds/BUILD b/plugin/pkg/admission/defaulttolerationseconds/BUILD new file mode 100644 index 0000000000..dbd842fcd2 --- /dev/null +++ b/plugin/pkg/admission/defaulttolerationseconds/BUILD @@ -0,0 +1,47 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["admission_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/api:go_default_library", + "//pkg/api/v1:go_default_library", + "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apiserver/pkg/admission", + ], +) + +go_library( + name = "go_default_library", + srcs = ["admission.go"], + tags = ["automanaged"], + deps = [ + "//pkg/api/v1:go_default_library", + "//vendor:github.com/golang/glog", + "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1", + "//vendor:k8s.io/apiserver/pkg/admission", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/plugin/pkg/admission/defaulttolerationseconds/admission.go b/plugin/pkg/admission/defaulttolerationseconds/admission.go new file mode 100644 index 0000000000..0cd692cb8f --- /dev/null +++ b/plugin/pkg/admission/defaulttolerationseconds/admission.go @@ -0,0 +1,126 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaulttolerationseconds + +import ( + "flag" + "fmt" + "io" + + "github.com/golang/glog" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/admission" + "k8s.io/kubernetes/pkg/api/v1" +) + +var ( + defaultNotReadyTolerationSeconds = flag.Int64("default-not-ready-toleration-seconds", 300, + "Indicates the tolerationSeconds of the toleration for `notReady:NoExecute`"+ + " that is added by default to every pod that does not already have such a toleration.") + + defaultUnreachableTolerationSeconds = flag.Int64("default-unreachable-toleration-seconds", 300, + "Indicates the tolerationSeconds of the toleration for unreachable:NoExecute"+ + " that is added by default to every pod that does not already have such a toleration.") +) + +func init() { + admission.RegisterPlugin("DefaultTolerationSeconds", func(config io.Reader) (admission.Interface, error) { + return NewDefaultTolerationSeconds(), nil + }) +} + +// plugin contains the client used by the admission controller +// It will add default tolerations for every pod +// that tolerate taints `notReady:NoExecute` and `unreachable:NoExecute`, +// with tolerationSeconds of 300s. +// If the pod already specifies a toleration for taint `notReady:NoExecute` +// or `unreachable:NoExecute`, the plugin won't touch it. +type plugin struct { + *admission.Handler +} + +// NewDefaultTolerationSeconds creates a new instance of the DefaultTolerationSeconds admission controller +func NewDefaultTolerationSeconds() admission.Interface { + return &plugin{ + Handler: admission.NewHandler(admission.Create, admission.Update), + } +} + +func (p *plugin) Admit(attributes admission.Attributes) (err error) { + if attributes.GetResource().GroupResource() != v1.Resource("pods") { + return nil + } + + pod, ok := attributes.GetObject().(*v1.Pod) + if !ok { + glog.Errorf("expected pod but got %s", attributes.GetKind().Kind) + return nil + } + + tolerations, err := v1.GetPodTolerations(pod) + if err != nil { + glog.V(5).Infof("Invalid pod tolerations detected, but we will leave handling of this to validation phase") + return nil + } + + toleratesNodeNotReady := false + toleratesNodeUnreachable := false + for _, toleration := range tolerations { + if (toleration.Key == metav1.TaintNodeNotReady || len(toleration.Key) == 0) && + (toleration.Effect == v1.TaintEffectNoExecute || len(toleration.Effect) == 0) { + toleratesNodeNotReady = true + } + + if (toleration.Key == metav1.TaintNodeUnreachable || len(toleration.Key) == 0) && + (toleration.Effect == v1.TaintEffectNoExecute || len(toleration.Effect) == 0) { + toleratesNodeUnreachable = true + } + } + + // no change is required, return immediately + if toleratesNodeNotReady && toleratesNodeUnreachable { + return nil + } + + if !toleratesNodeNotReady { + _, err := v1.AddOrUpdateTolerationInPod(pod, &v1.Toleration{ + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: defaultNotReadyTolerationSeconds, + }) + if err != nil { + return admission.NewForbidden(attributes, + fmt.Errorf("failed to add default tolerations for taints `notReady:NoExecute` and `unreachable:NoExecute`, err: %v", err)) + } + } + + if !toleratesNodeUnreachable { + _, err := v1.AddOrUpdateTolerationInPod(pod, &v1.Toleration{ + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: defaultUnreachableTolerationSeconds, + }) + if err != nil { + return admission.NewForbidden(attributes, + fmt.Errorf("failed to add default tolerations for taints `notReady:NoExecute` and `unreachable:NoExecute`, err: %v", err)) + } + } + return nil +} diff --git a/plugin/pkg/admission/defaulttolerationseconds/admission_test.go b/plugin/pkg/admission/defaulttolerationseconds/admission_test.go new file mode 100644 index 0000000000..4e3c64fd93 --- /dev/null +++ b/plugin/pkg/admission/defaulttolerationseconds/admission_test.go @@ -0,0 +1,338 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package defaulttolerationseconds + +import ( + "encoding/json" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/admission" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/v1" +) + +func TestForgivenessAdmission(t *testing.T) { + var defaultTolerationSeconds int64 = 300 + + marshalTolerations := func(tolerations []v1.Toleration) string { + tolerationsData, _ := json.Marshal(tolerations) + return string(tolerationsData) + } + + genTolerationSeconds := func(s int64) *int64 { + return &s + } + + handler := NewDefaultTolerationSeconds() + tests := []struct { + description string + requestedPod v1.Pod + expectedPod v1.Pod + }{ + { + description: "pod has no tolerations, expect add tolerations for `notread:NoExecute` and `unreachable:NoExecute`", + requestedPod: v1.Pod{ + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + { + description: "pod has tolerations, but none is for taint `notread:NoExecute` or `unreachable:NoExecute`, expect add tolerations for `notread:NoExecute` and `unreachable:NoExecute`", + requestedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: "foo", + Operator: v1.TolerationOpEqual, + Value: "bar", + Effect: v1.TaintEffectNoSchedule, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: "foo", + Operator: v1.TolerationOpEqual, + Value: "bar", + Effect: v1.TaintEffectNoSchedule, + TolerationSeconds: genTolerationSeconds(700), + }, + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + { + description: "pod specified a toleration for taint `notReady:NoExecute`, expect add toleration for `unreachable:NoExecute`", + requestedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + { + description: "pod specified a toleration for taint `unreachable:NoExecute`, expect add toleration for `notReady:NoExecute`", + requestedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: &defaultTolerationSeconds, + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + { + description: "pod specified tolerations for both `notread:NoExecute` and `unreachable:NoExecute`, expect no change", + requestedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + { + description: "pod specified toleration for taint `unreachable`, expect add toleration for `notReady:NoExecute`", + requestedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Key: metav1.TaintNodeUnreachable, + Operator: v1.TolerationOpExists, + TolerationSeconds: genTolerationSeconds(700), + }, + { + Key: metav1.TaintNodeNotReady, + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoExecute, + TolerationSeconds: genTolerationSeconds(300), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + { + description: "pod has wildcard toleration for all kind of taints, expect no change", + requestedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Operator: v1.TolerationOpExists, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + expectedPod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{ + { + Operator: v1.TolerationOpExists, + TolerationSeconds: genTolerationSeconds(700), + }, + }), + }, + }, + Spec: v1.PodSpec{}, + }, + }, + } + + for _, test := range tests { + err := handler.Admit(admission.NewAttributesRecord(&test.requestedPod, nil, api.Kind("Pod").WithVersion("version"), "foo", "name", v1.Resource("pods").WithVersion("version"), "", "ignored", nil)) + if err != nil { + t.Errorf("[%s]: unexpected error %v for pod %+v", test.description, err, test.requestedPod) + } + + if !api.Semantic.DeepEqual(test.expectedPod.Annotations, test.requestedPod.Annotations) { + t.Errorf("[%s]: expected %#v got %#v", test.description, test.expectedPod.Annotations, test.requestedPod.Annotations) + } + } +} + +func TestHandles(t *testing.T) { + handler := NewDefaultTolerationSeconds() + tests := map[admission.Operation]bool{ + admission.Update: true, + admission.Create: true, + admission.Delete: false, + admission.Connect: false, + } + for op, expected := range tests { + result := handler.Handles(op) + if result != expected { + t.Errorf("Unexpected result for operation %s: %v\n", op, result) + } + } +}