github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add defaultTolerationSeconds admission controller

pull/6/head
Kevin 2017-02-14 22:47:02 +08:00
parent b2df7e5397
commit 83545a65f1
6 changed files with 562 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hack/.linted_packages
View File

@ -254,6 +254,7 @@ plugin/cmd/kube-scheduler
plugin/cmd/kube-scheduler/app/options
plugin/pkg/admission/admit
plugin/pkg/admission/alwayspullimages
plugin/pkg/admission/defaulttolerationseconds
plugin/pkg/admission/deny
plugin/pkg/admission/exec
plugin/pkg/admission/gc

49
pkg/api/v1/helpers.go
View File

@ -296,6 +296,45 @@ func GetPodTolerations(pod *Pod) ([]Toleration, error) {
return GetTolerationsFromPodAnnotations(pod.Annotations)
}
// Tries to add a toleration to annotations list. Returns true if something was updated
// false otherwise.
func AddOrUpdateTolerationInPod(pod *Pod, toleration *Toleration) (bool, error) {
podTolerations, err := GetPodTolerations(pod)
if err != nil {
return false, err
}
var newTolerations []*Toleration
updated := false
for i := range podTolerations {
if toleration.MatchToleration(&podTolerations[i]) {
if api.Semantic.DeepEqual(toleration, podTolerations[i]) {
return false, nil
}
newTolerations = append(newTolerations, toleration)
updated = true
continue
}
newTolerations = append(newTolerations, &podTolerations[i])
}
if !updated {
newTolerations = append(newTolerations, toleration)
}
tolerationsData, err := json.Marshal(newTolerations)
if err != nil {
return false, err
}
if pod.Annotations == nil {
pod.Annotations = make(map[string]string)
}
pod.Annotations[TolerationsAnnotationKey] = string(tolerationsData)
return true, nil
}
// GetTaintsFromNodeAnnotations gets the json serialized taints data from Pod.Annotations
// and converts it to the []Taint type in api.
func GetTaintsFromNodeAnnotations(annotations map[string]string) ([]Taint, error) {
@ -313,6 +352,16 @@ func GetNodeTaints(node *Node) ([]Taint, error) {
return GetTaintsFromNodeAnnotations(node.Annotations)
}
// MatchToleration checks if the toleration matches tolerationToMatch. Tolerations are unique by <key,effect,operator,value>,
// if the two tolerations have same <key,effect,operator,value> combination, regard as they match.
// TODO: uniqueness check for tolerations in api validations.
func (t *Toleration) MatchToleration(tolerationToMatch *Toleration) bool {
return t.Key == tolerationToMatch.Key &&
t.Effect == tolerationToMatch.Effect &&
t.Operator == tolerationToMatch.Operator &&
t.Value == tolerationToMatch.Value
}
// ToleratesTaint checks if the toleration tolerates the taint.
// The matching follows the rules below:
// (1) Empty toleration.effect means to match all taint effects,

1
plugin/BUILD
View File

@ -17,6 +17,7 @@ filegroup(
"//plugin/pkg/admission/admit:all-srcs",
"//plugin/pkg/admission/alwayspullimages:all-srcs",
"//plugin/pkg/admission/antiaffinity:all-srcs",
"//plugin/pkg/admission/defaulttolerationseconds:all-srcs",
"//plugin/pkg/admission/deny:all-srcs",
"//plugin/pkg/admission/exec:all-srcs",
"//plugin/pkg/admission/gc:all-srcs",

47
plugin/pkg/admission/defaulttolerationseconds/BUILD Normal file
View File

@ -0,0 +1,47 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = ["admission_test.go"],
library = ":go_default_library",
tags = ["automanaged"],
deps = [
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
"//vendor:k8s.io/apiserver/pkg/admission",
],
)
go_library(
name = "go_default_library",
srcs = ["admission.go"],
tags = ["automanaged"],
deps = [
"//pkg/api/v1:go_default_library",
"//vendor:github.com/golang/glog",
"//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
"//vendor:k8s.io/apiserver/pkg/admission",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

126
plugin/pkg/admission/defaulttolerationseconds/admission.go Normal file
View File

@ -0,0 +1,126 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package defaulttolerationseconds
import (
"flag"
"fmt"
"io"
"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/admission"
"k8s.io/kubernetes/pkg/api/v1"
)
var (
defaultNotReadyTolerationSeconds = flag.Int64("default-not-ready-toleration-seconds", 300,
"Indicates the tolerationSeconds of the toleration for `notReady:NoExecute`"+
" that is added by default to every pod that does not already have such a toleration.")
defaultUnreachableTolerationSeconds = flag.Int64("default-unreachable-toleration-seconds", 300,
"Indicates the tolerationSeconds of the toleration for unreachable:NoExecute"+
" that is added by default to every pod that does not already have such a toleration.")
)
func init() {
admission.RegisterPlugin("DefaultTolerationSeconds", func(config io.Reader) (admission.Interface, error) {
return NewDefaultTolerationSeconds(), nil
})
}
// plugin contains the client used by the admission controller
// It will add default tolerations for every pod
// that tolerate taints `notReady:NoExecute` and `unreachable:NoExecute`,
// with tolerationSeconds of 300s.
// If the pod already specifies a toleration for taint `notReady:NoExecute`
// or `unreachable:NoExecute`, the plugin won't touch it.
type plugin struct {
*admission.Handler
}
// NewDefaultTolerationSeconds creates a new instance of the DefaultTolerationSeconds admission controller
func NewDefaultTolerationSeconds() admission.Interface {
return &plugin{
Handler: admission.NewHandler(admission.Create, admission.Update),
}
}
func (p *plugin) Admit(attributes admission.Attributes) (err error) {
if attributes.GetResource().GroupResource() != v1.Resource("pods") {
return nil
}
pod, ok := attributes.GetObject().(*v1.Pod)
if !ok {
glog.Errorf("expected pod but got %s", attributes.GetKind().Kind)
return nil
}
tolerations, err := v1.GetPodTolerations(pod)
if err != nil {
glog.V(5).Infof("Invalid pod tolerations detected, but we will leave handling of this to validation phase")
return nil
}
toleratesNodeNotReady := false
toleratesNodeUnreachable := false
for _, toleration := range tolerations {
if (toleration.Key == metav1.TaintNodeNotReady || len(toleration.Key) == 0) &&
(toleration.Effect == v1.TaintEffectNoExecute || len(toleration.Effect) == 0) {
toleratesNodeNotReady = true
}
if (toleration.Key == metav1.TaintNodeUnreachable || len(toleration.Key) == 0) &&
(toleration.Effect == v1.TaintEffectNoExecute || len(toleration.Effect) == 0) {
toleratesNodeUnreachable = true
}
}
// no change is required, return immediately
if toleratesNodeNotReady && toleratesNodeUnreachable {
return nil
}
if !toleratesNodeNotReady {
_, err := v1.AddOrUpdateTolerationInPod(pod, &v1.Toleration{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: defaultNotReadyTolerationSeconds,
})
if err != nil {
return admission.NewForbidden(attributes,
fmt.Errorf("failed to add default tolerations for taints `notReady:NoExecute` and `unreachable:NoExecute`, err: %v", err))
}
}
if !toleratesNodeUnreachable {
_, err := v1.AddOrUpdateTolerationInPod(pod, &v1.Toleration{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: defaultUnreachableTolerationSeconds,
})
if err != nil {
return admission.NewForbidden(attributes,
fmt.Errorf("failed to add default tolerations for taints `notReady:NoExecute` and `unreachable:NoExecute`, err: %v", err))
}
}
return nil
}

338
plugin/pkg/admission/defaulttolerationseconds/admission_test.go Normal file
View File

@ -0,0 +1,338 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package defaulttolerationseconds
import (
"encoding/json"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/admission"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
)
func TestForgivenessAdmission(t *testing.T) {
var defaultTolerationSeconds int64 = 300
marshalTolerations := func(tolerations []v1.Toleration) string {
tolerationsData, _ := json.Marshal(tolerations)
return string(tolerationsData)
}
genTolerationSeconds := func(s int64) *int64 {
return &s
}
handler := NewDefaultTolerationSeconds()
tests := []struct {
description string
requestedPod v1.Pod
expectedPod v1.Pod
}{
{
description: "pod has no tolerations, expect add tolerations for `notread:NoExecute` and `unreachable:NoExecute`",
requestedPod: v1.Pod{
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: &defaultTolerationSeconds,
},
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: &defaultTolerationSeconds,
},
}),
},
},
Spec: v1.PodSpec{},
},
},
{
description: "pod has tolerations, but none is for taint `notread:NoExecute` or `unreachable:NoExecute`, expect add tolerations for `notread:NoExecute` and `unreachable:NoExecute`",
requestedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: "foo",
Operator: v1.TolerationOpEqual,
Value: "bar",
Effect: v1.TaintEffectNoSchedule,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: "foo",
Operator: v1.TolerationOpEqual,
Value: "bar",
Effect: v1.TaintEffectNoSchedule,
TolerationSeconds: genTolerationSeconds(700),
},
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: &defaultTolerationSeconds,
},
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: &defaultTolerationSeconds,
},
}),
},
},
Spec: v1.PodSpec{},
},
},
{
description: "pod specified a toleration for taint `notReady:NoExecute`, expect add toleration for `unreachable:NoExecute`",
requestedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: &defaultTolerationSeconds,
},
}),
},
},
Spec: v1.PodSpec{},
},
},
{
description: "pod specified a toleration for taint `unreachable:NoExecute`, expect add toleration for `notReady:NoExecute`",
requestedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: &defaultTolerationSeconds,
},
}),
},
},
Spec: v1.PodSpec{},
},
},
{
description: "pod specified tolerations for both `notread:NoExecute` and `unreachable:NoExecute`, expect no change",
requestedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
},
{
description: "pod specified toleration for taint `unreachable`, expect add toleration for `notReady:NoExecute`",
requestedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Key: metav1.TaintNodeUnreachable,
Operator: v1.TolerationOpExists,
TolerationSeconds: genTolerationSeconds(700),
},
{
Key: metav1.TaintNodeNotReady,
Operator: v1.TolerationOpExists,
Effect: v1.TaintEffectNoExecute,
TolerationSeconds: genTolerationSeconds(300),
},
}),
},
},
Spec: v1.PodSpec{},
},
},
{
description: "pod has wildcard toleration for all kind of taints, expect no change",
requestedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Operator: v1.TolerationOpExists,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
expectedPod: v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
v1.TolerationsAnnotationKey: marshalTolerations([]v1.Toleration{
{
Operator: v1.TolerationOpExists,
TolerationSeconds: genTolerationSeconds(700),
},
}),
},
},
Spec: v1.PodSpec{},
},
},
}
for _, test := range tests {
err := handler.Admit(admission.NewAttributesRecord(&test.requestedPod, nil, api.Kind("Pod").WithVersion("version"), "foo", "name", v1.Resource("pods").WithVersion("version"), "", "ignored", nil))
if err != nil {
t.Errorf("[%s]: unexpected error %v for pod %+v", test.description, err, test.requestedPod)
}
if !api.Semantic.DeepEqual(test.expectedPod.Annotations, test.requestedPod.Annotations) {
t.Errorf("[%s]: expected %#v got %#v", test.description, test.expectedPod.Annotations, test.requestedPod.Annotations)
}
}
}
func TestHandles(t *testing.T) {
handler := NewDefaultTolerationSeconds()
tests := map[admission.Operation]bool{
admission.Update: true,
admission.Create: true,
admission.Delete: false,
admission.Connect: false,
}
for op, expected := range tests {
result := handler.Handles(op)
if result != expected {
t.Errorf("Unexpected result for operation %s: %v\n", op, result)
}
}
}