diff --git a/pkg/kubeapiserver/options/plugins.go b/pkg/kubeapiserver/options/plugins.go index abcb5b4673..fbfefc598f 100644 --- a/pkg/kubeapiserver/options/plugins.go +++ b/pkg/kubeapiserver/options/plugins.go @@ -38,6 +38,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision" "k8s.io/kubernetes/plugin/pkg/admission/namespace/exists" "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" + "k8s.io/kubernetes/plugin/pkg/admission/nodetaint" "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector" "k8s.io/kubernetes/plugin/pkg/admission/podpreset" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" @@ -73,6 +74,7 @@ var AllOrderedPlugins = []string{ limitranger.PluginName, // LimitRanger serviceaccount.PluginName, // ServiceAccount noderestriction.PluginName, // NodeRestriction + nodetaint.PluginName, // TaintNodesByCondition alwayspullimages.PluginName, // AlwaysPullImages imagepolicy.PluginName, // ImagePolicyWebhook podsecuritypolicy.PluginName, // PodSecurityPolicy @@ -113,6 +115,7 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { autoprovision.Register(plugins) exists.Register(plugins) noderestriction.Register(plugins) + nodetaint.Register(plugins) label.Register(plugins) // DEPRECATED in favor of NewPersistentVolumeLabelController in CCM podnodeselector.Register(plugins) podpreset.Register(plugins) @@ -145,5 +148,9 @@ func DefaultOffAdmissionPlugins() sets.String { defaultOnPlugins.Insert(podpriority.PluginName) //PodPriority } + if utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition) { + defaultOnPlugins.Insert(nodetaint.PluginName) //TaintNodesByCondition + } + return sets.NewString(AllOrderedPlugins...).Difference(defaultOnPlugins) } diff --git a/plugin/pkg/admission/noderestriction/BUILD b/plugin/pkg/admission/noderestriction/BUILD index 5c3bd1e93d..b0383ce5d5 100644 --- a/plugin/pkg/admission/noderestriction/BUILD +++ b/plugin/pkg/admission/noderestriction/BUILD @@ -19,6 +19,7 @@ go_library( "//pkg/auth/nodeidentifier:go_default_library", "//pkg/features:go_default_library", "//pkg/kubelet/apis:go_default_library", + "//plugin/pkg/admission/nodetaint:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library", diff --git a/plugin/pkg/admission/nodetaint/admission.go b/plugin/pkg/admission/nodetaint/admission.go new file mode 100644 index 0000000000..71cce0d9e2 --- /dev/null +++ b/plugin/pkg/admission/nodetaint/admission.go @@ -0,0 +1,104 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetaint + +import ( + "fmt" + "io" + "k8s.io/apiserver/pkg/admission" + utilfeature "k8s.io/apiserver/pkg/util/feature" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" +) + +const ( + // PluginName is the name of the plugin. + PluginName = "TaintNodesByCondition" + // TaintNodeNotReady is the not-ready label as specified in the API. + TaintNodeNotReady = "node.kubernetes.io/not-ready" +) + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + return NewPlugin(), nil + }) +} + +// NewPlugin creates a new NodeTaint admission plugin. +// This plugin identifies requests from nodes +func NewPlugin() *Plugin { + return &Plugin{ + Handler: admission.NewHandler(admission.Create), + features: utilfeature.DefaultFeatureGate, + } +} + +// Plugin holds state for and implements the admission plugin. +type Plugin struct { + *admission.Handler + // allows overriding for testing + features utilfeature.FeatureGate +} + +var ( + _ = admission.Interface(&Plugin{}) +) + +var ( + nodeResource = api.Resource("nodes") +) + +// Admit is the main function that checks node identity and adds taints as needed. +func (p *Plugin) Admit(a admission.Attributes) error { + // If TaintNodesByCondition is not enabled, we don't need to do anything. + if !p.features.Enabled(features.TaintNodesByCondition) { + return nil + } + + // Our job is just to taint nodes. + if a.GetResource().GroupResource() != nodeResource || a.GetSubresource() != "" { + return nil + } + + node, ok := a.GetObject().(*api.Node) + if !ok { + return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject())) + } + + // Taint node with NotReady taint at creation if TaintNodesByCondition is + // enabled. This is needed to make sure that nodes are added to the cluster + // with the NotReady taint. Otherwise, a new node may receive the taint with + // some delay causing pods to be scheduled on a not-ready node. + // Node controller will remove the taint when the node becomes ready. + addNotReadyTaint(node) + return nil +} + +func addNotReadyTaint(node *api.Node) { + notReadyTaint := api.Taint{ + Key: TaintNodeNotReady, + Effect: api.TaintEffectNoSchedule, + } + for _, taint := range node.Spec.Taints { + if taint.MatchTaint(notReadyTaint) { + // the taint already exists. + return + } + } + node.Spec.Taints = append(node.Spec.Taints, notReadyTaint) +} diff --git a/plugin/pkg/admission/nodetaint/admission_test.go b/plugin/pkg/admission/nodetaint/admission_test.go new file mode 100644 index 0000000000..28c5a2da8c --- /dev/null +++ b/plugin/pkg/admission/nodetaint/admission_test.go @@ -0,0 +1,113 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetaint + +import ( + "reflect" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/authentication/user" + utilfeature "k8s.io/apiserver/pkg/util/feature" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/features" +) + +var ( + enableTaintNodesByCondition = utilfeature.NewFeatureGate() + disableTaintNodesByCondition = utilfeature.NewFeatureGate() +) + +func init() { + if err := enableTaintNodesByCondition.Add(map[utilfeature.Feature]utilfeature.FeatureSpec{features.TaintNodesByCondition: {Default: true}}); err != nil { + panic(err) + } + if err := disableTaintNodesByCondition.Add(map[utilfeature.Feature]utilfeature.FeatureSpec{features.TaintNodesByCondition: {Default: false}}); err != nil { + panic(err) + } +} + +func Test_nodeTaints(t *testing.T) { + var ( + mynode = &user.DefaultInfo{Name: "system:node:mynode", Groups: []string{"system:nodes"}} + resource = api.Resource("nodes").WithVersion("v1") + notReadyTaint = api.Taint{Key: TaintNodeNotReady, Effect: api.TaintEffectNoSchedule} + notReadyCondition = api.NodeCondition{Type: api.NodeReady, Status: api.ConditionFalse} + myNodeObjMeta = metav1.ObjectMeta{Name: "mynode"} + myNodeObj = api.Node{ObjectMeta: myNodeObjMeta} + myTaintedNodeObj = api.Node{ObjectMeta: myNodeObjMeta, + Spec: api.NodeSpec{Taints: []api.Taint{notReadyTaint}}} + myUnreadyNodeObj = api.Node{ObjectMeta: myNodeObjMeta, + Status: api.NodeStatus{Conditions: []api.NodeCondition{notReadyCondition}}} + nodeKind = api.Kind("Node").WithVersion("v1") + ) + tests := []struct { + name string + node api.Node + oldNode api.Node + features utilfeature.FeatureGate + operation admission.Operation + expectedTaints []api.Taint + }{ + { + name: "notReady taint is added on creation", + node: myNodeObj, + features: enableTaintNodesByCondition, + operation: admission.Create, + expectedTaints: []api.Taint{notReadyTaint}, + }, + { + name: "NotReady taint is not added when TaintNodesByCondition is disabled", + node: myNodeObj, + features: disableTaintNodesByCondition, + operation: admission.Create, + expectedTaints: nil, + }, + { + name: "already tainted node is not tainted again", + node: myTaintedNodeObj, + features: enableTaintNodesByCondition, + operation: admission.Create, + expectedTaints: []api.Taint{notReadyTaint}, + }, + { + name: "NotReady taint is added to an unready node as well", + node: myUnreadyNodeObj, + features: enableTaintNodesByCondition, + operation: admission.Create, + expectedTaints: []api.Taint{notReadyTaint}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attributes := admission.NewAttributesRecord(&tt.node, &tt.oldNode, nodeKind, myNodeObj.Namespace, myNodeObj.Name, resource, "", tt.operation, false, mynode) + c := NewPlugin() + if tt.features != nil { + c.features = tt.features + } + err := c.Admit(attributes) + if err != nil { + t.Errorf("nodePlugin.Admit() error = %v", err) + } + node, _ := attributes.GetObject().(*api.Node) + if !reflect.DeepEqual(node.Spec.Taints, tt.expectedTaints) { + t.Errorf("Unexpected Node taints. Got %v\nExpected: %v", node.Spec.Taints, tt.expectedTaints) + } + }) + } +} diff --git a/test/integration/auth/node_test.go b/test/integration/auth/node_test.go index 7b701c22ca..50490d200d 100644 --- a/test/integration/auth/node_test.go +++ b/test/integration/auth/node_test.go @@ -88,7 +88,7 @@ func TestNodeAuthorizer(t *testing.T) { "--enable-admission-plugins", "NodeRestriction", // The "default" SA is not installed, causing the ServiceAccount plugin to retry for ~1s per // API request. - "--disable-admission-plugins", "ServiceAccount", + "--disable-admission-plugins", "ServiceAccount,TaintNodesByCondition", }, framework.SharedEtcd()) defer server.TearDownFn()