Merge pull request #53025 from mtaufen/feature-gate-map

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Make feature gates loadable from a map[string]bool

Command line flag API remains the same. This allows ComponentConfig             
structures (e.g. KubeletConfiguration) to express the map structure             
behind feature gates in a natural way when written as JSON or YAML.             
                                                                                
For example:                                                                    
                                                                                
KubeletConfiguration Before:
```
apiVersion: kubeletconfig/v1alpha1
kind: KubeletConfiguration
featureGates: "DynamicKubeletConfig=true,Accelerators=true"
```

KubeletConfiguration After:
```
apiVersion: kubeletconfig/v1alpha1
kind: KubeletConfiguration
featureGates:
  DynamicKubeletConfig: true
  Accelerators: true
```

Fixes: #53024

```release-note
The Kubelet's feature gates are now specified as a map when provided via a JSON or YAML KubeletConfiguration, rather than as a string of key-value pairs.
```

/cc @mikedanese @jlowdermilk @smarterclayton

cmd/kubelet/app/options/options.go

@@ -312,9 +312,8 @@ func AddKubeletConfigFlags(fs *pflag.FlagSet, c *kubeletconfig.KubeletConfigurat
 	fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
 	fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to 0.")
 	fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
-	fs.StringVar(&c.FeatureGates, "feature-gates", c.FeatureGates, "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
+	fs.Var(flag.MapStringBool(c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
 		"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
-
 	fs.StringVar(&c.KubeletCgroups, "kubelet-cgroups", c.KubeletCgroups, "Optional absolute name of cgroups to create and run the Kubelet in.")
 	fs.StringVar(&c.SystemCgroups, "system-cgroups", c.SystemCgroups, "Optional absolute name of cgroups in which to place all non-kernel processes that are not already inside a cgroup under `/`. Empty for no container. Rolling back the flag requires a reboot.")
 

cmd/kubelet/app/server.go

@@ -238,7 +238,7 @@ func makeEventRecorder(kubeDeps *kubelet.Dependencies, nodeName types.NodeName)
 
 func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
 	// Set global feature gates based on the value on the initial KubeletServer
-	err = utilfeature.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
+	err = utilfeature.DefaultFeatureGate.SetFromMap(s.KubeletConfiguration.FeatureGates)
 	if err != nil {
 		return err
 	}

cmd/kubelet/kubelet.go

@@ -65,7 +65,7 @@ func main() {
 
 	// TODO(mtaufen): won't need this this once dynamic config is GA
 	// set feature gates so we can check if dynamic config is enabled
-	if err := utilfeature.DefaultFeatureGate.Set(defaultConfig.FeatureGates); err != nil {
+	if err := utilfeature.DefaultFeatureGate.SetFromMap(defaultConfig.FeatureGates); err != nil {
 		die(err)
 	}
 	// validate the initial KubeletFlags, to make sure the dynamic-config-related flags aren't used unless the feature gate is on

pkg/kubelet/apis/kubeletconfig/types.go

@@ -331,9 +331,8 @@ type KubeletConfiguration struct {
 	// Whitelist of unsafe sysctls or sysctl patterns (ending in *).
 	// +optional
 	AllowedUnsafeSysctls []string
-	// featureGates is a string of comma-separated key=value pairs that describe feature
-	// gates for alpha/experimental features.
-	FeatureGates string
+	// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
+	FeatureGates map[string]bool
 	// Tells the Kubelet to fail to start if swap is enabled on the node.
 	FailSwapOn bool
 	// This flag, if set, enables a check prior to mount operations to verify that the required components

pkg/kubelet/apis/kubeletconfig/v1alpha1/defaults.go

@@ -275,6 +275,9 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 			obj.RemoteRuntimeEndpoint = "tcp://localhost:3735"
 		}
 	}
+	if obj.FeatureGates == nil {
+		obj.FeatureGates = make(map[string]bool)
+	}
 }
 
 func boolVar(b bool) *bool {

pkg/kubelet/apis/kubeletconfig/v1alpha1/types.go

@@ -317,9 +317,8 @@ type KubeletConfiguration struct {
 	// Resource isolation might be lacking and pod might influence each other on the same node.
 	// +optional
 	AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"`
-	// featureGates is a string of comma-separated key=value pairs that describe feature
-	// gates for alpha/experimental features.
-	FeatureGates string `json:"featureGates,omitempty"`
+	// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
+	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 	// Tells the Kubelet to fail to start if swap is enabled on the node.
 	FailSwapOn bool `json:"failSwapOn,omitempty"`
 	// This flag, if set, enables a check prior to mount operations to verify that the required components

pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.conversion.go

@@ -288,7 +288,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_kubeletconfig_KubeletConfigura
 		return err
 	}
 	out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
-	out.FeatureGates = in.FeatureGates
+	out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
 	out.FailSwapOn = in.FailSwapOn
 	out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
 	out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes
@@ -453,7 +453,7 @@ func autoConvert_kubeletconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigura
 		return err
 	}
 	out.AllowedUnsafeSysctls = *(*[]string)(unsafe.Pointer(&in.AllowedUnsafeSysctls))
-	out.FeatureGates = in.FeatureGates
+	out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
 	out.FailSwapOn = in.FailSwapOn
 	out.ExperimentalCheckNodeCapabilitiesBeforeMount = in.ExperimentalCheckNodeCapabilitiesBeforeMount
 	out.KeepTerminatedPodVolumes = in.KeepTerminatedPodVolumes

pkg/kubelet/apis/kubeletconfig/v1alpha1/zz_generated.deepcopy.go

@@ -436,6 +436,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.FeatureGates != nil {
+		in, out := &in.FeatureGates, &out.FeatureGates
+		*out = make(map[string]bool, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.SystemReserved != nil {
 		in, out := &in.SystemReserved, &out.SystemReserved
 		*out = make(map[string]string, len(*in))

pkg/kubelet/apis/kubeletconfig/zz_generated.deepcopy.go

@@ -193,6 +193,13 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.FeatureGates != nil {
+		in, out := &in.FeatureGates, &out.FeatureGates
+		*out = make(map[string]bool, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.SystemReserved != nil {
 		in, out := &in.SystemReserved, &out.SystemReserved
 		*out = make(ConfigurationMap, len(*in))

staging/src/k8s.io/apiserver/pkg/util/feature/feature_gate.go

@@ -77,6 +77,8 @@ type FeatureGate interface {
 	// Set parses and stores flag gates for known features
 	// from a string like feature1=true,feature2=false,...
 	Set(value string) error
+	// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
+	SetFromMap(m map[string]bool) error
 	// Enabled returns true if the key is enabled.
 	Enabled(key Feature) bool
 	// Add adds features to the featureGate.
@@ -133,7 +135,7 @@ func NewFeatureGate() *featureGate {
 	return f
 }
 
-// Set Parses a string of the form "key1=value1,key2=value2,..." into a
+// Set parses a string of the form "key1=value1,key2=value2,..." into a
 // map[string]bool of known keys or returns an error.
 func (f *featureGate) Set(value string) error {
 	f.lock.Lock()
@@ -183,6 +185,42 @@ func (f *featureGate) Set(value string) error {
 	return nil
 }
 
+// SetFromMap stores flag gates for known features from a map[string]bool or returns an error
+func (f *featureGate) SetFromMap(m map[string]bool) error {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	// Copy existing state
+	known := map[Feature]FeatureSpec{}
+	for k, v := range f.known.Load().(map[Feature]FeatureSpec) {
+		known[k] = v
+	}
+	enabled := map[Feature]bool{}
+	for k, v := range f.enabled.Load().(map[Feature]bool) {
+		enabled[k] = v
+	}
+
+	for k, v := range m {
+		k := Feature(k)
+		_, ok := known[k]
+		if !ok {
+			return fmt.Errorf("unrecognized key: %s", k)
+		}
+		enabled[k] = v
+		// Handle "special" features like "all alpha gates"
+		if fn, found := f.special[k]; found {
+			fn(known, enabled, v)
+		}
+	}
+
+	// Persist changes
+	f.known.Store(known)
+	f.enabled.Store(enabled)
+
+	glog.Infof("feature gates: %v", f.enabled)
+	return nil
+}
+
 // String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
 func (f *featureGate) String() string {
 	pairs := []string{}

staging/src/k8s.io/apiserver/pkg/util/flag/BUILD

@@ -8,9 +8,16 @@ load(
 
 go_test(
     name = "go_default_test",
-    srcs = ["namedcertkey_flag_test.go"],
+    srcs = [
+        "map_string_bool_test.go",
+        "namedcertkey_flag_test.go",
+    ],
     library = ":go_default_library",
-    deps = ["//vendor/github.com/spf13/pflag:go_default_library"],
+    deps = [
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+    ],
 )
 
 go_library(
@@ -18,6 +25,7 @@ go_library(
     srcs = [
         "configuration_map.go",
         "flags.go",
+        "map_string_bool.go",
         "namedcertkey_flag.go",
         "string_flag.go",
         "tristate.go",

staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flag
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type MapStringBool map[string]bool
+
+// String implements github.com/spf13/pflag.Value
+func (m MapStringBool) String() string {
+	pairs := []string{}
+	for k, v := range m {
+		pairs = append(pairs, fmt.Sprintf("%s=%t", k, v))
+	}
+	sort.Strings(pairs)
+	return strings.Join(pairs, ",")
+}
+
+// Set implements github.com/spf13/pflag.Value
+func (m MapStringBool) Set(value string) error {
+	for _, s := range strings.Split(value, ",") {
+		if len(s) == 0 {
+			continue
+		}
+		arr := strings.SplitN(s, "=", 2)
+		if len(arr) != 2 {
+			return fmt.Errorf("malformed pair, expect string=bool")
+		}
+		k := strings.TrimSpace(arr[0])
+		v := strings.TrimSpace(arr[1])
+		boolValue, err := strconv.ParseBool(v)
+		if err != nil {
+			return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err)
+		}
+		m[k] = boolValue
+	}
+	return nil
+}
+
+// Type implements github.com/spf13/pflag.Value
+func (MapStringBool) Type() string {
+	return "mapStringBool"
+}

staging/src/k8s.io/apiserver/pkg/util/flag/map_string_bool_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flag
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStringMapStringBool(t *testing.T) {
+	cases := []struct {
+		desc   string
+		m      MapStringBool
+		expect string
+	}{
+		{"empty", MapStringBool{}, ""},
+		{"one key", MapStringBool{"one": true}, "one=true"},
+		{"two keys", MapStringBool{"one": true, "two": false}, "one=true,two=false"},
+	}
+	for _, c := range cases {
+		t.Run(c.desc, func(t *testing.T) {
+			str := c.m.String()
+			assert.Equal(t, c.expect, str)
+		})
+	}
+}
+
+func TestSetMapStringBool(t *testing.T) {
+	cases := []struct {
+		desc   string
+		val    string
+		expect MapStringBool
+		err    string
+	}{
+		{"empty", "", MapStringBool{}, ""},
+		{"one key", "one=true", MapStringBool{"one": true}, ""},
+		{"two keys", "one=true,two=false", MapStringBool{"one": true, "two": false}, ""},
+		{"two keys with space", "one=true, two=false", MapStringBool{"one": true, "two": false}, ""},
+		{"empty key", "=true", MapStringBool{"": true}, ""},
+		{"missing value", "one", MapStringBool{}, "malformed pair, expect string=bool"},
+		{"non-boolean value", "one=foo", MapStringBool{}, `invalid value of one: foo, err: strconv.ParseBool: parsing "foo": invalid syntax`},
+	}
+	for _, c := range cases {
+		t.Run(c.desc, func(t *testing.T) {
+			m := MapStringBool{}
+			err := m.Set(c.val)
+			if c.err != "" {
+				require.EqualError(t, err, c.err)
+			} else {
+				require.NoError(t, err)
+			}
+			assert.Equal(t, c.expect, m)
+		})
+	}
+}

test/e2e_node/BUILD

@@ -27,6 +27,7 @@ go_library(
     }),
     deps = [
         "//pkg/api/v1/pod:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubelet/apis/cri:go_default_library",
         "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig:go_default_library",
@@ -113,6 +114,7 @@ go_test(
     deps = [
         "//pkg/api:go_default_library",
         "//pkg/api/v1/node:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubelet:go_default_library",
         "//pkg/kubelet/apis/kubeletconfig:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",

test/e2e_node/cpu_manager_test.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
 	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
@@ -35,10 +36,6 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-const (
-	cpuManagerFeatureGate = "CPUManager=true"
-)
-
 // Helper for makeCPUManagerPod().
 type ctnAttribute struct {
 	ctnName    string
@@ -148,11 +145,7 @@ func enableCPUManagerInKubelet(f *framework.Framework) (oldCfg *kubeletconfig.Ku
 	newCfg := oldCfg.DeepCopy()
 
 	// Enable CPU Manager using feature gate.
-	if newCfg.FeatureGates != "" {
-		newCfg.FeatureGates = fmt.Sprintf("%s,%s", cpuManagerFeatureGate, newCfg.FeatureGates)
-	} else {
-		newCfg.FeatureGates = cpuManagerFeatureGate
-	}
+	newCfg.FeatureGates[string(features.CPUManager)] = true
 
 	// Set the CPU Manager policy to static.
 	newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)

test/e2e_node/critical_pod_test.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeapi "k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -43,7 +44,7 @@ var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive]", func() {
 
 	Context("when we need to admit a critical pod", func() {
 		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			initialConfig.FeatureGates += ", ExperimentalCriticalPodAnnotation=true"
+			initialConfig.FeatureGates[string(features.ExperimentalCriticalPodAnnotation)] = true
 		})
 
 		It("should be able to create and delete a critical pod", func() {

test/e2e_node/eviction_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
@@ -133,10 +134,7 @@ var _ = framework.KubeDescribe("LocalStorageAllocatableEviction [Slow] [Serial]
 			initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey}
 			initialConfig.CgroupsPerQOS = true
 			initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false
-			if initialConfig.FeatureGates != "" {
-				initialConfig.FeatureGates += ","
-			}
-			initialConfig.FeatureGates += "LocalStorageCapacityIsolation=true"
+			initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
 			// set evictionHard to be very small, so that only the allocatable eviction threshold triggers
 			initialConfig.EvictionHard = "nodefs.available<1"
 			initialConfig.EvictionMinimumReclaim = ""
@@ -221,10 +219,7 @@ var _ = framework.KubeDescribe("LocalStorageCapacityIsolationEviction [Slow] [Se
 	evictionTestTimeout := 10 * time.Minute
 	Context(fmt.Sprintf(testContextFmt, "evictions due to pod local storage violations"), func() {
 		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			if initialConfig.FeatureGates != "" {
-				initialConfig.FeatureGates += ","
-			}
-			initialConfig.FeatureGates += "LocalStorageCapacityIsolation=true"
+			initialConfig.FeatureGates[string(features.LocalStorageCapacityIsolation)] = true
 			initialConfig.EvictionHard = ""
 		})
 		sizeLimit := resource.MustParse("100Mi")

test/e2e_node/gpu_device_plugin.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	"k8s.io/kubernetes/test/e2e/framework"
 
@@ -45,7 +46,7 @@ var _ = framework.KubeDescribe("NVIDIA GPU Device Plugin [Feature:GPUDevicePlugi
 	Context("DevicePlugin", func() {
 		By("Enabling support for Device Plugin")
 		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			initialConfig.FeatureGates += "," + devicePluginFeatureGate
+			initialConfig.FeatureGates[string(features.DevicePlugins)] = true
 		})
 
 		BeforeEach(func() {

test/e2e_node/gpus.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	"k8s.io/kubernetes/test/e2e/framework"
 
@@ -31,8 +32,6 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-const acceleratorsFeatureGate = "Accelerators=true"
-
 func getGPUsAvailable(f *framework.Framework) int64 {
 	nodeList, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
 	framework.ExpectNoError(err, "getting node list")
@@ -91,14 +90,11 @@ var _ = framework.KubeDescribe("GPU [Serial]", func() {
 				}
 			}()
 
+			// Enable Accelerators
 			oldCfg, err = getCurrentKubeletConfig()
 			framework.ExpectNoError(err)
 			newCfg := oldCfg.DeepCopy()
-			if newCfg.FeatureGates != "" {
-				newCfg.FeatureGates = fmt.Sprintf("%s,%s", acceleratorsFeatureGate, newCfg.FeatureGates)
-			} else {
-				newCfg.FeatureGates = acceleratorsFeatureGate
-			}
+			newCfg.FeatureGates[string(features.Accelerators)] = true
 			framework.ExpectNoError(setKubeletConfiguration(f, newCfg))
 
 			By("Waiting for GPUs to become available on the local node")

test/e2e_node/util.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
@@ -123,7 +124,11 @@ func isKubeletConfigEnabled(f *framework.Framework) (bool, error) {
 	if err != nil {
 		return false, fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
 	}
-	return strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true"), nil
+	v, ok := cfgz.FeatureGates[string(features.DynamicKubeletConfig)]
+	if !ok {
+		return false, nil
+	}
+	return v, nil
 }
 
 // Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart