Merge pull request #32530 from mtaufen/dynamic-settings-tests

Automatic merge from submit-queue

Utility functions for using dynamic Kubelet configuration from a test

/cc @vishh @dchen1107

test/e2e_node/BUILD

@@ -27,6 +27,8 @@ go_library(
         "//pkg/api:go_default_library",
         "//pkg/api/errors:go_default_library",
         "//pkg/api/unversioned:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/dockertools:go_default_library",
@@ -77,8 +79,6 @@ go_test(
         "//pkg/api/errors:go_default_library",
         "//pkg/api/resource:go_default_library",
         "//pkg/api/unversioned:go_default_library",
-        "//pkg/apis/componentconfig:go_default_library",
-        "//pkg/apis/componentconfig/v1alpha1:go_default_library",
         "//pkg/client/cache:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",

test/e2e_node/dynamic_kubelet_configuration_test.go

@@ -17,20 +17,13 @@ limitations under the License.
 package e2e_node
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
 	"time"
 
 	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/api"
+
-	"k8s.io/kubernetes/pkg/apis/componentconfig"
-	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 )
 
 // This test is marked [Disruptive] because the Kubelet temporarily goes down as part of of this test.
@@ -39,14 +32,9 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube
 
 	Context("When a configmap called `kubelet-<node-name>` is added to the `kube-system` namespace", func() {
 		It("The Kubelet on that node should restart to take up the new config", func() {
-			const (
-				restartGap = 40 * time.Second
-			)
-
 			// Get the current KubeletConfiguration (known to be valid) by
 			// querying the configz endpoint for the current node.
-			resp := pollConfigz(2*time.Minute, 5*time.Second)
+			kubeCfg, err := getCurrentKubeletConfig()
-			kubeCfg, err := decodeConfigz(resp)
 			framework.ExpectNoError(err)
 			glog.Infof("KubeletConfiguration - Initial values: %+v", *kubeCfg)
 
@@ -60,138 +48,15 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube
 			kubeCfg.FileCheckFrequency.Duration = newFileCheckFrequency
 
 			// Use the new config to create a new kube-<node-name> configmap in `kube-system` namespace.
-			_, err = createConfigMap(f, kubeCfg)
+			// Note: setKubeletConfiguration will return an error if the Kubelet does not present the
+			//       modified configuration via /configz when it comes back up.
+			err = setKubeletConfiguration(f, kubeCfg)
 			framework.ExpectNoError(err)
 
-			// Give the Kubelet time to see that there is new config and restart. If we don't do this,
-			// the Kubelet will still have the old config when we poll, and the test will fail.
-			time.Sleep(restartGap)
-
-			// Use configz to get the new config.
-			resp = pollConfigz(2*time.Minute, 5*time.Second)
-			kubeCfg, err = decodeConfigz(resp)
-			framework.ExpectNoError(err)
-			glog.Infof("KubeletConfiguration - After modification of FileCheckFrequency: %+v", *kubeCfg)
-
-			// We expect to see the new value in the new config.
-			Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(newFileCheckFrequency))
-
 			// Change the config back to what it originally was.
 			kubeCfg.FileCheckFrequency.Duration = oldFileCheckFrequency
-			_, err = updateConfigMap(f, kubeCfg)
+			err = setKubeletConfiguration(f, kubeCfg)
 			framework.ExpectNoError(err)
-
-			// Give the Kubelet time to see that there is new config and restart. If we don't do this,
-			// the Kubelet will still have the old config when we poll, and the test will fail.
-			time.Sleep(restartGap)
-
-			// User configz to get the new config.
-			resp = pollConfigz(2*time.Minute, 5*time.Second)
-			kubeCfg, err = decodeConfigz(resp)
-			framework.ExpectNoError(err)
-			glog.Infof("KubeletConfiguration - After restoration of FileCheckFrequency: %+v", *kubeCfg)
-
-			// We expect to see the original value restored in the new config.
-			Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(oldFileCheckFrequency))
 		})
 	})
 })
-
-// This function either causes the test to fail, or it returns a status 200 response.
-func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
-	endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
-	client := &http.Client{}
-	req, err := http.NewRequest("GET", endpoint, nil)
-	framework.ExpectNoError(err)
-	req.Header.Add("Accept", "application/json")
-
-	var resp *http.Response
-	Eventually(func() bool {
-		resp, err = client.Do(req)
-		if err != nil {
-			glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
-			return false
-		}
-		if resp.StatusCode != 200 {
-			glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
-			return false
-		}
-		return true
-	}, timeout, pollInterval).Should(Equal(true))
-	return resp
-}
-
-// Decodes the http response  from /configz and returns a componentconfig.KubeletConfiguration (internal type).
-func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) {
-	// This hack because /configz reports the following structure:
-	// {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}}
-	type configzWrapper struct {
-		ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"`
-	}
-
-	configz := configzWrapper{}
-	kubeCfg := componentconfig.KubeletConfiguration{}
-
-	contentsBytes, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	err = json.Unmarshal(contentsBytes, &configz)
-	if err != nil {
-		return nil, err
-	}
-
-	api.Scheme.Default(&configz.ComponentConfig)
-
-	err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return &kubeCfg, nil
-}
-
-// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
-func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
-	kubeCfgExt := v1alpha1.KubeletConfiguration{}
-	api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
-
-	bytes, err := json.Marshal(kubeCfgExt)
-	framework.ExpectNoError(err)
-
-	cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(&api.ConfigMap{
-		ObjectMeta: api.ObjectMeta{
-			Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName),
-		},
-		Data: map[string]string{
-			"kubelet.config": string(bytes),
-		},
-	})
-	if err != nil {
-		return nil, err
-	}
-	return cmap, nil
-}
-
-// Similar to createConfigMap, except this updates an existing ConfigMap.
-func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
-	kubeCfgExt := v1alpha1.KubeletConfiguration{}
-	api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
-
-	bytes, err := json.Marshal(kubeCfgExt)
-	framework.ExpectNoError(err)
-
-	cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(&api.ConfigMap{
-		ObjectMeta: api.ObjectMeta{
-			Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName),
-		},
-		Data: map[string]string{
-			"kubelet.config": string(bytes),
-		},
-	})
-	if err != nil {
-		return nil, err
-	}
-	return cmap, nil
-}

test/e2e_node/util.go

@@ -22,9 +22,21 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"reflect"
 	"strings"
+	"time"
 
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/pkg/api"
+	k8serr "k8s.io/kubernetes/pkg/api/errors"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	v1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
+	// utilconfig "k8s.io/kubernetes/pkg/util/config"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/gomega"
 )
 
 // TODO(random-liu): Get this automatically from kubelet flag.
@@ -60,3 +72,163 @@ func getNodeSummary() (*stats.Summary, error) {
 	}
 	return &summary, nil
 }
+
+// Returns the current KubeletConfiguration
+func getCurrentKubeletConfig() (*componentconfig.KubeletConfiguration, error) {
+	resp := pollConfigz(5*time.Minute, 5*time.Second)
+	kubeCfg, err := decodeConfigz(resp)
+	if err != nil {
+		return nil, err
+	}
+	return kubeCfg, nil
+}
+
+// Queries the API server for a Kubelet configuration for the node described by framework.TestContext.NodeName
+func getCurrentKubeletConfigMap(f *framework.Framework) (*api.ConfigMap, error) {
+	return f.ClientSet.Core().ConfigMaps("kube-system").Get(fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName))
+}
+
+// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
+// with the new configuration. Returns an error if the configuration after waiting 40 seconds
+// doesn't match what you attempted to set, or if the dynamic configuration feature is disabled.
+func setKubeletConfiguration(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) error {
+	const (
+		restartGap = 30 * time.Second
+	)
+
+	// Make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
+	cfgz, err := getCurrentKubeletConfig()
+	if err != nil {
+		return fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
+	}
+	if !strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true") {
+		return fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
+			"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
+			"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
+	}
+
+	// Check whether a configmap for KubeletConfiguration already exists
+	_, err = getCurrentKubeletConfigMap(f)
+
+	if k8serr.IsNotFound(err) {
+		_, err := createConfigMap(f, kubeCfg)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	} else {
+		// The configmap exists, update it instead of creating it.
+		_, err := updateConfigMap(f, kubeCfg)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Wait for the Kubelet to restart.
+	time.Sleep(restartGap)
+
+	// Retrieve the new config and compare it to the one we attempted to set
+	newKubeCfg, err := getCurrentKubeletConfig()
+	if err != nil {
+		return err
+	}
+
+	// Return an error if the desired config is not in use by now
+	if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
+		return fmt.Errorf("either the Kubelet did not restart or it did not present the modified configuration via /configz after restarting.")
+	}
+	return nil
+}
+
+// Causes the test to fail, or returns a status 200 response from the /configz endpoint
+func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
+	endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", endpoint, nil)
+	framework.ExpectNoError(err)
+	req.Header.Add("Accept", "application/json")
+
+	var resp *http.Response
+	Eventually(func() bool {
+		resp, err = client.Do(req)
+		if err != nil {
+			glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
+			return false
+		}
+		if resp.StatusCode != 200 {
+			glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
+			return false
+		}
+		return true
+	}, timeout, pollInterval).Should(Equal(true))
+	return resp
+}
+
+// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type).
+func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) {
+	// This hack because /configz reports the following structure:
+	// {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}}
+	type configzWrapper struct {
+		ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"`
+	}
+
+	configz := configzWrapper{}
+	kubeCfg := componentconfig.KubeletConfiguration{}
+
+	contentsBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(contentsBytes, &configz)
+	if err != nil {
+		return nil, err
+	}
+
+	err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &kubeCfg, nil
+}
+
+// Constructs a Kubelet ConfigMap targeting the current node running the node e2e tests
+func makeKubeletConfigMap(nodeName string, kubeCfg *componentconfig.KubeletConfiguration) *api.ConfigMap {
+	kubeCfgExt := v1alpha1.KubeletConfiguration{}
+	api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
+
+	bytes, err := json.Marshal(kubeCfgExt)
+	framework.ExpectNoError(err)
+
+	cmap := &api.ConfigMap{
+		ObjectMeta: api.ObjectMeta{
+			Name: fmt.Sprintf("kubelet-%s", nodeName),
+		},
+		Data: map[string]string{
+			"kubelet.config": string(bytes),
+		},
+	}
+	return cmap
+}
+
+// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
+func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
+	cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
+	cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(cmap)
+	if err != nil {
+		return nil, err
+	}
+	return cmap, nil
+}
+
+// Similar to createConfigMap, except this updates an existing ConfigMap.
+func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
+	cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
+	cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(cmap)
+	if err != nil {
+		return nil, err
+	}
+	return cmap, nil
+}