github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #32530 from mtaufen/dynamic-settings-tests 

Automatic merge from submit-queue

Utility functions for using dynamic Kubelet configuration from a test

/cc @vishh @dchen1107
pull/6/head
Kubernetes Submit Queue 2016-11-04 20:24:03 -07:00 committed by GitHub
parent 8a2c639bfb 5190a7d72d
commit 56526043d5
3 changed files with 180 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
test/e2e_node/BUILD
View File

@ -27,6 +27,8 @@ go_library(
"//pkg/api:go_default_library",
"//pkg/api/errors:go_default_library",
"//pkg/api/unversioned:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/dockertools:go_default_library",
@ -77,8 +79,6 @@ go_test(
"//pkg/api/errors:go_default_library",
"//pkg/api/resource:go_default_library",
"//pkg/api/unversioned:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/client/cache:go_default_library",
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/kubelet/api/v1alpha1/stats:go_default_library",

147
test/e2e_node/dynamic_kubelet_configuration_test.go
View File

@ -17,20 +17,13 @@ limitations under the License.
package e2e_node
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
// This test is marked [Disruptive] because the Kubelet temporarily goes down as part of of this test.
@ -39,14 +32,9 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube
Context("When a configmap called `kubelet-<node-name>` is added to the `kube-system` namespace", func() {
It("The Kubelet on that node should restart to take up the new config", func() {
const (
restartGap = 40 * time.Second
)
// Get the current KubeletConfiguration (known to be valid) by
// querying the configz endpoint for the current node.
resp := pollConfigz(2*time.Minute, 5*time.Second)
kubeCfg, err := decodeConfigz(resp)
kubeCfg, err := getCurrentKubeletConfig()
framework.ExpectNoError(err)
glog.Infof("KubeletConfiguration - Initial values: %+v", *kubeCfg)
@ -60,138 +48,15 @@ var _ = framework.KubeDescribe("DynamicKubeletConfiguration [Feature:DynamicKube
kubeCfg.FileCheckFrequency.Duration = newFileCheckFrequency
// Use the new config to create a new kube-<node-name> configmap in `kube-system` namespace.
_, err = createConfigMap(f, kubeCfg)
// Note: setKubeletConfiguration will return an error if the Kubelet does not present the
// modified configuration via /configz when it comes back up.
err = setKubeletConfiguration(f, kubeCfg)
framework.ExpectNoError(err)
// Give the Kubelet time to see that there is new config and restart. If we don't do this,
// the Kubelet will still have the old config when we poll, and the test will fail.
time.Sleep(restartGap)
// Use configz to get the new config.
resp = pollConfigz(2*time.Minute, 5*time.Second)
kubeCfg, err = decodeConfigz(resp)
framework.ExpectNoError(err)
glog.Infof("KubeletConfiguration - After modification of FileCheckFrequency: %+v", *kubeCfg)
// We expect to see the new value in the new config.
Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(newFileCheckFrequency))
// Change the config back to what it originally was.
kubeCfg.FileCheckFrequency.Duration = oldFileCheckFrequency
_, err = updateConfigMap(f, kubeCfg)
err = setKubeletConfiguration(f, kubeCfg)
framework.ExpectNoError(err)
// Give the Kubelet time to see that there is new config and restart. If we don't do this,
// the Kubelet will still have the old config when we poll, and the test will fail.
time.Sleep(restartGap)
// User configz to get the new config.
resp = pollConfigz(2*time.Minute, 5*time.Second)
kubeCfg, err = decodeConfigz(resp)
framework.ExpectNoError(err)
glog.Infof("KubeletConfiguration - After restoration of FileCheckFrequency: %+v", *kubeCfg)
// We expect to see the original value restored in the new config.
Expect(kubeCfg.FileCheckFrequency.Duration).To(Equal(oldFileCheckFrequency))
})
})
})
// This function either causes the test to fail, or it returns a status 200 response.
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
client := &http.Client{}
req, err := http.NewRequest("GET", endpoint, nil)
framework.ExpectNoError(err)
req.Header.Add("Accept", "application/json")
var resp *http.Response
Eventually(func() bool {
resp, err = client.Do(req)
if err != nil {
glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
return false
}
if resp.StatusCode != 200 {
glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
return false
}
return true
}, timeout, pollInterval).Should(Equal(true))
return resp
}
// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type).
func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) {
// This hack because /configz reports the following structure:
// {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}}
type configzWrapper struct {
ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"`
}
configz := configzWrapper{}
kubeCfg := componentconfig.KubeletConfiguration{}
contentsBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(contentsBytes, &configz)
if err != nil {
return nil, err
}
api.Scheme.Default(&configz.ComponentConfig)
err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
if err != nil {
return nil, err
}
return &kubeCfg, nil
}
// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
kubeCfgExt := v1alpha1.KubeletConfiguration{}
api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
bytes, err := json.Marshal(kubeCfgExt)
framework.ExpectNoError(err)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(&api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName),
},
Data: map[string]string{
"kubelet.config": string(bytes),
},
})
if err != nil {
return nil, err
}
return cmap, nil
}
// Similar to createConfigMap, except this updates an existing ConfigMap.
func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
kubeCfgExt := v1alpha1.KubeletConfiguration{}
api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
bytes, err := json.Marshal(kubeCfgExt)
framework.ExpectNoError(err)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(&api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName),
},
Data: map[string]string{
"kubelet.config": string(bytes),
},
})
if err != nil {
return nil, err
}
return cmap, nil
}

172
test/e2e_node/util.go
View File

@ -22,9 +22,21 @@ import (
"fmt"
"io/ioutil"
"net/http"
"reflect"
"strings"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
k8serr "k8s.io/kubernetes/pkg/api/errors"
"k8s.io/kubernetes/pkg/apis/componentconfig"
v1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
// utilconfig "k8s.io/kubernetes/pkg/util/config"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/gomega"
)
// TODO(random-liu): Get this automatically from kubelet flag.
@ -60,3 +72,163 @@ func getNodeSummary() (*stats.Summary, error) {
}
return &summary, nil
}
// Returns the current KubeletConfiguration
func getCurrentKubeletConfig() (*componentconfig.KubeletConfiguration, error) {
resp := pollConfigz(5*time.Minute, 5*time.Second)
kubeCfg, err := decodeConfigz(resp)
if err != nil {
return nil, err
}
return kubeCfg, nil
}
// Queries the API server for a Kubelet configuration for the node described by framework.TestContext.NodeName
func getCurrentKubeletConfigMap(f *framework.Framework) (*api.ConfigMap, error) {
return f.ClientSet.Core().ConfigMaps("kube-system").Get(fmt.Sprintf("kubelet-%s", framework.TestContext.NodeName))
}
// Creates or updates the configmap for KubeletConfiguration, waits for the Kubelet to restart
// with the new configuration. Returns an error if the configuration after waiting 40 seconds
// doesn't match what you attempted to set, or if the dynamic configuration feature is disabled.
func setKubeletConfiguration(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) error {
const (
restartGap = 30 * time.Second
)
// Make sure Dynamic Kubelet Configuration feature is enabled on the Kubelet we are about to reconfigure
cfgz, err := getCurrentKubeletConfig()
if err != nil {
return fmt.Errorf("could not determine whether 'DynamicKubeletConfig' feature is enabled, err: %v", err)
}
if !strings.Contains(cfgz.FeatureGates, "DynamicKubeletConfig=true") {
return fmt.Errorf("The Dynamic Kubelet Configuration feature is not enabled.\n" +
"Pass --feature-gates=DynamicKubeletConfig=true to the Kubelet to enable this feature.\n" +
"For `make test-e2e-node`, you can set `TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'`.")
}
// Check whether a configmap for KubeletConfiguration already exists
_, err = getCurrentKubeletConfigMap(f)
if k8serr.IsNotFound(err) {
_, err := createConfigMap(f, kubeCfg)
if err != nil {
return err
}
} else if err != nil {
return err
} else {
// The configmap exists, update it instead of creating it.
_, err := updateConfigMap(f, kubeCfg)
if err != nil {
return err
}
}
// Wait for the Kubelet to restart.
time.Sleep(restartGap)
// Retrieve the new config and compare it to the one we attempted to set
newKubeCfg, err := getCurrentKubeletConfig()
if err != nil {
return err
}
// Return an error if the desired config is not in use by now
if !reflect.DeepEqual(*kubeCfg, *newKubeCfg) {
return fmt.Errorf("either the Kubelet did not restart or it did not present the modified configuration via /configz after restarting.")
}
return nil
}
// Causes the test to fail, or returns a status 200 response from the /configz endpoint
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/proxy/nodes/%s/configz", framework.TestContext.NodeName)
client := &http.Client{}
req, err := http.NewRequest("GET", endpoint, nil)
framework.ExpectNoError(err)
req.Header.Add("Accept", "application/json")
var resp *http.Response
Eventually(func() bool {
resp, err = client.Do(req)
if err != nil {
glog.Errorf("Failed to get /configz, retrying. Error: %v", err)
return false
}
if resp.StatusCode != 200 {
glog.Errorf("/configz response status not 200, retrying. Response was: %+v", resp)
return false
}
return true
}, timeout, pollInterval).Should(Equal(true))
return resp
}
// Decodes the http response from /configz and returns a componentconfig.KubeletConfiguration (internal type).
func decodeConfigz(resp *http.Response) (*componentconfig.KubeletConfiguration, error) {
// This hack because /configz reports the following structure:
// {"componentconfig": {the JSON representation of v1alpha1.KubeletConfiguration}}
type configzWrapper struct {
ComponentConfig v1alpha1.KubeletConfiguration `json:"componentconfig"`
}
configz := configzWrapper{}
kubeCfg := componentconfig.KubeletConfiguration{}
contentsBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = json.Unmarshal(contentsBytes, &configz)
if err != nil {
return nil, err
}
err = api.Scheme.Convert(&configz.ComponentConfig, &kubeCfg, nil)
if err != nil {
return nil, err
}
return &kubeCfg, nil
}
// Constructs a Kubelet ConfigMap targeting the current node running the node e2e tests
func makeKubeletConfigMap(nodeName string, kubeCfg *componentconfig.KubeletConfiguration) *api.ConfigMap {
kubeCfgExt := v1alpha1.KubeletConfiguration{}
api.Scheme.Convert(kubeCfg, &kubeCfgExt, nil)
bytes, err := json.Marshal(kubeCfgExt)
framework.ExpectNoError(err)
cmap := &api.ConfigMap{
ObjectMeta: api.ObjectMeta{
Name: fmt.Sprintf("kubelet-%s", nodeName),
},
Data: map[string]string{
"kubelet.config": string(bytes),
},
}
return cmap
}
// Uses KubeletConfiguration to create a `kubelet-<node-name>` ConfigMap in the "kube-system" namespace.
func createConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Create(cmap)
if err != nil {
return nil, err
}
return cmap, nil
}
// Similar to createConfigMap, except this updates an existing ConfigMap.
func updateConfigMap(f *framework.Framework, kubeCfg *componentconfig.KubeletConfiguration) (*api.ConfigMap, error) {
cmap := makeKubeletConfigMap(framework.TestContext.NodeName, kubeCfg)
cmap, err := f.ClientSet.Core().ConfigMaps("kube-system").Update(cmap)
if err != nil {
return nil, err
}
return cmap, nil
}