From a40b2cbe10528eefd85427bc6930f0d411fe9db0 Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Fri, 26 Aug 2016 14:04:21 -0700 Subject: [PATCH 1/2] feature-gates flag plumbing for node e2e tests This gives the node e2e test binary a --feature-gates flag that populates a FeatureGates field on the test context. The value of this field is forwarded to the kubelet's --feature-gates flag and is also used to populate the global DefaultFeatureGate object so that statically-linked components see the same feature gate settings as provided via the flag. This means that you can set feature gates via the TEST_ARGS environment variable when running node e2e tests. For example: TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' --- test/e2e/framework/test_context.go | 3 +++ test/e2e_node/services/services.go | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 448e6b858b..c117f5ff7d 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -69,6 +69,8 @@ type TestContextType struct { DumpLogsOnFailure bool // If the garbage collector is enabled in the kube-apiserver and kube-controller-manager. GarbageCollectorEnabled bool + // FeatureGates is a set of key=value pairs that describe feature gates for alpha/experimental features. + FeatureGates string // Node e2e specific test context NodeTestContextType } @@ -123,6 +125,7 @@ func RegisterCommonFlags() { flag.StringVar(&TestContext.Host, "host", "http://127.0.0.1:8080", "The host, or apiserver, to connect to") flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.") flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") + flag.StringVar(&TestContext.FeatureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") } // Register flags specific to the cluster e2e test suite. diff --git a/test/e2e_node/services/services.go b/test/e2e_node/services/services.go index 1b3eb07ab2..038d1b5d28 100644 --- a/test/e2e_node/services/services.go +++ b/test/e2e_node/services/services.go @@ -36,6 +36,7 @@ import ( "github.com/golang/glog" "github.com/kardianos/osext" + utilconfig "k8s.io/kubernetes/pkg/util/config" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/build" ) @@ -76,6 +77,10 @@ func (e *E2EServices) Start() error { // TODO(random-liu): Add sudo after we statically link apiserver and etcd, because apiserver needs // sudo. We can't add sudo now, because etcd may not be in PATH of root. startCmd := exec.Command(testBin, + // TODO(mtaufen): Flags e.g. that target the TestContext need to be manually forwarded to the + // test binary when we start it up in run-services mode. This is not ideal. + // Very unintuitive because it prevents any falgs NOT manually forwarded here + // from being set via TEST_ARGS when running tests from the command line. "--run-services-mode", "--server-start-timeout", serverStartTimeout.String(), "--report-dir", framework.TestContext.ReportDir, @@ -87,6 +92,7 @@ func (e *E2EServices) Start() error { // "--cgroups-per-qos="+strconv.FormatBool(framework.TestContext.CgroupsPerQOS), "--manifest-path", framework.TestContext.ManifestPath, "--eviction-hard", framework.TestContext.EvictionHard, + "--feature-gates", framework.TestContext.FeatureGates, "--logtostderr", ) e.services = newServer("services", startCmd, nil, nil, getHealthCheckURLs(), servicesLogFile, false) @@ -114,6 +120,9 @@ func (e *E2EServices) Stop() error { // RunE2EServices actually start the e2e services. This function is used to // start e2e services in current process. This is only used in run-services-mode. func RunE2EServices() { + // Populate global DefaultFeatureGate with value from TestContext.FeatureGates. + // This way, statically-linked components see the same feature gate config as the test context. + utilconfig.DefaultFeatureGate.Set(framework.TestContext.FeatureGates) e := newE2EService() if err := e.run(); err != nil { glog.Fatalf("Failed to run e2e services: %v", err) @@ -375,7 +384,7 @@ func (es *e2eService) startKubeletServer() (*server, error) { "--pod-cidr=10.180.0.0/24", // Assign a fixed CIDR to the node because there is no node controller. "--eviction-hard", framework.TestContext.EvictionHard, "--eviction-pressure-transition-period", "30s", - "--feature-gates", "DynamicKubeletConfig=true", // TODO(mtaufen): Eventually replace with a value from the framework.TestContext + "--feature-gates", framework.TestContext.FeatureGates, ) if framework.TestContext.CgroupsPerQOS { // TODO: enable this when the flag is stable and available in kubelet. From af0a0c636738a41706cb74575ab16c1b1313d73b Mon Sep 17 00:00:00 2001 From: Michael Taufen Date: Wed, 31 Aug 2016 13:47:55 -0700 Subject: [PATCH 2/2] Enable dynamic kubelet configuration for node e2e Jenkins serial tests This commit enables the dynamic kubelet configuration feature for the node e2e Jenkins serial tests, which is where the test for dynamic kubelet configuration currently runs. --- test/e2e_node/jenkins/jenkins-serial.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e_node/jenkins/jenkins-serial.properties b/test/e2e_node/jenkins/jenkins-serial.properties index 7e45c1a28a..cb79454eeb 100644 --- a/test/e2e_node/jenkins/jenkins-serial.properties +++ b/test/e2e_node/jenkins/jenkins-serial.properties @@ -7,6 +7,6 @@ GINKGO_FLAGS='--focus="\[Serial\]" --skip="\[Flaky\]|\[Benchmark\]"' SETUP_NODE=false # DISABLED --cgroups-per-qos flag until feature stabilized. #TEST_ARGS=--cgroups-per-qos=false -TEST_ARGS= +TEST_ARGS='--feature-gates=DynamicKubeletConfig=true' PARALLELISM=1 TIMEOUT=3h