e2e node framework can generate a base kubelet config file

This allows the e2e node test framework to generate a kubelet config
file containing the defaults it would typically pass to a test via
flags, rather than passing these defaults as flags.

test/e2e_node/jenkins/jenkins-ci-ubuntu.properties

@@ -6,7 +6,8 @@ GCE_ZONE=us-central1-f
 GCE_PROJECT=k8s-jkns-ubuntu-node
 CLEANUP=true
 GINKGO_FLAGS='--skip="\[Flaky\]|\[Serial\]"'
-KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
+TEST_ARGS='--feature-gates=KubeletConfigFile=true --generate-kubelet-config-file=true'
+KUBELET_ARGS=''
 TIMEOUT=1h
 # Use the system spec defined in test/e2e_node/system/specs/gke.yaml.
 SYSTEM_SPEC_NAME=gke

test/e2e_node/jenkins/jenkins-ci.properties

@@ -4,5 +4,6 @@ GCE_ZONE=us-central1-f
 GCE_PROJECT=k8s-jkns-ci-node-e2e
 CLEANUP=true
 GINKGO_FLAGS='--skip="\[Flaky\]|\[Serial\]"'
-KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
+TEST_ARGS='--feature-gates=KubeletConfigFile=true --generate-kubelet-config-file=true'
+KUBELET_ARGS=''
 TIMEOUT=1h

test/e2e_node/jenkins/jenkins-flaky.properties

@@ -4,8 +4,8 @@ GCE_ZONE=us-central1-f
 GCE_PROJECT=k8s-jkns-ci-node-e2e
 CLEANUP=true
 GINKGO_FLAGS='--focus="\[Flaky\]"'
-TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,LocalStorageCapacityIsolation=true,PodPriority=true'
-KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
+TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,LocalStorageCapacityIsolation=true,PodPriority=true,KubeletConfigFile=true --generate-kubelet-config-file=true'
+KUBELET_ARGS=''
 PARALLELISM=1
 TIMEOUT=3h
 

test/e2e_node/jenkins/jenkins-pull.properties

@@ -4,5 +4,6 @@ GCE_ZONE=us-central1-f
 GCE_PROJECT=k8s-jkns-pr-node-e2e
 CLEANUP=true
 GINKGO_FLAGS='--skip="\[Flaky\]|\[Slow\]|\[Serial\]" --flakeAttempts=2'
-KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
+TEST_ARGS='--feature-gates=KubeletConfigFile=true --generate-kubelet-config-file=true'
+KUBELET_ARGS=''
 

test/e2e_node/jenkins/jenkins-serial-ubuntu.properties

@@ -6,8 +6,8 @@ GCE_ZONE=us-central1-f
 GCE_PROJECT=k8s-jkns-ubuntu-node-serial
 CLEANUP=true
 GINKGO_FLAGS='--focus="\[Serial\]" --skip="\[Flaky\]|\[Benchmark\]"'
-TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'
-KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
+TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,KubeletConfigFile=true --generate-kubelet-config-file=true'
+KUBELET_ARGS=''
 PARALLELISM=1
 TIMEOUT=3h
 # Use the system spec defined at test/e2e_node/system/specs/gke.yaml.

test/e2e_node/jenkins/jenkins-serial.properties

@@ -4,7 +4,7 @@ GCE_ZONE=us-west1-b
 GCE_PROJECT=k8s-jkns-ci-node-e2e
 CLEANUP=true
 GINKGO_FLAGS='--focus="\[Serial\]" --skip="\[Flaky\]|\[Benchmark\]"'
-TEST_ARGS='--feature-gates=DynamicKubeletConfig=true'
-KUBELET_ARGS='--cgroups-per-qos=true --cgroup-root=/'
+TEST_ARGS='--feature-gates=DynamicKubeletConfig=true,KubeletConfigFile=true --generate-kubelet-config-file=true'
+KUBELET_ARGS=''
 PARALLELISM=1
 TIMEOUT=3h

test/e2e_node/services/BUILD

@@ -22,9 +22,13 @@ go_library(
     deps = [
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-apiserver/app/options:go_default_library",
+        "//cmd/kubelet/app/options:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/controller/namespace:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e_node/builder:go_default_library",
         "//vendor/github.com/coreos/etcd/etcdserver:go_default_library",
@@ -33,8 +37,12 @@ go_library(
         "//vendor/github.com/coreos/etcd/pkg/types:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/kardianos/osext:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",

test/e2e_node/services/kubelet.go

@@ -25,11 +25,20 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/golang/glog"
+	"github.com/spf13/pflag"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/kubernetes/cmd/kubelet/app/options"
 	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e_node/builder"
 )
@@ -62,11 +71,14 @@ func (a *args) Set(value string) error {
 var kubeletArgs args
 var kubeletContainerized bool
 var hyperkubeImage string
+var genKubeletConfigFile bool
 
 func init() {
 	flag.Var(&kubeletArgs, "kubelet-flags", "Kubelet flags passed to kubelet, this will override default kubelet flags in the test. Flags specified in multiple kubelet-flags will be concatenate.")
 	flag.BoolVar(&kubeletContainerized, "kubelet-containerized", false, "Run kubelet in a docker container")
 	flag.StringVar(&hyperkubeImage, "hyperkube-image", "", "Docker image with containerized kubelet")
+	flag.BoolVar(&genKubeletConfigFile, "generate-kubelet-config-file", false, "The test runner will generate a Kubelet config file containing test defaults instead of passing default flags to the Kubelet. "+
+		"If you use this test framework feature, ensure that the KubeletConfigFile feature gate is enabled.")
 }
 
 // RunKubelet starts kubelet and waits for termination signal. Once receives the
@@ -112,6 +124,12 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		return nil, err
 	}
 
+	// KubeletConfiguration file path
+	kubeletConfigPath, err := kubeletConfigCWDPath()
+	if err != nil {
+		return nil, err
+	}
+
 	// Create pod manifest path
 	manifestPath, err := createPodManifestDirectory()
 	if err != nil {
@@ -122,6 +140,58 @@ func (e *E2EServices) startKubelet() (*server, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// PLEASE NOTE: If you set new KubeletConfiguration values or stop setting values here,
+	// you must also update the flag names in kubeletConfigFlags!
+	kubeletConfigFlags := []string{}
+
+	// set up the default kubeletconfiguration
+	kc, err := options.NewKubeletConfiguration()
+	if err != nil {
+		return nil, err
+	}
+
+	kc.CgroupRoot = "/"
+	kubeletConfigFlags = append(kubeletConfigFlags, "cgroup-root")
+
+	kc.VolumeStatsAggPeriod = metav1.Duration{Duration: 10 * time.Second} // Aggregate volumes frequently so tests don't need to wait as long
+	kubeletConfigFlags = append(kubeletConfigFlags, "volume-stats-agg-period")
+
+	kc.AllowPrivileged = true
+	kubeletConfigFlags = append(kubeletConfigFlags, "allow-privileged")
+
+	kc.SerializeImagePulls = false
+	kubeletConfigFlags = append(kubeletConfigFlags, "serialize-image-pulls")
+
+	kc.PodManifestPath = manifestPath
+	kubeletConfigFlags = append(kubeletConfigFlags, "pod-manifest-path")
+
+	kc.FileCheckFrequency = metav1.Duration{Duration: 10 * time.Second} // Check file frequently so tests won't wait too long
+	kubeletConfigFlags = append(kubeletConfigFlags, "file-check-frequency")
+
+	// Assign a fixed CIDR to the node because there is no node controller.
+	// Note: this MUST be in sync with with the IP in
+	// - cluster/gce/config-test.sh and
+	// - test/e2e_node/conformance/run_test.sh.
+	kc.PodCIDR = "10.100.0.0/24"
+	kubeletConfigFlags = append(kubeletConfigFlags, "pod-cidr")
+
+	kc.EvictionPressureTransitionPeriod = metav1.Duration{Duration: 30 * time.Second}
+	kubeletConfigFlags = append(kubeletConfigFlags, "eviction-pressure-transition-period")
+
+	kc.EvictionHard = map[string]string{
+		"memory.available":  "250Mi",
+		"nodefs.available":  "10%",
+		"nodefs.inodesFree": "5%",
+	}
+	kubeletConfigFlags = append(kubeletConfigFlags, "eviction-hard")
+
+	kc.EvictionMinimumReclaim = map[string]string{
+		"nodefs.available":  "5%",
+		"nodefs.inodesFree": "5%",
+	}
+	kubeletConfigFlags = append(kubeletConfigFlags, "eviction-minimum-reclaim")
+
 	var killCommand, restartCommand *exec.Cmd
 	var isSystemd bool
 	// Apply default kubelet flags.
@@ -151,12 +221,22 @@ func (e *E2EServices) startKubelet() (*server, error) {
 				"-v", "/var/lib/kubelet:/var/lib/kubelet:rw,rslave",
 				"-v", "/var/log:/var/log",
 				"-v", manifestPath+":"+manifestPath+":rw",
-				hyperkubeImage, "/hyperkube", "kubelet",
-				"--containerized",
 			)
+
+			// if we will generate a kubelet config file, we need to mount that path into the container too
+			if genKubeletConfigFile {
+				cmdArgs = append(cmdArgs, "-v", filepath.Dir(kubeletConfigPath)+":"+filepath.Dir(kubeletConfigPath)+":ro")
+			}
+
+			cmdArgs = append(cmdArgs, hyperkubeImage, "/hyperkube", "kubelet", "--containerized")
 			kubeconfigPath = "/etc/kubernetes/kubeconfig"
 		} else {
-			cmdArgs = append(cmdArgs, systemdRun, "--unit="+unitName, "--slice=runtime.slice", "--remain-after-exit", builder.GetKubeletServerBin())
+			cmdArgs = append(cmdArgs,
+				systemdRun,
+				"--unit="+unitName,
+				"--slice=runtime.slice",
+				"--remain-after-exit",
+				builder.GetKubeletServerBin())
 		}
 
 		killCommand = exec.Command("systemctl", "kill", unitName)
@@ -165,41 +245,24 @@ func (e *E2EServices) startKubelet() (*server, error) {
 			Name:              "kubelet.log",
 			JournalctlCommand: []string{"-u", unitName},
 		}
-		cmdArgs = append(cmdArgs,
-			"--kubelet-cgroups=/kubelet.slice",
-			"--cgroup-root=/",
-		)
+
+		kc.KubeletCgroups = "/kubelet.slice"
+		kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups")
 	} else {
 		cmdArgs = append(cmdArgs, builder.GetKubeletServerBin())
-		cmdArgs = append(cmdArgs,
-			// TODO(random-liu): Get rid of this docker specific thing.
-			"--runtime-cgroups=/docker-daemon",
-			"--kubelet-cgroups=/kubelet",
-			"--cgroup-root=/",
-			"--system-cgroups=/system",
-		)
+		// TODO(random-liu): Get rid of this docker specific thing.
+		cmdArgs = append(cmdArgs, "--runtime-cgroups=/docker-daemon")
+
+		kc.KubeletCgroups = "/kubelet"
+		kubeletConfigFlags = append(kubeletConfigFlags, "kubelet-cgroups")
+
+		kc.SystemCgroups = "/system"
+		kubeletConfigFlags = append(kubeletConfigFlags, "system-cgroups")
 	}
 	cmdArgs = append(cmdArgs,
 		"--kubeconfig", kubeconfigPath,
-		"--address", "0.0.0.0",
-		"--port", kubeletPort,
-		"--read-only-port", kubeletReadOnlyPort,
 		"--root-dir", KubeletRootDirectory,
-		"--volume-stats-agg-period", "10s", // Aggregate volumes frequently so tests don't need to wait as long
-		"--allow-privileged", "true",
-		"--serialize-image-pulls", "false",
-		"--pod-manifest-path", manifestPath,
-		"--file-check-frequency", "10s", // Check file frequently so tests won't wait too long
 		"--docker-disable-shared-pid=false",
-		// Assign a fixed CIDR to the node because there is no node controller.
-		//
-		// Note: this MUST be in sync with with the IP in
-		// - cluster/gce/config-test.sh and
-		// - test/e2e_node/conformance/run_test.sh.
-		"--pod-cidr", "10.100.0.0/24",
-		"--eviction-pressure-transition-period", "30s",
-		"--eviction-hard", "memory.available<250Mi,nodefs.available<10%,nodefs.inodesFree<5%", // The hard eviction thresholds.
-		"--eviction-minimum-reclaim", "nodefs.available=5%,nodefs.inodesFree=5%", // The minimum reclaimed resources after eviction.
 		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
 	)
 
@@ -207,6 +270,7 @@ func (e *E2EServices) startKubelet() (*server, error) {
 	// by kubelet-flags.
 	if framework.TestContext.FeatureGates != "" {
 		cmdArgs = append(cmdArgs, "--feature-gates", framework.TestContext.FeatureGates)
+		utilflag.NewMapStringBool(&kc.FeatureGates).Set(framework.TestContext.FeatureGates)
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
@@ -239,6 +303,18 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		cmdArgs = append(cmdArgs, "--hostname-override", framework.TestContext.NodeName)
 	}
 
+	// Write config file or flags, depending on whether --generate-kubelet-config-file was provided
+	if genKubeletConfigFile {
+		if err := writeKubeletConfigFile(kc, kubeletConfigPath); err != nil {
+			return nil, err
+		}
+		// add the flag to load config from a file
+		cmdArgs = append(cmdArgs, "--config", kubeletConfigPath)
+	} else {
+		// generate command line flags from the default config, since --generate-kubelet-config-file was not provided
+		addKubeletConfigFlags(&cmdArgs, kc, kubeletConfigFlags)
+	}
+
 	// Override the default kubelet flags.
 	cmdArgs = append(cmdArgs, kubeletArgs...)
 
@@ -260,6 +336,61 @@ func (e *E2EServices) startKubelet() (*server, error) {
 	return server, server.start()
 }
 
+// addKubeletConfigFlags adds the flags we care about from the provided kubelet configuration object
+func addKubeletConfigFlags(cmdArgs *[]string, kc *kubeletconfig.KubeletConfiguration, flags []string) {
+	fs := pflag.NewFlagSet("kubelet", pflag.ExitOnError)
+	options.AddKubeletConfigFlags(fs, kc)
+	for _, name := range flags {
+		*cmdArgs = append(*cmdArgs, "--"+name, fs.Lookup(name).Value.String())
+	}
+}
+
+// writeKubeletConfigFile writes the kubelet config file based on the args and returns the filename
+func writeKubeletConfigFile(internal *kubeletconfig.KubeletConfiguration, path string) error {
+	// extract the KubeletConfiguration and convert to versioned
+	versioned := &v1alpha1.KubeletConfiguration{}
+	scheme, _, err := scheme.NewSchemeAndCodecs()
+	if err != nil {
+		return err
+	}
+	if err := scheme.Convert(internal, versioned, nil); err != nil {
+		return err
+	}
+	// encode
+	encoder, err := newKubeletConfigJSONEncoder()
+	if err != nil {
+		return err
+	}
+	data, err := runtime.Encode(encoder, versioned)
+	if err != nil {
+		return err
+	}
+	// create the directory, if it does not exist
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	// write the file
+	if err := ioutil.WriteFile(path, data, 0755); err != nil {
+		return err
+	}
+	return nil
+}
+
+func newKubeletConfigJSONEncoder() (runtime.Encoder, error) {
+	_, kubeletCodecs, err := scheme.NewSchemeAndCodecs()
+	if err != nil {
+		return nil, err
+	}
+
+	mediaType := "application/json"
+	info, ok := runtime.SerializerInfoForMediaType(kubeletCodecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return nil, fmt.Errorf("unsupported media type %q", mediaType)
+	}
+	return kubeletCodecs.EncoderForVersion(info.Serializer, v1alpha1.SchemeGroupVersion), nil
+}
+
 // createPodManifestDirectory creates pod manifest directory.
 func createPodManifestDirectory() (string, error) {
 	cwd, err := os.Getwd()
@@ -316,6 +447,15 @@ func kubeconfigCWDPath() (string, error) {
 	return filepath.Join(cwd, "kubeconfig"), nil
 }
 
+func kubeletConfigCWDPath() (string, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("failed to get current working directory: %v", err)
+	}
+	// DO NOT name this file "kubelet" - you will overwrite the the kubelet binary and be very confused :)
+	return filepath.Join(cwd, "kubelet-config"), nil
+}
+
 // like createKubeconfig, but creates kubeconfig at current-working-directory/kubeconfig
 // returns a fully-qualified path to the kubeconfig file
 func createKubeconfigCWD() (string, error) {