Merge pull request #47983 from yguo0905/memcg

Automatic merge from submit-queue (batch tested with PRs 48092, 47894, 47983)

Enables memcg notification in cluster/node e2e tests

Ref: https://github.com/kubernetes/kubernetes/issues/42676

This PR sets Kubelet flag `--experimental-kernel-memcg-notification=true` when running cluster/node e2e tests on COS and Ubuntu images.

Tested:
```
e2e-node-cos:
I0623 00:09:06.641776    1080 server.go:147] Starting server "kubelet" with command "/usr/bin/systemd-run --unit=kubelet-777178888.service --slice=runtime.slice --remain-after-exit /tmp/node-e2e-20170622T170739/kubelet --kubelet-cgroups=/kubelet.slice --cgroup-root=/ --api-servers http://localhost:8080 --address 0.0.0.0 --port 10250 --read-only-port 10255 --volume-stats-agg-period 10s --allow-privileged true --serialize-image-pulls false --pod-manifest-path /tmp/node-e2e-20170622T170739/pod-manifest571288056 --file-check-frequency 10s --pod-cidr 10.100.0.0/24 --eviction-pressure-transition-period 30s --feature-gates  --eviction-hard memory.available<250Mi,nodefs.available<10%%,nodefs.inodesFree<5%% --eviction-minimum-reclaim nodefs.available=5%%,nodefs.inodesFree=5%% --v 4 --logtostderr --network-plugin=kubenet --cni-bin-dir /tmp/node-e2e-20170622T170739/cni/bin --cni-conf-dir /tmp/node-e2e-20170622T170739/cni/net.d --hostname-override tmp-node-e2e-bfe5799d-cos-stable-59-9460-64-0 --experimental-mounter-path=/tmp/node-e2e-20170622T170739/cluster/gce/gci/mounter/mounter --experimental-kernel-memcg-notification=true"

e2e-node-ubuntu:
I0623 00:03:28.526984    2279 server.go:147] Starting server "kubelet" with command "/usr/bin/systemd-run --unit=kubelet-1407651753.service --slice=runtime.slice --remain-after-exit /tmp/node-e2e-20170622T170203/kubelet --kubelet-cgroups=/kubelet.slice --cgroup-root=/ --api-servers http://localhost:8080 --address 0.0.0.0 --port 10250 --read-only-port 10255 --volume-stats-agg-period 10s --allow-privileged true --serialize-image-pulls false --pod-manifest-path /tmp/node-e2e-20170622T170203/pod-manifest083943734 --file-check-frequency 10s --pod-cidr 10.100.0.0/24 --eviction-pressure-transition-period 30s --feature-gates  --eviction-hard memory.available<250Mi,nodefs.available<10%%,nodefs.inodesFree<5%% --eviction-minimum-reclaim nodefs.available=5%%,nodefs.inodesFree=5%% --v 4 --logtostderr --network-plugin=kubenet --cni-bin-dir /tmp/node-e2e-20170622T170203/cni/bin --cni-conf-dir /tmp/node-e2e-20170622T170203/cni/net.d --hostname-override tmp-node-e2e-e48cdd73-ubuntu-gke-1604-xenial-v20170420-1 --experimental-kernel-memcg-notification=true"

e2e-node-containervm:
I0623 00:14:35.392383    2774 server.go:147] Starting server "kubelet" with command "/tmp/node-e2e-20170622T171318/kubelet --runtime-cgroups=/docker-daemon --kubelet-cgroups=/kubelet --cgroup-root=/ --system-cgroups=/system --api-servers http://localhost:8080 --address 0.0.0.0 --port 10250 --read-only-port 10255 --volume-stats-agg-period 10s --allow-privileged true --serialize-image-pulls false --pod-manifest-path /tmp/node-e2e-20170622T171318/pod-manifest507536807 --file-check-frequency 10s --pod-cidr 10.100.0.0/24 --eviction-pressure-transition-period 30s --feature-gates  --eviction-hard memory.available<250Mi,nodefs.available<10%,nodefs.inodesFree<5% --eviction-minimum-reclaim nodefs.available=5%,nodefs.inodesFree=5% --v 4 --logtostderr --network-plugin=kubenet --cni-bin-dir /tmp/node-e2e-20170622T171318/cni/bin --cni-conf-dir /tmp/node-e2e-20170622T171318/cni/net.d --hostname-override tmp-node-e2e-9e3fdd7c-e2e-node-containervm-v20161208-image"

e2e-cos:
Jun 23 17:54:38 e2e-test-ygg-minion-group-t5r0 kubelet[2005]: I0623 17:54:38.646374    2005 flags.go:52] FLAG: --experimental-kernel-memcg-notification="true"

e2e-ubuntu:
Jun 23 18:25:27 e2e-test-ygg-minion-group-19qp kubelet[1547]: I0623 18:25:27.722253    1547 flags.go:52] FLAG: --experimental-kernel-memcg-notification="true"

e2e-containervm:
I0623 18:55:51.886632    3385 flags.go:52] FLAG: --experimental-kernel-memcg-notification="false"
```

**Release note**:
```
None
```

/sig node
/area node-e2e
/assign @dchen1107 @dashpole

cluster/common.sh

@@ -696,6 +696,16 @@ EOF
   if [ -n "${KUBELET_TEST_ARGS:-}" ]; then
       cat >>$file <<EOF
 KUBELET_TEST_ARGS: $(yaml-quote ${KUBELET_TEST_ARGS})
+EOF
+  fi
+  if [ -n "${NODE_KUBELET_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+NODE_KUBELET_TEST_ARGS: $(yaml-quote ${NODE_KUBELET_TEST_ARGS})
+EOF
+  fi
+  if [ -n "${MASTER_KUBELET_TEST_ARGS:-}" ]; then
+      cat >>$file <<EOF
+MASTER_KUBELET_TEST_ARGS: $(yaml-quote ${MASTER_KUBELET_TEST_ARGS})
 EOF
   fi
   if [ -n "${KUBELET_TEST_LOG_LEVEL:-}" ]; then

cluster/gce/config-test.sh

@@ -150,10 +150,10 @@ TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m
 TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
 
 KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false --outofdisk-transition-frequency=0 ${TEST_CLUSTER_API_CONTENT_TYPE}"
-if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then
+if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
   NODE_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
 fi
-if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then
+if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then
   MASTER_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
 fi
 APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}"

cluster/gce/gci/configure-helper.sh

@@ -880,7 +880,7 @@ function start-kubelet {
     flags+=" --port=${KUBELET_PORT}"
   fi
   if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
-    flags+="${MASTER_KUBELET_TEST_ARGS:-}"
+    flags+=" ${MASTER_KUBELET_TEST_ARGS:-}"
     flags+=" --enable-debugging-handlers=false"
     flags+=" --hairpin-mode=none"
     if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then
@@ -895,7 +895,7 @@ function start-kubelet {
       flags+=" --pod-cidr=${MASTER_IP_RANGE}"
     fi
   else # For nodes
-    flags+="${NODE_KUBELET_TEST_ARGS:-}"
+    flags+=" ${NODE_KUBELET_TEST_ARGS:-}"
     flags+=" --enable-debugging-handlers=true"
     flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig"
     flags+=" --require-kubeconfig"

test/e2e_node/remote/node_e2e.go

@@ -108,19 +108,9 @@ func tarAddCOSMounter(tar string) error {
 	return nil
 }
 
-// updateCOSKubeletFlags updates kubelet flags to set gci mounter path, and enables memcg notifications. This will only take effect for
-// GCI/COS image.
-func updateCOSKubeletFlags(args, host, workspace string) (string, error) {
-	// Determine if tests will run on a GCI/COS node.
-	output, err := SSH(host, "cat", "/etc/os-release")
-	if err != nil {
-		return args, fmt.Errorf("issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
-	}
-	if !strings.Contains(output, "ID=gci") && !strings.Contains(output, "ID=cos") {
-		// This is not a GCI/COS image
-		return args, nil
-	}
-
+// prependCOSMounterFlag prepends the flag for setting the GCI mounter path to
+// args and returns the result.
+func prependCOSMounterFlag(args, host, workspace string) (string, error) {
 	// If we are testing on a GCI/COS node, we chmod 544 the mounter and specify a different mounter path in the test args.
 	// We do this here because the local var `workspace` tells us which /tmp/node-e2e-%d is relevant to the current test run.
 
@@ -140,16 +130,38 @@ func updateCOSKubeletFlags(args, host, workspace string) (string, error) {
 	// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
 	// here will tell us to update the remote test runner.
 	mounterPath := filepath.Join(workspace, localCOSMounterPath)
-	output, err = SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
+	output, err := SSH(host, "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
 	if err != nil {
 		return args, fmt.Errorf("unabled to chmod 544 GCI/COS mounter script. Err: %v, Output:\n%s", err, output)
 	}
 	// Insert args at beginning of test args, so any values from command line take precedence
-	args = "--kubelet-flags=--experimental-kernel-memcg-notification=true " + args
 	args = fmt.Sprintf("--kubelet-flags=--experimental-mounter-path=%s ", mounterPath) + args
 	return args, nil
 }
 
+// prependMemcgNotificationFlag prepends the flag for enabling memcg
+// notification to args and returns the result.
+func prependMemcgNotificationFlag(args string) string {
+	return "--kubelet-flags=--experimental-kernel-memcg-notification=true " + args
+}
+
+// updateOSSpecificKubeletFlags updates the Kubelet args with OS specific
+// settings.
+func updateOSSpecificKubeletFlags(args, host, workspace string) (string, error) {
+	output, err := SSH(host, "cat", "/etc/os-release")
+	if err != nil {
+		return "", fmt.Errorf("issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
+	}
+	switch {
+	case strings.Contains(output, "ID=gci"), strings.Contains(output, "ID=cos"):
+		args = prependMemcgNotificationFlag(args)
+		return prependCOSMounterFlag(args, host, workspace)
+	case strings.Contains(output, "ID=ubuntu"):
+		return prependMemcgNotificationFlag(args), nil
+	}
+	return args, nil
+}
+
 // RunTest runs test on the node.
 func (n *NodeE2ERemote) RunTest(host, workspace, results, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) {
 	// Install the cni plugins and add a basic CNI configuration.
@@ -165,7 +177,7 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, junitFilePrefix, testA
 	// Kill any running node processes
 	cleanupNodeProcesses(host)
 
-	testArgs, err := updateCOSKubeletFlags(testArgs, host, workspace)
+	testArgs, err := updateOSSpecificKubeletFlags(testArgs, host, workspace)
 	if err != nil {
 		return "", err
 	}