Merge pull request #35161 from mtaufen/mike-klet-cmount-node-e2e

Automatic merge from submit-queue

e2e node plumbing and bundling for GCI mounter

**Note:** The code in this PR only bundles the mounter and modifies `--mounter-path` if it can find `cluster/gce/gci/mounter` in the K8s source dir when building the test bundle.

This bundles the mounter script for GCI with the node e2e tests and allows the `--mounter-path` to be passed to the Kubelet via the node test framework. The node test runner will detect when we are running on a remote GCI node and add the appropriate `--mounter-path` to the `testArgs`. 

It also includes a simple node test that mounts a tmpfs volume. This will exercise the Kubelet's mounter code path. 

**ITEM OF NOTE:** To get the k8s root dir (in order to copy the mount script into the tarball), I changed `getK8sRootDir` -> `GetK8sRootDir` in `test/e2e_node/build/build.go`. Based on the comment above that function (and the fact that it was private to begin with), I'm not sure this is the best way to do things:
```
// TODO: Dedup / merge this with comparable utilities in e2e/util.go
```
On the other hand, the `e2e/util.go` file mentioned in that comment doesn't exist anymore. This should be resolved before this PR is merged.

build/lib/release.sh

@@ -320,6 +320,7 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${dst_dir}/"
   cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
+  cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
   cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
   cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
   local objects

cluster/gce/gci/configure.sh

@@ -99,7 +99,7 @@ function split-commas {
 }
 
 # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
-# and places them into suitable directories. Files are placed in /home/kubernetes. 
+# and places them into suitable directories. Files are placed in /home/kubernetes.
 function install-kube-binary-config {
   cd "${KUBE_HOME}"
   local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") )
@@ -171,6 +171,7 @@ function install-kube-binary-config {
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
   cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
+  cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
   cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
   chmod -R 755 "${kube_bin}"
 

cluster/gce/gci/master.yaml

@@ -34,6 +34,7 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

cluster/gce/gci/mounter/mounter

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -o pipefail
+
+sudo /bin/mount "$@"

cluster/gce/gci/node.yaml

@@ -34,6 +34,7 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

pkg/util/mount/mount_linux.go

@@ -101,15 +101,15 @@ func isBind(options []string) (bool, []string) {
 
 // doMount runs the mount command.
 func doMount(mountCmd string, source string, target string, fstype string, options []string) error {
-	glog.V(5).Infof("Mounting %s %s %s %v", source, target, fstype, options)
+	glog.V(5).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd)
 	mountArgs := makeMountArgs(source, target, fstype, options)
 
 	command := exec.Command(mountCmd, mountArgs...)
 	output, err := command.CombinedOutput()
 	if err != nil {
-		glog.Errorf("Mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n", err, source, target, fstype, options, string(output))
-		return fmt.Errorf("mount failed: %v\nMounting arguments: %s %s %s %v\nOutput: %s\n",
-			err, source, target, fstype, options, string(output))
+		glog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s %s %s %v\nOutput: %s\n", err, mountCmd, source, target, fstype, options, string(output))
+		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s %s %s %v\nOutput: %s\n",
+			err, mountCmd, source, target, fstype, options, string(output))
 	}
 	return err
 }

test/e2e/framework/test_context.go

@@ -114,6 +114,8 @@ type NodeTestContextType struct {
 	PrepullImages bool
 	// RuntimeIntegrationType indicates how runtime is integrated with Kubelet. This is mainly used for CRI validation test.
 	RuntimeIntegrationType string
+	// MounterPath is the path to the program to run to perform a mount
+	MounterPath string
 }
 
 type CloudConfig struct {
@@ -209,6 +211,7 @@ func RegisterNodeFlags() {
 	flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.")
 	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
 	flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.")
+	flag.StringVar(&TestContext.MounterPath, "mounter-path", "", "Path of mounter binary. Leave empty to use the default mount.")
 }
 
 // overwriteFlagsWithViperConfig finds and writes values to flags using viper as input.

test/e2e_node/build/build.go

@@ -38,7 +38,7 @@ var buildTargets = []string{
 
 func BuildGo() error {
 	glog.Infof("Building k8s binaries...")
-	k8sRoot, err := getK8sRootDir()
+	k8sRoot, err := GetK8sRootDir()
 	if err != nil {
 		return fmt.Errorf("failed to locate kubernetes root directory %v.", err)
 	}
@@ -87,7 +87,7 @@ func getK8sBin(bin string) (string, error) {
 }
 
 // TODO: Dedup / merge this with comparable utilities in e2e/util.go
-func getK8sRootDir() (string, error) {
+func GetK8sRootDir() (string, error) {
 	// Get the directory of the current executable
 	_, testExec, _, _ := runtime.Caller(0)
 	path := filepath.Dir(testExec)
@@ -102,7 +102,7 @@ func getK8sRootDir() (string, error) {
 }
 
 func GetK8sBuildOutputDir() (string, error) {
-	k8sRoot, err := getK8sRootDir()
+	k8sRoot, err := GetK8sRootDir()
 	if err != nil {
 		return "", err
 	}

test/e2e_node/remote/remote.go

@@ -113,8 +113,34 @@ func CreateTestArchive() (string, error) {
 		}
 	}
 
+	// Include the GCI mounter in the deployed tarball
+	k8sDir, err := build.GetK8sRootDir()
+	if err != nil {
+		return "", fmt.Errorf("Could not find K8s root dir! Err: %v", err)
+	}
+	localSource := "cluster/gce/gci/mounter/mounter"
+	source := filepath.Join(k8sDir, localSource)
+
+	// Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves
+	if _, err := os.Stat(source); err != nil {
+		return "", fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err)
+	}
+
+	bindir := "cluster/gce/gci/mounter"
+	bin := "mounter"
+	destdir := filepath.Join(tardir, bindir)
+	dest := filepath.Join(destdir, bin)
+	out, err := exec.Command("mkdir", "-p", filepath.Join(tardir, bindir)).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to create directory %q for GCI mounter script. Err: %v. Output:\n%s", destdir, err, out)
+	}
+	out, err = exec.Command("cp", source, dest).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to copy GCI mounter script to the archive bin. Err: %v. Output:\n%s", err, out)
+	}
+
 	// Build the tar
-	out, err := exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput()
+	out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput()
 	if err != nil {
 		return "", fmt.Errorf("failed to build tar %v.  Output:\n%s", err, out)
 	}
@@ -213,6 +239,44 @@ func RunRemote(archive string, host string, cleanup bool, junitFilePrefix string
 		// Exit failure with the error
 		return "", false, err
 	}
+
+	// If we are testing on a GCI node, we chmod 544 the mounter and specify a different mounter path in the test args.
+	// We do this here because the local var `tmp` tells us which /tmp/gcloud-e2e-%d is relevant to the current test run.
+
+	// Determine if the GCI mounter script exists locally.
+	k8sDir, err := build.GetK8sRootDir()
+	if err != nil {
+		return "", false, fmt.Errorf("Could not find K8s root dir! Err: %v", err)
+	}
+	localSource := "cluster/gce/gci/mounter/mounter"
+	source := filepath.Join(k8sDir, localSource)
+
+	// Require the GCI mounter script, we want to make sure the remote test runner stays up to date if the mounter file moves
+	if _, err = os.Stat(source); err != nil {
+		return "", false, fmt.Errorf("Could not find GCI mounter script at %q! If this script has been (re)moved, please update the e2e node remote test runner accordingly! Err: %v", source, err)
+	}
+
+	// Determine if tests will run on a GCI node.
+	output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", "'cat /etc/os-release'")
+	if err != nil {
+		glog.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
+		return "", false, fmt.Errorf("Issue detecting node's OS via node's /etc/os-release. Err: %v, Output:\n%s", err, output)
+	}
+	if strings.Contains(output, "ID=gci") {
+		glog.Infof("GCI node and GCI mounter both detected, modifying --mounter-path accordingly")
+
+		// Note this implicitly requires the script to be where we expect in the tarball, so if that location changes the error
+		// here will tell us to update the remote test runner.
+		mounterPath := filepath.Join(tmp, "cluster/gce/gci/mounter/mounter")
+		output, err = RunSshCommand("ssh", GetHostnameOrIp(host), "--", "sh", "-c", fmt.Sprintf("'chmod 544 %s'", mounterPath))
+		if err != nil {
+			glog.Errorf("Unable to chmod 544 GCI mounter script. Err: %v, Output:\n%s", err, output)
+			return "", false, err
+		}
+		// Insert args at beginning of testArgs, so any values from command line take precedence
+		testArgs = fmt.Sprintf("--mounter-path=%s ", mounterPath) + testArgs
+	}
+
 	// Run the tests
 	cmd = getSshCommand(" && ",
 		fmt.Sprintf("cd %s", tmp),

test/e2e_node/services/services.go

@@ -211,7 +211,9 @@ func (e *E2EServices) startKubelet() (*server, error) {
 		"--eviction-pressure-transition-period", "30s",
 		"--feature-gates", framework.TestContext.FeatureGates,
 		"--v", LOG_VERBOSITY_LEVEL, "--logtostderr",
+		"--mounter-path", framework.TestContext.MounterPath,
 	)
+
 	if framework.TestContext.RuntimeIntegrationType != "" {
 		cmdArgs = append(cmdArgs, "--experimental-runtime-integration-type",
 			framework.TestContext.RuntimeIntegrationType) // Whether to use experimental cri integration.

test/e2e_node/simple_mount.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/unversioned"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = framework.KubeDescribe("SimpleMount", func() {
+	f := framework.NewDefaultFramework("simple-mount-test")
+
+	// This is a very simple test that exercises the Kubelet's mounter code path.
+	// If the mount fails, the pod will not be able to run, and CreateSync will timeout.
+	It("should be able to mount an emptydir on a container", func() {
+		pod := &api.Pod{
+			TypeMeta: unversioned.TypeMeta{
+				Kind:       "Pod",
+				APIVersion: "v1",
+			},
+			ObjectMeta: api.ObjectMeta{
+				Name: "simple-mount-pod",
+			},
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name:  "simple-mount-container",
+						Image: framework.GetPauseImageNameForHostArch(),
+						VolumeMounts: []api.VolumeMount{
+							{
+								Name:      "simply-mounted-volume",
+								MountPath: "/opt/",
+							},
+						},
+					},
+				},
+				Volumes: []api.Volume{
+					{
+						Name: "simply-mounted-volume",
+						VolumeSource: api.VolumeSource{
+							EmptyDir: &api.EmptyDirVolumeSource{
+								Medium: "Memory",
+							},
+						},
+					},
+				},
+			},
+		}
+		podClient := f.PodClient()
+		pod = podClient.CreateSync(pod)
+
+	})
+})