Use chroot for containerized mounts

This PR is to modify the containerized mounter script to use chroot
instead of rkt fly. This will avoid the problem of possible large number
of mounts caused by rkt containers if they are not cleaned up.

cluster/gce/gci/configure-helper.sh

@@ -604,7 +604,7 @@ function start-kubelet {
   flags+=" --cluster-dns=${DNS_SERVER_IP}"
   flags+=" --cluster-domain=${DNS_DOMAIN}"
   flags+=" --pod-manifest-path=/etc/kubernetes/manifests"
-  flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter"
+  flags+=" --experimental-mounter-path=${CONTAINERIZED_MOUNTER_HOME}/mounter"
   flags+=" --experimental-check-node-capabilities-before-mount=true"
 
   if [[ -n "${KUBELET_PORT:-}" ]]; then
@@ -880,6 +880,18 @@ function compute-master-manifest-variables {
   fi
 }
 
+# A helper function that bind mounts kubelet dirs for running mount in a chroot
+function prepare-mounter-rootfs {
+  echo "Prepare containerized mounter"
+  mount --bind "${CONTAINERIZED_MOUNTER_HOME}" "${CONTAINERIZED_MOUNTER_HOME}"
+  mount -o remount,exec "${CONTAINERIZED_MOUNTER_HOME}"
+  CONTAINERIZED_MOUNTER_ROOTFS="${CONTAINERIZED_MOUNTER_HOME}/rootfs"
+  mount --rbind /var/lib/kubelet/ "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
+  mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
+  mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc"
+  mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev"
+}
+
 # A helper function for removing salt configuration and comments from a file.
 # This is mainly for preparing a manifest file.
 #
@@ -1431,15 +1443,11 @@ function override-kubectl {
     echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
 }
 
-function pre-warm-mounter {
-    echo "prewarming mounter"
-    ${KUBE_HOME}/bin/mounter &> /dev/null
-}
-
 ########### Main Function ###########
 echo "Start to configure instance for kubernetes"
 
 KUBE_HOME="/home/kubernetes"
+CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
 if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
   echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
   exit 1
@@ -1483,7 +1491,6 @@ fi
 
 override-kubectl
 # Run the containerized mounter once to pre-cache the container image.
-pre-warm-mounter
 assemble-docker-flags
 load-docker-images
 start-kubelet
@@ -1514,4 +1521,5 @@ else
   fi
 fi
 reset-motd
+prepare-mounter-rootfs
 echo "Done for the configuration for kubernetes"

cluster/gce/gci/configure.sh

@@ -115,19 +115,18 @@ function split-commas {
 }
 
 function install-gci-mounter-tools {
-    local -r rkt_version="v1.18.0"
+    CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
-    local -r gci_mounter_version="v2"
+    mkdir "${CONTAINERIZED_MOUNTER_HOME}"
-    local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
+    chmod a+x "${CONTAINERIZED_MOUNTER_HOME}"
-    local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091"
+    mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs"
-    local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561"
+    local -r mounter_tar_sha="8003b798cf33c7f91320cd6ee5cec4fa22244571"
-    download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt"
+    download-or-bust "${mounter_tar_sha}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar"
-    download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci"
+    cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${CONTAINERIZED_MOUNTER_HOME}/mounter"
-    download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci"
+    chmod a+x "${CONTAINERIZED_MOUNTER_HOME}/mounter"
-    local -r rkt_dst="${KUBE_HOME}/bin/"
+    mv "${KUBE_HOME}/mounter.tar" /tmp/mounter.tar
-    mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt"
+    tar xvf /tmp/mounter.tar -C "${CONTAINERIZED_MOUNTER_HOME}/rootfs"
-    mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci"
+    rm /tmp/mounter.tar
-    mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci"
+    mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs/var/lib/kubelet"
-    chmod a+x "${rkt_dst}/rkt"
 }
 
 # Install node problem detector binary.
@@ -222,7 +221,6 @@ function install-kube-binary-config {
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
   cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
-  cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
   cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
   chmod -R 755 "${kube_bin}"
 

cluster/gce/gci/master.yaml

@@ -34,7 +34,6 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

cluster/gce/gci/mounter/mounter.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	// Location of the mount file to use
+	chrootCmd        = "chroot"
+	mountCmd         = "mount"
+	rootfs           = "rootfs"
+	nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n"
+	rpcBindCmd       = "/sbin/rpcbind"
+	defaultRootfs    = "/home/kubernetes/containerized_mounter/rootfs"
+)
+
+func main() {
+
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n")
+		return
+	}
+	path, _ := filepath.Split(os.Args[0])
+	rootfsPath := filepath.Join(path, rootfs)
+	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
+		rootfsPath = defaultRootfs
+	}
+	command := os.Args[1]
+	switch command {
+	case mountCmd:
+		mountErr := mountInChroot(rootfsPath, os.Args[2:])
+		if mountErr != nil {
+			fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr)
+			os.Exit(1)
+		}
+	default:
+		fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd)
+		os.Exit(1)
+
+	}
+}
+
+// MountInChroot is to run mount within chroot with the passing root directory
+func mountInChroot(rootfsPath string, args []string) error {
+	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
+		return fmt.Errorf("Path <%s> does not exist.\n", rootfsPath)
+	}
+	args = append([]string{rootfsPath, mountCmd}, args...)
+	output, err := exec.Command(chrootCmd, args...).CombinedOutput()
+	if err == nil {
+		return err
+	}
+
+	if !strings.EqualFold(string(output), nfsRPCBindErrMsg) {
+		// Mount failed but not because of RPC bind error
+		return fmt.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s\n", err, chrootCmd, args, string(output))
+	}
+
+	// Mount failed because it is NFS V3 and we need to run rpcBind
+	output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err)
+	}
+
+	// Rpcbind is running, try mounting again
+	output, err = exec.Command(chrootCmd, args...).CombinedOutput()
+
+	if err != nil {
+		return fmt.Errorf("Mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err)
+	}
+
+	return nil
+}

cluster/gce/gci/node.yaml

@@ -34,7 +34,6 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
-      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

cluster/gce/trusty/configure.sh

@@ -137,7 +137,6 @@ install_kube_binary_config() {
     # This should be the case of GCI.
     readonly kube_bin="${kube_home}/bin"
     mkdir -p "${kube_bin}"
-    mount --bind "${kube_bin}" "${kube_bin}"
     mount -o remount,rw,exec "${kube_bin}"
     cp "${src_dir}/kubelet" "${kube_bin}"
     cp "${src_dir}/kubectl" "${kube_bin}"

cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest

@@ -27,7 +27,7 @@ spec:
     command:
     - /bin/sh
     - -c
-    - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
+    - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.8 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
     securityContext:
       privileged: true
     volumeMounts:

pkg/util/mount/mount.go

@@ -98,11 +98,6 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string,
 // It provides options to override the default mounter behavior.
 // mounterPath allows using an alternative to `/bin/mount` for mounting.
 func New(mounterPath string) Interface {
-	// If mounter-path flag is not set, use default mount path
-	if mounterPath == "" {
-		mounterPath = defaultMountCommand
-	}
-
 	return &Mounter{
 		mounterPath: mounterPath,
 	}

pkg/util/mount/mount_linux.go

@@ -63,23 +63,23 @@ type Mounter struct {
 // currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
 // required, call Mount with an empty string list or nil.
 func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
-	// Path to mounter binary. Set to mount accessible via $PATH by default.
+	// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
 	// All Linux distros are expected to be shipped with a mount utility that an support bind mounts.
-	mounterPath := defaultMountCommand
+	mounterPath := ""
 	bind, bindRemountOpts := isBind(options)
 	if bind {
-		err := doMount(mounterPath, source, target, fstype, []string{"bind"})
+		err := doMount(mounterPath, defaultMountCommand, source, target, fstype, []string{"bind"})
 		if err != nil {
 			return err
 		}
-		return doMount(mounterPath, source, target, fstype, bindRemountOpts)
+		return doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts)
 	}
 	// The list of filesystems that require containerized mounter on GCI image cluster
-	fsTypesNeedMounter := sets.NewString("nfs", "glusterfs")
+	fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs")
 	if fsTypesNeedMounter.Has(fstype) {
 		mounterPath = mounter.mounterPath
 	}
-	return doMount(mounterPath, source, target, fstype, options)
+	return doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
 }
 
 // isBind detects whether a bind mount is being requested and makes the remount options to
@@ -107,10 +107,13 @@ func isBind(options []string) (bool, []string) {
 	return bind, bindRemountOpts
 }
 
-// doMount runs the mount command.
+// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
-func doMount(mountCmd string, source string, target string, fstype string, options []string) error {
+func doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error {
-	glog.V(4).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd)
 	mountArgs := makeMountArgs(source, target, fstype, options)
+	if len(mounterPath) > 0 {
+		mountArgs = append([]string{mountCmd}, mountArgs...)
+		mountCmd = mounterPath
+	}
 
 	glog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs)
 	command := exec.Command(mountCmd, mountArgs...)

test/e2e/common/volumes.go

@@ -393,6 +393,36 @@ var _ = framework.KubeDescribe("GCP Volumes", func() {
 		})
 	})
 
+	framework.KubeDescribe("NFSv3", func() {
+		It("should be mountable for NFSv3 [Volume]", func() {
+			config := VolumeTestConfig{
+				namespace:   namespace.Name,
+				prefix:      "nfs",
+				serverImage: "gcr.io/google_containers/volume-nfs:0.8",
+				serverPorts: []int{2049},
+			}
+
+			defer func() {
+				if clean {
+					volumeTestCleanup(f, config)
+				}
+			}()
+			pod := startVolumeServer(f, config)
+			serverIP := pod.Status.PodIP
+			framework.Logf("NFS server IP address: %v", serverIP)
+
+			volume := v1.VolumeSource{
+				NFS: &v1.NFSVolumeSource{
+					Server:   serverIP,
+					Path:     "/exports",
+					ReadOnly: true,
+				},
+			}
+			// Must match content of test/images/volume-tester/nfs/index.html
+			testVolumeClient(f, config, volume, nil, "Hello from NFS!")
+		})
+	})
+
 	////////////////////////////////////////////////////////////////////////
 	// Gluster
 	////////////////////////////////////////////////////////////////////////
@@ -488,10 +518,10 @@ func isTestEnabled(c clientset.Interface) bool {
 	}
 
 	// For cluster e2e test, because nodeName is empty, retrieve the node objects from api server
-	// and check their images. Only run NFSv4 and GlusterFS if nodes are using GCI image for now.
+	// and check their images. Only run NFS and GlusterFS tests if nodes are using GCI image for now.
 	nodes := framework.GetReadySchedulableNodesOrDie(c)
 	for _, node := range nodes.Items {
-		if !strings.Contains(node.Status.NodeInfo.OSImage, "Google Container-VM") {
+		if !strings.Contains(node.Status.NodeInfo.OSImage, "Container-Optimized OS") {
 			return false
 		}
 	}

test/e2e_node/jenkins/gci-init.yaml

@@ -3,10 +3,15 @@
 runcmd:
   - mount /tmp /tmp -o remount,exec,suid
   - usermod -a -G docker jenkins
-  - mkdir -p /home/kubernetes/bin/
+  - mkdir -p /var/lib/kubelet
-  - mount -B /home/kubernetes/bin /home/kubernetes/bin
+  - mkdir -p /home/kubernetes/containerized_mounter/rootfs
-  - mount -B -o remount,exec /home/kubernetes/bin 
+  - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/
-  - wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/rkt -O /home/kubernetes/bin/rkt
+  - mount -o remount, exec /home/kubernetes/containerized_mounter/
-  - wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/stage1-fly.aci -O /home/kubernetes/bin/stage1-fly.aci
+  - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar
-  - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-v2.aci -O /home/kubernetes/bin/gci-mounter-v2.aci
+  - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs
-  - chmod a+x /home/kubernetes/bin/rkt
+  - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
+  - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc
+  - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev
+  - rm /tmp/mounter.tar