diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c80a73b96c..d7696a79ae 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -652,7 +652,7 @@ function start-kubelet { flags+=" --cluster-dns=${DNS_SERVER_IP}" flags+=" --cluster-domain=${DNS_DOMAIN}" flags+=" --pod-manifest-path=/etc/kubernetes/manifests" - flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter" + flags+=" --experimental-mounter-path=${CONTAINERIZED_MOUNTER_HOME}/mounter" flags+=" --experimental-check-node-capabilities-before-mount=true" if [[ -n "${KUBELET_PORT:-}" ]]; then @@ -928,6 +928,18 @@ function compute-master-manifest-variables { fi } +# A helper function that bind mounts kubelet dirs for running mount in a chroot +function prepare-mounter-rootfs { + echo "Prepare containerized mounter" + mount --bind "${CONTAINERIZED_MOUNTER_HOME}" "${CONTAINERIZED_MOUNTER_HOME}" + mount -o remount,exec "${CONTAINERIZED_MOUNTER_HOME}" + CONTAINERIZED_MOUNTER_ROOTFS="${CONTAINERIZED_MOUNTER_HOME}/rootfs" + mount --rbind /var/lib/kubelet/ "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet" + mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet" + mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc" + mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev" +} + # A helper function for removing salt configuration and comments from a file. # This is mainly for preparing a manifest file. # @@ -1479,15 +1491,11 @@ function override-kubectl { echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh } -function pre-warm-mounter { - echo "prewarming mounter" - ${KUBE_HOME}/bin/mounter &> /dev/null -} - ########### Main Function ########### echo "Start to configure instance for kubernetes" KUBE_HOME="/home/kubernetes" +CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter" if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization." exit 1 @@ -1534,7 +1542,6 @@ fi override-kubectl # Run the containerized mounter once to pre-cache the container image. -pre-warm-mounter assemble-docker-flags load-docker-images start-kubelet @@ -1565,4 +1572,5 @@ else fi fi reset-motd +prepare-mounter-rootfs echo "Done for the configuration for kubernetes" diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh index d998b3b718..9dc8dbdd83 100644 --- a/cluster/gce/gci/configure.sh +++ b/cluster/gce/gci/configure.sh @@ -115,19 +115,18 @@ function split-commas { } function install-gci-mounter-tools { - local -r rkt_version="v1.18.0" - local -r gci_mounter_version="v2" - local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967" - local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091" - local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561" - download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt" - download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci" - download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci" - local -r rkt_dst="${KUBE_HOME}/bin/" - mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt" - mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci" - mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci" - chmod a+x "${rkt_dst}/rkt" + CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter" + mkdir "${CONTAINERIZED_MOUNTER_HOME}" + chmod a+x "${CONTAINERIZED_MOUNTER_HOME}" + mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs" + local -r mounter_tar_sha="8003b798cf33c7f91320cd6ee5cec4fa22244571" + download-or-bust "${mounter_tar_sha}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar" + cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${CONTAINERIZED_MOUNTER_HOME}/mounter" + chmod a+x "${CONTAINERIZED_MOUNTER_HOME}/mounter" + mv "${KUBE_HOME}/mounter.tar" /tmp/mounter.tar + tar xvf /tmp/mounter.tar -C "${CONTAINERIZED_MOUNTER_HOME}/rootfs" + rm /tmp/mounter.tar + mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs/var/lib/kubelet" } # Install node problem detector binary. @@ -222,7 +221,6 @@ function install-kube-binary-config { xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@" fi cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh" - cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter" cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh" chmod -R 755 "${kube_bin}" diff --git a/cluster/gce/gci/master.yaml b/cluster/gce/gci/master.yaml index 78b39164ae..845973df84 100644 --- a/cluster/gce/gci/master.yaml +++ b/cluster/gce/gci/master.yaml @@ -34,7 +34,6 @@ write_files: Type=oneshot RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter ExecStart=/home/kubernetes/bin/configure-helper.sh [Install] diff --git a/cluster/gce/gci/mounter/mounter b/cluster/gce/gci/mounter/mounter index 2fba3255f2..963296e03d 100755 Binary files a/cluster/gce/gci/mounter/mounter and b/cluster/gce/gci/mounter/mounter differ diff --git a/cluster/gce/gci/mounter/mounter.go b/cluster/gce/gci/mounter/mounter.go new file mode 100644 index 0000000000..ef37ce0601 --- /dev/null +++ b/cluster/gce/gci/mounter/mounter.go @@ -0,0 +1,93 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + // Location of the mount file to use + chrootCmd = "chroot" + mountCmd = "mount" + rootfs = "rootfs" + nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n" + rpcBindCmd = "/sbin/rpcbind" + defaultRootfs = "/home/kubernetes/containerized_mounter/rootfs" +) + +func main() { + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n") + return + } + path, _ := filepath.Split(os.Args[0]) + rootfsPath := filepath.Join(path, rootfs) + if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { + rootfsPath = defaultRootfs + } + command := os.Args[1] + switch command { + case mountCmd: + mountErr := mountInChroot(rootfsPath, os.Args[2:]) + if mountErr != nil { + fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr) + os.Exit(1) + } + default: + fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd) + os.Exit(1) + + } +} + +// MountInChroot is to run mount within chroot with the passing root directory +func mountInChroot(rootfsPath string, args []string) error { + if _, err := os.Stat(rootfsPath); os.IsNotExist(err) { + return fmt.Errorf("Path <%s> does not exist.\n", rootfsPath) + } + args = append([]string{rootfsPath, mountCmd}, args...) + output, err := exec.Command(chrootCmd, args...).CombinedOutput() + if err == nil { + return err + } + + if !strings.EqualFold(string(output), nfsRPCBindErrMsg) { + // Mount failed but not because of RPC bind error + return fmt.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s\n", err, chrootCmd, args, string(output)) + } + + // Mount failed because it is NFS V3 and we need to run rpcBind + output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput() + if err != nil { + return fmt.Errorf("Mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err) + } + + // Rpcbind is running, try mounting again + output, err = exec.Command(chrootCmd, args...).CombinedOutput() + + if err != nil { + return fmt.Errorf("Mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err) + } + + return nil +} diff --git a/cluster/gce/gci/node.yaml b/cluster/gce/gci/node.yaml index d0cc12e752..de59fcac27 100644 --- a/cluster/gce/gci/node.yaml +++ b/cluster/gce/gci/node.yaml @@ -34,7 +34,6 @@ write_files: Type=oneshot RemainAfterExit=yes ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh - ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter ExecStart=/home/kubernetes/bin/configure-helper.sh [Install] diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index d301e919db..8ddbfbf57d 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -137,7 +137,6 @@ install_kube_binary_config() { # This should be the case of GCI. readonly kube_bin="${kube_home}/bin" mkdir -p "${kube_bin}" - mount --bind "${kube_bin}" "${kube_bin}" mount -o remount,rw,exec "${kube_bin}" cp "${src_dir}/kubelet" "${kube_bin}" cp "${src_dir}/kubectl" "${kube_bin}" diff --git a/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest b/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest index 2bda14166f..75b0c15c20 100644 --- a/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest +++ b/cluster/saltbase/salt/e2e-image-puller/e2e-image-puller.manifest @@ -27,7 +27,7 @@ spec: command: - /bin/sh - -c - - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;" + - "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.8 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;" securityContext: privileged: true volumeMounts: diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 8796d6a52d..44058042d5 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -98,11 +98,6 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, // It provides options to override the default mounter behavior. // mounterPath allows using an alternative to `/bin/mount` for mounting. func New(mounterPath string) Interface { - // If mounter-path flag is not set, use default mount path - if mounterPath == "" { - mounterPath = defaultMountCommand - } - return &Mounter{ mounterPath: mounterPath, } diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 26fbb7460c..366101be22 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -63,23 +63,23 @@ type Mounter struct { // currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is // required, call Mount with an empty string list or nil. func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { - // Path to mounter binary. Set to mount accessible via $PATH by default. + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. // All Linux distros are expected to be shipped with a mount utility that an support bind mounts. - mounterPath := defaultMountCommand + mounterPath := "" bind, bindRemountOpts := isBind(options) if bind { - err := doMount(mounterPath, source, target, fstype, []string{"bind"}) + err := doMount(mounterPath, defaultMountCommand, source, target, fstype, []string{"bind"}) if err != nil { return err } - return doMount(mounterPath, source, target, fstype, bindRemountOpts) + return doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts) } // The list of filesystems that require containerized mounter on GCI image cluster - fsTypesNeedMounter := sets.NewString("nfs", "glusterfs") + fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs") if fsTypesNeedMounter.Has(fstype) { mounterPath = mounter.mounterPath } - return doMount(mounterPath, source, target, fstype, options) + return doMount(mounterPath, defaultMountCommand, source, target, fstype, options) } // isBind detects whether a bind mount is being requested and makes the remount options to @@ -107,10 +107,13 @@ func isBind(options []string) (bool, []string) { return bind, bindRemountOpts } -// doMount runs the mount command. -func doMount(mountCmd string, source string, target string, fstype string, options []string) error { - glog.V(4).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd) +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +func doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { mountArgs := makeMountArgs(source, target, fstype, options) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountCmd = mounterPath + } glog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) command := exec.Command(mountCmd, mountArgs...) diff --git a/test/e2e/common/volumes.go b/test/e2e/common/volumes.go index 8daad193a2..35b3b35286 100644 --- a/test/e2e/common/volumes.go +++ b/test/e2e/common/volumes.go @@ -393,6 +393,36 @@ var _ = framework.KubeDescribe("GCP Volumes", func() { }) }) + framework.KubeDescribe("NFSv3", func() { + It("should be mountable for NFSv3 [Volume]", func() { + config := VolumeTestConfig{ + namespace: namespace.Name, + prefix: "nfs", + serverImage: "gcr.io/google_containers/volume-nfs:0.8", + serverPorts: []int{2049}, + } + + defer func() { + if clean { + volumeTestCleanup(f, config) + } + }() + pod := startVolumeServer(f, config) + serverIP := pod.Status.PodIP + framework.Logf("NFS server IP address: %v", serverIP) + + volume := v1.VolumeSource{ + NFS: &v1.NFSVolumeSource{ + Server: serverIP, + Path: "/exports", + ReadOnly: true, + }, + } + // Must match content of test/images/volume-tester/nfs/index.html + testVolumeClient(f, config, volume, nil, "Hello from NFS!") + }) + }) + //////////////////////////////////////////////////////////////////////// // Gluster //////////////////////////////////////////////////////////////////////// @@ -488,10 +518,10 @@ func isTestEnabled(c clientset.Interface) bool { } // For cluster e2e test, because nodeName is empty, retrieve the node objects from api server - // and check their images. Only run NFSv4 and GlusterFS if nodes are using GCI image for now. + // and check their images. Only run NFS and GlusterFS tests if nodes are using GCI image for now. nodes := framework.GetReadySchedulableNodesOrDie(c) for _, node := range nodes.Items { - if !strings.Contains(node.Status.NodeInfo.OSImage, "Google Container-VM") { + if !strings.Contains(node.Status.NodeInfo.OSImage, "Container-Optimized OS") { return false } } diff --git a/test/e2e_node/jenkins/gci-init.yaml b/test/e2e_node/jenkins/gci-init.yaml index 8b3cb6c7bc..42bebb4908 100644 --- a/test/e2e_node/jenkins/gci-init.yaml +++ b/test/e2e_node/jenkins/gci-init.yaml @@ -3,10 +3,15 @@ runcmd: - mount /tmp /tmp -o remount,exec,suid - usermod -a -G docker jenkins - - mkdir -p /home/kubernetes/bin/ - - mount -B /home/kubernetes/bin /home/kubernetes/bin - - mount -B -o remount,exec /home/kubernetes/bin - - wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/rkt -O /home/kubernetes/bin/rkt - - wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/stage1-fly.aci -O /home/kubernetes/bin/stage1-fly.aci - - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-v2.aci -O /home/kubernetes/bin/gci-mounter-v2.aci - - chmod a+x /home/kubernetes/bin/rkt + - mkdir -p /var/lib/kubelet + - mkdir -p /home/kubernetes/containerized_mounter/rootfs + - mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/ + - mount -o remount, exec /home/kubernetes/containerized_mounter/ + - wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar + - tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs + - mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet + - mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet + - mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet + - mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc + - mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev + - rm /tmp/mounter.tar \ No newline at end of file