From f2438cacf56dd1d71b7a77c1d29c9d8afaa874f4 Mon Sep 17 00:00:00 2001 From: Travis Rhoden Date: Tue, 26 Feb 2019 12:14:55 -0700 Subject: [PATCH] Copy mount files to pkg/volume/util/subpath Files in pkg/util/mount that contain significant code implementation for subpaths are moved to a new package at pkg/volume/util/subpath. This move is done in order to preserve git history. --- .../util/subpath}/mount_linux.go | 0 .../util/subpath}/mount_linux_test.go | 0 .../util/subpath}/mount_unsupported.go | 0 .../util/subpath}/mount_windows.go | 0 .../util/subpath}/mount_windows_test.go | 0 .../util/subpath}/nsenter_mount.go | 0 .../util/subpath}/nsenter_mount_test.go | 0 pkg/volume/util/subpath/subpath_linux.go | 1329 +++++++++++ pkg/volume/util/subpath/subpath_linux_test.go | 2054 +++++++++++++++++ pkg/volume/util/subpath/subpath_nsenter.go | 470 ++++ .../util/subpath/subpath_nsenter_test.go | 724 ++++++ .../util/subpath/subpath_unsupported.go | 139 ++ pkg/volume/util/subpath/subpath_windows.go | 647 ++++++ .../util/subpath/subpath_windows_test.go | 827 +++++++ 14 files changed, 6190 insertions(+) rename pkg/{util/mount => volume/util/subpath}/mount_linux.go (100%) rename pkg/{util/mount => volume/util/subpath}/mount_linux_test.go (100%) rename pkg/{util/mount => volume/util/subpath}/mount_unsupported.go (100%) rename pkg/{util/mount => volume/util/subpath}/mount_windows.go (100%) rename pkg/{util/mount => volume/util/subpath}/mount_windows_test.go (100%) rename pkg/{util/mount => volume/util/subpath}/nsenter_mount.go (100%) rename pkg/{util/mount => volume/util/subpath}/nsenter_mount_test.go (100%) create mode 100644 pkg/volume/util/subpath/subpath_linux.go create mode 100644 pkg/volume/util/subpath/subpath_linux_test.go create mode 100644 pkg/volume/util/subpath/subpath_nsenter.go create mode 100644 pkg/volume/util/subpath/subpath_nsenter_test.go create mode 100644 pkg/volume/util/subpath/subpath_unsupported.go create mode 100644 pkg/volume/util/subpath/subpath_windows.go create mode 100644 pkg/volume/util/subpath/subpath_windows_test.go diff --git a/pkg/util/mount/mount_linux.go b/pkg/volume/util/subpath/mount_linux.go similarity index 100% rename from pkg/util/mount/mount_linux.go rename to pkg/volume/util/subpath/mount_linux.go diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/volume/util/subpath/mount_linux_test.go similarity index 100% rename from pkg/util/mount/mount_linux_test.go rename to pkg/volume/util/subpath/mount_linux_test.go diff --git a/pkg/util/mount/mount_unsupported.go b/pkg/volume/util/subpath/mount_unsupported.go similarity index 100% rename from pkg/util/mount/mount_unsupported.go rename to pkg/volume/util/subpath/mount_unsupported.go diff --git a/pkg/util/mount/mount_windows.go b/pkg/volume/util/subpath/mount_windows.go similarity index 100% rename from pkg/util/mount/mount_windows.go rename to pkg/volume/util/subpath/mount_windows.go diff --git a/pkg/util/mount/mount_windows_test.go b/pkg/volume/util/subpath/mount_windows_test.go similarity index 100% rename from pkg/util/mount/mount_windows_test.go rename to pkg/volume/util/subpath/mount_windows_test.go diff --git a/pkg/util/mount/nsenter_mount.go b/pkg/volume/util/subpath/nsenter_mount.go similarity index 100% rename from pkg/util/mount/nsenter_mount.go rename to pkg/volume/util/subpath/nsenter_mount.go diff --git a/pkg/util/mount/nsenter_mount_test.go b/pkg/volume/util/subpath/nsenter_mount_test.go similarity index 100% rename from pkg/util/mount/nsenter_mount_test.go rename to pkg/volume/util/subpath/nsenter_mount_test.go diff --git a/pkg/volume/util/subpath/subpath_linux.go b/pkg/volume/util/subpath/subpath_linux.go new file mode 100644 index 0000000000..c66039e5e0 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_linux.go @@ -0,0 +1,1329 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" + utilexec "k8s.io/utils/exec" + utilio "k8s.io/utils/io" + utilpath "k8s.io/utils/path" +) + +const ( + // How many times to retry for a consistent read of /proc/mounts. + maxListTries = 3 + // Number of fields per line in /proc/mounts as per the fstab man page. + expectedNumFieldsPerLine = 6 + // At least number of fields per line in /proc//mountinfo. + expectedAtLeastNumFieldsPerMountInfo = 10 + // Location of the mount file to use + procMountsPath = "/proc/mounts" + // Location of the mountinfo file + procMountInfoPath = "/proc/self/mountinfo" + // 'fsck' found errors and corrected them + fsckErrorsCorrected = 1 + // 'fsck' found errors but exited without correcting them + fsckErrorsUncorrected = 4 + + // place for subpath mounts + // TODO: pass in directory using kubelet_getters instead + containerSubPathDirectoryName = "volume-subpaths" + // syscall.Openat flags used to traverse directories not following symlinks + nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW + // flags for getting file descriptor without following the symlink + openFDFlags = unix.O_NOFOLLOW | unix.O_PATH +) + +// Mounter provides the default implementation of mount.Interface +// for the linux platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string + withSystemd bool +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + withSystemd: detectSystemd(), + } +} + +// Mount mounts source to target as fstype with given options. 'source' and 'fstype' must +// be an empty string in case it's not required, e.g. for remount, or for auto filesystem +// type, where kernel handles fstype for you. The mount 'options' is a list of options, +// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is +// required, call Mount with an empty string list or nil. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + // Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty. + // All Linux distros are expected to be shipped with a mount utility that a support bind mounts. + mounterPath := "" + bind, bindOpts, bindRemountOpts := isBind(options) + if bind { + err := mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindOpts) + if err != nil { + return err + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts) + } + // The list of filesystems that require containerized mounter on GCI image cluster + fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs") + if fsTypesNeedMounter.Has(fstype) { + mounterPath = mounter.mounterPath + } + return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options) +} + +// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. +func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { + mountArgs := makeMountArgs(source, target, fstype, options) + if len(mounterPath) > 0 { + mountArgs = append([]string{mountCmd}, mountArgs...) + mountCmd = mounterPath + } + + if m.withSystemd { + // Try to run mount via systemd-run --scope. This will escape the + // service where kubelet runs and any fuse daemons will be started in a + // specific scope. kubelet service than can be restarted without killing + // these fuse daemons. + // + // Complete command line (when mounterPath is not used): + // systemd-run --description=... --scope -- mount -t + // + // Expected flow: + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet service can be restarted and the fuse daemon survives. + // * When the fuse daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + // + // systemd-mount is not used because it's too new for older distros + // (CentOS 7, Debian Jessie). + mountCmd, mountArgs = addSystemdScope("systemd-run", target, mountCmd, mountArgs) + } else { + // No systemd-run on the host (or we failed to check it), assume kubelet + // does not run as a systemd service. + // No code here, mountCmd and mountArgs are already populated. + } + + klog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs) + command := exec.Command(mountCmd, mountArgs...) + output, err := command.CombinedOutput() + if err != nil { + args := strings.Join(mountArgs, " ") + klog.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", err, mountCmd, args, string(output)) + return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %s\nOutput: %s\n", + err, mountCmd, args, string(output)) + } + return err +} + +// detectSystemd returns true if OS runs with systemd as init. When not sure +// (permission errors, ...), it returns false. +// There may be different ways how to detect systemd, this one makes sure that +// systemd-runs (needed by Mount()) works. +func detectSystemd() bool { + if _, err := exec.LookPath("systemd-run"); err != nil { + klog.V(2).Infof("Detected OS without systemd") + return false + } + // Try to run systemd-run --scope /bin/true, that should be enough + // to make sure that systemd is really running and not just installed, + // which happens when running in a container with a systemd-based image + // but with different pid 1. + cmd := exec.Command("systemd-run", "--description=Kubernetes systemd probe", "--scope", "true") + output, err := cmd.CombinedOutput() + if err != nil { + klog.V(2).Infof("Cannot run systemd-run, assuming non-systemd OS") + klog.V(4).Infof("systemd-run failed with: %v", err) + klog.V(4).Infof("systemd-run output: %s", string(output)) + return false + } + klog.V(2).Infof("Detected OS with systemd") + return true +} + +// makeMountArgs makes the arguments to the mount(8) command. +func makeMountArgs(source, target, fstype string, options []string) []string { + // Build mount command as follows: + // mount [-t $fstype] [-o $options] [$source] $target + mountArgs := []string{} + if len(fstype) > 0 { + mountArgs = append(mountArgs, "-t", fstype) + } + if len(options) > 0 { + mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) + } + if len(source) > 0 { + mountArgs = append(mountArgs, source) + } + mountArgs = append(mountArgs, target) + + return mountArgs +} + +// addSystemdScope adds "system-run --scope" to given command line +func addSystemdScope(systemdRunPath, mountName, command string, args []string) (string, []string) { + descriptionArg := fmt.Sprintf("--description=Kubernetes transient mount for %s", mountName) + systemdRunArgs := []string{descriptionArg, "--scope", "--", command} + return systemdRunPath, append(systemdRunArgs, args...) +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("Unmounting %s", target) + command := exec.Command("umount", target) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("Unmount failed: %v\nUnmounting arguments: %s\nOutput: %s\n", err, target, string(output)) + } + return nil +} + +// List returns a list of all mounted filesystems. +func (*Mounter) List() ([]MountPoint, error) { + return listProcMounts(procMountsPath) +} + +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) + return ((mp.Path == dir) || (mp.Path == deletedDir)) +} + +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return isNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +// It is fast but not necessarily ALWAYS correct. If the path is in fact +// a bind mount from one part of a mount to another it will not be detected. +// mkdir /tmp/a /tmp/b; mount --bin /tmp/a /tmp/b; IsLikelyNotMountPoint("/tmp/b") +// will return true. When in fact /tmp/b is a mount point. If this situation +// if of interest to you, don't use this function... +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Stat(file) + if err != nil { + return true, err + } + rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/"))) + if err != nil { + return true, err + } + // If the directory has a different device as parent, then it is a mountpoint. + if stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev { + return false, nil + } + + return true, nil +} + +// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. +// If pathname is not a device, log and return false with nil error. +// If open returns errno EBUSY, return true with nil error. +// If open returns nil, return false with nil error. +// Otherwise, return false with error +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return exclusiveOpenFailsOnDevice(pathname) +} + +// PathIsDevice uses FileInfo returned from os.Stat to check if path refers +// to a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + pathType, err := mounter.GetFileType(pathname) + isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev + return isDevice, err +} + +func exclusiveOpenFailsOnDevice(pathname string) (bool, error) { + var isDevice bool + finfo, err := os.Stat(pathname) + if os.IsNotExist(err) { + isDevice = false + } + // err in call to os.Stat + if err != nil { + return false, fmt.Errorf( + "PathIsDevice failed for path %q: %v", + pathname, + err) + } + // path refers to a device + if finfo.Mode()&os.ModeDevice != 0 { + isDevice = true + } + + if !isDevice { + klog.Errorf("Path %q is not referring to a device.", pathname) + return false, nil + } + fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL, 0) + // If the device is in use, open will return an invalid fd. + // When this happens, it is expected that Close will fail and throw an error. + defer unix.Close(fd) + if errno == nil { + // device not in use + return false, nil + } else if errno == unix.EBUSY { + // device is in use + return true, nil + } + // error during call to Open + return false, errno +} + +//GetDeviceNameFromMount: given a mount point, find the device name from its global mount point +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// getDeviceNameFromMount find the device name from /proc/mounts in which +// the mount path reference should match the given plugin directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + klog.V(4).Infof("Directory %s is not mounted", mountPath) + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := path.Join(pluginDir, MountsInGlobalPDPath) + for _, ref := range refs { + if strings.HasPrefix(ref, basemountPath) { + volumeID, err := filepath.Rel(basemountPath, ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +func listProcMounts(mountFilePath string) ([]MountPoint, error) { + content, err := utilio.ConsistentRead(mountFilePath, maxListTries) + if err != nil { + return nil, err + } + return parseProcMounts(content) +} + +func parseProcMounts(content []byte) ([]MountPoint, error) { + out := []MountPoint{} + lines := strings.Split(string(content), "\n") + for _, line := range lines { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) != expectedNumFieldsPerLine { + return nil, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line) + } + + mp := MountPoint{ + Device: fields[0], + Path: fields[1], + Type: fields[2], + Opts: strings.Split(fields[3], ","), + } + + freq, err := strconv.Atoi(fields[4]) + if err != nil { + return nil, err + } + mp.Freq = freq + + pass, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, err + } + mp.Pass = pass + + out = append(out, mp) + } + return out, nil +} + +func (mounter *Mounter) MakeRShared(path string) error { + return doMakeRShared(path, procMountInfoPath) +} + +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +func (mounter *Mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (mounter *Mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) +} + +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// formatAndMount uses unix utils to format and mount the given disk +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + readOnly := false + for _, option := range options { + if option == "ro" { + readOnly = true + break + } + } + + options = append(options, "defaults") + + if !readOnly { + // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. + klog.V(4).Infof("Checking for issues with fsck on disk: %s", source) + args := []string{"-a", source} + out, err := mounter.Exec.Run("fsck", args...) + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + klog.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + klog.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s.", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + klog.Infof("`fsck` error %s", string(out)) + } + } + } + + // Try to mount the disk + klog.V(4).Infof("Attempting to mount disk: %s %s %s", fstype, source, target) + mountErr := mounter.Interface.Mount(source, target, fstype, options) + if mountErr != nil { + // Mount failed. This indicates either that the disk is unformatted or + // it contains an unexpected filesystem. + existingFormat, err := mounter.GetDiskFormat(source) + if err != nil { + return err + } + if existingFormat == "" { + if readOnly { + // Don't attempt to format if mounting as readonly, return an error to reflect this. + return errors.New("failed to mount unformatted volume as read only") + } + + // Disk is unformatted so format it. + args := []string{source} + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + } + klog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + _, err := mounter.Exec.Run("mkfs."+fstype, args...) + if err == nil { + // the disk has been formatted successfully try to mount it again. + klog.Infof("Disk successfully formatted (mkfs): %s - %s %s", fstype, source, target) + return mounter.Interface.Mount(source, target, fstype, options) + } + klog.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", source, fstype, target, options, err) + return err + } else { + // Disk is already formatted and failed to mount + if len(fstype) == 0 || fstype == existingFormat { + // This is mount error + return mountErr + } else { + // Block device is formatted with unexpected filesystem, let the user know + return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr) + } + } + } + return mountErr +} + +// GetDiskFormat uses 'blkid' to see if the given disk is unformated +func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) { + args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} + klog.V(4).Infof("Attempting to determine if disk %q is formatted using blkid with args: (%v)", disk, args) + dataOut, err := mounter.Exec.Run("blkid", args...) + output := string(dataOut) + klog.V(4).Infof("Output: %q, err: %v", output, err) + + if err != nil { + if exit, ok := err.(utilexec.ExitError); ok { + if exit.ExitStatus() == 2 { + // Disk device is unformatted. + // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was + // not found, or no (specified) devices could be identified, an + // exit code of 2 is returned. + return "", nil + } + } + klog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return "", err + } + + var fstype, pttype string + + lines := strings.Split(output, "\n") + for _, l := range lines { + if len(l) <= 0 { + // Ignore empty line. + continue + } + cs := strings.Split(l, "=") + if len(cs) != 2 { + return "", fmt.Errorf("blkid returns invalid output: %s", output) + } + // TYPE is filesystem type, and PTTYPE is partition table type, according + // to https://www.kernel.org/pub/linux/utils/util-linux/v2.21/libblkid-docs/. + if cs[0] == "TYPE" { + fstype = cs[1] + } else if cs[0] == "PTTYPE" { + pttype = cs[1] + } + } + + if len(pttype) > 0 { + klog.V(4).Infof("Disk %s detected partition table type: %s", disk, pttype) + // Returns a special non-empty string as filesystem type, then kubelet + // will not format it. + return "unknown data, probably partitions", nil + } + + return fstype, nil +} + +// isShared returns true, if given path is on a mount point that has shared +// mount propagation. +func isShared(mount string, mountInfoPath string) (bool, error) { + info, err := findMountInfo(mount, mountInfoPath) + if err != nil { + return false, err + } + + // parse optional parameters + for _, opt := range info.optionalFields { + if strings.HasPrefix(opt, "shared:") { + return true, nil + } + } + return false, nil +} + +// This represents a single line in /proc//mountinfo. +type mountInfo struct { + // Unique ID for the mount (maybe reused after umount). + id int + // The ID of the parent mount (or of self for the root of this mount namespace's mount tree). + parentID int + // The value of `st_dev` for files on this filesystem. + majorMinor string + // The pathname of the directory in the filesystem which forms the root of this mount. + root string + // Mount source, filesystem-specific information. e.g. device, tmpfs name. + source string + // Mount point, the pathname of the mount point. + mountPoint string + // Optional fieds, zero or more fields of the form "tag[:value]". + optionalFields []string + // The filesystem type in the form "type[.subtype]". + fsType string + // Per-mount options. + mountOptions []string + // Per-superblock options. + superOptions []string +} + +// parseMountInfo parses /proc/xxx/mountinfo. +func parseMountInfo(filename string) ([]mountInfo, error) { + content, err := utilio.ConsistentRead(filename, maxListTries) + if err != nil { + return []mountInfo{}, err + } + contentStr := string(content) + infos := []mountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + // See `man proc` for authoritative description of format of the file. + fields := strings.Fields(line) + if len(fields) < expectedAtLeastNumFieldsPerMountInfo { + return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line) + } + id, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, err + } + parentID, err := strconv.Atoi(fields[1]) + if err != nil { + return nil, err + } + info := mountInfo{ + id: id, + parentID: parentID, + majorMinor: fields[2], + root: fields[3], + mountPoint: fields[4], + mountOptions: strings.Split(fields[5], ","), + } + // All fields until "-" are "optional fields". + i := 6 + for ; i < len(fields) && fields[i] != "-"; i++ { + info.optionalFields = append(info.optionalFields, fields[i]) + } + // Parse the rest 3 fields. + i += 1 + if len(fields)-i < 3 { + return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i) + } + info.fsType = fields[i] + info.source = fields[i+1] + info.superOptions = strings.Split(fields[i+2], ",") + infos = append(infos, info) + } + return infos, nil +} + +func findMountInfo(path, mountInfoPath string) (mountInfo, error) { + infos, err := parseMountInfo(mountInfoPath) + if err != nil { + return mountInfo{}, err + } + + // process /proc/xxx/mountinfo in backward order and find the first mount + // point that is prefix of 'path' - that's the mount where path resides + var info *mountInfo + for i := len(infos) - 1; i >= 0; i-- { + if PathWithinBase(path, infos[i].mountPoint) { + info = &infos[i] + break + } + } + if info == nil { + return mountInfo{}, fmt.Errorf("cannot find mount point for %q", path) + } + return *info, nil +} + +// doMakeRShared is common implementation of MakeRShared on Linux. It checks if +// path is shared and bind-mounts it as rshared if needed. mountCmd and +// mountArgs are expected to contain mount-like command, doMakeRShared will add +// '--bind ' and '--make-rshared ' to mountArgs. +func doMakeRShared(path string, mountInfoFilename string) error { + shared, err := isShared(path, mountInfoFilename) + if err != nil { + return err + } + if shared { + klog.V(4).Infof("Directory %s is already on a shared mount", path) + return nil + } + + klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path) + // mount --bind /var/lib/kubelet /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil { + return fmt.Errorf("failed to bind-mount %s: %v", path, err) + } + + // mount --make-rshared /var/lib/kubelet + if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil { + return fmt.Errorf("failed to make %s rshared: %v", path, err) + } + + return nil +} + +// getSELinuxSupport is common implementation of GetSELinuxSupport on Linux. +func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) { + info, err := findMountInfo(path, mountInfoFilename) + if err != nil { + return false, err + } + + // "seclabel" can be both in mount options and super options. + for _, opt := range info.superOptions { + if opt == "seclabel" { + return true, nil + } + } + for _, opt := range info.mountOptions { + if opt == "seclabel" { + return true, nil + } + } + return false, nil +} + +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + newHostPath, err = doBindSubPath(mounter, subPath) + + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +// This implementation is shared between Linux and NsEnterMounter +func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) { + if !PathWithinBase(subpath.Path, subpath.VolumePath) { + return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath) + } + fd, err := doSafeOpen(subpath.Path, subpath.VolumePath) + if err != nil { + return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err) + } + return fd, nil +} + +// prepareSubpathTarget creates target for bind-mount of subpath. It returns +// "true" when the target already exists and something is mounted there. +// Given Subpath must have all paths with already resolved symlinks and with +// paths relevant to kubelet (when it runs in a container). +// This function is called also by NsEnterMounter. It works because +// /var/lib/kubelet is mounted from the host into the container with Kubelet as +// /var/lib/kubelet too. +func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) { + // Early check for already bind-mounted subpath. + bindPathTarget := getSubpathBindTarget(subpath) + notMount, err := mounter.IsNotMountPoint(bindPathTarget) + if err != nil { + if !os.IsNotExist(err) { + return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err) + } + // Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet. + notMount = true + } + if !notMount { + // It's already mounted + klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) + return true, bindPathTarget, nil + } + + // bindPathTarget is in /var/lib/kubelet and thus reachable without any + // translation even to containerized kubelet. + bindParent := filepath.Dir(bindPathTarget) + err = os.MkdirAll(bindParent, 0750) + if err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err) + } + + t, err := os.Lstat(subpath.Path) + if err != nil { + return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err) + } + + if t.Mode()&os.ModeDir > 0 { + if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) { + return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err) + } + } else { + // "/bin/touch ". + // A file is enough for all possible targets (symlink, device, pipe, + // socket, ...), bind-mounting them into a file correctly changes type + // of the target file. + if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil { + return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err) + } + } + return false, bindPathTarget, nil +} + +func getSubpathBindTarget(subpath Subpath) string { + // containerName is DNS label, i.e. safe as a directory name. + return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex)) +} + +func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) { + // Linux, kubelet runs on the host: + // - safely open the subpath + // - bind-mount /proc//fd/ to subpath target + // User can't change /proc//fd/ to point to a bad place. + + // Evaluate all symlinks here once for all subsequent functions. + newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) + } + newPath, err := filepath.EvalSymlinks(subpath.Path) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) + } + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath) + subpath.VolumePath = newVolumePath + subpath.Path = newPath + + fd, err := safeOpenSubPath(mounter, subpath) + if err != nil { + return "", err + } + defer syscall.Close(fd) + + alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) + if err != nil { + return "", err + } + if alreadyMounted { + return bindPathTarget, nil + } + + success := false + defer func() { + // Cleanup subpath on error + if !success { + klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + } + } + }() + + kubeletPid := os.Getpid() + mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) + + // Do the bind mount + options := []string{"bind"} + klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) + if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil { + return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) + } + success = true + + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + return bindPathTarget, nil +} + +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(mounter, podDir, volumeName) +} + +// This implementation is shared between Linux and NsEnterMounter +func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { + // scan /var/lib/kubelet/pods//volume-subpaths//* + subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) + klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir) + + containerDirs, err := ioutil.ReadDir(subPathDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("error reading %s: %s", subPathDir, err) + } + + for _, containerDir := range containerDirs { + if !containerDir.IsDir() { + klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) + continue + } + klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) + + // scan /var/lib/kubelet/pods//volume-subpaths///* + fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) + err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error { + if path == fullContainerDirPath { + // Skip top level directory + return nil + } + + // pass through errors and let doCleanSubPath handle them + if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err) + } + + // Whole container has been processed, remove its directory. + if err := os.Remove(fullContainerDirPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) + } + klog.V(5).Infof("Removed %s", fullContainerDirPath) + } + // Whole pod volume subpaths have been cleaned up, remove its subpath directory. + if err := os.Remove(subPathDir); err != nil { + return fmt.Errorf("error deleting %s: %s", subPathDir, err) + } + klog.V(5).Infof("Removed %s", subPathDir) + + // Remove entire subpath directory if it's the last one + podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) + if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { + return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) + } + klog.V(5).Infof("Removed %s", podSubPathDir) + return nil +} + +// doCleanSubPath tears down the single subpath bind mount +func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { + // process /var/lib/kubelet/pods//volume-subpaths/// + klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) + fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) + + if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil { + return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err) + } + + klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath) + return nil +} + +// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty +func cleanSubPath(mounter Interface, subpath Subpath) error { + containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) + + // Clean subdir bindmount + if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) { + return err + } + + // Recusively remove directories if empty + if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil { + return err + } + + return nil +} + +// removeEmptyDirs works backwards from endDir to baseDir and removes each directory +// if it is empty. It stops once it encounters a directory that has content +func removeEmptyDirs(baseDir, endDir string) error { + if !PathWithinBase(endDir, baseDir) { + return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) + } + + for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) { + s, err := os.Stat(curDir) + if err != nil { + if os.IsNotExist(err) { + klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) + continue + } + return fmt.Errorf("error stat %q: %v", curDir, err) + } + if !s.IsDir() { + return fmt.Errorf("path %q not a directory", curDir) + } + + err = os.Remove(curDir) + if os.IsExist(err) { + klog.V(5).Infof("Directory %q not empty, not removing", curDir) + break + } else if err != nil { + return fmt.Errorf("error removing directory %q: %v", curDir, err) + } + klog.V(5).Infof("Removed directory %q", curDir) + } + return nil +} + +func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + if _, err := os.Stat(pathname); os.IsNotExist(err) { + return []string{}, nil + } else if err != nil { + return nil, err + } + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return searchMountPoints(realpath, procMountInfoPath) +} + +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + return getSELinuxSupport(pathname, procMountInfoPath) +} + +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return 0, err + } + return getFSGroup(realpath) +} + +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + return getMode(pathname) +} + +// This implementation is shared between Linux and NsEnterMounter +func getFSGroup(pathname string) (int64, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return int64(info.Sys().(*syscall.Stat_t).Gid), nil +} + +// This implementation is shared between Linux and NsEnterMounter +func getMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} + +// This implementation is shared between Linux and NsEnterMounter. Both pathname +// and base must be either already resolved symlinks or thet will be resolved in +// kubelet's mount namespace (in case it runs containerized). +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", existingPath, err) + } + if !PathWithinBase(fullExistingPath, base) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + parentFD, err := doSafeOpen(fullExistingPath, base) + if err != nil { + return fmt.Errorf("cannot open directory %s: %s", existingPath, err) + } + childFD := -1 + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + } + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink. + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) + if err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + // Dive into the created directory + childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0) + if err != nil { + return fmt.Errorf("cannot open %s: %s", currentPath, err) + } + // We can be sure that childFD is safe to use. It could be changed + // by user after Mkdirat() and before Openat(), however: + // - it could not be changed to symlink - we use nofollowFlags + // - it could be changed to a file (or device, pipe, socket, ...) + // but either subsequent Mkdirat() fails or we mount this file + // to user's container. Security is no violated in both cases + // and user either gets error or the file that it can already access. + + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + parentFD = childFD + childFD = -1 + } + + // Everything was created. mkdirat(..., perm) above was affected by current + // umask and we must apply the right permissions to the last directory + // (that's the one that will be available to the container as subpath) + // so user can read/write it. This is the behavior of previous code. + // TODO: chmod all created directories, not just the last one. + // parentFD is the last created directory. + + // Translate perm (os.FileMode) to uint32 that fchmod() expects + kernelPerm := uint32(perm & os.ModePerm) + if perm&os.ModeSetgid > 0 { + kernelPerm |= syscall.S_ISGID + } + if perm&os.ModeSetuid > 0 { + kernelPerm |= syscall.S_ISUID + } + if perm&os.ModeSticky > 0 { + kernelPerm |= syscall.S_ISVTX + } + if err = syscall.Fchmod(parentFD, kernelPerm); err != nil { + return fmt.Errorf("chmod %q failed: %s", currentPath, err) + } + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + dirs := strings.Split(rel, string(filepath.Separator)) + + // Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks. + // This should be faster than looping through all dirs and calling os.Stat() + // on each of them, as the symlinks are resolved only once with OpenAt(). + currentPath := base + fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0) + if err != nil { + return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err) + } + defer func() { + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + }() + for i, dir := range dirs { + // Using O_PATH here will prevent hangs in case user replaces directory with + // fifo + childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0) + if err != nil { + if os.IsNotExist(err) { + return currentPath, dirs[i:], nil + } + return base, nil, err + } + if err = syscall.Close(fd); err != nil { + klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + fd = childFD + currentPath = filepath.Join(currentPath, dir) + } + return pathname, []string{}, nil +} + +// This implementation is shared between Linux and NsEnterMounter +// Open path and return its fd. +// Symlinks are disallowed (pathname must already resolve symlinks), +// and the path must be within the base directory. +func doSafeOpen(pathname string, base string) (int, error) { + pathname = filepath.Clean(pathname) + base = filepath.Clean(base) + + // Calculate segments to follow + subpath, err := filepath.Rel(base, pathname) + if err != nil { + return -1, err + } + segments := strings.Split(subpath, string(filepath.Separator)) + + // Assumption: base is the only directory that we have under control. + // Base dir is not allowed to be a symlink. + parentFD, err := syscall.Open(base, nofollowFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open directory %s: %s", base, err) + } + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) + } + } + }() + + childFD := -1 + defer func() { + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := base + + // Follow the segments one by one using openat() to make + // sure the user cannot change already existing directories into symlinks. + for _, seg := range segments { + currentPath = filepath.Join(currentPath, seg) + if !PathWithinBase(currentPath, base) { + return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) + } + + klog.V(5).Infof("Opening path %s", currentPath) + childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) + } + + var deviceStat unix.Stat_t + err := unix.Fstat(childFD, &deviceStat) + if err != nil { + return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err) + } + fileFmt := deviceStat.Mode & syscall.S_IFMT + if fileFmt == syscall.S_IFLNK { + return -1, fmt.Errorf("Unexpected symlink found %s", currentPath) + } + + // Close parentFD + if err = syscall.Close(parentFD); err != nil { + return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err) + } + // Set child to new parent + parentFD = childFD + childFD = -1 + } + + // We made it to the end, return this fd, don't close it + finalFD := parentFD + parentFD = -1 + + return finalFD, nil +} + +// searchMountPoints finds all mount references to the source, returns a list of +// mountpoints. +// This function assumes source cannot be device. +// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting, +// it's possible to mount a non-root path of a filesystem, so we need to use +// root path and major:minor to represent mount source uniquely. +// This implementation is shared between Linux and NsEnterMounter +func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) { + mis, err := parseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + + mountID := 0 + rootPath := "" + majorMinor := "" + + // Finding the underlying root path and major:minor if possible. + // We need search in backward order because it's possible for later mounts + // to overlap earlier mounts. + for i := len(mis) - 1; i >= 0; i-- { + if hostSource == mis[i].mountPoint || PathWithinBase(hostSource, mis[i].mountPoint) { + // If it's a mount point or path under a mount point. + mountID = mis[i].id + rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint)) + majorMinor = mis[i].majorMinor + break + } + } + + if rootPath == "" || majorMinor == "" { + return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource) + } + + var refs []string + for i := range mis { + if mis[i].id == mountID { + // Ignore mount entry for mount source itself. + continue + } + if mis[i].root == rootPath && mis[i].majorMinor == majorMinor { + refs = append(refs, mis[i].mountPoint) + } + } + + return refs, nil +} diff --git a/pkg/volume/util/subpath/subpath_linux_test.go b/pkg/volume/util/subpath/subpath_linux_test.go new file mode 100644 index 0000000000..e2669dd0f7 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_linux_test.go @@ -0,0 +1,2054 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "syscall" + "testing" + + "k8s.io/utils/exec" + + "k8s.io/klog" +) + +func TestReadProcMountsFrom(t *testing.T) { + successCase := + `/dev/0 /path/to/0 type0 flags 0 0 +/dev/1 /path/to/1 type1 flags 1 1 +/dev/2 /path/to/2 type2 flags,1,2=3 2 2 +` + // NOTE: readProcMountsFrom has been updated to using fnv.New32a() + mounts, err := parseProcMounts([]byte(successCase)) + if err != nil { + t.Errorf("expected success, got %v", err) + } + if len(mounts) != 3 { + t.Fatalf("expected 3 mounts, got %d", len(mounts)) + } + mp := MountPoint{"/dev/0", "/path/to/0", "type0", []string{"flags"}, 0, 0} + if !mountPointsEqual(&mounts[0], &mp) { + t.Errorf("got unexpected MountPoint[0]: %#v", mounts[0]) + } + mp = MountPoint{"/dev/1", "/path/to/1", "type1", []string{"flags"}, 1, 1} + if !mountPointsEqual(&mounts[1], &mp) { + t.Errorf("got unexpected MountPoint[1]: %#v", mounts[1]) + } + mp = MountPoint{"/dev/2", "/path/to/2", "type2", []string{"flags", "1", "2=3"}, 2, 2} + if !mountPointsEqual(&mounts[2], &mp) { + t.Errorf("got unexpected MountPoint[2]: %#v", mounts[2]) + } + + errorCases := []string{ + "/dev/0 /path/to/mount\n", + "/dev/1 /path/to/mount type flags a 0\n", + "/dev/2 /path/to/mount type flags 0 b\n", + } + for _, ec := range errorCases { + _, err := parseProcMounts([]byte(ec)) + if err == nil { + t.Errorf("expected error") + } + } +} + +func mountPointsEqual(a, b *MountPoint) bool { + if a.Device != b.Device || a.Path != b.Path || a.Type != b.Type || !reflect.DeepEqual(a.Opts, b.Opts) || a.Pass != b.Pass || a.Freq != b.Freq { + return false + } + return true +} + +func TestGetMountRefs(t *testing.T) { + fm := &FakeMounter{ + MountPoints: []MountPoint{ + {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd"}, + {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2"}, + }, + } + + tests := []struct { + mountPath string + expectedRefs []string + }{ + { + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod", + []string{ + "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd", + }, + }, + { + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1", + []string{ + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2", + "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2", + }, + }, + { + "/var/fake/directory/that/doesnt/exist", + []string{}, + }, + } + + for i, test := range tests { + if refs, err := fm.GetMountRefs(test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { + t.Errorf("%d. getMountRefs(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs) + } + } +} + +func setEquivalent(set1, set2 []string) bool { + map1 := make(map[string]bool) + map2 := make(map[string]bool) + for _, s := range set1 { + map1[s] = true + } + for _, s := range set2 { + map2[s] = true + } + + for s := range map1 { + if !map2[s] { + return false + } + } + for s := range map2 { + if !map1[s] { + return false + } + } + return true +} + +func TestGetDeviceNameFromMount(t *testing.T) { + fm := &FakeMounter{ + MountPoints: []MountPoint{ + {Device: "/dev/disk/by-path/prefix-lun-1", + Path: "/mnt/111"}, + {Device: "/dev/disk/by-path/prefix-lun-1", + Path: "/mnt/222"}, + }, + } + + tests := []struct { + mountPath string + expectedDevice string + expectedRefs int + }{ + { + "/mnt/222", + "/dev/disk/by-path/prefix-lun-1", + 2, + }, + } + + for i, test := range tests { + if device, refs, err := GetDeviceNameFromMount(fm, test.mountPath); err != nil || test.expectedRefs != refs || test.expectedDevice != device { + t.Errorf("%d. GetDeviceNameFromMount(%s) = (%s, %d), %v; expected (%s,%d), nil", i, test.mountPath, device, refs, err, test.expectedDevice, test.expectedRefs) + } + } +} + +func TestGetMountRefsByDev(t *testing.T) { + fm := &FakeMounter{ + MountPoints: []MountPoint{ + {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd"}, + {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1"}, + {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2"}, + }, + } + + tests := []struct { + mountPath string + expectedRefs []string + }{ + { + "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd", + []string{ + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd-in-pod", + }, + }, + { + "/var/lib/kubelet/plugins/kubernetes.io/gce-pd/mounts/gce-pd2", + []string{ + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod1", + "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~gce-pd/gce-pd2-in-pod2", + }, + }, + } + + for i, test := range tests { + + if refs, err := getMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { + t.Errorf("%d. getMountRefsByDev(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs) + } + } +} + +func writeFile(content string) (string, string, error) { + tempDir, err := ioutil.TempDir("", "mounter_shared_test") + if err != nil { + return "", "", err + } + filename := filepath.Join(tempDir, "mountinfo") + err = ioutil.WriteFile(filename, []byte(content), 0600) + if err != nil { + os.RemoveAll(tempDir) + return "", "", err + } + return tempDir, filename, nil +} + +func TestIsSharedSuccess(t *testing.T) { + successMountInfo := + `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +76 62 8:1 / /boot rw,relatime shared:29 - ext4 /dev/sda1 rw,seclabel,data=ordered +78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel +80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw +82 62 0:43 / /var/lib/foo rw,relatime shared:32 - tmpfs tmpfs rw +83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw +227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +` + tempDir, filename, err := writeFile(successMountInfo) + if err != nil { + t.Fatalf("cannot create temporary file: %v", err) + } + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + path string + expectedResult bool + }{ + { + // /var/lib/kubelet is a directory on mount '/' that is shared + // This is the most common case. + "shared", + "/var/lib/kubelet", + true, + }, + { + // 8a2a... is a directory on mount /var/lib/docker/devicemapper + // that is private. + "private", + "/var/lib/docker/devicemapper/mnt/8a2a5c19eefb06d6f851dfcb240f8c113427f5b49b19658b5c60168e88267693/", + false, + }, + { + // 'directory' is a directory on mount + // /var/lib/docker/devicemapper/test/shared that is shared, but one + // of its parent is private. + "nested-shared", + "/var/lib/docker/devicemapper/test/shared/my/test/directory", + true, + }, + { + // /var/lib/foo is a mount point and it's shared + "shared-mount", + "/var/lib/foo", + true, + }, + { + // /var/lib/bar is a mount point and it's private + "private-mount", + "/var/lib/bar", + false, + }, + } + for _, test := range tests { + ret, err := isShared(test.path, filename) + if err != nil { + t.Errorf("test %s got unexpected error: %v", test.name, err) + } + if ret != test.expectedResult { + t.Errorf("test %s expected %v, got %v", test.name, test.expectedResult, ret) + } + } +} + +func TestIsSharedFailure(t *testing.T) { + errorTests := []struct { + name string + content string + }{ + { + // the first line is too short + name: "too-short-line", + content: `62 0 253:0 / / rw,relatime +76 62 8:1 / /boot rw,relatime shared:29 - ext4 /dev/sda1 rw,seclabel,data=ordered +78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel +80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw +227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +`, + }, + { + // there is no root mount + name: "no-root-mount", + content: `76 62 8:1 / /boot rw,relatime shared:29 - ext4 /dev/sda1 rw,seclabel,data=ordered +78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel +80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw +227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +`, + }, + } + for _, test := range errorTests { + tempDir, filename, err := writeFile(test.content) + if err != nil { + t.Fatalf("cannot create temporary file: %v", err) + } + defer os.RemoveAll(tempDir) + + _, err = isShared("/", filename) + if err == nil { + t.Errorf("test %q: expected error, got none", test.name) + } + } +} + +func TestPathWithinBase(t *testing.T) { + tests := []struct { + name string + fullPath string + basePath string + expected bool + }{ + { + name: "good subpath", + fullPath: "/a/b/c", + basePath: "/a", + expected: true, + }, + { + name: "good subpath 2", + fullPath: "/a/b/c", + basePath: "/a/b", + expected: true, + }, + { + name: "good subpath end slash", + fullPath: "/a/b/c/", + basePath: "/a/b", + expected: true, + }, + { + name: "good subpath backticks", + fullPath: "/a/b/../c", + basePath: "/a", + expected: true, + }, + { + name: "good subpath equal", + fullPath: "/a/b/c", + basePath: "/a/b/c", + expected: true, + }, + { + name: "good subpath equal 2", + fullPath: "/a/b/c/", + basePath: "/a/b/c", + expected: true, + }, + { + name: "good subpath root", + fullPath: "/a", + basePath: "/", + expected: true, + }, + { + name: "bad subpath parent", + fullPath: "/a/b/c", + basePath: "/a/b/c/d", + expected: false, + }, + { + name: "bad subpath outside", + fullPath: "/b/c", + basePath: "/a/b/c", + expected: false, + }, + { + name: "bad subpath prefix", + fullPath: "/a/b/cd", + basePath: "/a/b/c", + expected: false, + }, + { + name: "bad subpath backticks", + fullPath: "/a/../b", + basePath: "/a", + expected: false, + }, + { + name: "configmap subpath", + fullPath: "/var/lib/kubelet/pods/uuid/volumes/kubernetes.io~configmap/config/..timestamp/file.txt", + basePath: "/var/lib/kubelet/pods/uuid/volumes/kubernetes.io~configmap/config", + expected: true, + }, + } + for _, test := range tests { + if PathWithinBase(test.fullPath, test.basePath) != test.expected { + t.Errorf("test %q failed: expected %v", test.name, test.expected) + } + + } +} + +func TestSafeMakeDir(t *testing.T) { + defaultPerm := os.FileMode(0750) + os.ModeDir + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + path string + checkPath string + perm os.FileMode + expectError bool + }{ + { + "directory-does-not-exist", + func(base string) error { + return nil + }, + "test/directory", + "test/directory", + defaultPerm, + false, + }, + { + "directory-with-sgid", + func(base string) error { + return nil + }, + "test/directory", + "test/directory", + os.FileMode(0777) + os.ModeDir + os.ModeSetgid, + false, + }, + { + "directory-with-suid", + func(base string) error { + return nil + }, + "test/directory", + "test/directory", + os.FileMode(0777) + os.ModeDir + os.ModeSetuid, + false, + }, + { + "directory-with-sticky-bit", + func(base string) error { + return nil + }, + "test/directory", + "test/directory", + os.FileMode(0777) + os.ModeDir + os.ModeSticky, + false, + }, + { + "directory-exists", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) + }, + "test/directory", + "test/directory", + defaultPerm, + false, + }, + { + "create-base", + func(base string) error { + return nil + }, + "", + "", + defaultPerm, + false, + }, + { + "escape-base-using-dots", + func(base string) error { + return nil + }, + "..", + "", + defaultPerm, + true, + }, + { + "escape-base-using-dots-2", + func(base string) error { + return nil + }, + "test/../../..", + "", + defaultPerm, + true, + }, + { + "follow-symlinks", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil { + return err + } + return os.Symlink("destination", filepath.Join(base, "test")) + }, + "test/directory", + "destination/directory", + defaultPerm, + false, + }, + { + "follow-symlink-loop", + func(base string) error { + return os.Symlink("test", filepath.Join(base, "test")) + }, + "test/directory", + "", + defaultPerm, + true, + }, + { + "follow-symlink-multiple follow", + func(base string) error { + /* test1/dir points to test2 and test2/dir points to test1 */ + if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil { + return err + } + return nil + }, + "test1/dir/dir/dir/dir/dir/dir/dir/foo", + "test2/foo", + defaultPerm, + false, + }, + { + "danglink-symlink", + func(base string) error { + return os.Symlink("non-existing", filepath.Join(base, "test")) + }, + "test/directory", + "", + defaultPerm, + true, + }, + { + "non-directory", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test/directory", + "", + defaultPerm, + true, + }, + { + "non-directory-final", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test", + "", + defaultPerm, + true, + }, + { + "escape-with-relative-symlink", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil { + return err + } + return os.Symlink("../exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + "", + defaultPerm, + false, + }, + { + "escape-with-relative-symlink-not-exists", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + return os.Symlink("../not-exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + "", + defaultPerm, + true, + }, + { + "escape-with-symlink", + func(base string) error { + return os.Symlink("/", filepath.Join(base, "test")) + }, + "test/directory", + "", + defaultPerm, + true, + }, + } + + for i := range tests { + test := tests[i] + t.Run(test.name, func(t *testing.T) { + base, err := ioutil.TempDir("", "safe-make-dir-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + defer os.RemoveAll(base) + test.prepare(base) + pathToCreate := filepath.Join(base, test.path) + err = doSafeMakeDir(pathToCreate, base, test.perm) + if err != nil && !test.expectError { + t.Fatal(err) + } + if err != nil { + t.Logf("got error: %s", err) + } + if err == nil && test.expectError { + t.Fatalf("expected error, got none") + } + + if test.checkPath != "" { + st, err := os.Stat(filepath.Join(base, test.checkPath)) + if err != nil { + t.Fatalf("cannot read path %s", test.checkPath) + } + actualMode := st.Mode() + if actualMode != test.perm { + if actualMode^test.perm == os.ModeSetgid && test.perm&os.ModeSetgid == 0 { + // when TMPDIR is a kubernetes emptydir, the sticky gid bit is set due to fsgroup + t.Logf("masking bit from %o", actualMode) + } else { + t.Errorf("expected permissions %o, got %o (%b)", test.perm, actualMode, test.perm^actualMode) + } + } + } + }) + } +} + +func TestRemoveEmptyDirs(t *testing.T) { + defaultPerm := os.FileMode(0750) + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + // Function that validates directory structure after the test + validate func(base string) error + baseDir string + endDir string + expectError bool + }{ + { + name: "all-empty", + prepare: func(base string) error { + return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm) + }, + validate: func(base string) error { + return validateDirEmpty(filepath.Join(base, "a")) + }, + baseDir: "a", + endDir: "a/b/c", + expectError: false, + }, + { + name: "dir-not-empty", + prepare: func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm); err != nil { + return err + } + return os.Mkdir(filepath.Join(base, "a/b/d"), defaultPerm) + }, + validate: func(base string) error { + if err := validateDirNotExists(filepath.Join(base, "a/b/c")); err != nil { + return err + } + return validateDirExists(filepath.Join(base, "a/b")) + }, + baseDir: "a", + endDir: "a/b/c", + expectError: false, + }, + { + name: "path-not-within-base", + prepare: func(base string) error { + return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm) + }, + validate: func(base string) error { + return validateDirExists(filepath.Join(base, "a")) + }, + baseDir: "a", + endDir: "b/c", + expectError: true, + }, + { + name: "path-already-deleted", + prepare: func(base string) error { + return nil + }, + validate: func(base string) error { + return nil + }, + baseDir: "a", + endDir: "a/b/c", + expectError: false, + }, + { + name: "path-not-dir", + prepare: func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "a/b"), defaultPerm); err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(base, "a/b", "c"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + if err := validateDirExists(filepath.Join(base, "a/b")); err != nil { + return err + } + return validateFileExists(filepath.Join(base, "a/b/c")) + }, + baseDir: "a", + endDir: "a/b/c", + expectError: true, + }, + } + + for _, test := range tests { + klog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "remove-empty-dirs-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + if err = test.prepare(base); err != nil { + os.RemoveAll(base) + t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) + } + + err = removeEmptyDirs(filepath.Join(base, test.baseDir), filepath.Join(base, test.endDir)) + if err != nil && !test.expectError { + t.Errorf("test %q failed: %v", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("test %q failed: expected error, got success", test.name) + } + + if err = test.validate(base); err != nil { + t.Errorf("test %q failed validation: %v", test.name, err) + } + + os.RemoveAll(base) + } +} + +func TestCleanSubPaths(t *testing.T) { + defaultPerm := os.FileMode(0750) + testVol := "vol1" + + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) ([]MountPoint, error) + // Function that validates directory structure after the test + validate func(base string) error + expectError bool + }{ + { + name: "not-exists", + prepare: func(base string) ([]MountPoint, error) { + return nil, nil + }, + validate: func(base string) error { + return nil + }, + expectError: false, + }, + { + name: "subpath-not-mount", + prepare: func(base string) ([]MountPoint, error) { + return nil, os.MkdirAll(filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0"), defaultPerm) + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + expectError: false, + }, + { + name: "subpath-file", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + return nil, ioutil.WriteFile(filepath.Join(path, "0"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + expectError: false, + }, + { + name: "subpath-container-not-dir", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol) + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + return nil, ioutil.WriteFile(filepath.Join(path, "container1"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + return validateDirExists(filepath.Join(base, containerSubPathDirectoryName, testVol)) + }, + expectError: true, + }, + { + name: "subpath-multiple-container-not-dir", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol) + if err := os.MkdirAll(filepath.Join(path, "container1"), defaultPerm); err != nil { + return nil, err + } + return nil, ioutil.WriteFile(filepath.Join(path, "container2"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + path := filepath.Join(base, containerSubPathDirectoryName, testVol) + if err := validateDirNotExists(filepath.Join(path, "container1")); err != nil { + return err + } + return validateFileExists(filepath.Join(path, "container2")) + }, + expectError: true, + }, + { + name: "subpath-mount", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + mounts := []MountPoint{{Device: "/dev/sdb", Path: path}} + return mounts, nil + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + }, + { + name: "subpath-mount-multiple", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") + path2 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "1") + path3 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container2", "1") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + if err := os.MkdirAll(path2, defaultPerm); err != nil { + return nil, err + } + if err := os.MkdirAll(path3, defaultPerm); err != nil { + return nil, err + } + mounts := []MountPoint{ + {Device: "/dev/sdb", Path: path}, + {Device: "/dev/sdb", Path: path3}, + } + return mounts, nil + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + }, + { + name: "subpath-mount-multiple-vols", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") + path2 := filepath.Join(base, containerSubPathDirectoryName, "vol2", "container1", "1") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + if err := os.MkdirAll(path2, defaultPerm); err != nil { + return nil, err + } + mounts := []MountPoint{ + {Device: "/dev/sdb", Path: path}, + } + return mounts, nil + }, + validate: func(base string) error { + baseSubdir := filepath.Join(base, containerSubPathDirectoryName) + if err := validateDirNotExists(filepath.Join(baseSubdir, testVol)); err != nil { + return err + } + return validateDirExists(baseSubdir) + }, + }, + } + + for _, test := range tests { + klog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "clean-subpaths-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + mounts, err := test.prepare(base) + if err != nil { + os.RemoveAll(base) + t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) + } + + fm := &FakeMounter{MountPoints: mounts} + + err = doCleanSubPaths(fm, base, testVol) + if err != nil && !test.expectError { + t.Errorf("test %q failed: %v", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("test %q failed: expected error, got success", test.name) + } + if err = test.validate(base); err != nil { + t.Errorf("test %q failed validation: %v", test.name, err) + } + + os.RemoveAll(base) + } +} + +var ( + testVol = "vol1" + testPod = "pod0" + testContainer = "container0" + testSubpath = 1 +) + +func setupFakeMounter(testMounts []string) *FakeMounter { + mounts := []MountPoint{} + for _, mountPoint := range testMounts { + mounts = append(mounts, MountPoint{Device: "/foo", Path: mountPoint}) + } + return &FakeMounter{MountPoints: mounts} +} + +func getTestPaths(base string) (string, string) { + return filepath.Join(base, testVol), + filepath.Join(base, testPod, containerSubPathDirectoryName, testVol, testContainer, strconv.Itoa(testSubpath)) +} + +func TestBindSubPath(t *testing.T) { + defaultPerm := os.FileMode(0750) + + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) ([]string, string, string, error) + expectError bool + }{ + { + name: "subpath-dir", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + }, + { + name: "subpath-dir-symlink", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + if err := os.MkdirAll(subpath, defaultPerm); err != nil { + return nil, "", "", err + } + subpathLink := filepath.Join(volpath, "dirLink") + return nil, volpath, subpath, os.Symlink(subpath, subpathLink) + }, + expectError: false, + }, + { + name: "subpath-file", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "file0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, defaultPerm) + }, + expectError: false, + }, + { + name: "subpath-not-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "file0") + return nil, volpath, subpath, nil + }, + expectError: true, + }, + { + name: "subpath-outside", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, os.Symlink(base, subpath) + }, + expectError: true, + }, + { + name: "subpath-symlink-child-outside", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + subpath := filepath.Join(subpathDir, "child0") + if err := os.MkdirAll(subpathDir, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, os.Symlink(base, subpath) + }, + expectError: true, + }, + { + name: "subpath-child-outside-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + child := filepath.Join(base, "child0") + subpath := filepath.Join(subpathDir, "child0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + // touch file outside + if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil { + return nil, "", "", err + } + + // create symlink for subpath dir + return nil, volpath, subpath, os.Symlink(base, subpathDir) + }, + expectError: true, + }, + { + name: "subpath-child-outside-not-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + subpath := filepath.Join(subpathDir, "child0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + // create symlink for subpath dir + return nil, volpath, subpath, os.Symlink(base, subpathDir) + }, + expectError: true, + }, + { + name: "subpath-child-outside-exists-middle-dir-symlink", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + symlinkDir := filepath.Join(subpathDir, "linkDir0") + child := filepath.Join(base, "child0") + subpath := filepath.Join(symlinkDir, "child0") + if err := os.MkdirAll(subpathDir, defaultPerm); err != nil { + return nil, "", "", err + } + // touch file outside + if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil { + return nil, "", "", err + } + + // create symlink for middle dir + return nil, volpath, subpath, os.Symlink(base, symlinkDir) + }, + expectError: true, + }, + { + name: "subpath-backstepping", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + symlinkBase := filepath.Join(volpath, "..") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + + // create symlink for subpath + return nil, volpath, subpath, os.Symlink(symlinkBase, subpath) + }, + expectError: true, + }, + { + name: "subpath-mountdir-already-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, subpathMount := getTestPaths(base) + if err := os.MkdirAll(subpathMount, defaultPerm); err != nil { + return nil, "", "", err + } + + subpath := filepath.Join(volpath, "dir0") + return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + }, + { + name: "subpath-mount-already-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, subpathMount := getTestPaths(base) + mounts := []string{subpathMount} + if err := os.MkdirAll(subpathMount, defaultPerm); err != nil { + return nil, "", "", err + } + + subpath := filepath.Join(volpath, "dir0") + return mounts, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + }, + { + name: "mount-unix-socket", + prepare: func(base string) ([]string, string, string, error) { + volpath, subpathMount := getTestPaths(base) + mounts := []string{subpathMount} + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + + socketFile, socketCreateError := createSocketFile(volpath) + + return mounts, volpath, socketFile, socketCreateError + }, + expectError: false, + }, + { + name: "subpath-mounting-fifo", + prepare: func(base string) ([]string, string, string, error) { + volpath, subpathMount := getTestPaths(base) + mounts := []string{subpathMount} + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + + testFifo := filepath.Join(volpath, "mount_test.fifo") + err := syscall.Mkfifo(testFifo, 0) + return mounts, volpath, testFifo, err + }, + expectError: false, + }, + } + + for _, test := range tests { + klog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "bind-subpath-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + + mounts, volPath, subPath, err := test.prepare(base) + if err != nil { + os.RemoveAll(base) + t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) + } + + fm := setupFakeMounter(mounts) + + subpath := Subpath{ + VolumeMountIndex: testSubpath, + Path: subPath, + VolumeName: testVol, + VolumePath: volPath, + PodDir: filepath.Join(base, "pod0"), + ContainerName: testContainer, + } + + _, subpathMount := getTestPaths(base) + bindPathTarget, err := doBindSubPath(fm, subpath) + if test.expectError { + if err == nil { + t.Errorf("test %q failed: expected error, got success", test.name) + } + if bindPathTarget != "" { + t.Errorf("test %q failed: expected empty bindPathTarget, got %v", test.name, bindPathTarget) + } + if err = validateDirNotExists(subpathMount); err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } + } + if !test.expectError { + if err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } + if bindPathTarget != subpathMount { + t.Errorf("test %q failed: expected bindPathTarget %v, got %v", test.name, subpathMount, bindPathTarget) + } + if err = validateFileExists(subpathMount); err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } + } + + os.RemoveAll(base) + } +} + +func TestParseMountInfo(t *testing.T) { + info := + `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel +80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw +82 62 0:43 / /var/lib/foo rw,relatime shared:32 - tmpfs tmpfs rw +83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw +227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +76 17 8:1 / /mnt/stateful_partition rw,nosuid,nodev,noexec,relatime - ext4 /dev/sda1 rw,commit=30,data=ordered +80 17 8:1 /var /var rw,nosuid,nodev,noexec,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +189 80 8:1 /var/lib/kubelet /var/lib/kubelet rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +818 77 8:40 / /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered +819 78 8:48 / /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +900 100 8:48 /dir1 /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volume-subpaths/vol1/subpath1/0 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +901 101 8:1 /dir1 /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volume-subpaths/vol1/subpath1/1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +902 102 8:1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volumes/kubernetes.io~empty-dir/vol1/dir1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volume-subpaths/vol1/subpath1/0 rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +903 103 8:1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volumes/kubernetes.io~empty-dir/vol2/dir1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volume-subpaths/vol1/subpath1/1 rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +178 25 253:0 /etc/bar /var/lib/kubelet/pods/12345/volume-subpaths/vol1/subpath1/0 rw,relatime shared:1 - ext4 /dev/sdb2 rw,errors=remount-ro,data=ordered +698 186 0:41 /tmp1/dir1 /var/lib/kubelet/pods/41135147-e697-11e7-9342-42010a800002/volume-subpaths/vol1/subpath1/0 rw shared:26 - tmpfs tmpfs rw +918 77 8:50 / /var/lib/kubelet/pods/2345/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered +919 78 8:58 / /var/lib/kubelet/pods/2345/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +920 100 8:50 /dir1 /var/lib/kubelet/pods/2345/volume-subpaths/vol1/subpath1/0 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered +150 23 1:58 / /media/nfs_vol rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +151 24 1:58 / /media/nfs_bindmount rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs/foo rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +134 23 0:58 / /var/lib/kubelet/pods/43219158-e5e1-11e7-a392-0e858b8eaf40/volumes/kubernetes.io~nfs/nfs1 rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +187 23 0:58 / /var/lib/kubelet/pods/1fc5ea21-eff4-11e7-ac80-0e858b8eaf40/volumes/kubernetes.io~nfs/nfs2 rw,relatime shared:96 - nfs4 172.18.4.223:/srv/nfs2 rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +188 24 0:58 / /var/lib/kubelet/pods/43219158-e5e1-11e7-a392-0e858b8eaf40/volume-subpaths/nfs1/subpath1/0 rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs/foo rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +347 60 0:71 / /var/lib/kubelet/pods/13195d46-f9fa-11e7-bbf1-5254007a695a/volumes/kubernetes.io~nfs/vol2 rw,relatime shared:170 - nfs 172.17.0.3:/exports/2 rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.17.0.3,mountvers=3,mountport=20048,mountproto=udp,local_lock=none,addr=172.17.0.3 +222 24 253:0 /tmp/src /mnt/dst rw,relatime shared:1 - ext4 /dev/mapper/vagrant--vg-root rw,errors=remount-ro,data=ordered +28 18 0:24 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:9 - tmpfs tmpfs ro,mode=755 +29 28 0:25 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd +31 28 0:27 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:13 - cgroup cgroup rw,cpuset +32 28 0:28 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:14 - cgroup cgroup rw,cpu,cpuacct +33 28 0:29 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:15 - cgroup cgroup rw,freezer +34 28 0:30 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,net_cls,net_prio +35 28 0:31 / /sys/fs/cgroup/pids rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,pids +36 28 0:32 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,devices +37 28 0:33 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,hugetlb +38 28 0:34 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:20 - cgroup cgroup rw,blkio +39 28 0:35 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:21 - cgroup cgroup rw,memory +40 28 0:36 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:22 - cgroup cgroup rw,perf_event +` + tempDir, filename, err := writeFile(info) + if err != nil { + t.Fatalf("cannot create temporary file: %v", err) + } + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + id int + expectedInfo mountInfo + }{ + { + "simple bind mount", + 189, + mountInfo{ + id: 189, + parentID: 80, + majorMinor: "8:1", + root: "/var/lib/kubelet", + source: "/dev/sda1", + mountPoint: "/var/lib/kubelet", + optionalFields: []string{"shared:30"}, + fsType: "ext4", + mountOptions: []string{"rw", "relatime"}, + superOptions: []string{"rw", "commit=30", "data=ordered"}, + }, + }, + { + "bind mount a directory", + 222, + mountInfo{ + id: 222, + parentID: 24, + majorMinor: "253:0", + root: "/tmp/src", + source: "/dev/mapper/vagrant--vg-root", + mountPoint: "/mnt/dst", + optionalFields: []string{"shared:1"}, + fsType: "ext4", + mountOptions: []string{"rw", "relatime"}, + superOptions: []string{"rw", "errors=remount-ro", "data=ordered"}, + }, + }, + { + "more than one optional fields", + 224, + mountInfo{ + id: 224, + parentID: 62, + majorMinor: "253:0", + root: "/var/lib/docker/devicemapper/test/shared", + source: "/dev/mapper/ssd-root", + mountPoint: "/var/lib/docker/devicemapper/test/shared", + optionalFields: []string{"master:1", "shared:44"}, + fsType: "ext4", + mountOptions: []string{"rw", "relatime"}, + superOptions: []string{"rw", "seclabel", "data=ordered"}, + }, + }, + { + "cgroup-mountpoint", + 28, + mountInfo{ + id: 28, + parentID: 18, + majorMinor: "0:24", + root: "/", + source: "tmpfs", + mountPoint: "/sys/fs/cgroup", + optionalFields: []string{"shared:9"}, + fsType: "tmpfs", + mountOptions: []string{"ro", "nosuid", "nodev", "noexec"}, + superOptions: []string{"ro", "mode=755"}, + }, + }, + { + "cgroup-subsystem-systemd-mountpoint", + 29, + mountInfo{ + id: 29, + parentID: 28, + majorMinor: "0:25", + root: "/", + source: "cgroup", + mountPoint: "/sys/fs/cgroup/systemd", + optionalFields: []string{"shared:10"}, + fsType: "cgroup", + mountOptions: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}, + superOptions: []string{"rw", "xattr", "release_agent=/lib/systemd/systemd-cgroups-agent", "name=systemd"}, + }, + }, + { + "cgroup-subsystem-cpuset-mountpoint", + 31, + mountInfo{ + id: 31, + parentID: 28, + majorMinor: "0:27", + root: "/", + source: "cgroup", + mountPoint: "/sys/fs/cgroup/cpuset", + optionalFields: []string{"shared:13"}, + fsType: "cgroup", + mountOptions: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}, + superOptions: []string{"rw", "cpuset"}, + }, + }, + } + + infos, err := parseMountInfo(filename) + if err != nil { + t.Fatalf("Cannot parse %s: %s", filename, err) + } + + for _, test := range tests { + found := false + for _, info := range infos { + if info.id == test.id { + found = true + if !reflect.DeepEqual(info, test.expectedInfo) { + t.Errorf("Test case %q:\n expected: %+v\n got: %+v", test.name, test.expectedInfo, info) + } + break + } + } + if !found { + t.Errorf("Test case %q: mountPoint %d not found", test.name, test.id) + } + } +} + +func TestGetSELinuxSupport(t *testing.T) { + info := + `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel +83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw +227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +150 23 1:58 / /media/nfs_vol rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +` + tempDir, filename, err := writeFile(info) + if err != nil { + t.Fatalf("cannot create temporary file: %v", err) + } + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + mountPoint string + expectedResult bool + }{ + { + "ext4 on /", + "/", + true, + }, + { + "tmpfs on /var/lib/bar", + "/var/lib/bar", + false, + }, + { + "nfsv4", + "/media/nfs_vol", + false, + }, + } + + for _, test := range tests { + out, err := getSELinuxSupport(test.mountPoint, filename) + if err != nil { + t.Errorf("Test %s failed with error: %s", test.name, err) + } + if test.expectedResult != out { + t.Errorf("Test %s failed: expected %v, got %v", test.name, test.expectedResult, out) + } + } +} + +func TestSafeOpen(t *testing.T) { + defaultPerm := os.FileMode(0750) + + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + path string + expectError bool + }{ + { + "directory-does-not-exist", + func(base string) error { + return nil + }, + "test/directory", + true, + }, + { + "directory-exists", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) + }, + "test/directory", + false, + }, + { + "escape-base-using-dots", + func(base string) error { + return nil + }, + "..", + true, + }, + { + "escape-base-using-dots-2", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test"), 0750) + }, + "test/../../..", + true, + }, + { + "symlink", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil { + return err + } + return os.Symlink("destination", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "symlink-nested", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir1/dir2"), defaultPerm); err != nil { + return err + } + return os.Symlink("dir1", filepath.Join(base, "dir1/dir2/test")) + }, + "test", + true, + }, + { + "symlink-loop", + func(base string) error { + return os.Symlink("test", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "symlink-not-exists", + func(base string) error { + return os.Symlink("non-existing", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "non-directory", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test/directory", + true, + }, + { + "non-directory-final", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test", + false, + }, + { + "escape-with-relative-symlink", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil { + return err + } + return os.Symlink("../exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + true, + }, + { + "escape-with-relative-symlink-not-exists", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + return os.Symlink("../not-exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + true, + }, + { + "escape-with-symlink", + func(base string) error { + return os.Symlink("/", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "mount-unix-socket", + func(base string) error { + socketFile, socketError := createSocketFile(base) + + if socketError != nil { + return fmt.Errorf("Error preparing socket file %s with %v", socketFile, socketError) + } + return nil + }, + "mt.sock", + false, + }, + { + "mounting-unix-socket-in-middle", + func(base string) error { + testSocketFile, socketError := createSocketFile(base) + + if socketError != nil { + return fmt.Errorf("Error preparing socket file %s with %v", testSocketFile, socketError) + } + return nil + }, + "mt.sock/bar", + true, + }, + } + + for _, test := range tests { + klog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "safe-open-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + + test.prepare(base) + pathToCreate := filepath.Join(base, test.path) + fd, err := doSafeOpen(pathToCreate, base) + if err != nil && !test.expectError { + t.Errorf("test %q: %s", test.name, err) + } + if err != nil { + klog.Infof("got error: %s", err) + } + if err == nil && test.expectError { + t.Errorf("test %q: expected error, got none", test.name) + } + + syscall.Close(fd) + os.RemoveAll(base) + } +} + +func createSocketFile(socketDir string) (string, error) { + testSocketFile := filepath.Join(socketDir, "mt.sock") + + // Switch to volume path and create the socket file + // socket file can not have length of more than 108 character + // and hence we must use relative path + oldDir, _ := os.Getwd() + + err := os.Chdir(socketDir) + if err != nil { + return "", err + } + defer func() { + os.Chdir(oldDir) + }() + _, socketCreateError := net.Listen("unix", "mt.sock") + return testSocketFile, socketCreateError +} + +func TestFindExistingPrefix(t *testing.T) { + defaultPerm := os.FileMode(0750) + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + path string + expectedPath string + expectedDirs []string + expectError bool + }{ + { + "directory-does-not-exist", + func(base string) error { + return nil + }, + "directory", + "", + []string{"directory"}, + false, + }, + { + "directory-exists", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) + }, + "test/directory", + "test/directory", + []string{}, + false, + }, + { + "follow-symlinks", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "destination/directory"), defaultPerm); err != nil { + return err + } + return os.Symlink("destination", filepath.Join(base, "test")) + }, + "test/directory", + "test/directory", + []string{}, + false, + }, + { + "follow-symlink-loop", + func(base string) error { + return os.Symlink("test", filepath.Join(base, "test")) + }, + "test/directory", + "", + nil, + true, + }, + { + "follow-symlink-multiple follow", + func(base string) error { + /* test1/dir points to test2 and test2/dir points to test1 */ + if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil { + return err + } + return nil + }, + "test1/dir/dir/foo/bar", + "test1/dir/dir", + []string{"foo", "bar"}, + false, + }, + { + "danglink-symlink", + func(base string) error { + return os.Symlink("non-existing", filepath.Join(base, "test")) + }, + // OS returns IsNotExist error both for dangling symlink and for + // non-existing directory. + "test/directory", + "", + []string{"test", "directory"}, + false, + }, + { + "with-fifo-in-middle", + func(base string) error { + testFifo := filepath.Join(base, "mount_test.fifo") + return syscall.Mkfifo(testFifo, 0) + }, + "mount_test.fifo/directory", + "", + nil, + true, + }, + } + + for _, test := range tests { + klog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "find-prefix-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + test.prepare(base) + path := filepath.Join(base, test.path) + existingPath, dirs, err := findExistingPrefix(base, path) + if err != nil && !test.expectError { + t.Errorf("test %q: %s", test.name, err) + } + if err != nil { + klog.Infof("got error: %s", err) + } + if err == nil && test.expectError { + t.Errorf("test %q: expected error, got none", test.name) + } + + fullExpectedPath := filepath.Join(base, test.expectedPath) + if existingPath != fullExpectedPath { + t.Errorf("test %q: expected path %q, got %q", test.name, fullExpectedPath, existingPath) + } + if !reflect.DeepEqual(dirs, test.expectedDirs) { + t.Errorf("test %q: expected dirs %v, got %v", test.name, test.expectedDirs, dirs) + } + os.RemoveAll(base) + } +} + +func TestGetFileType(t *testing.T) { + mounter := Mounter{"fake/path", false} + + testCase := []struct { + name string + expectedType FileType + setUp func() (string, string, error) + }{ + { + "Directory Test", + FileTypeDirectory, + func() (string, string, error) { + tempDir, err := ioutil.TempDir("", "test-get-filetype-") + return tempDir, tempDir, err + }, + }, + { + "File Test", + FileTypeFile, + func() (string, string, error) { + tempFile, err := ioutil.TempFile("", "test-get-filetype") + if err != nil { + return "", "", err + } + tempFile.Close() + return tempFile.Name(), tempFile.Name(), nil + }, + }, + { + "Socket Test", + FileTypeSocket, + func() (string, string, error) { + tempDir, err := ioutil.TempDir("", "test-get-filetype-") + if err != nil { + return "", "", err + } + tempSocketFile, err := createSocketFile(tempDir) + return tempSocketFile, tempDir, err + }, + }, + { + "Block Device Test", + FileTypeBlockDev, + func() (string, string, error) { + tempDir, err := ioutil.TempDir("", "test-get-filetype-") + if err != nil { + return "", "", err + } + + tempBlockFile := filepath.Join(tempDir, "test_blk_dev") + outputBytes, err := exec.New().Command("mknod", tempBlockFile, "b", "89", "1").CombinedOutput() + if err != nil { + err = fmt.Errorf("%v: %s ", err, outputBytes) + } + return tempBlockFile, tempDir, err + }, + }, + { + "Character Device Test", + FileTypeCharDev, + func() (string, string, error) { + tempDir, err := ioutil.TempDir("", "test-get-filetype-") + if err != nil { + return "", "", err + } + + tempCharFile := filepath.Join(tempDir, "test_char_dev") + outputBytes, err := exec.New().Command("mknod", tempCharFile, "c", "89", "1").CombinedOutput() + if err != nil { + err = fmt.Errorf("%v: %s ", err, outputBytes) + } + return tempCharFile, tempDir, err + }, + }, + } + + for idx, tc := range testCase { + path, cleanUpPath, err := tc.setUp() + if err != nil { + // Locally passed, but upstream CI is not friendly to create such device files + // Leave "Operation not permitted" out, which can be covered in an e2e test + if isOperationNotPermittedError(err) { + continue + } + t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) + } + if len(cleanUpPath) > 0 { + defer os.RemoveAll(cleanUpPath) + } + + fileType, err := mounter.GetFileType(path) + if err != nil { + t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) + } + if fileType != tc.expectedType { + t.Fatalf("[%d-%s] expected %s, but got %s", idx, tc.name, tc.expectedType, fileType) + } + } +} + +func isOperationNotPermittedError(err error) bool { + if strings.Contains(err.Error(), "Operation not permitted") { + return true + } + return false +} + +func TestSearchMountPoints(t *testing.T) { + base := ` +19 25 0:18 / /sys rw,nosuid,nodev,noexec,relatime shared:7 - sysfs sysfs rw +20 25 0:4 / /proc rw,nosuid,nodev,noexec,relatime shared:12 - proc proc rw +21 25 0:6 / /dev rw,nosuid,relatime shared:2 - devtmpfs udev rw,size=4058156k,nr_inodes=1014539,mode=755 +22 21 0:14 / /dev/pts rw,nosuid,noexec,relatime shared:3 - devpts devpts rw,gid=5,mode=620,ptmxmode=000 +23 25 0:19 / /run rw,nosuid,noexec,relatime shared:5 - tmpfs tmpfs rw,size=815692k,mode=755 +25 0 252:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered +26 19 0:12 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:8 - securityfs securityfs rw +27 21 0:21 / /dev/shm rw,nosuid,nodev shared:4 - tmpfs tmpfs rw +28 23 0:22 / /run/lock rw,nosuid,nodev,noexec,relatime shared:6 - tmpfs tmpfs rw,size=5120k +29 19 0:23 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:9 - tmpfs tmpfs ro,mode=755 +30 29 0:24 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd +31 19 0:25 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:11 - pstore pstore rw +32 29 0:26 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:13 - cgroup cgroup rw,devices +33 29 0:27 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:14 - cgroup cgroup rw,freezer +34 29 0:28 / /sys/fs/cgroup/pids rw,nosuid,nodev,noexec,relatime shared:15 - cgroup cgroup rw,pids +35 29 0:29 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,blkio +36 29 0:30 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,memory +37 29 0:31 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,perf_event +38 29 0:32 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,hugetlb +39 29 0:33 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:20 - cgroup cgroup rw,cpu,cpuacct +40 29 0:34 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:21 - cgroup cgroup rw,cpuset +41 29 0:35 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime shared:22 - cgroup cgroup rw,net_cls,net_prio +58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordere +` + + testcases := []struct { + name string + source string + mountInfos string + expectedRefs []string + expectedErr error + }{ + { + "dir", + "/mnt/disks/vol1", + base, + nil, + nil, + }, + { + "dir-used", + "/mnt/disks/vol1", + base + ` +56 25 252:0 /mnt/disks/vol1 /var/lib/kubelet/pods/1890aef5-5a60-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered +57 25 0:45 / /mnt/disks/vol rw,relatime shared:36 - tmpfs tmpfs rw +`, + []string{"/var/lib/kubelet/pods/1890aef5-5a60-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, + nil, + }, + { + "tmpfs-vol", + "/mnt/disks/vol1", + base + `120 25 0:76 / /mnt/disks/vol1 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k +`, + nil, + nil, + }, + { + "tmpfs-vol-used-by-two-pods", + "/mnt/disks/vol1", + base + `120 25 0:76 / /mnt/disks/vol1 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k +196 25 0:76 / /var/lib/kubelet/pods/ade3ac21-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k +228 25 0:76 / /var/lib/kubelet/pods/ac60532d-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585 rw,relatime shared:41 - tmpfs vol1 rw,size=10000k +`, + []string{ + "/var/lib/kubelet/pods/ade3ac21-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585", + "/var/lib/kubelet/pods/ac60532d-5a5b-11e8-8559-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-8f263585", + }, + nil, + }, + { + "tmpfs-subdir-used-indirectly-via-bindmount-dir-by-one-pod", + "/mnt/vol1/foo", + base + `177 25 0:46 / /mnt/data rw,relatime shared:37 - tmpfs data rw +190 25 0:46 /vol1 /mnt/vol1 rw,relatime shared:37 - tmpfs data rw +191 25 0:46 /vol2 /mnt/vol2 rw,relatime shared:37 - tmpfs data rw +62 25 0:46 /vol1/foo /var/lib/kubelet/pods/e25f2f01-5b06-11e8-8694-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:37 - tmpfs data rw +`, + []string{"/var/lib/kubelet/pods/e25f2f01-5b06-11e8-8694-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, + nil, + }, + { + "dir-bindmounted", + "/mnt/disks/vol2", + base + `342 25 252:0 /mnt/disks/vol2 /mnt/disks/vol2 rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered +`, + nil, + nil, + }, + { + "dir-bindmounted-used-by-one-pod", + "/mnt/disks/vol2", + base + `342 25 252:0 /mnt/disks/vol2 /mnt/disks/vol2 rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered +77 25 252:0 /mnt/disks/vol2 /var/lib/kubelet/pods/f30dc360-5a5d-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-1fb30a1c rw,relatime shared:1 - ext4 /dev/mapper/ubuntu--vg-root rw,errors=remount-ro,data=ordered +`, + []string{"/var/lib/kubelet/pods/f30dc360-5a5d-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-1fb30a1c"}, + nil, + }, + { + "blockfs", + "/mnt/disks/blkvol1", + base + `58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered +`, + nil, + nil, + }, + { + "blockfs-used-by-one-pod", + "/mnt/disks/blkvol1", + base + `58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered +62 25 7:1 / /var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered +`, + []string{"/var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, + nil, + }, + { + "blockfs-used-by-two-pods", + "/mnt/disks/blkvol1", + base + `58 25 7:1 / /mnt/disks/blkvol1 rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered +62 25 7:1 / /var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered +95 25 7:1 / /var/lib/kubelet/pods/4854a48b-5a64-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test rw,relatime shared:38 - ext4 /dev/loop1 rw,data=ordered +`, + []string{"/var/lib/kubelet/pods/f19fe4e2-5a63-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test", + "/var/lib/kubelet/pods/4854a48b-5a64-11e8-962f-000c29bb0377/volumes/kubernetes.io~local-volume/local-pv-test"}, + nil, + }, + } + tmpFile, err := ioutil.TempFile("", "test-get-filetype") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpFile.Name()) + defer tmpFile.Close() + for _, v := range testcases { + tmpFile.Truncate(0) + tmpFile.Seek(0, 0) + tmpFile.WriteString(v.mountInfos) + tmpFile.Sync() + refs, err := searchMountPoints(v.source, tmpFile.Name()) + if !reflect.DeepEqual(refs, v.expectedRefs) { + t.Errorf("test %q: expected Refs: %#v, got %#v", v.name, v.expectedRefs, refs) + } + if !reflect.DeepEqual(err, v.expectedErr) { + t.Errorf("test %q: expected err: %v, got %v", v.name, v.expectedErr, err) + } + } +} diff --git a/pkg/volume/util/subpath/subpath_nsenter.go b/pkg/volume/util/subpath/subpath_nsenter.go new file mode 100644 index 0000000000..ce5d825da7 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_nsenter.go @@ -0,0 +1,470 @@ +// +build linux + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + + "golang.org/x/sys/unix" + "k8s.io/klog" + "k8s.io/utils/nsenter" + utilpath "k8s.io/utils/path" +) + +const ( + // hostProcMountsPath is the default mount path for rootfs + hostProcMountsPath = "/rootfs/proc/1/mounts" + // hostProcMountinfoPath is the default mount info path for rootfs + hostProcMountinfoPath = "/rootfs/proc/1/mountinfo" +) + +// Currently, all docker containers receive their own mount namespaces. +// NsenterMounter works by executing nsenter to run commands in +// the host's mount namespace. +type NsenterMounter struct { + ne *nsenter.Nsenter + // rootDir is location of /var/lib/kubelet directory. + rootDir string +} + +// NewNsenterMounter creates a new mounter for kubelet that runs as a container. +func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter { + return &NsenterMounter{ + rootDir: rootDir, + ne: ne, + } +} + +// NsenterMounter implements mount.Interface +var _ = Interface(&NsenterMounter{}) + +// Mount runs mount(8) in the host's root mount namespace. Aside from this +// aspect, Mount has the same semantics as the mounter returned by mount.New() +func (n *NsenterMounter) Mount(source string, target string, fstype string, options []string) error { + bind, bindOpts, bindRemountOpts := isBind(options) + + if bind { + err := n.doNsenterMount(source, target, fstype, bindOpts) + if err != nil { + return err + } + return n.doNsenterMount(source, target, fstype, bindRemountOpts) + } + + return n.doNsenterMount(source, target, fstype, options) +} + +// doNsenterMount nsenters the host's mount namespace and performs the +// requested mount. +func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error { + klog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options) + cmd, args := n.makeNsenterArgs(source, target, fstype, options) + outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput() + if len(outputBytes) != 0 { + klog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes)) + } + return err +} + +// makeNsenterArgs makes a list of argument to nsenter in order to do the +// requested mount. +func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) { + mountCmd := n.ne.AbsHostPath("mount") + mountArgs := makeMountArgs(source, target, fstype, options) + + if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd { + // Complete command line: + // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t + // Expected flow is: + // * nsenter breaks out of container's mount namespace and executes + // host's systemd-run. + // * systemd-run creates a transient scope (=~ cgroup) and executes its + // argument (/bin/mount) there. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // (systemd-run --scope finishes at this point, returning mount's exit + // code and stdout/stderr - thats one of --scope benefits). + // * systemd keeps the fuse daemon running in the scope (i.e. in its own + // cgroup) until the fuse daemon dies (another --scope benefit). + // Kubelet container can be restarted and the fuse daemon survives. + // * When the daemon dies (e.g. during unmount) systemd removes the + // scope automatically. + mountCmd, mountArgs = addSystemdScope(systemdRunPath, target, mountCmd, mountArgs) + } else { + // Fall back to simple mount when the host has no systemd. + // Complete command line: + // nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/mount -t + // Expected flow is: + // * nsenter breaks out of container's mount namespace and executes host's /bin/mount. + // * mount does its job, forks a fuse daemon if necessary and finishes. + // * Any fuse daemon runs in cgroup of kubelet docker container, + // restart of kubelet container will kill it! + + // No code here, mountCmd and mountArgs use /bin/mount + } + + return mountCmd, mountArgs +} + +// Unmount runs umount(8) in the host's mount namespace. +func (n *NsenterMounter) Unmount(target string) error { + args := []string{target} + // No need to execute systemd-run here, it's enough that unmount is executed + // in the host's mount namespace. It will finish appropriate fuse daemon(s) + // running in any scope. + klog.V(5).Infof("nsenter unmount args: %v", args) + outputBytes, err := n.ne.Exec("umount", args).CombinedOutput() + if len(outputBytes) != 0 { + klog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes)) + } + return err +} + +// List returns a list of all mounted filesystems in the host's mount namespace. +func (*NsenterMounter) List() ([]MountPoint, error) { + return listProcMounts(hostProcMountsPath) +} + +func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) { + return isNotMountPoint(m, dir) +} + +func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool { + deletedDir := fmt.Sprintf("%s\\040(deleted)", dir) + return (mp.Path == dir) || (mp.Path == deletedDir) +} + +// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt +// in the host's root mount namespace. +func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) { + file, err := filepath.Abs(file) + if err != nil { + return true, err + } + + // Check the directory exists + if _, err = os.Stat(file); os.IsNotExist(err) { + klog.V(5).Infof("findmnt: directory %s does not exist", file) + return true, err + } + + // Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts + resolvedFile, err := n.EvalHostSymlinks(file) + if err != nil { + return true, err + } + + // Add --first-only option: since we are testing for the absence of a mountpoint, it is sufficient to get only + // the first of multiple possible mountpoints using --first-only. + // Also add fstype output to make sure that the output of target file will give the full path + // TODO: Need more refactoring for this function. Track the solution with issue #26996 + args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", resolvedFile} + klog.V(5).Infof("nsenter findmnt args: %v", args) + out, err := n.ne.Exec("findmnt", args).CombinedOutput() + if err != nil { + klog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err) + // Different operating systems behave differently for paths which are not mount points. + // On older versions (e.g. 2.20.1) we'd get error, on newer ones (e.g. 2.26.2) we'd get "/". + // It's safer to assume that it's not a mount point. + return true, nil + } + mountTarget, err := parseFindMnt(string(out)) + if err != nil { + return false, err + } + + klog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget) + + if mountTarget == resolvedFile { + klog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile) + return false, nil + } + klog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile) + return true, nil +} + +// parse output of "findmnt -o target,fstype" and return just the target +func parseFindMnt(out string) (string, error) { + // cut trailing newline + out = strings.TrimSuffix(out, "\n") + // cut everything after the last space - it's the filesystem type + i := strings.LastIndex(out, " ") + if i == -1 { + return "", fmt.Errorf("error parsing findmnt output, expected at least one space: %q", out) + } + return out[:i], nil +} + +// DeviceOpened checks if block device in use by calling Open with O_EXCL flag. +// Returns true if open returns errno EBUSY, and false if errno is nil. +// Returns an error if errno is any error other than EBUSY. +// Returns with error if pathname is not a device. +func (n *NsenterMounter) DeviceOpened(pathname string) (bool, error) { + return exclusiveOpenFailsOnDevice(pathname) +} + +// PathIsDevice uses FileInfo returned from os.Stat to check if path refers +// to a device. +func (n *NsenterMounter) PathIsDevice(pathname string) (bool, error) { + pathType, err := n.GetFileType(pathname) + isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev + return isDevice, err +} + +//GetDeviceNameFromMount given a mount point, find the volume id from checking /proc/mounts +func (n *NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(n, mountPath, pluginDir) +} + +func (n *NsenterMounter) MakeRShared(path string) error { + return doMakeRShared(path, hostProcMountinfoPath) +} + +func (mounter *NsenterMounter) GetFileType(pathname string) (FileType, error) { + var pathType FileType + outputBytes, err := mounter.ne.Exec("stat", []string{"-L", "--printf=%F", pathname}).CombinedOutput() + if err != nil { + if strings.Contains(string(outputBytes), "No such file") { + err = fmt.Errorf("%s does not exist", pathname) + } else { + err = fmt.Errorf("stat %s error: %v", pathname, string(outputBytes)) + } + return pathType, err + } + + switch string(outputBytes) { + case "socket": + return FileTypeSocket, nil + case "character special file": + return FileTypeCharDev, nil + case "block special file": + return FileTypeBlockDev, nil + case "directory": + return FileTypeDirectory, nil + case "regular file": + return FileTypeFile, nil + } + + return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device") +} + +func (mounter *NsenterMounter) MakeDir(pathname string) error { + args := []string{"-p", pathname} + if _, err := mounter.ne.Exec("mkdir", args).CombinedOutput(); err != nil { + return err + } + return nil +} + +func (mounter *NsenterMounter) MakeFile(pathname string) error { + args := []string{pathname} + if _, err := mounter.ne.Exec("touch", args).CombinedOutput(); err != nil { + return err + } + return nil +} + +func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) { + // Resolve the symlinks but allow the target not to exist. EvalSymlinks + // would return an generic error when the target does not exist. + hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */) + if err != nil { + return false, err + } + kubeletpath := mounter.ne.KubeletPath(hostPath) + return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath) +} + +func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) { + return mounter.ne.EvalSymlinks(pathname, true) +} + +func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(mounter, podDir, volumeName) +} + +func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + // Bind-mount the subpath to avoid using symlinks in subpaths. + newHostPath, err = doNsEnterBindSubPath(mounter, subPath) + + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + fullSubdirPath := filepath.Join(base, subdir) + evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err) + } + evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath) + + evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + evaluatedBase = filepath.Clean(evaluatedBase) + + rootDir := filepath.Clean(mounter.rootDir) + if PathWithinBase(evaluatedBase, rootDir) { + // Base is in /var/lib/kubelet. This directory is shared between the + // container with kubelet and the host. We don't need to add '/rootfs'. + // This is useful when /rootfs is mounted as read-only - we can still + // create subpaths for paths in /var/lib/kubelet. + return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm) + } + + // Base is somewhere on the host's filesystem. Add /rootfs and try to make + // the directory there. + // This requires /rootfs to be writable. + kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath) + kubeletBase := mounter.ne.KubeletPath(evaluatedBase) + return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm) +} + +func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) { + exists, err := mounter.ExistsPath(pathname) + if err != nil { + return nil, err + } + if !exists { + return []string{}, nil + } + hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) + if err != nil { + return nil, err + } + return searchMountPoints(hostpath, hostProcMountinfoPath) +} + +func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) { + // Linux, kubelet runs in a container: + // - safely open the subpath + // - bind-mount the subpath to target (this can be unsafe) + // - check that we mounted the right thing by comparing device ID and inode + // of the subpath (via safely opened fd) and the target (that's under our + // control) + + // Evaluate all symlinks here once for all subsequent functions. + evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err) + } + evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/) + if err != nil { + return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err) + } + klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath) + subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath) + subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath) + + // Check the subpath is correct and open it + fd, err := safeOpenSubPath(mounter, subpath) + if err != nil { + return "", err + } + defer syscall.Close(fd) + + alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath) + if err != nil { + return "", err + } + if alreadyMounted { + return bindPathTarget, nil + } + + success := false + defer func() { + // Cleanup subpath on error + if !success { + klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + } + } + }() + + // Leap of faith: optimistically expect that nobody has modified previously + // expanded evalSubPath with evil symlinks and bind-mount it. + // Mount is done on the host! don't use kubelet path! + klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget) + if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { + return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err) + } + + // Check that the bind-mount target is the same inode and device as the + // source that we keept open, i.e. we mounted the right thing. + err = checkDeviceInode(fd, bindPathTarget) + if err != nil { + return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err) + } + + success = true + klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + return bindPathTarget, nil +} + +// checkDeviceInode checks that opened file and path represent the same file. +func checkDeviceInode(fd int, path string) error { + var srcStat, dstStat unix.Stat_t + err := unix.Fstat(fd, &srcStat) + if err != nil { + return fmt.Errorf("error running fstat on subpath FD: %v", err) + } + + err = unix.Stat(path, &dstStat) + if err != nil { + return fmt.Errorf("error running fstat on %s: %v", path, err) + } + + if srcStat.Dev != dstStat.Dev { + return fmt.Errorf("different device number") + } + if srcStat.Ino != dstStat.Ino { + return fmt.Errorf("different inode") + } + return nil +} + +func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) { + hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) + if err != nil { + return -1, err + } + kubeletpath := mounter.ne.KubeletPath(hostPath) + return getFSGroup(kubeletpath) +} + +func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) { + return getSELinuxSupport(pathname, hostProcMountsPath) +} + +func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) { + hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */) + if err != nil { + return 0, err + } + kubeletpath := mounter.ne.KubeletPath(hostPath) + return getMode(kubeletpath) +} diff --git a/pkg/volume/util/subpath/subpath_nsenter_test.go b/pkg/volume/util/subpath/subpath_nsenter_test.go new file mode 100644 index 0000000000..1ed4bd222b --- /dev/null +++ b/pkg/volume/util/subpath/subpath_nsenter_test.go @@ -0,0 +1,724 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "io/ioutil" + "os" + "os/user" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" + "k8s.io/utils/nsenter" +) + +func TestParseFindMnt(t *testing.T) { + tests := []struct { + input string + target string + expectError bool + }{ + { + // standard mount name, e.g. for AWS + "/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389 ext4\n", + "/var/lib/kubelet/plugins/kubernetes.io/aws-ebs/mounts/aws/us-east-1d/vol-020f82b0759f72389", + false, + }, + { + // mount name with space, e.g. vSphere + "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n", + "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[datastore1] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk", + false, + }, + { + // hypotetic mount with several spaces + "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk ext2\n", + "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[ d a t a s t o r e 1 ] kubevols/kubernetes-dynamic-pvc-4aacaa9b-6ba5-11e7-8f64-0050569f1b82.vmdk", + false, + }, + { + // invalid output - no filesystem type + "/var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/blabla", + "", + true, + }, + } + + for i, test := range tests { + target, err := parseFindMnt(test.input) + if test.expectError && err == nil { + t.Errorf("test %d expected error, got nil", i) + } + if !test.expectError && err != nil { + t.Errorf("test %d returned error: %s", i, err) + } + if target != test.target { + t.Errorf("test %d expected %q, got %q", i, test.target, target) + } + } +} + +func TestCheckDeviceInode(t *testing.T) { + testDir, err := ioutil.TempDir("", "nsenter-mounter-device-") + if err != nil { + t.Fatalf("Cannot create temporary directory: %s", err) + } + defer os.RemoveAll(testDir) + + tests := []struct { + name string + srcPath string + dstPath string + expectError string + }{ + { + name: "the same file", + srcPath: filepath.Join(testDir, "1"), + dstPath: filepath.Join(testDir, "1"), + expectError: "", + }, + { + name: "different file on the same FS", + srcPath: filepath.Join(testDir, "2.1"), + dstPath: filepath.Join(testDir, "2.2"), + expectError: "different inode", + }, + { + name: "different file on different device", + srcPath: filepath.Join(testDir, "3"), + // /proc is always on a different "device" than /tmp (or $TEMP) + dstPath: "/proc/self/status", + expectError: "different device", + }, + } + + for _, test := range tests { + if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil { + t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err) + continue + } + + // Don't create dst if it exists + if _, err := os.Stat(test.dstPath); os.IsNotExist(err) { + if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil { + t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err) + continue + } + } else if err != nil { + t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err) + continue + } + + fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644) + if err != nil { + t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err) + continue + } + + err = checkDeviceInode(fd, test.dstPath) + + if test.expectError == "" && err != nil { + t.Errorf("Test %q: expected no error, got %s", test.name, err) + } + if test.expectError != "" { + if err == nil { + t.Errorf("Test %q: expected error, got none", test.name) + } else { + if !strings.Contains(err.Error(), test.expectError) { + t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err) + } + } + } + } +} + +func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) { + rootfsPath = filepath.Join(tmpdir, "rootfs") + if err := os.Mkdir(rootfsPath, 0755); err != nil { + return nil, "", "", err + } + ne, err := nsenter.NewFakeNsenter(rootfsPath) + if err != nil { + return nil, "", "", err + } + + varlibPath = filepath.Join(tmpdir, "/var/lib/kubelet") + if err := os.MkdirAll(varlibPath, 0755); err != nil { + return nil, "", "", err + } + + return NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, nil +} + +func TestNsenterExistsFile(t *testing.T) { + var isRoot bool + usr, err := user.Current() + if err == nil { + isRoot = usr.Username == "root" + } else { + switch err.(type) { + case user.UnknownUserIdError: + // Root should be always known, this is some random UID + isRoot = false + default: + t.Fatal(err) + } + } + + tests := []struct { + name string + prepare func(base, rootfs string) (string, error) + expectedOutput bool + expectError bool + }{ + { + name: "simple existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/file + path := filepath.Join(base, "file") + if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { + return "", err + } + // In kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, path, 0644); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: true, + }, + { + name: "simple non-existing file", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "file") + return path, nil + }, + expectedOutput: false, + }, + { + name: "simple non-accessible file", + prepare: func(base, rootfs string) (string, error) { + // On the host: + // create /base/dir/file, then make the dir inaccessible + dir := filepath.Join(base, "dir") + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + path := filepath.Join(dir, "file") + if err := ioutil.WriteFile(path, []byte{}, 0); err != nil { + return "", err + } + if err := os.Chmod(dir, 0644); err != nil { + return "", err + } + + // In kubelet: do the same with /rootfs/base/dir/file + rootfsPath, err := writeRootfsFile(rootfs, path, 0777) + if err != nil { + return "", err + } + rootfsDir := filepath.Dir(rootfsPath) + if err := os.Chmod(rootfsDir, 0644); err != nil { + return "", err + } + + return path, nil + }, + expectedOutput: isRoot, // ExistsPath success when running as root + expectError: !isRoot, // ExistsPath must fail when running as not-root + }, + { + name: "relative symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink("file", path); err != nil { + return "", err + } + // In kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: true, + }, + { + name: "absolute symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> /base/file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink(file, path); err != nil { + return "", err + } + // In kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0644); err != nil { + return "", err + } + + return path, nil + }, + expectedOutput: true, + }, + { + name: "relative symlink to non-existing file", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "link") + if err := os.Symlink("file", path); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: false, + }, + { + name: "absolute symlink to non-existing file", + prepare: func(base, rootfs string) (string, error) { + file := filepath.Join(base, "file") + path := filepath.Join(base, "link") + if err := os.Symlink(file, path); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: false, + }, + { + name: "symlink loop", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "link") + if err := os.Symlink(path, path); err != nil { + return "", err + } + return path, nil + }, + expectedOutput: false, + // TODO: realpath -m is not able to detect symlink loop. Should we care? + expectError: false, + }, + } + + for _, test := range tests { + tmpdir, err := ioutil.TempDir("", "nsenter-exists-file") + if err != nil { + t.Error(err) + continue + } + defer os.RemoveAll(tmpdir) + + testBase := filepath.Join(tmpdir, "base") + if err := os.Mkdir(testBase, 0755); err != nil { + t.Error(err) + continue + } + + mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t) + if err != nil { + t.Error(err) + continue + } + + path, err := test.prepare(testBase, rootfs) + if err != nil { + t.Error(err) + continue + } + + out, err := mounter.ExistsPath(path) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("Test %q: expected error, got none", test.name) + } + + if out != test.expectedOutput { + t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out) + } + } +} + +func TestNsenterGetMode(t *testing.T) { + tests := []struct { + name string + prepare func(base, rootfs string) (string, error) + expectedMode os.FileMode + expectError bool + }{ + { + name: "simple file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/file + path := filepath.Join(base, "file") + if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil { + return "", err + } + + // Prepare a different file as /rootfs/base/file (="the host + // visible from container") to check that NsEnterMounter calls + // stat on this file and not on /base/file. + // Visible from kubelet: /rootfs/base/file + if _, err := writeRootfsFile(rootfs, path, 0777); err != nil { + return "", err + } + + return path, nil + }, + expectedMode: 0777, + }, + { + name: "non-existing file", + prepare: func(base, rootfs string) (string, error) { + path := filepath.Join(base, "file") + return path, nil + }, + expectedMode: 0, + expectError: true, + }, + { + name: "absolute symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> /base/file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink(file, path); err != nil { + return "", err + } + + // Visible from kubelet: + // /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0747); err != nil { + return "", err + } + + return path, nil + }, + expectedMode: 0747, + }, + { + name: "relative symlink to existing file", + prepare: func(base, rootfs string) (string, error) { + // On the host: /base/link -> file + file := filepath.Join(base, "file") + if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil { + return "", err + } + path := filepath.Join(base, "link") + if err := os.Symlink("file", path); err != nil { + return "", err + } + + // Visible from kubelet: + // /rootfs/base/file + if _, err := writeRootfsFile(rootfs, file, 0647); err != nil { + return "", err + } + + return path, nil + }, + expectedMode: 0647, + }, + } + + for _, test := range tests { + tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") + if err != nil { + t.Error(err) + continue + } + defer os.RemoveAll(tmpdir) + + testBase := filepath.Join(tmpdir, "base") + if err := os.Mkdir(testBase, 0755); err != nil { + t.Error(err) + continue + } + + mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t) + if err != nil { + t.Error(err) + continue + } + + path, err := test.prepare(testBase, rootfs) + if err != nil { + t.Error(err) + continue + } + + mode, err := mounter.GetMode(path) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("Test %q: expected error, got none", test.name) + } + + if mode != test.expectedMode { + t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode) + } + } +} + +func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) { + fullPath := filepath.Join(rootfs, path) + dir := filepath.Dir(fullPath) + if err := os.MkdirAll(dir, 0755); err != nil { + return "", err + } + if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil { + return "", err + } + // Use chmod, io.WriteFile is affected by umask + if err := os.Chmod(fullPath, mode); err != nil { + return "", err + } + return fullPath, nil +} + +func TestNsenterSafeMakeDir(t *testing.T) { + tests := []struct { + name string + prepare func(base, rootfs, varlib string) (expectedDir string, err error) + subdir string + expectError bool + // If true, "base" directory for SafeMakeDir will be /var/lib/kubelet + baseIsVarLib bool + }{ + { + name: "simple directory", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // expected to be created in /roots/ + expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure") + return expectedDir, nil + }, + }, + { + name: "simple existing directory", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: directory exists + hostPath := filepath.Join(base, "some/subdirectory/structure") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + // In rootfs: directory exists + kubeletPath := filepath.Join(rootfs, hostPath) + if err := os.MkdirAll(kubeletPath, 0755); err != nil { + return "", err + } + // expected to be created in /roots/ + expectedDir = kubeletPath + return expectedDir, nil + }, + }, + { + name: "absolute symlink into safe place", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /base/other/subdirectory exists, /base/some is link to /base/other + hostPath := filepath.Join(base, "other/subdirectory") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + somePath := filepath.Join(base, "some") + otherPath := filepath.Join(base, "other") + if err := os.Symlink(otherPath, somePath); err != nil { + return "", err + } + + // In rootfs: /base/other/subdirectory exists + kubeletPath := filepath.Join(rootfs, hostPath) + if err := os.MkdirAll(kubeletPath, 0755); err != nil { + return "", err + } + // expected 'structure' to be created + expectedDir = filepath.Join(rootfs, hostPath, "structure") + return expectedDir, nil + }, + }, + { + name: "relative symlink into safe place", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /base/other/subdirectory exists, /base/some is link to other + hostPath := filepath.Join(base, "other/subdirectory") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + somePath := filepath.Join(base, "some") + if err := os.Symlink("other", somePath); err != nil { + return "", err + } + + // In rootfs: /base/other/subdirectory exists + kubeletPath := filepath.Join(rootfs, hostPath) + if err := os.MkdirAll(kubeletPath, 0755); err != nil { + return "", err + } + // expected 'structure' to be created + expectedDir = filepath.Join(rootfs, hostPath, "structure") + return expectedDir, nil + }, + }, + { + name: "symlink into unsafe place", + // evaluated in base + subdir: "some/subdirectory/structure", + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /base/some is link to /bin/other + somePath := filepath.Join(base, "some") + if err := os.Symlink("/bin", somePath); err != nil { + return "", err + } + return "", nil + }, + expectError: true, + }, + { + name: "simple directory in /var/lib/kubelet", + // evaluated in varlib + subdir: "some/subdirectory/structure", + baseIsVarLib: true, + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // expected to be created in /base/var/lib/kubelet, not in /rootfs! + expectedDir = filepath.Join(varlib, "some/subdirectory/structure") + return expectedDir, nil + }, + }, + { + name: "safe symlink in /var/lib/kubelet", + // evaluated in varlib + subdir: "some/subdirectory/structure", + baseIsVarLib: true, + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other + hostPath := filepath.Join(varlib, "other/subdirectory") + if err := os.MkdirAll(hostPath, 0755); err != nil { + return "", err + } + somePath := filepath.Join(varlib, "some") + if err := os.Symlink("other", somePath); err != nil { + return "", err + } + + // expected to be created in /base/var/lib/kubelet, not in /rootfs! + expectedDir = filepath.Join(varlib, "other/subdirectory/structure") + return expectedDir, nil + }, + }, + { + name: "unsafe symlink in /var/lib/kubelet", + // evaluated in varlib + subdir: "some/subdirectory/structure", + baseIsVarLib: true, + prepare: func(base, rootfs, varlib string) (expectedDir string, err error) { + // On the host: /varlib/some is link to /bin + somePath := filepath.Join(varlib, "some") + if err := os.Symlink("/bin", somePath); err != nil { + return "", err + } + + return "", nil + }, + expectError: true, + }, + } + for _, test := range tests { + tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-") + if err != nil { + t.Error(err) + continue + } + defer os.RemoveAll(tmpdir) + + mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t) + if err != nil { + t.Error(err) + continue + } + // Prepare base directory for the test + testBase := filepath.Join(tmpdir, "base") + if err := os.Mkdir(testBase, 0755); err != nil { + t.Error(err) + continue + } + // Prepare base directory also in /rootfs + rootfsBase := filepath.Join(rootfs, testBase) + if err := os.MkdirAll(rootfsBase, 0755); err != nil { + t.Error(err) + continue + } + + expectedDir := "" + if test.prepare != nil { + expectedDir, err = test.prepare(testBase, rootfs, varlib) + if err != nil { + t.Error(err) + continue + } + } + + if test.baseIsVarLib { + // use /var/lib/kubelet as the test base so we can test creating + // subdirs there directly in /var/lib/kubenet and not in + // /rootfs/var/lib/kubelet + testBase = varlib + } + + err = mounter.SafeMakeDir(test.subdir, testBase, 0755) + if err != nil && !test.expectError { + t.Errorf("Test %q: unexpected error: %s", test.name, err) + } + if test.expectError { + if err == nil { + t.Errorf("Test %q: expected error, got none", test.name) + } else { + if !strings.Contains(err.Error(), "is outside of allowed base") { + t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err) + } + } + } + + if expectedDir != "" { + _, err := os.Stat(expectedDir) + if err != nil { + t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err) + } + } + } +} diff --git a/pkg/volume/util/subpath/subpath_unsupported.go b/pkg/volume/util/subpath/subpath_unsupported.go new file mode 100644 index 0000000000..918d58cd24 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_unsupported.go @@ -0,0 +1,139 @@ +// +build !linux,!windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "errors" + "os" +) + +type Mounter struct { + mounterPath string +} + +var unsupportedErr = errors.New("util/mount on this platform is not supported") + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + return unsupportedErr +} + +func (mounter *Mounter) Unmount(target string) error { + return unsupportedErr +} + +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, unsupportedErr +} + +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + return (mp.Path == dir) +} + +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return isNotMountPoint(mounter, dir) +} + +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, unsupportedErr +} + +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return "", unsupportedErr +} + +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + return "", unsupportedErr +} + +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, unsupportedErr +} + +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return true, unsupportedErr +} + +func (mounter *Mounter) MakeRShared(path string) error { + return unsupportedErr +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + return mounter.Interface.Mount(source, target, fstype, options) +} + +func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { + return true, unsupportedErr +} + +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return FileType("fake"), unsupportedErr +} + +func (mounter *Mounter) MakeDir(pathname string) error { + return unsupportedErr +} + +func (mounter *Mounter) MakeFile(pathname string) error { + return unsupportedErr +} + +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return true, errors.New("not implemented") +} + +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return "", unsupportedErr +} + +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, unsupportedErr +} + +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return unsupportedErr +} + +func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return unsupportedErr +} + +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} + +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + return false, errors.New("not implemented") +} + +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + return 0, errors.New("not implemented") +} diff --git a/pkg/volume/util/subpath/subpath_windows.go b/pkg/volume/util/subpath/subpath_windows.go new file mode 100644 index 0000000000..917d0ce511 --- /dev/null +++ b/pkg/volume/util/subpath/subpath_windows.go @@ -0,0 +1,647 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + + "k8s.io/klog" + + utilpath "k8s.io/utils/path" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount : mounts source to target with given options. +// currently only supports cifs(smb), bind mount(for disk) +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + target = normalizeWindowsPath(target) + + if source == "tmpfs" { + klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + options, source, target, fstype) + bindSource := source + + // tell it's going to mount azure disk or azure file according to options + if bind, _, _ := isBind(options); bind { + // mount azure disk + bindSource = normalizeWindowsPath(source) + } else { + if len(options) < 2 { + klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + options, len(options), source, target) + return nil + } + + // currently only cifs mount is supported + if strings.ToLower(fstype) != "cifs" { + return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) + } + + if output, err := newSMBMapping(options[0], options[1], source); err != nil { + if isSMBMappingExist(source) { + klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source) + if output, err := removeSMBMapping(source); err != nil { + return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) + } + if output, err := newSMBMapping(options[0], options[1], source); err != nil { + return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output) + } + } else { + return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output) + } + } + } + + if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { + klog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + return err + } + + return nil +} + +// do the SMB mount with username, password, remotepath +// return (output, error) +func newSMBMapping(username, password, remotepath string) (string, error) { + if username == "" || password == "" || remotepath == "" { + return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath) + } + + // use PowerShell Environment Variables to store user input string to prevent command line injection + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 + cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` + cmd := exec.Command("powershell", "/c", cmdLine) + cmd.Env = append(os.Environ(), + fmt.Sprintf("smbuser=%s", username), + fmt.Sprintf("smbpassword=%s", password), + fmt.Sprintf("smbremotepath=%s", remotepath)) + + output, err := cmd.CombinedOutput() + return string(output), err +} + +// check whether remotepath is already mounted +func isSMBMappingExist(remotepath string) bool { + cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + _, err := cmd.CombinedOutput() + return err == nil +} + +// remove SMB mapping +func removeSMBMapping(remotepath string) (string, error) { + cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) + cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) + output, err := cmd.CombinedOutput() + return string(output), err +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + klog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = normalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + klog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, nil +} + +// IsMountPointMatch determines if the mountpoint matches the dir +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + return mp.Path == dir +} + +// IsNotMountPoint determines if a directory is a mountpoint. +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return isNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + // If current file is a symlink, then it is a mountpoint. + if stat.Mode()&os.ModeSymlink != 0 { + target, err := os.Readlink(file) + if err != nil { + return true, fmt.Errorf("readlink error: %v", err) + } + exists, err := mounter.ExistsPath(target) + if err != nil { + return true, err + } + return !exists, nil + } + + return true, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// getDeviceNameFromMount find the device(drive) name in which +// the mount path reference should match the given plugin directory. In case no mount path reference +// matches, returns the volume name taken from its given mountPath +func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) { + refs, err := mounter.GetMountRefs(mountPath) + if err != nil { + klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err) + return "", err + } + if len(refs) == 0 { + return "", fmt.Errorf("directory %s is not mounted", mountPath) + } + basemountPath := normalizeWindowsPath(path.Join(pluginDir, MountsInGlobalPDPath)) + for _, ref := range refs { + if strings.Contains(ref, basemountPath) { + volumeID, err := filepath.Rel(normalizeWindowsPath(basemountPath), ref) + if err != nil { + klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err) + return "", err + } + return volumeID, nil + } + } + + return path.Base(mountPath), nil +} + +// DeviceOpened determines if the device is in use elsewhere +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, nil +} + +// PathIsDevice determines if a path is a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return false, nil +} + +// MakeRShared checks that given path is on a mount with 'rshared' mount +// propagation. Empty implementation here. +func (mounter *Mounter) MakeRShared(path string) error { + return nil +} + +// GetFileType checks for sockets/block/character devices +func (mounter *Mounter) GetFileType(pathname string) (FileType, error) { + return getFileType(pathname) +} + +// MakeFile creates a new directory +func (mounter *Mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// MakeFile creates an empty file +func (mounter *Mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + defer f.Close() + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +// ExistsPath checks whether the path exists +func (mounter *Mounter) ExistsPath(pathname string) (bool, error) { + return utilpath.Exists(utilpath.CheckFollowSymlink, pathname) +} + +// EvalHostSymlinks returns the path name after evaluating symlinks +func (mounter *Mounter) EvalHostSymlinks(pathname string) (string, error) { + return filepath.EvalSymlinks(pathname) +} + +// check whether hostPath is within volume path +// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started +func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(hostPath) == 0 { + return []uintptr{}, nil + } + + finalSubPath, err := filepath.EvalSymlinks(hostPath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err) + } + finalVolumePath, err := filepath.EvalSymlinks(volumePath) + if err != nil { + return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err) + } + + return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath) +} + +// lock all intermediate subPath directories and check they are all within volumePath +// volumePath & subPath should not contain any symlink, otherwise it will return error +func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) { + if len(volumePath) == 0 || len(subPath) == 0 { + return []uintptr{}, nil + } + + // get relative path to volumePath + relSubPath, err := filepath.Rel(volumePath, subPath) + if err != nil { + return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err) + } + if startsWithBackstep(relSubPath) { + return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath) + } + + if relSubPath == "." { + // volumePath and subPath are equal + return []uintptr{}, nil + } + + fileHandles := []uintptr{} + var errorResult error + + currentFullPath := volumePath + dirs := strings.Split(relSubPath, string(os.PathSeparator)) + for _, dir := range dirs { + // lock intermediate subPath directory first + currentFullPath = filepath.Join(currentFullPath, dir) + handle, err := lockPath(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err) + break + } + fileHandles = append(fileHandles, handle) + + // make sure intermediate subPath directory does not contain symlink any more + stat, err := os.Lstat(currentFullPath) + if err != nil { + errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err) + break + } + if stat.Mode()&os.ModeSymlink != 0 { + errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath) + break + } + + if !PathWithinBase(currentFullPath, volumePath) { + errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath) + break + } + } + + return fileHandles, errorResult +} + +// unlockPath unlock directories +func unlockPath(fileHandles []uintptr) { + if fileHandles != nil { + for _, handle := range fileHandles { + syscall.CloseHandle(syscall.Handle(handle)) + } + } +} + +// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path +func lockPath(path string) (uintptr, error) { + if len(path) == 0 { + return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return uintptr(syscall.InvalidHandle), err + } + access := uint32(syscall.GENERIC_READ) + sharemode := uint32(syscall.FILE_SHARE_READ) + createmode := uint32(syscall.OPEN_EXISTING) + flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT) + fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0) + return uintptr(fd), err +} + +// Lock all directories in subPath and check they're not symlinks. +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path) + + // Unlock the directories when the container starts + cleanupAction = func() { + unlockPath(handles) + } + return subPath.Path, cleanupAction, err +} + +// No bind-mounts for subpaths are necessary on Windows +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + // Try to mount the disk + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + klog.Errorf("diskMount: formatAndMount failed, err: %v", err) + return err + } + + if len(fstype) == 0 { + // Use 'NTFS' as the default + fstype = "NTFS" + } + + // format disk if it is unformatted(raw) + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+ + " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype) + if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil { + return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output)) + } + klog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype) + + driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) + if err != nil { + return err + } + driverPath := driveLetter + ":" + target = normalizeWindowsPath(target) + klog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) + if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil { + klog.Errorf("mklink failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +func normalizeWindowsPath(path string) string { + normalizedPath := strings.Replace(path, "/", "\\", -1) + if strings.HasPrefix(normalizedPath, "\\") { + normalizedPath = "c:" + normalizedPath + } + return normalizedPath +} + +// ValidateDiskNumber : disk number should be a number in [0, 99] +func ValidateDiskNumber(disk string) error { + diskNum, err := strconv.Atoi(disk) + if err != nil { + return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) + } + + if diskNum < 0 || diskNum > 99 { + return fmt.Errorf("disk number out of range: %q", disk) + } + + return nil +} + +// Get drive letter according to windows disk number +func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) { + cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum) + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output)) + } + if len(string(output)) < 1 { + return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty") + } + return string(output)[:1], nil +} + +// getAllParentLinks walks all symbolic links and return all the parent targets recursively +func getAllParentLinks(path string) ([]string, error) { + const maxIter = 255 + links := []string{} + for { + links = append(links, path) + if len(links) > maxIter { + return links, fmt.Errorf("unexpected length of parent links: %v", links) + } + + fi, err := os.Lstat(path) + if err != nil { + return links, fmt.Errorf("Lstat: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + break + } + + path, err = os.Readlink(path) + if err != nil { + return links, fmt.Errorf("Readlink error: %v", err) + } + } + + return links, nil +} + +// GetMountRefs : empty implementation here since there is no place to query all mount points on Windows +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + if _, err := os.Stat(normalizeWindowsPath(pathname)); os.IsNotExist(err) { + return []string{}, nil + } else if err != nil { + return nil, err + } + return []string{pathname}, nil +} + +// Note that on windows, it always returns 0. We actually don't set FSGroup on +// windows platform, see SetVolumeOwnership implementation. +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return 0, nil +} + +func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) { + // Windows does not support SELinux. + return false, nil +} + +func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return info.Mode(), nil +} + +// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. +func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error { + realBase, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("error resolving symlinks in %s: %s", base, err) + } + + realFullPath := filepath.Join(realBase, subdir) + return doSafeMakeDir(realFullPath, realBase, perm) +} + +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + klog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !PathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + klog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + if len(toCreate) == 0 { + return nil + } + + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening existing directory %s: %s", existingPath, err) + } + fullBasePath, err := filepath.EvalSymlinks(base) + if err != nil { + return fmt.Errorf("cannot read link %s: %s", base, err) + } + if !PathWithinBase(fullExistingPath, fullBasePath) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err) + } + + // lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom) + fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath) + defer unlockPath(fileHandles) + if err != nil { + return err + } + + klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink by lock that directory immediately + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + klog.V(4).Infof("Creating %s", dir) + if err := os.Mkdir(currentPath, perm); err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + handle, err := lockPath(currentPath) + if err != nil { + return fmt.Errorf("cannot lock path %s: %s", currentPath, err) + } + defer syscall.CloseHandle(syscall.Handle(handle)) + // make sure newly created directory does not contain symlink after lock + stat, err := os.Lstat(currentPath) + if err != nil { + return fmt.Errorf("Lstat(%q) error: %v", currentPath, err) + } + if stat.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath) + } + } + + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + + if startsWithBackstep(rel) { + return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base) + } + + if rel == "." { + // base and pathname are equal + return pathname, []string{}, nil + } + + dirs := strings.Split(rel, string(filepath.Separator)) + + parent := base + currentPath := base + for i, dir := range dirs { + parent = currentPath + currentPath = filepath.Join(parent, dir) + if _, err := os.Lstat(currentPath); err != nil { + if os.IsNotExist(err) { + return parent, dirs[i:], nil + } + return base, nil, err + } + } + + return pathname, []string{}, nil +} diff --git a/pkg/volume/util/subpath/subpath_windows_test.go b/pkg/volume/util/subpath/subpath_windows_test.go new file mode 100644 index 0000000000..e918f3f9ef --- /dev/null +++ b/pkg/volume/util/subpath/subpath_windows_test.go @@ -0,0 +1,827 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNormalizeWindowsPath(t *testing.T) { + path := `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4/volumes/kubernetes.io~azure-disk` + normalizedPath := normalizeWindowsPath(path) + if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` { + t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) + } + + path = `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` + normalizedPath = normalizeWindowsPath(path) + if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` { + t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) + } + + path = `/` + normalizedPath = normalizeWindowsPath(path) + if normalizedPath != `c:\` { + t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) + } +} + +func TestValidateDiskNumber(t *testing.T) { + diskNum := "0" + if err := ValidateDiskNumber(diskNum); err != nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } + + diskNum = "99" + if err := ValidateDiskNumber(diskNum); err != nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } + + diskNum = "ab" + if err := ValidateDiskNumber(diskNum); err == nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } + + diskNum = "100" + if err := ValidateDiskNumber(diskNum); err == nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } +} + +func makeLink(link, target string) error { + if output, err := exec.Command("cmd", "/c", "mklink", "/D", link, target).CombinedOutput(); err != nil { + return fmt.Errorf("mklink failed: %v, link(%q) target(%q) output: %q", err, link, target, string(output)) + } + return nil +} + +func removeLink(link string) error { + if output, err := exec.Command("cmd", "/c", "rmdir", link).CombinedOutput(); err != nil { + return fmt.Errorf("rmdir failed: %v, output: %q", err, string(output)) + } + return nil +} + +func setEquivalent(set1, set2 []string) bool { + map1 := make(map[string]bool) + map2 := make(map[string]bool) + for _, s := range set1 { + map1[s] = true + } + for _, s := range set2 { + map2[s] = true + } + + for s := range map1 { + if !map2[s] { + return false + } + } + for s := range map2 { + if !map1[s] { + return false + } + } + return true +} + +// this func must run in admin mode, otherwise it will fail +func TestGetMountRefs(t *testing.T) { + tests := []struct { + mountPath string + expectedRefs []string + }{ + { + mountPath: `c:\windows`, + expectedRefs: []string{`c:\windows`}, + }, + { + mountPath: `c:\doesnotexist`, + expectedRefs: []string{}, + }, + } + + mounter := Mounter{"fake/path"} + + for _, test := range tests { + if refs, err := mounter.GetMountRefs(test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { + t.Errorf("getMountRefs(%q) = %v, error: %v; expected %v", test.mountPath, refs, err, test.expectedRefs) + } + } +} + +func TestDoSafeMakeDir(t *testing.T) { + base, err := ioutil.TempDir("", "TestDoSafeMakeDir") + if err != nil { + t.Fatalf(err.Error()) + } + + defer os.RemoveAll(base) + + testingVolumePath := filepath.Join(base, "testingVolumePath") + os.MkdirAll(testingVolumePath, 0755) + defer os.RemoveAll(testingVolumePath) + + tests := []struct { + volumePath string + subPath string + expectError bool + symlinkTarget string + }{ + { + volumePath: testingVolumePath, + subPath: ``, + expectError: true, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `x`), + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\d`), + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `symlink`), + expectError: false, + symlinkTarget: base, + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `symlink\c\d`), + expectError: true, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `symlink\y926`), + expectError: true, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\symlink`), + expectError: false, + symlinkTarget: base, + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\x\symlink`), + expectError: false, + symlinkTarget: filepath.Join(testingVolumePath, `a`), + }, + } + + for _, test := range tests { + if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 { + // make all parent sub directories + if parent := filepath.Dir(test.subPath); parent != "." { + os.MkdirAll(parent, 0755) + } + + // make last element as symlink + linkPath := test.subPath + if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) { + if err := makeLink(linkPath, test.symlinkTarget); err != nil { + t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err)) + } + } + } + + err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755)) + if test.expectError { + assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath) + continue + } + assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath) + if _, err := os.Stat(test.subPath); os.IsNotExist(err) { + t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath) + } + } +} + +func TestLockAndCheckSubPath(t *testing.T) { + base, err := ioutil.TempDir("", "TestLockAndCheckSubPath") + if err != nil { + t.Fatalf(err.Error()) + } + + defer os.RemoveAll(base) + + testingVolumePath := filepath.Join(base, "testingVolumePath") + + tests := []struct { + volumePath string + subPath string + expectedHandleCount int + expectError bool + symlinkTarget string + }{ + { + volumePath: `c:\`, + subPath: ``, + expectedHandleCount: 0, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: ``, + subPath: `a`, + expectedHandleCount: 0, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a`), + expectedHandleCount: 1, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\d`), + expectedHandleCount: 4, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `symlink`), + expectedHandleCount: 0, + expectError: true, + symlinkTarget: base, + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`), + expectedHandleCount: 0, + expectError: true, + symlinkTarget: base, + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`), + expectedHandleCount: 2, + expectError: false, + symlinkTarget: filepath.Join(testingVolumePath, `a\b`), + }, + } + + for _, test := range tests { + if len(test.volumePath) > 0 && len(test.subPath) > 0 { + os.MkdirAll(test.volumePath, 0755) + if len(test.symlinkTarget) == 0 { + // make all intermediate sub directories + os.MkdirAll(test.subPath, 0755) + } else { + // make all parent sub directories + if parent := filepath.Dir(test.subPath); parent != "." { + os.MkdirAll(parent, 0755) + } + + // make last element as symlink + linkPath := test.subPath + if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) { + if err := makeLink(linkPath, test.symlinkTarget); err != nil { + t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err)) + } + } + } + } + + fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath) + unlockPath(fileHandles) + assert.Equal(t, test.expectedHandleCount, len(fileHandles)) + if test.expectError { + assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) + continue + } + assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) + } + + // remove dir will happen after closing all file handles + assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) +} + +func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) { + base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink") + if err != nil { + t.Fatalf(err.Error()) + } + + defer os.RemoveAll(base) + + testingVolumePath := filepath.Join(base, "testingVolumePath") + + tests := []struct { + volumePath string + subPath string + expectedHandleCount int + expectError bool + symlinkTarget string + }{ + { + volumePath: `c:\`, + subPath: ``, + expectedHandleCount: 0, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: ``, + subPath: `a`, + expectedHandleCount: 0, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a`), + expectedHandleCount: 1, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\d`), + expectedHandleCount: 4, + expectError: false, + symlinkTarget: "", + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `symlink`), + expectedHandleCount: 1, + expectError: true, + symlinkTarget: base, + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`), + expectedHandleCount: 4, + expectError: true, + symlinkTarget: base, + }, + { + volumePath: testingVolumePath, + subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`), + expectedHandleCount: 5, + expectError: true, + symlinkTarget: filepath.Join(testingVolumePath, `a\b`), + }, + } + + for _, test := range tests { + if len(test.volumePath) > 0 && len(test.subPath) > 0 { + os.MkdirAll(test.volumePath, 0755) + if len(test.symlinkTarget) == 0 { + // make all intermediate sub directories + os.MkdirAll(test.subPath, 0755) + } else { + // make all parent sub directories + if parent := filepath.Dir(test.subPath); parent != "." { + os.MkdirAll(parent, 0755) + } + + // make last element as symlink + linkPath := test.subPath + if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) { + if err := makeLink(linkPath, test.symlinkTarget); err != nil { + t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err)) + } + } + } + } + + fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath) + unlockPath(fileHandles) + assert.Equal(t, test.expectedHandleCount, len(fileHandles)) + if test.expectError { + assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) + continue + } + assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath) + } + + // remove dir will happen after closing all file handles + assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) +} + +func TestFindExistingPrefix(t *testing.T) { + base, err := ioutil.TempDir("", "TestFindExistingPrefix") + if err != nil { + t.Fatalf(err.Error()) + } + + defer os.RemoveAll(base) + + testingVolumePath := filepath.Join(base, "testingVolumePath") + + tests := []struct { + base string + pathname string + expectError bool + expectedExistingPath string + expectedToCreateDirs []string + createSubPathBeforeTest bool + }{ + { + base: `c:\tmp\a`, + pathname: `c:\tmp\b`, + expectError: true, + expectedExistingPath: "", + expectedToCreateDirs: []string{}, + createSubPathBeforeTest: false, + }, + { + base: ``, + pathname: `c:\tmp\b`, + expectError: true, + expectedExistingPath: "", + expectedToCreateDirs: []string{}, + createSubPathBeforeTest: false, + }, + { + base: `c:\tmp\a`, + pathname: `d:\tmp\b`, + expectError: true, + expectedExistingPath: "", + expectedToCreateDirs: []string{}, + createSubPathBeforeTest: false, + }, + { + base: testingVolumePath, + pathname: testingVolumePath, + expectError: false, + expectedExistingPath: testingVolumePath, + expectedToCreateDirs: []string{}, + createSubPathBeforeTest: false, + }, + { + base: testingVolumePath, + pathname: filepath.Join(testingVolumePath, `a\b`), + expectError: false, + expectedExistingPath: filepath.Join(testingVolumePath, `a\b`), + expectedToCreateDirs: []string{}, + createSubPathBeforeTest: true, + }, + { + base: testingVolumePath, + pathname: filepath.Join(testingVolumePath, `a\b\c\`), + expectError: false, + expectedExistingPath: filepath.Join(testingVolumePath, `a\b`), + expectedToCreateDirs: []string{`c`}, + createSubPathBeforeTest: false, + }, + { + base: testingVolumePath, + pathname: filepath.Join(testingVolumePath, `a\b\c\d`), + expectError: false, + expectedExistingPath: filepath.Join(testingVolumePath, `a\b`), + expectedToCreateDirs: []string{`c`, `d`}, + createSubPathBeforeTest: false, + }, + } + + for _, test := range tests { + if test.createSubPathBeforeTest { + os.MkdirAll(test.pathname, 0755) + } + + existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname) + if test.expectError { + assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname) + continue + } + assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname) + + assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q", + test.base, test.pathname, existingPath, test.expectedExistingPath) + + assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q", + test.base, test.pathname, toCreate, test.expectedToCreateDirs) + + } + // remove dir will happen after closing all file handles + assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath) +} + +func TestPathWithinBase(t *testing.T) { + tests := []struct { + fullPath string + basePath string + expectedResult bool + }{ + { + fullPath: `c:\tmp\a\b\c`, + basePath: `c:\tmp`, + expectedResult: true, + }, + { + fullPath: `c:\tmp1`, + basePath: `c:\tmp2`, + expectedResult: false, + }, + { + fullPath: `c:\tmp`, + basePath: `c:\tmp`, + expectedResult: true, + }, + { + fullPath: `c:\tmp`, + basePath: `c:\tmp\a\b\c`, + expectedResult: false, + }, + { + fullPath: `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config\..timestamp\file.txt`, + basePath: `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config`, + expectedResult: true, + }, + } + + for _, test := range tests { + result := PathWithinBase(test.fullPath, test.basePath) + assert.Equal(t, result, test.expectedResult, "Expect result not equal with PathWithinBase(%s, %s) return: %q, expected: %q", + test.fullPath, test.basePath, result, test.expectedResult) + } +} + +func TestGetFileType(t *testing.T) { + mounter := New("fake/path") + + testCase := []struct { + name string + expectedType FileType + setUp func() (string, string, error) + }{ + { + "Directory Test", + FileTypeDirectory, + func() (string, string, error) { + tempDir, err := ioutil.TempDir("", "test-get-filetype-") + return tempDir, tempDir, err + }, + }, + { + "File Test", + FileTypeFile, + func() (string, string, error) { + tempFile, err := ioutil.TempFile("", "test-get-filetype") + if err != nil { + return "", "", err + } + tempFile.Close() + return tempFile.Name(), tempFile.Name(), nil + }, + }, + } + + for idx, tc := range testCase { + path, cleanUpPath, err := tc.setUp() + if err != nil { + t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) + } + if len(cleanUpPath) > 0 { + defer os.RemoveAll(cleanUpPath) + } + + fileType, err := mounter.GetFileType(path) + if err != nil { + t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err) + } + if fileType != tc.expectedType { + t.Fatalf("[%d-%s] expected %s, but got %s", idx, tc.name, tc.expectedType, fileType) + } + } +} + +func TestIsLikelyNotMountPoint(t *testing.T) { + mounter := Mounter{"fake/path"} + + tests := []struct { + fileName string + targetLinkName string + setUp func(base, fileName, targetLinkName string) error + expectedResult bool + expectError bool + }{ + { + "Dir", + "", + func(base, fileName, targetLinkName string) error { + return os.Mkdir(filepath.Join(base, fileName), 0750) + }, + true, + false, + }, + { + "InvalidDir", + "", + func(base, fileName, targetLinkName string) error { + return nil + }, + true, + true, + }, + { + "ValidSymLink", + "targetSymLink", + func(base, fileName, targetLinkName string) error { + targeLinkPath := filepath.Join(base, targetLinkName) + if err := os.Mkdir(targeLinkPath, 0750); err != nil { + return err + } + + filePath := filepath.Join(base, fileName) + if err := makeLink(filePath, targeLinkPath); err != nil { + return err + } + return nil + }, + false, + false, + }, + { + "InvalidSymLink", + "targetSymLink2", + func(base, fileName, targetLinkName string) error { + targeLinkPath := filepath.Join(base, targetLinkName) + if err := os.Mkdir(targeLinkPath, 0750); err != nil { + return err + } + + filePath := filepath.Join(base, fileName) + if err := makeLink(filePath, targeLinkPath); err != nil { + return err + } + return removeLink(targeLinkPath) + }, + true, + false, + }, + } + + for _, test := range tests { + base, err := ioutil.TempDir("", test.fileName) + if err != nil { + t.Fatalf(err.Error()) + } + + defer os.RemoveAll(base) + + if err := test.setUp(base, test.fileName, test.targetLinkName); err != nil { + t.Fatalf("unexpected error in setUp(%s, %s): %v", test.fileName, test.targetLinkName, err) + } + + filePath := filepath.Join(base, test.fileName) + result, err := mounter.IsLikelyNotMountPoint(filePath) + assert.Equal(t, result, test.expectedResult, "Expect result not equal with IsLikelyNotMountPoint(%s) return: %q, expected: %q", + filePath, result, test.expectedResult) + + if test.expectError { + assert.NotNil(t, err, "Expect error during IsLikelyNotMountPoint(%s)", filePath) + } else { + assert.Nil(t, err, "Expect error is nil during IsLikelyNotMountPoint(%s)", filePath) + } + } +} + +func TestFormatAndMount(t *testing.T) { + fakeMounter := ErrorMounter{&FakeMounter{}, 0, nil} + execCallback := func(cmd string, args ...string) ([]byte, error) { + for j := range args { + if strings.Contains(args[j], "Get-Disk -Number") { + return []byte("0"), nil + } + + if strings.Contains(args[j], "Get-Partition -DiskNumber") { + return []byte("0"), nil + } + + if strings.Contains(args[j], "mklink") { + return nil, nil + } + } + return nil, fmt.Errorf("Unexpected cmd %s, args %v", cmd, args) + } + fakeExec := NewFakeExec(execCallback) + + mounter := SafeFormatAndMount{ + Interface: &fakeMounter, + Exec: fakeExec, + } + + tests := []struct { + device string + target string + fstype string + mountOptions []string + expectError bool + }{ + { + "0", + "disk", + "NTFS", + []string{}, + false, + }, + { + "0", + "disk", + "", + []string{}, + false, + }, + { + "invalidDevice", + "disk", + "NTFS", + []string{}, + true, + }, + } + + for _, test := range tests { + base, err := ioutil.TempDir("", test.device) + if err != nil { + t.Fatalf(err.Error()) + } + defer os.RemoveAll(base) + + target := filepath.Join(base, test.target) + err = mounter.FormatAndMount(test.device, target, test.fstype, test.mountOptions) + if test.expectError { + assert.NotNil(t, err, "Expect error during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions) + } else { + assert.Nil(t, err, "Expect error is nil during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions) + } + } +} + +func TestNewSMBMapping(t *testing.T) { + tests := []struct { + username string + password string + remotepath string + expectError bool + }{ + { + "", + "password", + `\\remotepath`, + true, + }, + { + "username", + "", + `\\remotepath`, + true, + }, + { + "username", + "password", + "", + true, + }, + } + + for _, test := range tests { + _, err := newSMBMapping(test.username, test.password, test.remotepath) + if test.expectError { + assert.NotNil(t, err, "Expect error during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath) + } else { + assert.Nil(t, err, "Expect error is nil during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath) + } + } +}