From 3748197876a6a1f174a38ede3dcfc3b9ec3e2c2f Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Fri, 20 Apr 2018 15:26:31 +0800 Subject: [PATCH] Add more volume types in e2e and fix part of them. - Add dir-link/dir-bindmounted/dir-link-bindmounted/blockfs volume types for e2e tests. - Return error if we cannot resolve volume path. - Add GetFSGroup/GetMountRefs methods for mount.Interface. - Fix fsGroup related e2e tests partially. --- .../cm/container_manager_linux_test.go | 9 + pkg/util/mount/exec_mount.go | 8 + pkg/util/mount/exec_mount_test.go | 9 + pkg/util/mount/exec_mount_unsupported.go | 8 + pkg/util/mount/fake.go | 14 ++ pkg/util/mount/mount.go | 21 ++- pkg/util/mount/mount_linux.go | 25 +++ pkg/util/mount/mount_linux_test.go | 2 +- pkg/util/mount/mount_unsupported.go | 8 + pkg/util/mount/mount_windows.go | 14 ++ pkg/util/mount/nsenter_mount.go | 16 ++ pkg/util/mount/nsenter_mount_unsupported.go | 8 + pkg/util/nsenter/nsenter.go | 23 +++ pkg/util/removeall/removeall_test.go | 8 + pkg/volume/host_path/host_path_test.go | 9 + pkg/volume/local/local.go | 6 +- pkg/volume/rbd/rbd.go | 2 +- pkg/volume/volume_linux.go | 14 -- pkg/volume/volume_unsupported.go | 4 - test/e2e/storage/persistent_volumes-local.go | 166 ++++++++++++++++-- 20 files changed, 331 insertions(+), 43 deletions(-) diff --git a/pkg/kubelet/cm/container_manager_linux_test.go b/pkg/kubelet/cm/container_manager_linux_test.go index 3bbec062cb..2e9bb61b64 100644 --- a/pkg/kubelet/cm/container_manager_linux_test.go +++ b/pkg/kubelet/cm/container_manager_linux_test.go @@ -19,6 +19,7 @@ limitations under the License. package cm import ( + "errors" "fmt" "io/ioutil" "os" @@ -107,6 +108,14 @@ func (mi *fakeMountInterface) SafeMakeDir(_, _ string, _ os.FileMode) error { return nil } +func (mi *fakeMountInterface) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (mi *fakeMountInterface) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} + func fakeContainerMgrMountInt() mount.Interface { return &fakeMountInterface{ []mount.MountPoint{ diff --git a/pkg/util/mount/exec_mount.go b/pkg/util/mount/exec_mount.go index 574174af85..3145d71dd5 100644 --- a/pkg/util/mount/exec_mount.go +++ b/pkg/util/mount/exec_mount.go @@ -151,3 +151,11 @@ func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error { func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return m.wrappedMounter.SafeMakeDir(pathname, base, perm) } + +func (m *execMounter) GetMountRefs(pathname string) ([]string, error) { + return m.wrappedMounter.GetMountRefs(pathname) +} + +func (m *execMounter) GetFSGroup(pathname string) (int64, error) { + return m.wrappedMounter.GetFSGroup(pathname) +} diff --git a/pkg/util/mount/exec_mount_test.go b/pkg/util/mount/exec_mount_test.go index 248f49631b..6adf44cbef 100644 --- a/pkg/util/mount/exec_mount_test.go +++ b/pkg/util/mount/exec_mount_test.go @@ -19,6 +19,7 @@ limitations under the License. package mount import ( + "errors" "fmt" "os" "reflect" @@ -163,3 +164,11 @@ func (fm *fakeMounter) CleanSubPaths(podDir string, volumeName string) error { func (fm *fakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return nil } + +func (fm *fakeMounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (fm *fakeMounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} diff --git a/pkg/util/mount/exec_mount_unsupported.go b/pkg/util/mount/exec_mount_unsupported.go index 666a57063e..5274a3938a 100644 --- a/pkg/util/mount/exec_mount_unsupported.go +++ b/pkg/util/mount/exec_mount_unsupported.go @@ -98,3 +98,11 @@ func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) erro func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return nil } + +func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (mounter *execMounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} diff --git a/pkg/util/mount/fake.go b/pkg/util/mount/fake.go index 4bc32ff21c..dd4e9e6404 100644 --- a/pkg/util/mount/fake.go +++ b/pkg/util/mount/fake.go @@ -17,6 +17,7 @@ limitations under the License. package mount import ( + "errors" "os" "path/filepath" "sync" @@ -208,3 +209,16 @@ func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error { func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return nil } + +func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + // Ignore error in FakeMounter, because we actually didn't create files. + realpath = pathname + } + return getMountRefsByDev(f, realpath) +} + +func (f *FakeMounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("GetFSGroup not implemented") +} diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index 244710064b..42c1d6ba66 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -108,6 +108,12 @@ type Interface interface { // subpath starts. On the other hand, Interface.CleanSubPaths must be called // when the pod finishes. PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) + // GetMountRefs finds all mount references to the path, returns a + // list of paths. Path could be a mountpoint path, device or a normal + // directory (for bind mount). + GetMountRefs(pathname string) ([]string, error) + // GetFSGroup returns FSGroup of the path. + GetFSGroup(pathname string) (int64, error) } type Subpath struct { @@ -166,22 +172,19 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string, return mounter.formatAndMount(source, target, fstype, options) } -// GetMountRefsByDev finds all references to the device provided +// getMountRefsByDev finds all references to the device provided // by mountPath; returns a list of paths. -func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { +// Note that mountPath should be path after the evaluation of any symblolic links. +func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { mps, err := mounter.List() if err != nil { return nil, err } - slTarget, err := filepath.EvalSymlinks(mountPath) - if err != nil { - slTarget = mountPath - } // Finding the device mounted to mountPath diskDev := "" for i := range mps { - if slTarget == mps[i].Path { + if mountPath == mps[i].Path { diskDev = mps[i].Device break } @@ -190,8 +193,8 @@ func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) { // Find all references to the device. var refs []string for i := range mps { - if mps[i].Device == diskDev || mps[i].Device == slTarget { - if mps[i].Path != slTarget { + if mps[i].Device == diskDev || mps[i].Device == mountPath { + if mps[i].Path != mountPath { refs = append(refs, mps[i].Path) } } diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index 2eac05a7cc..cc8434dd25 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -921,6 +921,31 @@ func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMo return doSafeMakeDir(pathname, base, perm) } +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return getMountRefsByDev(mounter, realpath) +} + +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return 0, err + } + return getFSGroup(realpath) +} + +// This implementation is shared between Linux and NsEnterMounter +func getFSGroup(pathname string) (int64, error) { + info, err := os.Stat(pathname) + if err != nil { + return 0, err + } + return int64(info.Sys().(*syscall.Stat_t).Gid), nil +} + // This implementation is shared between Linux and NsEnterMounter func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { glog.V(4).Infof("Creating directory %q within base %q", pathname, base) diff --git a/pkg/util/mount/mount_linux_test.go b/pkg/util/mount/mount_linux_test.go index d6b3c497c2..ffba769bc9 100644 --- a/pkg/util/mount/mount_linux_test.go +++ b/pkg/util/mount/mount_linux_test.go @@ -203,7 +203,7 @@ func TestGetMountRefsByDev(t *testing.T) { for i, test := range tests { - if refs, err := GetMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { + if refs, err := getMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) { t.Errorf("%d. getMountRefsByDev(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs) } } diff --git a/pkg/util/mount/mount_unsupported.go b/pkg/util/mount/mount_unsupported.go index 9a2f8418e7..87b8e33482 100644 --- a/pkg/util/mount/mount_unsupported.go +++ b/pkg/util/mount/mount_unsupported.go @@ -126,3 +126,11 @@ func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return unsupportedErr } + +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} diff --git a/pkg/util/mount/mount_windows.go b/pkg/util/mount/mount_windows.go index 0384ba179f..e469b8a414 100644 --- a/pkg/util/mount/mount_windows.go +++ b/pkg/util/mount/mount_windows.go @@ -438,6 +438,20 @@ func getAllParentLinks(path string) ([]string, error) { return links, nil } +func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) { + realpath, err := filepath.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return getMountRefsByDev(mounter, realpath) +} + +// Note that on windows, it always returns 0. We actually don't set FSGroup on +// windows platform, see SetVolumeOwnership implementation. +func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) { + return 0, nil +} + // SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks. func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return doSafeMakeDir(pathname, base, perm) diff --git a/pkg/util/mount/nsenter_mount.go b/pkg/util/mount/nsenter_mount.go index b887b1a3ef..dfe2a51561 100644 --- a/pkg/util/mount/nsenter_mount.go +++ b/pkg/util/mount/nsenter_mount.go @@ -323,3 +323,19 @@ func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { return doSafeMakeDir(pathname, base, perm) } + +func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) { + hostpath, err := mounter.ne.EvalSymlinks(pathname) + if err != nil { + return nil, err + } + return getMountRefsByDev(mounter, hostpath) +} + +func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) { + kubeletpath, err := mounter.ne.KubeletPath(pathname) + if err != nil { + return 0, err + } + return getFSGroup(kubeletpath) +} diff --git a/pkg/util/mount/nsenter_mount_unsupported.go b/pkg/util/mount/nsenter_mount_unsupported.go index 4b29f0ba42..5f82b8c094 100644 --- a/pkg/util/mount/nsenter_mount_unsupported.go +++ b/pkg/util/mount/nsenter_mount_unsupported.go @@ -98,3 +98,11 @@ func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { return nil } + +func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (*NsenterMounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} diff --git a/pkg/util/nsenter/nsenter.go b/pkg/util/nsenter/nsenter.go index 32fbc08487..290db65ef6 100644 --- a/pkg/util/nsenter/nsenter.go +++ b/pkg/util/nsenter/nsenter.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "k8s.io/utils/exec" @@ -122,3 +123,25 @@ func (ne *Nsenter) SupportsSystemd() (string, bool) { systemdRunPath, hasSystemd := ne.paths["systemd-run"] return systemdRunPath, hasSystemd } + +// EvalSymlinks returns the path name on the host after evaluating symlinks on the +// host. +func (ne *Nsenter) EvalSymlinks(pathname string) (string, error) { + args := []string{"-m", pathname} + outBytes, err := ne.Exec("realpath", args).CombinedOutput() + if err != nil { + glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err) + return "", err + } + return strings.TrimSpace(string(outBytes)), nil +} + +// KubeletPath returns the path name that can be accessed by containerized +// kubelet, after evaluating symlinks on the host. +func (ne *Nsenter) KubeletPath(pathname string) (string, error) { + hostpath, err := ne.EvalSymlinks(pathname) + if err != nil { + return "", err + } + return filepath.Join(hostRootFsPath, hostpath), nil +} diff --git a/pkg/util/removeall/removeall_test.go b/pkg/util/removeall/removeall_test.go index 99eaf7c46e..ad0efac8d1 100644 --- a/pkg/util/removeall/removeall_test.go +++ b/pkg/util/removeall/removeall_test.go @@ -91,6 +91,14 @@ func (mounter *fakeMounter) SafeMakeDir(_, _ string, _ os.FileMode) error { return nil } +func (mounter *fakeMounter) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (mounter *fakeMounter) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} + func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { name := path.Base(file) if strings.HasPrefix(name, "mount") { diff --git a/pkg/volume/host_path/host_path_test.go b/pkg/volume/host_path/host_path_test.go index b5220ca47e..b48c026444 100644 --- a/pkg/volume/host_path/host_path_test.go +++ b/pkg/volume/host_path/host_path_test.go @@ -17,6 +17,7 @@ limitations under the License. package host_path import ( + "errors" "fmt" "os" "testing" @@ -388,6 +389,14 @@ func (fftc *fakeFileTypeChecker) SafeMakeDir(_, _ string, _ os.FileMode) error { return nil } +func (fftc *fakeFileTypeChecker) GetMountRefs(pathname string) ([]string, error) { + return nil, errors.New("not implemented") +} + +func (fftc *fakeFileTypeChecker) GetFSGroup(pathname string) (int64, error) { + return -1, errors.New("not implemented") +} + func setUp() error { err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755)) if err != nil { diff --git a/pkg/volume/local/local.go b/pkg/volume/local/local.go index f80089e976..a3be4dd61d 100644 --- a/pkg/volume/local/local.go +++ b/pkg/volume/local/local.go @@ -273,7 +273,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if !notMnt { return nil } - refs, err := mount.GetMountRefsByDev(m.mounter, m.globalPath) + refs, err := m.mounter.GetMountRefs(m.globalPath) if fsGroup != nil { if err != nil { glog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err) @@ -282,11 +282,11 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if len(refs) > 0 { fsGroupNew := int64(*fsGroup) - fsGroupSame, fsGroupOld, err := volume.IsSameFSGroup(m.globalPath, fsGroupNew) + fsGroupOld, err := m.mounter.GetFSGroup(m.globalPath) if err != nil { return fmt.Errorf("failed to check fsGroup for %s (%v)", m.globalPath, err) } - if !fsGroupSame { + if fsGroupNew != fsGroupOld { m.plugin.recorder.Eventf(m.pod, v1.EventTypeWarning, events.WarnAlreadyMountedVolume, "The requested fsGroup is %d, but the volume %s has GID %d. The volume may not be shareable.", fsGroupNew, m.volName, fsGroupOld) } } diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 16ff2bdecd..eb0dda09fa 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -509,7 +509,7 @@ func (plugin *rbdPlugin) newUnmapperInternal(volName string, podUID types.UID, m } func (plugin *rbdPlugin) getDeviceNameFromOldMountPath(mounter mount.Interface, mountPath string) (string, error) { - refs, err := mount.GetMountRefsByDev(mounter, mountPath) + refs, err := mounter.GetMountRefs(mountPath) if err != nil { return "", err } diff --git a/pkg/volume/volume_linux.go b/pkg/volume/volume_linux.go index d67ee4a95a..ef1f45208c 100644 --- a/pkg/volume/volume_linux.go +++ b/pkg/volume/volume_linux.go @@ -89,17 +89,3 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error { return nil }) } - -// IsSameFSGroup is called only for requests to mount an already mounted -// volume. It checks if fsGroup of new mount request is the same or not. -// It returns false if it not the same. It also returns current Gid of a path -// provided for dir variable. -func IsSameFSGroup(dir string, fsGroup int64) (bool, int, error) { - info, err := os.Stat(dir) - if err != nil { - glog.Errorf("Error getting stats for %s (%v)", dir, err) - return false, 0, err - } - s := info.Sys().(*syscall.Stat_t) - return int(s.Gid) == int(fsGroup), int(s.Gid), nil -} diff --git a/pkg/volume/volume_unsupported.go b/pkg/volume/volume_unsupported.go index 46a6aeaf0a..45a6cc5ca7 100644 --- a/pkg/volume/volume_unsupported.go +++ b/pkg/volume/volume_unsupported.go @@ -21,7 +21,3 @@ package volume func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error { return nil } - -func IsSameFSGroup(dir string, fsGroup int64) (bool, int, error) { - return true, int(fsGroup), nil -} diff --git a/test/e2e/storage/persistent_volumes-local.go b/test/e2e/storage/persistent_volumes-local.go index 612fe7f5bd..ee5b80d57b 100644 --- a/test/e2e/storage/persistent_volumes-local.go +++ b/test/e2e/storage/persistent_volumes-local.go @@ -63,26 +63,44 @@ type localVolumeType string const ( // default local volume type, aka a directory DirectoryLocalVolumeType localVolumeType = "dir" + // like DirectoryLocalVolumeType but it's a symbolic link to directory + DirectoryLinkLocalVolumeType localVolumeType = "dir-link" + // like DirectoryLocalVolumeType but bind mounted + DirectoryBindMountedLocalVolumeType localVolumeType = "dir-bindmounted" + // like DirectoryLocalVolumeType but it's a symbolic link to self bind mounted directory + // Note that bind mounting at symbolic link actually mounts at directory it + // links to. + DirectoryLinkBindMountedLocalVolumeType localVolumeType = "dir-link-bindmounted" // creates a tmpfs and mounts it TmpfsLocalVolumeType localVolumeType = "tmpfs" // tests based on local ssd at /mnt/disks/by-uuid/ GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs" // Creates a local file, formats it, and maps it as a block device. BlockLocalVolumeType localVolumeType = "block" + // Creates a local file, formats it, and mounts it to use as local volume. + BlockFsLocalVolumeType localVolumeType = "blockfs" ) var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{ - GCELocalSSDVolumeType: setupLocalVolumeGCELocalSSD, - TmpfsLocalVolumeType: setupLocalVolumeTmpfs, - DirectoryLocalVolumeType: setupLocalVolumeDirectory, - BlockLocalVolumeType: setupLocalVolumeBlock, + GCELocalSSDVolumeType: setupLocalVolumeGCELocalSSD, + TmpfsLocalVolumeType: setupLocalVolumeTmpfs, + DirectoryLocalVolumeType: setupLocalVolumeDirectory, + DirectoryLinkLocalVolumeType: setupLocalVolumeDirectoryLink, + DirectoryBindMountedLocalVolumeType: setupLocalVolumeDirectoryBindMounted, + DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted, + BlockLocalVolumeType: setupLocalVolumeBlock, + BlockFsLocalVolumeType: setupLocalVolumeBlockFs, } var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){ - GCELocalSSDVolumeType: cleanupLocalVolumeGCELocalSSD, - TmpfsLocalVolumeType: cleanupLocalVolumeTmpfs, - DirectoryLocalVolumeType: cleanupLocalVolumeDirectory, - BlockLocalVolumeType: cleanupLocalVolumeBlock, + GCELocalSSDVolumeType: cleanupLocalVolumeGCELocalSSD, + TmpfsLocalVolumeType: cleanupLocalVolumeTmpfs, + DirectoryLocalVolumeType: cleanupLocalVolumeDirectory, + DirectoryLinkLocalVolumeType: cleanupLocalVolumeDirectoryLink, + DirectoryBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryBindMounted, + DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted, + BlockLocalVolumeType: cleanupLocalVolumeBlock, + BlockFsLocalVolumeType: cleanupLocalVolumeBlockFs, } type localTestVolume struct { @@ -186,8 +204,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { } }) - localVolumeTypes := []localVolumeType{DirectoryLocalVolumeType, TmpfsLocalVolumeType, GCELocalSSDVolumeType, BlockLocalVolumeType} - for _, tempTestVolType := range localVolumeTypes { + for tempTestVolType := range setupLocalVolumeMap { // New variable required for gingko test closures testVolType := tempTestVolType @@ -266,8 +283,21 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) Context("Set fsGroup for local volume", func() { + BeforeEach(func() { + if testVolType == BlockLocalVolumeType { + framework.Skipf("We don't set fsGroup on block device, skipped.") + } + }) It("should set fsGroup for one pod", func() { + skipTypes := sets.NewString( + string(DirectoryBindMountedLocalVolumeType), + string(DirectoryLinkBindMountedLocalVolumeType), + ) + if skipTypes.Has(string(testVolType)) { + // TODO(cofyc): Test it when bug is fixed. + framework.Skipf("Skipped when volume type is %v", testVolType) + } By("Checking fsGroup is set") pod := createPodWithFsGroupTest(config, testVol, 1234, 1234) By("Deleting pod") @@ -275,6 +305,14 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) It("should set same fsGroup for two pods simultaneously", func() { + skipTypes := sets.NewString( + string(DirectoryBindMountedLocalVolumeType), + string(DirectoryLinkBindMountedLocalVolumeType), + ) + if skipTypes.Has(string(testVolType)) { + // TODO(cofyc): Test it when bug is fixed. + framework.Skipf("Skipped when volume type is %v", testVolType) + } fsGroup := int64(1234) By("Create first pod and check fsGroup is set") pod1 := createPodWithFsGroupTest(config, testVol, fsGroup, fsGroup) @@ -287,6 +325,14 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) It("should set different fsGroup for second pod if first pod is deleted", func() { + skipTypes := sets.NewString( + string(DirectoryBindMountedLocalVolumeType), + string(DirectoryLinkBindMountedLocalVolumeType), + ) + if skipTypes.Has(string(testVolType)) { + // TODO(cofyc): Test it when bug is fixed. + framework.Skipf("Skipped when volume type is %v", testVolType) + } fsGroup1, fsGroup2 := int64(1234), int64(4321) By("Create first pod and check fsGroup is set") pod1 := createPodWithFsGroupTest(config, testVol, fsGroup1, fsGroup1) @@ -300,7 +346,13 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() { }) It("should not set different fsGroups for two pods simultaneously", func() { - if testVolType == DirectoryLocalVolumeType { + skipTypes := sets.NewString( + string(DirectoryLocalVolumeType), + string(DirectoryLinkLocalVolumeType), + string(DirectoryBindMountedLocalVolumeType), + string(DirectoryLinkBindMountedLocalVolumeType), + ) + if skipTypes.Has(string(testVolType)) { // TODO(cofyc): Test it when bug is fixed. framework.Skipf("Skipped when volume type is %v", testVolType) } @@ -899,6 +951,39 @@ func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTes return setupWriteTestFile(hostDir, config, DirectoryLocalVolumeType, node) } +func setupLocalVolumeDirectoryLink(config *localTestConfig, node *v1.Node) *localTestVolume { + testDirName := "local-volume-test-" + string(uuid.NewUUID()) + hostDir := filepath.Join(hostBase, testDirName) + hostDirBackend := hostDir + "-backend" + cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir) + _, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node) + Expect(err).NotTo(HaveOccurred()) + // Populate volume with testFile containing testFileContent. + return setupWriteTestFile(hostDir, config, DirectoryLinkLocalVolumeType, node) +} + +func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume { + testDirName := "local-volume-test-" + string(uuid.NewUUID()) + hostDir := filepath.Join(hostBase, testDirName) + cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir) + _, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node) + Expect(err).NotTo(HaveOccurred()) + // Populate volume with testFile containing testFileContent. + return setupWriteTestFile(hostDir, config, DirectoryBindMountedLocalVolumeType, node) +} + +func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume { + testDirName := "local-volume-test-" + string(uuid.NewUUID()) + hostDir := filepath.Join(hostBase, testDirName) + hostDirBackend := hostDir + "-backend" + cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && ln -s %s %s", + hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir) + _, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node) + Expect(err).NotTo(HaveOccurred()) + // Populate volume with testFile containing testFileContent. + return setupWriteTestFile(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node) +} + func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume { testDirName := "local-volume-test-" + string(uuid.NewUUID()) hostDir := filepath.Join(hostBase, testDirName) @@ -911,6 +996,23 @@ func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVol return volume } +func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestVolume { + testDirName := "local-volume-test-" + string(uuid.NewUUID()) + hostDir := filepath.Join(hostBase, testDirName) + createAndMapBlockLocalVolume(config, hostDir, node) + loopDev := getBlockLoopDev(hostDir, node) + // format and mount at hostDir + // give others rwx for read/write testing + cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir) + _, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node) + Expect(err).NotTo(HaveOccurred()) + // Populate block volume with testFile containing testFileContent. + volume := setupWriteTestFile(hostDir, config, BlockFsLocalVolumeType, node) + volume.hostDir = hostDir + volume.loopDevDir = loopDev + return volume +} + // Determine the /dev/loopXXX device associated with this test, via its hostDir. func getBlockLoopDev(hostDir string, node *v1.Node) string { loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir) @@ -956,6 +1058,35 @@ func cleanupLocalVolumeDirectory(config *localTestConfig, volume *localTestVolum Expect(err).NotTo(HaveOccurred()) } +// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory. +func cleanupLocalVolumeDirectoryLink(config *localTestConfig, volume *localTestVolume) { + By("Removing the test directory") + hostDir := volume.hostDir + hostDirBackend := hostDir + "-backend" + removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend) + err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node) + Expect(err).NotTo(HaveOccurred()) +} + +// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory. +func cleanupLocalVolumeDirectoryBindMounted(config *localTestConfig, volume *localTestVolume) { + By("Removing the test directory") + hostDir := volume.hostDir + removeCmd := fmt.Sprintf("sudo umount %s && rm -r %s", hostDir, hostDir) + err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node) + Expect(err).NotTo(HaveOccurred()) +} + +// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory. +func cleanupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, volume *localTestVolume) { + By("Removing the test directory") + hostDir := volume.hostDir + hostDirBackend := hostDir + "-backend" + removeCmd := fmt.Sprintf("rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend) + err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node) + Expect(err).NotTo(HaveOccurred()) +} + // Deletes the PVC/PV and removes the test directory holding the block file. func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) { volume.hostDir = volume.loopDevDir @@ -966,6 +1097,19 @@ func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) { Expect(err).NotTo(HaveOccurred()) } +// Deletes the PVC/PV and removes the test directory holding the block file. +func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) { + // umount first + By("Umount blockfs mountpoint") + umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir) + err := framework.IssueSSHCommand(umountCmd, framework.TestContext.Provider, volume.node) + unmapBlockLocalVolume(config, volume.hostDir, volume.node) + By("Removing the test directory") + removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir) + err = framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node) + Expect(err).NotTo(HaveOccurred()) +} + func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig { pvcConfig := framework.PersistentVolumeClaimConfig{ AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},