Merge pull request #62903 from cofyc/fixfsgroupcheckinlocal

Automatic merge from submit-queue (batch tested with PRs 62657, 63278, 62903, 63375). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add more volume types in e2e and fix part of them.

**What this PR does / why we need it**:

- Add dir-link/dir-bindmounted/dir-link-bindmounted/bockfs volume types for e2e tests.
- Fix fsGroup related e2e tests partially.
- Return error if we cannot resolve volume path.
  - Because we should not fallback to volume path, if it's a symbolic link, we may get wrong results.

To safely set fsGroup on local volume, we need to implement these two methods correctly for all volume types both on the host and in container:

- get volume path kubelet can access
  - paths on the host and in container are different
- get mount references
  - for directories, we cannot use its mount source (device field) to identify mount references, because directories on same filesystem have same mount source (e.g. tmpfs), we need to check filesystem's major:minor and directory root path on it

Here is current status:

| | (A) volume-path (host) | (B) volume-path (container) | (C) mount-refs (host) | (D) mount-refs (container) |
| --- | --- | --- | --- | --- |
| (1) dir | OK | FAIL | FAIL | FAIL |
| (2) dir-link | OK | FAIL | FAIL | FAIL |
| (3) dir-bindmounted | OK | FAIL | FAIL | FAIL |
| (4) dir-link-bindmounted | OK | FAIL | FAIL | FAIL |
| (5) tmpfs| OK | FAIL | FAIL | FAIL |
| (6) blockfs| OK | FAIL | OK | FAIL |
| (7) block| NOTNEEDED | NOTNEEDED | NOTNEEDED | NOTNEEDED |
| (8) gce-localssd-scsi-fs| NOTTESTED | NOTTESTED | NOTTESTED | NOTTESTED |

- This PR uses `nsenter ... readlink` to resolve path in container as @msau42  @jsafrane [suggested](https://github.com/kubernetes/kubernetes/pull/61489#pullrequestreview-110032850). This fixes B1:B6 and D6, , the rest will be addressed in https://github.com/kubernetes/kubernetes/pull/62102.
- C5:D5 marked `FAIL` because `tmpfs` filesystems can share same mount source, we cannot rely on it to check mount references. e2e tests passes due to we use unique mount source string in tests.
- A7:D7 marked `NOTNEEDED` because we don't set fsGroup on block devices in local plugin. (TODO: Should we set fsGroup on block device?)
- A8:D8 marked `NOTTESTED` because I didn't test it, I leave it to `pull-kubernetes-e2e-gce`. I think it should be same as `blockfs`.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

pkg/kubelet/cm/container_manager_linux_test.go

@@ -19,6 +19,7 @@ limitations under the License.
 package cm
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -107,6 +108,14 @@ func (mi *fakeMountInterface) SafeMakeDir(_, _ string, _ os.FileMode) error {
 	return nil
 }
 
+func (mi *fakeMountInterface) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mi *fakeMountInterface) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
 func fakeContainerMgrMountInt() mount.Interface {
 	return &fakeMountInterface{
 		[]mount.MountPoint{

pkg/util/mount/exec_mount.go

@@ -151,3 +151,11 @@ func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
 func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
 }
+
+func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
+	return m.wrappedMounter.GetMountRefs(pathname)
+}
+
+func (m *execMounter) GetFSGroup(pathname string) (int64, error) {
+	return m.wrappedMounter.GetFSGroup(pathname)
+}

pkg/util/mount/exec_mount_test.go

@@ -19,6 +19,7 @@ limitations under the License.
 package mount
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"reflect"
@@ -163,3 +164,11 @@ func (fm *fakeMounter) CleanSubPaths(podDir string, volumeName string) error {
 func (fm *fakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return nil
 }
+
+func (fm *fakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (fm *fakeMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}

pkg/util/mount/exec_mount_unsupported.go

@@ -98,3 +98,11 @@ func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) erro
 func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return nil
 }
+
+func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mounter *execMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}

pkg/util/mount/fake.go

@@ -17,6 +17,7 @@ limitations under the License.
 package mount
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"sync"
@@ -208,3 +209,16 @@ func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
 func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return nil
 }
+
+func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		// Ignore error in FakeMounter, because we actually didn't create files.
+		realpath = pathname
+	}
+	return getMountRefsByDev(f, realpath)
+}
+
+func (f *FakeMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("GetFSGroup not implemented")
+}

pkg/util/mount/mount.go

@@ -108,6 +108,12 @@ type Interface interface {
 	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
 	// when the pod finishes.
 	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
+	// GetMountRefs finds all mount references to the path, returns a
+	// list of paths. Path could be a mountpoint path, device or a normal
+	// directory (for bind mount).
+	GetMountRefs(pathname string) ([]string, error)
+	// GetFSGroup returns FSGroup of the path.
+	GetFSGroup(pathname string) (int64, error)
 }
 
 type Subpath struct {
@@ -166,22 +172,19 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string,
 	return mounter.formatAndMount(source, target, fstype, options)
 }
 
-// GetMountRefsByDev finds all references to the device provided
+// getMountRefsByDev finds all references to the device provided
 // by mountPath; returns a list of paths.
-func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
+// Note that mountPath should be path after the evaluation of any symblolic links.
+func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
 	mps, err := mounter.List()
 	if err != nil {
 		return nil, err
 	}
-	slTarget, err := filepath.EvalSymlinks(mountPath)
-	if err != nil {
-		slTarget = mountPath
-	}
 
 	// Finding the device mounted to mountPath
 	diskDev := ""
 	for i := range mps {
-		if slTarget == mps[i].Path {
+		if mountPath == mps[i].Path {
 			diskDev = mps[i].Device
 			break
 		}
@@ -190,8 +193,8 @@ func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
 	// Find all references to the device.
 	var refs []string
 	for i := range mps {
-		if mps[i].Device == diskDev || mps[i].Device == slTarget {
-			if mps[i].Path != slTarget {
+		if mps[i].Device == diskDev || mps[i].Device == mountPath {
+			if mps[i].Path != mountPath {
 				refs = append(refs, mps[i].Path)
 			}
 		}

pkg/util/mount/mount_linux.go

@@ -921,6 +921,31 @@ func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMo
 	return doSafeMakeDir(pathname, base, perm)
 }
 
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return nil, err
+	}
+	return getMountRefsByDev(mounter, realpath)
+}
+
+func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return getFSGroup(realpath)
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+func getFSGroup(pathname string) (int64, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return int64(info.Sys().(*syscall.Stat_t).Gid), nil
+}
+
 // This implementation is shared between Linux and NsEnterMounter
 func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	glog.V(4).Infof("Creating directory %q within base %q", pathname, base)

pkg/util/mount/mount_linux_test.go

@@ -203,7 +203,7 @@ func TestGetMountRefsByDev(t *testing.T) {
 
 	for i, test := range tests {
 
-		if refs, err := GetMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) {
+		if refs, err := getMountRefsByDev(fm, test.mountPath); err != nil || !setEquivalent(test.expectedRefs, refs) {
 			t.Errorf("%d. getMountRefsByDev(%q) = %v, %v; expected %v, nil", i, test.mountPath, refs, err, test.expectedRefs)
 		}
 	}

pkg/util/mount/mount_unsupported.go

@@ -126,3 +126,11 @@ func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
 func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return unsupportedErr
 }
+
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}

pkg/util/mount/mount_windows.go

@@ -438,6 +438,20 @@ func getAllParentLinks(path string) ([]string, error) {
 	return links, nil
 }
 
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return nil, err
+	}
+	return getMountRefsByDev(mounter, realpath)
+}
+
+// Note that on windows, it always returns 0. We actually don't set FSGroup on
+// windows platform, see SetVolumeOwnership implementation.
+func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
+	return 0, nil
+}
+
 // SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
 func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return doSafeMakeDir(pathname, base, perm)

pkg/util/mount/nsenter_mount.go

@@ -327,3 +327,19 @@ func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath
 func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
 	return doSafeMakeDir(pathname, base, perm)
 }
+
+func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
+	hostpath, err := mounter.ne.EvalSymlinks(pathname)
+	if err != nil {
+		return nil, err
+	}
+	return getMountRefsByDev(mounter, hostpath)
+}
+
+func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
+	kubeletpath, err := mounter.ne.KubeletPath(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return getFSGroup(kubeletpath)
+}

pkg/util/mount/nsenter_mount_unsupported.go

@@ -98,3 +98,11 @@ func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string,
 func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
 	return nil
 }
+
+func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (*NsenterMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}

pkg/util/nsenter/nsenter.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"k8s.io/utils/exec"
 
@@ -124,3 +125,25 @@ func (ne *Nsenter) SupportsSystemd() (string, bool) {
 	systemdRunPath, ok := ne.paths["systemd-run"]
 	return systemdRunPath, ok && systemdRunPath != ""
 }
+
+// EvalSymlinks returns the path name on the host after evaluating symlinks on the
+// host.
+func (ne *Nsenter) EvalSymlinks(pathname string) (string, error) {
+	args := []string{"-m", pathname}
+	outBytes, err := ne.Exec("realpath", args).CombinedOutput()
+	if err != nil {
+		glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
+		return "", err
+	}
+	return strings.TrimSpace(string(outBytes)), nil
+}
+
+// KubeletPath returns the path name that can be accessed by containerized
+// kubelet, after evaluating symlinks on the host.
+func (ne *Nsenter) KubeletPath(pathname string) (string, error) {
+	hostpath, err := ne.EvalSymlinks(pathname)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(hostRootFsPath, hostpath), nil
+}

pkg/util/removeall/removeall_test.go

@@ -91,6 +91,14 @@ func (mounter *fakeMounter) SafeMakeDir(_, _ string, _ os.FileMode) error {
 	return nil
 }
 
+func (mounter *fakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mounter *fakeMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
 func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	name := path.Base(file)
 	if strings.HasPrefix(name, "mount") {

pkg/volume/host_path/host_path_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package host_path
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"testing"
@@ -388,6 +389,14 @@ func (fftc *fakeFileTypeChecker) SafeMakeDir(_, _ string, _ os.FileMode) error {
 	return nil
 }
 
+func (fftc *fakeFileTypeChecker) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (fftc *fakeFileTypeChecker) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
 func setUp() error {
 	err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755))
 	if err != nil {

pkg/volume/local/local.go

@@ -274,7 +274,7 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
 	if !notMnt {
 		return nil
 	}
-	refs, err := mount.GetMountRefsByDev(m.mounter, m.globalPath)
+	refs, err := m.mounter.GetMountRefs(m.globalPath)
 	if fsGroup != nil {
 		if err != nil {
 			glog.Errorf("cannot collect mounting information: %s %v", m.globalPath, err)
@@ -285,11 +285,11 @@ func (m *localVolumeMounter) SetUpAt(dir string, fsGroup *int64) error {
 		refs = m.filterPodMounts(refs)
 		if len(refs) > 0 {
 			fsGroupNew := int64(*fsGroup)
-			fsGroupSame, fsGroupOld, err := volume.IsSameFSGroup(m.globalPath, fsGroupNew)
+			fsGroupOld, err := m.mounter.GetFSGroup(m.globalPath)
 			if err != nil {
 				return fmt.Errorf("failed to check fsGroup for %s (%v)", m.globalPath, err)
 			}
-			if !fsGroupSame {
+			if fsGroupNew != fsGroupOld {
 				m.plugin.recorder.Eventf(m.pod, v1.EventTypeWarning, events.WarnAlreadyMountedVolume, "The requested fsGroup is %d, but the volume %s has GID %d. The volume may not be shareable.", fsGroupNew, m.volName, fsGroupOld)
 			}
 		}

pkg/volume/rbd/rbd.go

@@ -508,7 +508,7 @@ func (plugin *rbdPlugin) newUnmapperInternal(volName string, podUID types.UID, m
 }
 
 func (plugin *rbdPlugin) getDeviceNameFromOldMountPath(mounter mount.Interface, mountPath string) (string, error) {
-	refs, err := mount.GetMountRefsByDev(mounter, mountPath)
+	refs, err := mounter.GetMountRefs(mountPath)
 	if err != nil {
 		return "", err
 	}

pkg/volume/volume_linux.go

@@ -89,17 +89,3 @@ func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error {
 		return nil
 	})
 }
-
-// IsSameFSGroup is called only for requests to mount an already mounted
-// volume. It checks if fsGroup of new mount request is the same or not.
-// It returns false if it not the same. It also returns current Gid of a path
-// provided for dir variable.
-func IsSameFSGroup(dir string, fsGroup int64) (bool, int, error) {
-	info, err := os.Stat(dir)
-	if err != nil {
-		glog.Errorf("Error getting stats for %s (%v)", dir, err)
-		return false, 0, err
-	}
-	s := info.Sys().(*syscall.Stat_t)
-	return int(s.Gid) == int(fsGroup), int(s.Gid), nil
-}

pkg/volume/volume_unsupported.go

@@ -21,7 +21,3 @@ package volume
 func SetVolumeOwnership(mounter Mounter, fsGroup *int64) error {
 	return nil
 }
-
-func IsSameFSGroup(dir string, fsGroup int64) (bool, int, error) {
-	return true, int(fsGroup), nil
-}

test/e2e/storage/persistent_volumes-local.go

@@ -63,26 +63,44 @@ type localVolumeType string
 const (
 	// default local volume type, aka a directory
 	DirectoryLocalVolumeType localVolumeType = "dir"
+	// like DirectoryLocalVolumeType but it's a symbolic link to directory
+	DirectoryLinkLocalVolumeType localVolumeType = "dir-link"
+	// like DirectoryLocalVolumeType but bind mounted
+	DirectoryBindMountedLocalVolumeType localVolumeType = "dir-bindmounted"
+	// like DirectoryLocalVolumeType but it's a symbolic link to self bind mounted directory
+	// Note that bind mounting at symbolic link actually mounts at directory it
+	// links to.
+	DirectoryLinkBindMountedLocalVolumeType localVolumeType = "dir-link-bindmounted"
 	// creates a tmpfs and mounts it
 	TmpfsLocalVolumeType localVolumeType = "tmpfs"
 	// tests based on local ssd at /mnt/disks/by-uuid/
 	GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs"
 	// Creates a local file, formats it, and maps it as a block device.
 	BlockLocalVolumeType localVolumeType = "block"
+	// Creates a local file, formats it, and mounts it to use as local volume.
+	BlockFsLocalVolumeType localVolumeType = "blockfs"
 )
 
 var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{
 	GCELocalSSDVolumeType:                   setupLocalVolumeGCELocalSSD,
 	TmpfsLocalVolumeType:                    setupLocalVolumeTmpfs,
 	DirectoryLocalVolumeType:                setupLocalVolumeDirectory,
+	DirectoryLinkLocalVolumeType:            setupLocalVolumeDirectoryLink,
+	DirectoryBindMountedLocalVolumeType:     setupLocalVolumeDirectoryBindMounted,
+	DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted,
 	BlockLocalVolumeType:                    setupLocalVolumeBlock,
+	BlockFsLocalVolumeType:                  setupLocalVolumeBlockFs,
 }
 
 var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){
 	GCELocalSSDVolumeType:                   cleanupLocalVolumeGCELocalSSD,
 	TmpfsLocalVolumeType:                    cleanupLocalVolumeTmpfs,
 	DirectoryLocalVolumeType:                cleanupLocalVolumeDirectory,
+	DirectoryLinkLocalVolumeType:            cleanupLocalVolumeDirectoryLink,
+	DirectoryBindMountedLocalVolumeType:     cleanupLocalVolumeDirectoryBindMounted,
+	DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted,
 	BlockLocalVolumeType:                    cleanupLocalVolumeBlock,
+	BlockFsLocalVolumeType:                  cleanupLocalVolumeBlockFs,
 }
 
 type localTestVolume struct {
@@ -186,8 +204,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 		}
 	})
 
-	localVolumeTypes := []localVolumeType{DirectoryLocalVolumeType, TmpfsLocalVolumeType, GCELocalSSDVolumeType, BlockLocalVolumeType}
-	for _, tempTestVolType := range localVolumeTypes {
+	for tempTestVolType := range setupLocalVolumeMap {
 
 		// New variable required for gingko test closures
 		testVolType := tempTestVolType
@@ -266,8 +283,21 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 			})
 
 			Context("Set fsGroup for local volume", func() {
+				BeforeEach(func() {
+					if testVolType == BlockLocalVolumeType {
+						framework.Skipf("We don't set fsGroup on block device, skipped.")
+					}
+				})
 
 				It("should set fsGroup for one pod", func() {
+					skipTypes := sets.NewString(
+						string(DirectoryBindMountedLocalVolumeType),
+						string(DirectoryLinkBindMountedLocalVolumeType),
+					)
+					if skipTypes.Has(string(testVolType)) {
+						// TODO(cofyc): Test it when bug is fixed.
+						framework.Skipf("Skipped when volume type is %v", testVolType)
+					}
 					By("Checking fsGroup is set")
 					pod := createPodWithFsGroupTest(config, testVol, 1234, 1234)
 					By("Deleting pod")
@@ -275,6 +305,14 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 				})
 
 				It("should set same fsGroup for two pods simultaneously", func() {
+					skipTypes := sets.NewString(
+						string(DirectoryBindMountedLocalVolumeType),
+						string(DirectoryLinkBindMountedLocalVolumeType),
+					)
+					if skipTypes.Has(string(testVolType)) {
+						// TODO(cofyc): Test it when bug is fixed.
+						framework.Skipf("Skipped when volume type is %v", testVolType)
+					}
 					fsGroup := int64(1234)
 					By("Create first pod and check fsGroup is set")
 					pod1 := createPodWithFsGroupTest(config, testVol, fsGroup, fsGroup)
@@ -287,6 +325,14 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 				})
 
 				It("should set different fsGroup for second pod if first pod is deleted", func() {
+					skipTypes := sets.NewString(
+						string(DirectoryBindMountedLocalVolumeType),
+						string(DirectoryLinkBindMountedLocalVolumeType),
+					)
+					if skipTypes.Has(string(testVolType)) {
+						// TODO(cofyc): Test it when bug is fixed.
+						framework.Skipf("Skipped when volume type is %v", testVolType)
+					}
 					fsGroup1, fsGroup2 := int64(1234), int64(4321)
 					By("Create first pod and check fsGroup is set")
 					pod1 := createPodWithFsGroupTest(config, testVol, fsGroup1, fsGroup1)
@@ -300,7 +346,13 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
 				})
 
 				It("should not set different fsGroups for two pods simultaneously", func() {
-					if testVolType == DirectoryLocalVolumeType {
+					skipTypes := sets.NewString(
+						string(DirectoryLocalVolumeType),
+						string(DirectoryLinkLocalVolumeType),
+						string(DirectoryBindMountedLocalVolumeType),
+						string(DirectoryLinkBindMountedLocalVolumeType),
+					)
+					if skipTypes.Has(string(testVolType)) {
 						// TODO(cofyc): Test it when bug is fixed.
 						framework.Skipf("Skipped when volume type is %v", testVolType)
 					}
@@ -898,6 +950,39 @@ func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTes
 	return setupWriteTestFile(hostDir, config, DirectoryLocalVolumeType, node)
 }
 
+func setupLocalVolumeDirectoryLink(config *localTestConfig, node *v1.Node) *localTestVolume {
+	testDirName := "local-volume-test-" + string(uuid.NewUUID())
+	hostDir := filepath.Join(hostBase, testDirName)
+	hostDirBackend := hostDir + "-backend"
+	cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
+	_, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node)
+	Expect(err).NotTo(HaveOccurred())
+	// Populate volume with testFile containing testFileContent.
+	return setupWriteTestFile(hostDir, config, DirectoryLinkLocalVolumeType, node)
+}
+
+func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
+	testDirName := "local-volume-test-" + string(uuid.NewUUID())
+	hostDir := filepath.Join(hostBase, testDirName)
+	cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir)
+	_, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node)
+	Expect(err).NotTo(HaveOccurred())
+	// Populate volume with testFile containing testFileContent.
+	return setupWriteTestFile(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
+}
+
+func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
+	testDirName := "local-volume-test-" + string(uuid.NewUUID())
+	hostDir := filepath.Join(hostBase, testDirName)
+	hostDirBackend := hostDir + "-backend"
+	cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && ln -s %s %s",
+		hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir)
+	_, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node)
+	Expect(err).NotTo(HaveOccurred())
+	// Populate volume with testFile containing testFileContent.
+	return setupWriteTestFile(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
+}
+
 func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume {
 	testDirName := "local-volume-test-" + string(uuid.NewUUID())
 	hostDir := filepath.Join(hostBase, testDirName)
@@ -910,6 +995,23 @@ func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVol
 	return volume
 }
 
+func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestVolume {
+	testDirName := "local-volume-test-" + string(uuid.NewUUID())
+	hostDir := filepath.Join(hostBase, testDirName)
+	createAndMapBlockLocalVolume(config, hostDir, node)
+	loopDev := getBlockLoopDev(hostDir, node)
+	// format and mount at hostDir
+	// give others rwx for read/write testing
+	cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir)
+	_, err := framework.IssueSSHCommandWithResult(cmd, framework.TestContext.Provider, node)
+	Expect(err).NotTo(HaveOccurred())
+	// Populate block volume with testFile containing testFileContent.
+	volume := setupWriteTestFile(hostDir, config, BlockFsLocalVolumeType, node)
+	volume.hostDir = hostDir
+	volume.loopDevDir = loopDev
+	return volume
+}
+
 // Determine the /dev/loopXXX device associated with this test, via its hostDir.
 func getBlockLoopDev(hostDir string, node *v1.Node) string {
 	loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir)
@@ -955,6 +1057,35 @@ func cleanupLocalVolumeDirectory(config *localTestConfig, volume *localTestVolum
 	Expect(err).NotTo(HaveOccurred())
 }
 
+// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory.
+func cleanupLocalVolumeDirectoryLink(config *localTestConfig, volume *localTestVolume) {
+	By("Removing the test directory")
+	hostDir := volume.hostDir
+	hostDirBackend := hostDir + "-backend"
+	removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend)
+	err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory.
+func cleanupLocalVolumeDirectoryBindMounted(config *localTestConfig, volume *localTestVolume) {
+	By("Removing the test directory")
+	hostDir := volume.hostDir
+	removeCmd := fmt.Sprintf("sudo umount %s && rm -r %s", hostDir, hostDir)
+	err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+// Deletes the PVC/PV, and launches a pod with hostpath volume to remove the test directory.
+func cleanupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, volume *localTestVolume) {
+	By("Removing the test directory")
+	hostDir := volume.hostDir
+	hostDirBackend := hostDir + "-backend"
+	removeCmd := fmt.Sprintf("rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
+	err := framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node)
+	Expect(err).NotTo(HaveOccurred())
+}
+
 // Deletes the PVC/PV and removes the test directory holding the block file.
 func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) {
 	volume.hostDir = volume.loopDevDir
@@ -965,6 +1096,19 @@ func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
+// Deletes the PVC/PV and removes the test directory holding the block file.
+func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) {
+	// umount first
+	By("Umount blockfs mountpoint")
+	umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir)
+	err := framework.IssueSSHCommand(umountCmd, framework.TestContext.Provider, volume.node)
+	unmapBlockLocalVolume(config, volume.hostDir, volume.node)
+	By("Removing the test directory")
+	removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir)
+	err = framework.IssueSSHCommand(removeCmd, framework.TestContext.Provider, volume.node)
+	Expect(err).NotTo(HaveOccurred())
+}
+
 func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig {
 	pvcConfig := framework.PersistentVolumeClaimConfig{
 		AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},