Add e2e tests for multiAttach

test/e2e/framework/pv_util.go

@@ -93,6 +93,14 @@ type PersistentVolumeClaimConfig struct {
 	VolumeMode       *v1.PersistentVolumeMode
 }
 
+// NodeSelection specifies where to run a pod, using a combination of fixed node name,
+// node selector and/or affinity.
+type NodeSelection struct {
+	Name     string
+	Selector map[string]string
+	Affinity *v1.Affinity
+}
+
 // Clean up a pv and pvc in a single pv/pvc test case.
 // Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv.
 func PVPVCCleanup(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error {
@@ -874,14 +882,16 @@ func CreateNginxPod(client clientset.Interface, namespace string, nodeSelector m
 
 // create security pod with given claims
 func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, timeout time.Duration) (*v1.Pod, error) {
-	return CreateSecPodWithNodeName(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, "", timeout)
+	return CreateSecPodWithNodeSelection(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, NodeSelection{}, timeout)
 }
 
 // create security pod with given claims
-func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, nodeName string, timeout time.Duration) (*v1.Pod, error) {
+func CreateSecPodWithNodeSelection(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, node NodeSelection, timeout time.Duration) (*v1.Pod, error) {
 	pod := MakeSecPod(namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup)
-	// Setting nodeName
+	// Setting node
-	pod.Spec.NodeName = nodeName
+	pod.Spec.NodeName = node.Name
+	pod.Spec.NodeSelector = node.Selector
+	pod.Spec.Affinity = node.Affinity
 
 	pod, err := client.CoreV1().Pods(namespace).Create(pod)
 	if err != nil {
@@ -901,6 +911,44 @@ func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvcl
 	return pod, nil
 }
 
+// SetNodeAffinityRequirement sets affinity with specified operator to nodeName to nodeSelection
+func SetNodeAffinityRequirement(nodeSelection *NodeSelection, operator v1.NodeSelectorOperator, nodeName string) {
+	// Add node-anti-affinity.
+	if nodeSelection.Affinity == nil {
+		nodeSelection.Affinity = &v1.Affinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity == nil {
+		nodeSelection.Affinity.NodeAffinity = &v1.NodeAffinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
+	}
+	nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
+		v1.NodeSelectorTerm{
+			// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
+			// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
+			// So this might be cleaner:
+			// MatchFields: []v1.NodeSelectorRequirement{
+			// 	{Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{nodeName}},
+			// },
+			// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
+
+			MatchExpressions: []v1.NodeSelectorRequirement{
+				{Key: "kubernetes.io/hostname", Operator: operator, Values: []string{nodeName}},
+			},
+		})
+}
+
+// SetAffinity sets affinity to nodeName to nodeSelection
+func SetAffinity(nodeSelection *NodeSelection, nodeName string) {
+	SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpIn, nodeName)
+}
+
+// SetAntiAffinity sets anti-affinity to nodeName to nodeSelection
+func SetAntiAffinity(nodeSelection *NodeSelection, nodeName string) {
+	SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpNotIn, nodeName)
+}
+
 // Define and create a pod with a mounted PV.  Pod runs infinite loop until killed.
 func CreateClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
 	return CreatePod(c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")

test/e2e/storage/csi_mock_volume.go

@@ -153,12 +153,12 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 			scTest.AllowVolumeExpansion = true
 		}
 
-		nodeSelection := testsuites.NodeSelection{
+		nodeSelection := framework.NodeSelection{
 			// The mock driver only works when everything runs on a single node.
 			Name: nodeName,
 		}
 		if len(m.nodeLabel) > 0 {
-			nodeSelection = testsuites.NodeSelection{
+			nodeSelection = framework.NodeSelection{
 				Selector: m.nodeLabel,
 			}
 		}
@@ -177,11 +177,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 	createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
 		nodeName := m.config.ClientNodeName
-		nodeSelection := testsuites.NodeSelection{
+		nodeSelection := framework.NodeSelection{
 			Name: nodeName,
 		}
 		if len(m.nodeLabel) > 0 {
-			nodeSelection = testsuites.NodeSelection{
+			nodeSelection = framework.NodeSelection{
 				Selector: m.nodeLabel,
 			}
 		}
@@ -597,7 +597,7 @@ func checkNodeForLimits(nodeName string, attachKey v1.ResourceName, cs clientset
 	return int(attachLimit), waitErr
 }
 
-func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node testsuites.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
+func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node framework.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
 	class := newStorageClass(t, ns, "")
 	var err error
 	_, err = cs.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
@@ -659,7 +659,7 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t
 	return class, claim, pod
 }
 
-func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) {
+func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node framework.NodeSelection, ns string) (*v1.Pod, error) {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "pvc-volume-tester-",

test/e2e/storage/csi_volumes.go

@@ -47,6 +47,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
 	testsuites.InitSnapshottableTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 // This executes testSuites for csi volumes.
@@ -155,7 +156,7 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela
 		test.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 			// Ensure that a pod cannot be scheduled in an unsuitable zone.
 			pod := testsuites.StartInPodWithVolume(cs, namespace, claim.Name, "pvc-tester-unschedulable", "sleep 100000",
-				testsuites.NodeSelection{Selector: nodeSelector})
+				framework.NodeSelection{Selector: nodeSelector})
 			defer testsuites.StopPod(cs, pod)
 			framework.ExpectNoError(framework.WaitForPodNameUnschedulableInNamespace(cs, pod.Name, pod.Namespace), "pod should be unschedulable")
 		}

test/e2e/storage/drivers/in_tree.go

@@ -95,6 +95,7 @@ func InitNFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}
@@ -232,6 +233,7 @@ func InitGlusterFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}
@@ -586,6 +588,7 @@ func InitCephFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}

test/e2e/storage/in_tree_volumes.go

@@ -55,6 +55,7 @@ var testSuites = []func() testsuites.TestSuite{
 	testsuites.InitVolumeModeTestSuite,
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 // This executes testSuites for in-tree volumes.

test/e2e/storage/persistent_volumes.go

@@ -30,6 +30,7 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 )
@@ -308,68 +309,22 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() {
 	Describe("Default StorageClass", func() {
 		Context("pods that use multiple volumes", func() {
 
-			AfterEach(func() {
-				framework.DeleteAllStatefulSets(c, ns)
-			})
-
 			It("should be reschedulable [Slow]", func() {
 				// Only run on providers with default storageclass
 				framework.SkipUnlessProviderIs("openstack", "gce", "gke", "vsphere", "azure")
 
 				numVols := 4
-				ssTester := framework.NewStatefulSetTester(c)
 
-				By("Creating a StatefulSet pod to initialize data")
+				By("Creating pvcs")
-				writeCmd := "true"
+				claims := []*v1.PersistentVolumeClaim{}
-				for i := 0; i < numVols; i++ {
-					writeCmd += fmt.Sprintf("&& touch %v", getVolumeFile(i))
-				}
-				writeCmd += "&& sleep 10000"
-
-				probe := &v1.Probe{
-					Handler: v1.Handler{
-						Exec: &v1.ExecAction{
-							// Check that the last file got created
-							Command: []string{"test", "-f", getVolumeFile(numVols - 1)},
-						},
-					},
-					InitialDelaySeconds: 1,
-					PeriodSeconds:       1,
-				}
-
-				mounts := []v1.VolumeMount{}
-				claims := []v1.PersistentVolumeClaim{}
 				for i := 0; i < numVols; i++ {
 					pvc := framework.MakePersistentVolumeClaim(framework.PersistentVolumeClaimConfig{}, ns)
-					pvc.Name = getVolName(i)
+					claims = append(claims, pvc)
-					mounts = append(mounts, v1.VolumeMount{Name: pvc.Name, MountPath: getMountPath(i)})
-					claims = append(claims, *pvc)
 				}
 
-				spec := makeStatefulSetWithPVCs(ns, writeCmd, mounts, claims, probe)
+				By("Testing access to pvcs before and after pod recreation on differetn node")
-				ss, err := c.AppsV1().StatefulSets(ns).Create(spec)
+				testsuites.TestAccessMultipleVolumesAcrossPodRecreation(f, c, ns,
-				Expect(err).NotTo(HaveOccurred())
+					framework.NodeSelection{}, claims, false /* sameNode */)
-				ssTester.WaitForRunningAndReady(1, ss)
-
-				By("Deleting the StatefulSet but not the volumes")
-				// Scale down to 0 first so that the Delete is quick
-				ss, err = ssTester.Scale(ss, 0)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForStatusReplicas(ss, 0)
-				err = c.AppsV1().StatefulSets(ns).Delete(ss.Name, &metav1.DeleteOptions{})
-				Expect(err).NotTo(HaveOccurred())
-
-				By("Creating a new Statefulset and validating the data")
-				validateCmd := "true"
-				for i := 0; i < numVols; i++ {
-					validateCmd += fmt.Sprintf("&& test -f %v", getVolumeFile(i))
-				}
-				validateCmd += "&& sleep 10000"
-
-				spec = makeStatefulSetWithPVCs(ns, validateCmd, mounts, claims, probe)
-				ss, err = c.AppsV1().StatefulSets(ns).Create(spec)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForRunningAndReady(1, ss)
 			})
 		})
 	})

test/e2e/storage/regional_pd.go

@@ -116,7 +116,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
 				err = verifyZonesInPV(volume, sets.NewString(cloudZones...), true /* match */)
 				Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
 
-				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 			},
 		},
 		{
@@ -138,7 +138,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
 				err = verifyZonesInPV(volume, zones, false /* match */)
 				Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
 
-				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 			},
 		},
 	}

test/e2e/storage/testsuites/BUILD

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "base.go",
         "driveroperations.go",
+        "multivolume.go",
         "provisioning.go",
         "snapshottable.go",
         "subpath.go",
@@ -28,6 +29,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/e2e/framework:go_default_library",

test/e2e/storage/testsuites/base.go

@@ -188,7 +188,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
 		if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
 			pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, fsType, r.volume)
 			if pvSource != nil {
-				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false)
+				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false, pattern.VolMode)
 			}
 			r.volType = fmt.Sprintf("%s-preprovisionedPV", dInfo.Name)
 		}
@@ -205,7 +205,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
 
 			if r.sc != nil {
 				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPVFromDynamicProvisionSC(
-					f, dInfo.Name, claimSize, r.sc, false, nil)
+					f, dInfo.Name, claimSize, r.sc, false, pattern.VolMode)
 			}
 			r.volType = fmt.Sprintf("%s-dynamicPV", dInfo.Name)
 		}
@@ -269,6 +269,7 @@ func createVolumeSourceWithPVCPV(
 	pvSource *v1.PersistentVolumeSource,
 	volumeNodeAffinity *v1.VolumeNodeAffinity,
 	readOnly bool,
+	volMode v1.PersistentVolumeMode,
 ) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
 	pvConfig := framework.PersistentVolumeConfig{
 		NamePrefix:       fmt.Sprintf("%s-", name),
@@ -276,10 +277,16 @@ func createVolumeSourceWithPVCPV(
 		PVSource:         *pvSource,
 		NodeAffinity:     volumeNodeAffinity,
 	}
+
 	pvcConfig := framework.PersistentVolumeClaimConfig{
 		StorageClassName: &f.Namespace.Name,
 	}
 
+	if volMode != "" {
+		pvConfig.VolumeMode = &volMode
+		pvcConfig.VolumeMode = &volMode
+	}
+
 	framework.Logf("Creating PVC and PV")
 	pv, pvc, err := framework.CreatePVCPV(f.ClientSet, pvConfig, pvcConfig, f.Namespace.Name, false)
 	Expect(err).NotTo(HaveOccurred(), "PVC, PV creation failed")
@@ -302,7 +309,7 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
 	claimSize string,
 	sc *storagev1.StorageClass,
 	readOnly bool,
-	volMode *v1.PersistentVolumeMode,
+	volMode v1.PersistentVolumeMode,
 ) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
 	cs := f.ClientSet
 	ns := f.Namespace.Name
@@ -310,8 +317,8 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
 	By("creating a claim")
 	pvc := getClaim(claimSize, ns)
 	pvc.Spec.StorageClassName = &sc.Name
-	if volMode != nil {
+	if volMode != "" {
-		pvc.Spec.VolumeMode = volMode
+		pvc.Spec.VolumeMode = &volMode
 	}
 
 	var err error

test/e2e/storage/testsuites/driveroperations.go

@@ -22,6 +22,7 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apiserver/pkg/storage/names"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
 )
@@ -71,8 +72,8 @@ func GetStorageClass(
 			Kind: "StorageClass",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			// Name must be unique, so let's base it on namespace name
+			// Name must be unique, so let's base it on namespace name and use GenerateName
-			Name: ns + "-" + suffix,
+			Name: names.SimpleNameGenerator.GenerateName(ns + "-" + suffix),
 		},
 		Provisioner:       provisioner,
 		Parameters:        parameters,

test/e2e/storage/testsuites/multivolume.go

@@ -0,0 +1,474 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+type multiVolumeTestSuite struct {
+	tsInfo TestSuiteInfo
+}
+
+var _ TestSuite = &multiVolumeTestSuite{}
+
+// InitMultiVolumeTestSuite returns multiVolumeTestSuite that implements TestSuite interface
+func InitMultiVolumeTestSuite() TestSuite {
+	return &multiVolumeTestSuite{
+		tsInfo: TestSuiteInfo{
+			name: "multiVolume",
+			testPatterns: []testpatterns.TestPattern{
+				testpatterns.FsVolModePreprovisionedPV,
+				testpatterns.FsVolModeDynamicPV,
+				testpatterns.BlockVolModePreprovisionedPV,
+				testpatterns.BlockVolModeDynamicPV,
+			},
+		},
+	}
+}
+
+func (t *multiVolumeTestSuite) getTestSuiteInfo() TestSuiteInfo {
+	return t.tsInfo
+}
+
+func (t *multiVolumeTestSuite) defineTests(driver TestDriver, pattern testpatterns.TestPattern) {
+	type local struct {
+		config      *PerTestConfig
+		testCleanup func()
+
+		cs        clientset.Interface
+		ns        *v1.Namespace
+		driver    TestDriver
+		resources []*genericVolumeTestResource
+	}
+	var (
+		dInfo = driver.GetDriverInfo()
+		l     local
+	)
+
+	BeforeEach(func() {
+		// Check preconditions.
+		if pattern.VolMode == v1.PersistentVolumeBlock && !dInfo.Capabilities[CapBlock] {
+			framework.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolMode)
+		}
+	})
+
+	// This intentionally comes after checking the preconditions because it
+	// registers its own BeforeEach which creates the namespace. Beware that it
+	// also registers an AfterEach which renders f unusable. Any code using
+	// f must run inside an It or Context callback.
+	f := framework.NewDefaultFramework("multivolume")
+
+	init := func() {
+		l = local{}
+		l.ns = f.Namespace
+		l.cs = f.ClientSet
+		l.driver = driver
+
+		// Now do the more expensive test initialization.
+		l.config, l.testCleanup = driver.PrepareTest(f)
+	}
+
+	cleanup := func() {
+		for _, resource := range l.resources {
+			resource.cleanupResource()
+		}
+
+		if l.testCleanup != nil {
+			l.testCleanup()
+			l.testCleanup = nil
+		}
+	}
+
+	// This tests below configuration:
+	//          [pod1]                            same node       [pod2]
+	//      [   node1   ]                           ==>        [   node1   ]
+	//          /    \      <- same volume mode                   /    \
+	//   [volume1]  [volume2]                              [volume1]  [volume2]
+	It("should access to two volumes with the same volume mode and retain data across pod recreation on the same node", func() {
+		// Currently, multiple volumes are not generally available for pre-provisoined volume,
+		// because containerized storage servers, such as iSCSI and rbd, are just returning
+		// a static volume inside container, not actually creating a new volume per request.
+		if pattern.VolType == testpatterns.PreprovisionedPV {
+			framework.Skipf("This test doesn't work with pre-provisioned volume -- skipping")
+		}
+
+		init()
+		defer cleanup()
+
+		var pvcs []*v1.PersistentVolumeClaim
+		numVols := 2
+
+		for i := 0; i < numVols; i++ {
+			resource := createGenericVolumeTestResource(driver, l.config, pattern)
+			l.resources = append(l.resources, resource)
+			pvcs = append(pvcs, resource.pvc)
+		}
+
+		TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name,
+			framework.NodeSelection{Name: l.config.ClientNodeName}, pvcs, true /* sameNode */)
+	})
+
+	// This tests below configuration:
+	//          [pod1]                       different node       [pod2]
+	//      [   node1   ]                           ==>        [   node2   ]
+	//          /    \      <- same volume mode                   /    \
+	//   [volume1]  [volume2]                              [volume1]  [volume2]
+	It("should access to two volumes with the same volume mode and retain data across pod recreation on different node", func() {
+		// Currently, multiple volumes are not generally available for pre-provisoined volume,
+		// because containerized storage servers, such as iSCSI and rbd, are just returning
+		// a static volume inside container, not actually creating a new volume per request.
+		if pattern.VolType == testpatterns.PreprovisionedPV {
+			framework.Skipf("This test doesn't work with pre-provisioned volume -- skipping")
+		}
+
+		init()
+		defer cleanup()
+
+		// Check different-node test requirement
+		nodes := framework.GetReadySchedulableNodesOrDie(l.cs)
+		if len(nodes.Items) < 2 {
+			framework.Skipf("Number of available nodes is less than 2 - skipping")
+		}
+		if l.config.ClientNodeName != "" {
+			framework.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name)
+		}
+
+		var pvcs []*v1.PersistentVolumeClaim
+		numVols := 2
+
+		for i := 0; i < numVols; i++ {
+			resource := createGenericVolumeTestResource(driver, l.config, pattern)
+			l.resources = append(l.resources, resource)
+			pvcs = append(pvcs, resource.pvc)
+		}
+
+		TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name,
+			framework.NodeSelection{Name: l.config.ClientNodeName}, pvcs, false /* sameNode */)
+	})
+
+	// This tests below configuration (only <block, filesystem> pattern is tested):
+	//          [pod1]                            same node       [pod2]
+	//      [   node1   ]                          ==>        [   node1   ]
+	//          /    \      <- different volume mode             /    \
+	//   [volume1]  [volume2]                              [volume1]  [volume2]
+	It("should access to two volumes with different volume mode and retain data across pod recreation on the same node", func() {
+		if pattern.VolMode == v1.PersistentVolumeFilesystem {
+			framework.Skipf("Filesystem volume case should be covered by block volume case -- skipping")
+		}
+
+		// Currently, multiple volumes are not generally available for pre-provisoined volume,
+		// because containerized storage servers, such as iSCSI and rbd, are just returning
+		// a static volume inside container, not actually creating a new volume per request.
+		if pattern.VolType == testpatterns.PreprovisionedPV {
+			framework.Skipf("This test doesn't work with pre-provisioned volume -- skipping")
+		}
+
+		init()
+		defer cleanup()
+
+		var pvcs []*v1.PersistentVolumeClaim
+		numVols := 2
+
+		for i := 0; i < numVols; i++ {
+			curPattern := pattern
+			if i != 0 {
+				// 1st volume should be block and set filesystem for 2nd and later volumes
+				curPattern.VolMode = v1.PersistentVolumeFilesystem
+			}
+			resource := createGenericVolumeTestResource(driver, l.config, curPattern)
+			l.resources = append(l.resources, resource)
+			pvcs = append(pvcs, resource.pvc)
+		}
+
+		TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name,
+			framework.NodeSelection{Name: l.config.ClientNodeName}, pvcs, true /* sameNode */)
+	})
+
+	// This tests below configuration (only <block, filesystem> pattern is tested):
+	//          [pod1]                      different node       [pod2]
+	//      [   node1   ]                          ==>        [   node2   ]
+	//          /    \      <- different volume mode             /    \
+	//   [volume1]  [volume2]                              [volume1]  [volume2]
+	It("should access to two volumes with different volume mode and retain data across pod recreation on different node", func() {
+		if pattern.VolMode == v1.PersistentVolumeFilesystem {
+			framework.Skipf("Filesystem volume case should be covered by block volume case -- skipping")
+		}
+
+		// Currently, multiple volumes are not generally available for pre-provisoined volume,
+		// because containerized storage servers, such as iSCSI and rbd, are just returning
+		// a static volume inside container, not actually creating a new volume per request.
+		if pattern.VolType == testpatterns.PreprovisionedPV {
+			framework.Skipf("This test doesn't work with pre-provisioned volume -- skipping")
+		}
+
+		init()
+		defer cleanup()
+
+		// Check different-node test requirement
+		nodes := framework.GetReadySchedulableNodesOrDie(l.cs)
+		if len(nodes.Items) < 2 {
+			framework.Skipf("Number of available nodes is less than 2 - skipping")
+		}
+		if l.config.ClientNodeName != "" {
+			framework.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name)
+		}
+
+		var pvcs []*v1.PersistentVolumeClaim
+		numVols := 2
+
+		for i := 0; i < numVols; i++ {
+			curPattern := pattern
+			if i != 0 {
+				// 1st volume should be block and set filesystem for 2nd and later volumes
+				curPattern.VolMode = v1.PersistentVolumeFilesystem
+			}
+			resource := createGenericVolumeTestResource(driver, l.config, curPattern)
+			l.resources = append(l.resources, resource)
+			pvcs = append(pvcs, resource.pvc)
+		}
+
+		TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name,
+			framework.NodeSelection{Name: l.config.ClientNodeName}, pvcs, false /* sameNode */)
+	})
+
+	// This tests below configuration:
+	// [pod1] [pod2]
+	// [   node1   ]
+	//   \      /     <- same volume mode
+	//   [volume1]
+	It("should concurrently access the single volume from pods on the same node", func() {
+		init()
+		defer cleanup()
+
+		numPods := 2
+
+		if !l.driver.GetDriverInfo().Capabilities[CapMultiPODs] {
+			framework.Skipf("Driver %q does not support multiple concurrent pods - skipping", dInfo.Name)
+		}
+
+		// Create volume
+		resource := createGenericVolumeTestResource(l.driver, l.config, pattern)
+		l.resources = append(l.resources, resource)
+
+		// Test access to the volume from pods on different node
+		TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name,
+			framework.NodeSelection{Name: l.config.ClientNodeName}, resource.pvc, numPods, true /* sameNode */)
+	})
+
+	// This tests below configuration:
+	//        [pod1] [pod2]
+	// [   node1   ] [   node2   ]
+	//         \      /     <- same volume mode
+	//         [volume1]
+	It("should concurrently access the single volume from pods on different node", func() {
+		init()
+		defer cleanup()
+
+		numPods := 2
+
+		if !l.driver.GetDriverInfo().Capabilities[CapRWX] {
+			framework.Skipf("Driver %s doesn't support %v -- skipping", l.driver.GetDriverInfo().Name, CapRWX)
+		}
+
+		// Check different-node test requirement
+		nodes := framework.GetReadySchedulableNodesOrDie(l.cs)
+		if len(nodes.Items) < numPods {
+			framework.Skipf(fmt.Sprintf("Number of available nodes is less than %d - skipping", numPods))
+		}
+		if l.config.ClientNodeName != "" {
+			framework.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name)
+		}
+
+		// Create volume
+		resource := createGenericVolumeTestResource(l.driver, l.config, pattern)
+		l.resources = append(l.resources, resource)
+
+		// Test access to the volume from pods on different node
+		TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name,
+			framework.NodeSelection{Name: l.config.ClientNodeName}, resource.pvc, numPods, false /* sameNode */)
+	})
+}
+
+// testAccessMultipleVolumes tests access to multiple volumes from single pod on the specified node
+// If readSeedBase > 0, read test are done before write/read test assuming that there is already data written.
+func testAccessMultipleVolumes(f *framework.Framework, cs clientset.Interface, ns string,
+	node framework.NodeSelection, pvcs []*v1.PersistentVolumeClaim, readSeedBase int64, writeSeedBase int64) string {
+	By(fmt.Sprintf("Creating pod on %+v with multiple volumes", node))
+	pod, err := framework.CreateSecPodWithNodeSelection(cs, ns, pvcs,
+		false, "", false, false, framework.SELinuxLabel,
+		nil, node, framework.PodStartTimeout)
+	defer func() {
+		framework.ExpectNoError(framework.DeletePodWithWait(f, cs, pod))
+	}()
+	Expect(err).NotTo(HaveOccurred())
+
+	byteLen := 64
+	for i, pvc := range pvcs {
+		// CreateSecPodWithNodeSelection make volumes accessible via /mnt/volume({i} + 1)
+		index := i + 1
+		path := fmt.Sprintf("/mnt/volume%d", index)
+		By(fmt.Sprintf("Checking if the volume%d exists as expected volume mode (%s)", index, *pvc.Spec.VolumeMode))
+		utils.CheckVolumeModeOfPath(pod, *pvc.Spec.VolumeMode, path)
+
+		if readSeedBase > 0 {
+			By(fmt.Sprintf("Checking if read from the volume%d works properly", index))
+			utils.CheckReadFromPath(pod, *pvc.Spec.VolumeMode, path, byteLen, readSeedBase+int64(i))
+		}
+
+		By(fmt.Sprintf("Checking if write to the volume%d works properly", index))
+		utils.CheckWriteToPath(pod, *pvc.Spec.VolumeMode, path, byteLen, writeSeedBase+int64(i))
+
+		By(fmt.Sprintf("Checking if read from the volume%d works properly", index))
+		utils.CheckReadFromPath(pod, *pvc.Spec.VolumeMode, path, byteLen, writeSeedBase+int64(i))
+	}
+
+	pod, err = cs.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred(), "get pod")
+	return pod.Spec.NodeName
+}
+
+// TestAccessMultipleVolumesAcrossPodRecreation tests access to multiple volumes from single pod,
+// then recreate pod on the same or different node depending on requiresSameNode,
+// and recheck access to the volumes from the recreated pod
+func TestAccessMultipleVolumesAcrossPodRecreation(f *framework.Framework, cs clientset.Interface, ns string,
+	node framework.NodeSelection, pvcs []*v1.PersistentVolumeClaim, requiresSameNode bool) {
+
+	// No data is written in volume, so passing negative value
+	readSeedBase := int64(-1)
+	writeSeedBase := time.Now().UTC().UnixNano()
+	// Test access to multiple volumes on the specified node
+	nodeName := testAccessMultipleVolumes(f, cs, ns, node, pvcs, readSeedBase, writeSeedBase)
+
+	// Set affinity depending on requiresSameNode
+	if requiresSameNode {
+		framework.SetAffinity(&node, nodeName)
+	} else {
+		framework.SetAntiAffinity(&node, nodeName)
+	}
+
+	// Test access to multiple volumes again on the node updated above
+	// Setting previous writeSeed to current readSeed to check previous data is retained
+	readSeedBase = writeSeedBase
+	// Update writeSeed with new value
+	writeSeedBase = time.Now().UTC().UnixNano()
+	_ = testAccessMultipleVolumes(f, cs, ns, node, pvcs, readSeedBase, writeSeedBase)
+}
+
+// TestConcurrentAccessToSingleVolume tests access to a single volume from multiple pods,
+// then delete the last pod, and recheck access to the volume after pod deletion to check if other
+// pod deletion doesn't affect. Pods are deployed on the same node or different nodes depending on requiresSameNode.
+// Read/write check are done across pod, by check reading both what pod{n-1} and pod{n} wrote from pod{n}.
+func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Interface, ns string,
+	node framework.NodeSelection, pvc *v1.PersistentVolumeClaim, numPods int, requiresSameNode bool) {
+
+	var pods []*v1.Pod
+
+	// Create each pod with pvc
+	for i := 0; i < numPods; i++ {
+		index := i + 1
+		By(fmt.Sprintf("Creating pod%d with a volume on %+v", index, node))
+		pod, err := framework.CreateSecPodWithNodeSelection(cs, ns,
+			[]*v1.PersistentVolumeClaim{pvc},
+			false, "", false, false, framework.SELinuxLabel,
+			nil, node, framework.PodStartTimeout)
+		defer func() {
+			framework.ExpectNoError(framework.DeletePodWithWait(f, cs, pod))
+		}()
+		Expect(err).NotTo(HaveOccurred())
+		pod, err = cs.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{})
+		pods = append(pods, pod)
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get pod%d", index))
+		actualNodeName := pod.Spec.NodeName
+
+		// Set affinity depending on requiresSameNode
+		if requiresSameNode {
+			framework.SetAffinity(&node, actualNodeName)
+		} else {
+			framework.SetAntiAffinity(&node, actualNodeName)
+		}
+	}
+
+	var seed int64
+	byteLen := 64
+	path := "/mnt/volume1"
+	// Check if volume can be accessed from each pod
+	for i, pod := range pods {
+		index := i + 1
+		By(fmt.Sprintf("Checking if the volume in pod%d exists as expected volume mode (%s)", index, *pvc.Spec.VolumeMode))
+		utils.CheckVolumeModeOfPath(pod, *pvc.Spec.VolumeMode, path)
+
+		if i != 0 {
+			By(fmt.Sprintf("From pod%d, checking if reading the data that pod%d write works properly", index, index-1))
+			// For 1st pod, no one has written data yet, so pass the read check
+			utils.CheckReadFromPath(pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
+		}
+
+		// Update the seed and check if write/read works properly
+		seed = time.Now().UTC().UnixNano()
+
+		By(fmt.Sprintf("Checking if write to the volume in pod%d works properly", index))
+		utils.CheckWriteToPath(pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
+
+		By(fmt.Sprintf("Checking if read from the volume in pod%d works properly", index))
+		utils.CheckReadFromPath(pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
+	}
+
+	// Delete the last pod and remove from slice of pods
+	if len(pods) < 2 {
+		framework.Failf("Number of pods shouldn't be less than 2, but got %d", len(pods))
+	}
+	lastPod := pods[len(pods)-1]
+	framework.ExpectNoError(framework.DeletePodWithWait(f, cs, lastPod))
+	pods = pods[:len(pods)-1]
+
+	// Recheck if pv can be accessed from each pod after the last pod deletion
+	for i, pod := range pods {
+		index := i + 1
+		// index of pod and index of pvc match, because pods are created above way
+		By(fmt.Sprintf("Rechecking if the volume in pod%d exists as expected volume mode (%s)", index, *pvc.Spec.VolumeMode))
+		utils.CheckVolumeModeOfPath(pod, *pvc.Spec.VolumeMode, "/mnt/volume1")
+
+		if i == 0 {
+			// This time there should be data that last pod wrote, for 1st pod
+			By(fmt.Sprintf("From pod%d, rechecking if reading the data that last pod write works properly", index))
+		} else {
+			By(fmt.Sprintf("From pod%d, rechecking if reading the data that pod%d write works properly", index, index-1))
+		}
+		utils.CheckReadFromPath(pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
+
+		// Update the seed and check if write/read works properly
+		seed = time.Now().UTC().UnixNano()
+
+		By(fmt.Sprintf("Rechecking if write to the volume in pod%d works properly", index))
+		utils.CheckWriteToPath(pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
+
+		By(fmt.Sprintf("Rechecking if read from the volume in pod%d works properly", index))
+		utils.CheckReadFromPath(pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
+	}
+}

test/e2e/storage/testsuites/provisioning.go

@@ -18,7 +18,6 @@ package testsuites
 
 import (
 	"fmt"
-	"sync"
 	"time"
 
 	. "github.com/onsi/ginkgo"
@@ -180,7 +179,7 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 			framework.Skipf("need more than one node - skipping")
 		}
 		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-			PVMultiNodeCheck(l.cs, claim, volume, NodeSelection{Name: l.config.ClientNodeName})
+			PVMultiNodeCheck(l.cs, claim, volume, framework.NodeSelection{Name: l.config.ClientNodeName})
 		}
 		l.testCase.TestDynamicProvisioning()
 	})
@@ -214,62 +213,14 @@ func (p *provisioningTestSuite) defineTests(driver TestDriver, pattern testpatte
 
 		dc := l.config.Framework.DynamicClient
 		vsc := sDriver.GetSnapshotClass(l.config)
-		dataSource, cleanupFunc := prepareDataSourceForProvisioning(NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
+		dataSource, cleanupFunc := prepareDataSourceForProvisioning(framework.NodeSelection{Name: l.config.ClientNodeName}, l.cs, dc, l.pvc, l.sc, vsc)
 		defer cleanupFunc()
 
 		l.pvc.Spec.DataSource = dataSource
 		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 			By("checking whether the created volume has the pre-populated data")
 			command := fmt.Sprintf("grep '%s' /mnt/test/initialData", claim.Namespace)
-			RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, NodeSelection{Name: l.config.ClientNodeName})
+			RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, "pvc-snapshot-tester", command, framework.NodeSelection{Name: l.config.ClientNodeName})
-		}
-		l.testCase.TestDynamicProvisioning()
-	})
-
-	It("should allow concurrent writes on the same node", func() {
-		if !dInfo.Capabilities[CapMultiPODs] {
-			framework.Skipf("Driver %q does not support multiple concurrent pods - skipping", dInfo.Name)
-		}
-
-		init()
-		defer cleanup()
-
-		l.testCase.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-			// We start two pods concurrently on the same node,
-			// using the same PVC. Both wait for other to create a
-			// file before returning. The pods are forced onto the
-			// same node via pod affinity.
-			wg := sync.WaitGroup{}
-			wg.Add(2)
-			firstPodName := "pvc-tester-first"
-			secondPodName := "pvc-tester-second"
-			run := func(podName, command string) {
-				defer GinkgoRecover()
-				defer wg.Done()
-				node := NodeSelection{
-					Name: l.config.ClientNodeName,
-				}
-				if podName == secondPodName {
-					node.Affinity = &v1.Affinity{
-						PodAffinity: &v1.PodAffinity{
-							RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
-								{LabelSelector: &metav1.LabelSelector{
-									MatchLabels: map[string]string{
-										// Set by RunInPodWithVolume.
-										"app": firstPodName,
-									},
-								},
-									TopologyKey: "kubernetes.io/hostname",
-								},
-							},
-						},
-					}
-				}
-				RunInPodWithVolume(l.cs, claim.Namespace, claim.Name, podName, command, node)
-			}
-			go run(firstPodName, "touch /mnt/test/first && while ! [ -f /mnt/test/second ]; do sleep 1; done")
-			go run(secondPodName, "touch /mnt/test/second && while ! [ -f /mnt/test/first ]; do sleep 1; done")
-			wg.Wait()
 		}
 		l.testCase.TestDynamicProvisioning()
 	})
@@ -384,7 +335,7 @@ func (t StorageClassTest) TestDynamicProvisioning() *v1.PersistentVolume {
 // persistent across pods.
 //
 // This is a common test that can be called from a StorageClassTest.PvCheck.
-func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node NodeSelection) {
+func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node framework.NodeSelection) {
 	By(fmt.Sprintf("checking the created volume is writable and has the PV's mount options on node %+v", node))
 	command := "echo 'hello world' > /mnt/test/data"
 	// We give the first pod the secondary responsibility of checking the volume has
@@ -408,7 +359,7 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
 
 	By(fmt.Sprintf("checking the created volume is readable and retains data on the same node %q", actualNodeName))
 	command = "grep 'hello world' /mnt/test/data"
-	RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader", command, NodeSelection{Name: actualNodeName})
+	RunInPodWithVolume(client, claim.Namespace, claim.Name, "pvc-volume-tester-reader", command, framework.NodeSelection{Name: actualNodeName})
 }
 
 // PVMultiNodeCheck checks that a PV retains data when moved between nodes.
@@ -425,7 +376,7 @@ func PVWriteReadSingleNodeCheck(client clientset.Interface, claim *v1.Persistent
 // persistent across pods and across nodes.
 //
 // This is a common test that can be called from a StorageClassTest.PvCheck.
-func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node NodeSelection) {
+func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume, node framework.NodeSelection) {
 	Expect(node.Name).To(Equal(""), "this test only works when not locked onto a single node")
 
 	var pod *v1.Pod
@@ -446,30 +397,7 @@ func PVMultiNodeCheck(client clientset.Interface, claim *v1.PersistentVolumeClai
 
 	// Add node-anti-affinity.
 	secondNode := node
-	if secondNode.Affinity == nil {
+	framework.SetAntiAffinity(&secondNode, actualNodeName)
-		secondNode.Affinity = &v1.Affinity{}
-	}
-	if secondNode.Affinity.NodeAffinity == nil {
-		secondNode.Affinity.NodeAffinity = &v1.NodeAffinity{}
-	}
-	if secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
-		secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
-	}
-	secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(secondNode.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
-		v1.NodeSelectorTerm{
-			// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
-			// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
-			// So this might be cleaner:
-			// MatchFields: []v1.NodeSelectorRequirement{
-			// 	{Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{actualNodeName}},
-			// },
-			// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
-
-			MatchExpressions: []v1.NodeSelectorRequirement{
-				{Key: "kubernetes.io/hostname", Operator: v1.NodeSelectorOpNotIn, Values: []string{actualNodeName}},
-			},
-		})
-
 	By(fmt.Sprintf("checking the created volume is readable and retains data on another node %+v", secondNode))
 	command = "grep 'hello world' /mnt/test/data"
 	if framework.NodeOSDistroIs("windows") {
@@ -573,17 +501,9 @@ func (t StorageClassTest) TestBindingWaitForFirstConsumerMultiPVC(claims []*v1.P
 	return pvs, node
 }
 
-// NodeSelection specifies where to run a pod, using a combination of fixed node name,
-// node selector and/or affinity.
-type NodeSelection struct {
-	Name     string
-	Selector map[string]string
-	Affinity *v1.Affinity
-}
-
 // RunInPodWithVolume runs a command in a pod with given claim mounted to /mnt directory.
 // It starts, checks, collects output and stops it.
-func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node NodeSelection) {
+func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node framework.NodeSelection) {
 	pod := StartInPodWithVolume(c, ns, claimName, podName, command, node)
 	defer StopPod(c, pod)
 	framework.ExpectNoError(framework.WaitForPodSuccessInNamespaceSlow(c, pod.Name, pod.Namespace))
@@ -591,7 +511,7 @@ func RunInPodWithVolume(c clientset.Interface, ns, claimName, podName, command s
 
 // StartInPodWithVolume starts a command in a pod with given claim mounted to /mnt directory
 // The caller is responsible for checking the pod and deleting it.
-func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node NodeSelection) *v1.Pod {
+func StartInPodWithVolume(c clientset.Interface, ns, claimName, podName, command string, node framework.NodeSelection) *v1.Pod {
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Pod",
@@ -664,7 +584,7 @@ func verifyPVCsPending(client clientset.Interface, pvcs []*v1.PersistentVolumeCl
 }
 
 func prepareDataSourceForProvisioning(
-	node NodeSelection,
+	node framework.NodeSelection,
 	client clientset.Interface,
 	dynamicClient dynamic.Interface,
 	initClaim *v1.PersistentVolumeClaim,

test/e2e/storage/testsuites/testdriver.go

@@ -121,6 +121,8 @@ const (
 	// - https://github.com/container-storage-interface/spec/issues/178
 	// - NodeStageVolume in the spec
 	CapMultiPODs Capability = "multipods"
+
+	CapRWX Capability = "RWX" // support ReadWriteMany access modes
 )
 
 // DriverInfo represents static information about a TestDriver.

test/e2e/storage/testsuites/volumemode.go

@@ -182,9 +182,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 				framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
 
 				By("Creating pod")
-				pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
+				pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
 					false, "", false, false, framework.SELinuxLabel,
-					nil, l.config.ClientNodeName, framework.PodStartTimeout)
+					nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
 				defer func() {
 					framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
 				}()
@@ -213,9 +213,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 				framework.ExpectNoError(framework.WaitOnPVandPVC(l.cs, l.ns.Name, l.pv, l.pvc))
 
 				By("Creating pod")
-				pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
+				pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
 					false, "", false, false, framework.SELinuxLabel,
-					nil, l.config.ClientNodeName, framework.PodStartTimeout)
+					nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
 				defer func() {
 					framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
 				}()
@@ -273,9 +273,9 @@ func (t *volumeModeTestSuite) defineTests(driver TestDriver, pattern testpattern
 				Expect(err).NotTo(HaveOccurred())
 
 				By("Creating pod")
-				pod, err := framework.CreateSecPodWithNodeName(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
+				pod, err := framework.CreateSecPodWithNodeSelection(l.cs, l.ns.Name, []*v1.PersistentVolumeClaim{l.pvc},
 					false, "", false, false, framework.SELinuxLabel,
-					nil, l.config.ClientNodeName, framework.PodStartTimeout)
+					nil, framework.NodeSelection{Name: l.config.ClientNodeName}, framework.PodStartTimeout)
 				defer func() {
 					framework.ExpectNoError(framework.DeletePodWithWait(f, l.cs, pod))
 				}()

test/e2e/storage/utils/utils.go

@@ -17,7 +17,11 @@ limitations under the License.
 package utils
 
 import (
+	"crypto/sha256"
+	"encoding/base64"
 	"fmt"
+	"math/rand"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -483,3 +487,43 @@ func CheckReadWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path str
 		VerifyExecInPodFail(pod, fmt.Sprintf("dd if=/dev/urandom of=%s bs=64 count=1", path), 1)
 	}
 }
+
+func genBinDataFromSeed(len int, seed int64) []byte {
+	binData := make([]byte, len)
+	rand.Seed(seed)
+
+	len, err := rand.Read(binData)
+	if err != nil {
+		fmt.Printf("Error: %v", err)
+	}
+
+	return binData
+}
+
+func CheckReadFromPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
+	var pathForVolMode string
+	if volMode == v1.PersistentVolumeBlock {
+		pathForVolMode = path
+	} else {
+		pathForVolMode = filepath.Join(path, "file1.txt")
+	}
+
+	sum := sha256.Sum256(genBinDataFromSeed(len, seed))
+
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum", pathForVolMode, len))
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("dd if=%s bs=%d count=1 | sha256sum | grep -Fq %x", pathForVolMode, len, sum))
+}
+
+func CheckWriteToPath(pod *v1.Pod, volMode v1.PersistentVolumeMode, path string, len int, seed int64) {
+	var pathForVolMode string
+	if volMode == v1.PersistentVolumeBlock {
+		pathForVolMode = path
+	} else {
+		pathForVolMode = filepath.Join(path, "file1.txt")
+	}
+
+	encoded := base64.StdEncoding.EncodeToString(genBinDataFromSeed(len, seed))
+
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | sha256sum", encoded))
+	VerifyExecInPodSucceed(pod, fmt.Sprintf("echo %s | base64 -d | dd of=%s bs=%d count=1", encoded, pathForVolMode, len))
+}

test/e2e/storage/volume_provisioning.go

@@ -276,7 +276,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkGCEPD(volume, "pd-ssd")
 						Expect(err).NotTo(HaveOccurred(), "checkGCEPD pd-ssd")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -291,7 +291,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkGCEPD(volume, "pd-standard")
 						Expect(err).NotTo(HaveOccurred(), "checkGCEPD pd-standard")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// AWS
@@ -308,7 +308,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "gp2", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS gp2")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -324,7 +324,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "io1", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS io1")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -339,7 +339,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "sc1", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS sc1")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -354,7 +354,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "st1", false)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS st1")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -369,7 +369,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 						err := checkAWSEBS(volume, "gp2", true)
 						Expect(err).NotTo(HaveOccurred(), "checkAWSEBS gp2 encrypted")
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// OpenStack generic tests (works on all OpenStack deployments)
@@ -381,7 +381,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:      "1.5Gi",
 					ExpectedSize:   "2Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				{
@@ -395,7 +395,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:    "1.5Gi",
 					ExpectedSize: "2Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// vSphere generic test
@@ -407,7 +407,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:      "1.5Gi",
 					ExpectedSize:   "1.5Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 				// Azure
@@ -419,7 +419,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 					ClaimSize:      "1Gi",
 					ExpectedSize:   "1Gi",
 					PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
-						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+						testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 					},
 				},
 			}
@@ -480,7 +480,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
 				PvCheck: func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 					err := checkGCEPD(volume, "pd-standard")
 					Expect(err).NotTo(HaveOccurred(), "checkGCEPD")
-					testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+					testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 				},
 			}
 			test.Class = newStorageClass(test, ns, "reclaimpolicy")