fix and update local volume e2es

pull/8/head
NickrenREN 2018-05-11 19:00:10 +08:00
parent 8d3fdc5a32
commit 5bc823f141
1 changed files with 82 additions and 60 deletions

View File

@ -78,8 +78,12 @@ const (
GCELocalSSDVolumeType localVolumeType = "gce-localssd-scsi-fs"
// Creates a local file, formats it, and maps it as a block device.
BlockLocalVolumeType localVolumeType = "block"
// Creates a local file, formats it, and mounts it to use as local volume.
BlockFsLocalVolumeType localVolumeType = "blockfs"
// Creates a local file serving as the backing for block device., formats it,
// and mounts it to use as FS mode local volume.
BlockFsWithFormatLocalVolumeType localVolumeType = "blockfswithformat"
// Creates a local file serving as the backing for block device. do not format it manually,
// and mounts it to use as FS mode local volume.
BlockFsWithoutFormatLocalVolumeType localVolumeType = "blockfswithoutformat"
)
var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *localTestVolume{
@ -90,7 +94,8 @@ var setupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *v1.Node) *
DirectoryBindMountedLocalVolumeType: setupLocalVolumeDirectoryBindMounted,
DirectoryLinkBindMountedLocalVolumeType: setupLocalVolumeDirectoryLinkBindMounted,
BlockLocalVolumeType: setupLocalVolumeBlock,
BlockFsLocalVolumeType: setupLocalVolumeBlockFs,
BlockFsWithFormatLocalVolumeType: setupLocalVolumeBlockFsWithFormat,
BlockFsWithoutFormatLocalVolumeType: setupLocalVolumeBlockFsWithoutFormat,
}
var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTestVolume){
@ -101,7 +106,8 @@ var cleanupLocalVolumeMap = map[localVolumeType]func(*localTestConfig, *localTes
DirectoryBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryBindMounted,
DirectoryLinkBindMountedLocalVolumeType: cleanupLocalVolumeDirectoryLinkBindMounted,
BlockLocalVolumeType: cleanupLocalVolumeBlock,
BlockFsLocalVolumeType: cleanupLocalVolumeBlockFs,
BlockFsWithFormatLocalVolumeType: cleanupLocalVolumeBlockFsWithFormat,
BlockFsWithoutFormatLocalVolumeType: cleanupLocalVolumeBlockFsWithoutFormat,
}
type localTestVolume struct {
@ -247,6 +253,11 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
pod1, pod1Err = createLocalPod(config, testVol, nil)
Expect(pod1Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod1, config.node0.Name)
writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
})
AfterEach(func() {
@ -256,16 +267,16 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
It("should be able to mount volume and read from pod1", func() {
By("Reading in pod1")
// testFileContent was written during setupLocalVolume
// testFileContent was written in BeforeEach
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType)
})
It("should be able to mount volume and write from pod1", func() {
// testFileContent was written during setupLocalVolume
// testFileContent was written in BeforeEach
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVolType)
By("Writing in pod1")
writeCmd, _ := createWriteAndReadCmds(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVolType)
podRWCmdExec(pod1, writeCmd)
})
})
@ -346,12 +357,12 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
Context("Local volume that cannot be mounted [Slow]", func() {
// TODO:
// - check for these errors in unit tests intead
// - check for these errors in unit tests instead
It("should fail due to non-existent path", func() {
ep := &eventPatterns{
reason: "FailedMount",
pattern: make([]string, 2)}
ep.pattern = append(ep.pattern, "MountVolume.SetUp failed")
ep.pattern = append(ep.pattern, "MountVolume.NewMounter initialization failed")
testVol := &localTestVolume{
node: config.node0,
@ -461,7 +472,7 @@ var _ = utils.SIGDescribe("PersistentVolumes-local ", func() {
// Delete the persistent volume claim: file will be cleaned up and volume be re-created.
By("Deleting the persistent volume claim to clean up persistent volume and re-create one")
writeCmd, _ := createWriteAndReadCmds(volumePath, testFile, testFileContent, DirectoryLocalVolumeType)
writeCmd := createWriteCmd(volumePath, testFile, testFileContent, DirectoryLocalVolumeType)
err = issueNodeCommand(config, writeCmd, config.node0)
Expect(err).NotTo(HaveOccurred())
err = config.client.CoreV1().PersistentVolumeClaims(claim.Namespace).Delete(claim.Name, &metav1.DeleteOptions{})
@ -731,8 +742,6 @@ func testPodWithNodeConflict(config *localTestConfig, testVolType localVolumeTyp
err = framework.WaitForPodNameUnschedulableInNamespace(config.client, pod.Name, pod.Namespace)
Expect(err).NotTo(HaveOccurred())
cleanupLocalVolumes(config, []*localTestVolume{testVol})
}
type eventPatterns struct {
@ -766,7 +775,12 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
Expect(pod1Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod1, config.node0.Name)
// testFileContent was written during setupLocalVolume
writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
// testFileContent was written after creating pod1
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
By("Creating pod2 to read from the PV")
@ -774,16 +788,16 @@ func twoPodsReadWriteTest(config *localTestConfig, testVol *localTestVolume) {
Expect(pod2Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod2, config.node0.Name)
// testFileContent was written during setupLocalVolume
// testFileContent was written after creating pod1
testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
writeCmd = createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
By("Writing in pod2")
podRWCmdExec(pod2, writeCmd)
By("Reading in pod2")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType)
By("Reading in pod1")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod1, testVol.localVolumeType)
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
@ -798,14 +812,14 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum
Expect(pod1Err).NotTo(HaveOccurred())
verifyLocalPod(config, testVol, pod1, config.node0.Name)
// testFileContent was written during setupLocalVolume
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
writeCmd := createWriteCmd(volumeDir, testFile, testVol.hostDir /*writeTestFileContent*/, testVol.localVolumeType)
writeCmd := createWriteCmd(volumeDir, testFile, testFileContent, testVol.localVolumeType)
By("Writing in pod1")
podRWCmdExec(pod1, writeCmd)
// testFileContent was written after creating pod1
testReadFileContent(volumeDir, testFile, testFileContent, pod1, testVol.localVolumeType)
By("Deleting pod1")
framework.DeletePodOrFail(config.client, config.ns, pod1.Name)
@ -815,7 +829,7 @@ func twoPodsReadWriteSerialTest(config *localTestConfig, testVol *localTestVolum
verifyLocalPod(config, testVol, pod2, config.node0.Name)
By("Reading in pod2")
testReadFileContent(volumeDir, testFile, testVol.hostDir, pod2, testVol.localVolumeType)
testReadFileContent(volumeDir, testFile, testFileContent, pod2, testVol.localVolumeType)
By("Deleting pod2")
framework.DeletePodOrFail(config.client, config.ns, pod2.Name)
@ -885,11 +899,13 @@ func cleanupLocalVolumes(config *localTestConfig, volumes []*localTestVolume) {
}
}
func setupWriteTestFile(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume {
writeCmd, _ := createWriteAndReadCmds(hostDir, testFile, testFileContent, localVolumeType)
By(fmt.Sprintf("Creating test file on node %q in path %q", node.Name, hostDir))
err := issueNodeCommand(config, writeCmd, node)
Expect(err).NotTo(HaveOccurred())
func generateLocalTestVolume(hostDir string, config *localTestConfig, localVolumeType localVolumeType, node *v1.Node) *localTestVolume {
if localVolumeType != BlockLocalVolumeType && localVolumeType != BlockFsWithoutFormatLocalVolumeType {
mkdirCmd := fmt.Sprintf("mkdir -p %s", hostDir)
err := issueNodeCommand(config, mkdirCmd, node)
Expect(err).NotTo(HaveOccurred())
}
return &localTestVolume{
node: node,
hostDir: hostDir,
@ -901,8 +917,7 @@ func setupLocalVolumeTmpfs(config *localTestConfig, node *v1.Node) *localTestVol
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
createAndMountTmpfsLocalVolume(config, hostDir, node)
// populate volume with testFile containing testFileContent
return setupWriteTestFile(hostDir, config, TmpfsLocalVolumeType, node)
return generateLocalTestVolume(hostDir, config, TmpfsLocalVolumeType, node)
}
func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localTestVolume {
@ -910,15 +925,13 @@ func setupLocalVolumeGCELocalSSD(config *localTestConfig, node *v1.Node) *localT
Expect(err).NotTo(HaveOccurred())
dirName := strings.Fields(res)[0]
hostDir := "/mnt/disks/by-uuid/google-local-ssds-scsi-fs/" + dirName
// Populate volume with testFile containing testFileContent.
return setupWriteTestFile(hostDir, config, GCELocalSSDVolumeType, node)
return generateLocalTestVolume(hostDir, config, GCELocalSSDVolumeType, node)
}
func setupLocalVolumeDirectory(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
// Populate volume with testFile containing testFileContent.
return setupWriteTestFile(hostDir, config, DirectoryLocalVolumeType, node)
return generateLocalTestVolume(hostDir, config, DirectoryLocalVolumeType, node)
}
// launchNodeExecPodForLocalPV launches a hostexec pod for local PV and waits
@ -992,11 +1005,10 @@ func setupLocalVolumeDirectoryLink(config *localTestConfig, node *v1.Node) *loca
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
hostDirBackend := hostDir + "-backend"
cmd := fmt.Sprintf("mkdir %s && ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
cmd := fmt.Sprintf("mkdir %s && sudo ln -s %s %s", hostDirBackend, hostDirBackend, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred())
// Populate volume with testFile containing testFileContent.
return setupWriteTestFile(hostDir, config, DirectoryLinkLocalVolumeType, node)
return generateLocalTestVolume(hostDir, config, DirectoryLinkLocalVolumeType, node)
}
func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
@ -1005,20 +1017,18 @@ func setupLocalVolumeDirectoryBindMounted(config *localTestConfig, node *v1.Node
cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s", hostDir, hostDir, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred())
// Populate volume with testFile containing testFileContent.
return setupWriteTestFile(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
return generateLocalTestVolume(hostDir, config, DirectoryBindMountedLocalVolumeType, node)
}
func setupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
hostDirBackend := hostDir + "-backend"
cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && ln -s %s %s",
cmd := fmt.Sprintf("mkdir %s && sudo mount --bind %s %s && sudo ln -s %s %s",
hostDirBackend, hostDirBackend, hostDirBackend, hostDirBackend, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred())
// Populate volume with testFile containing testFileContent.
return setupWriteTestFile(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
return generateLocalTestVolume(hostDir, config, DirectoryLinkBindMountedLocalVolumeType, node)
}
func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVolume {
@ -1026,14 +1036,13 @@ func setupLocalVolumeBlock(config *localTestConfig, node *v1.Node) *localTestVol
hostDir := filepath.Join(hostBase, testDirName)
createAndMapBlockLocalVolume(config, hostDir, node)
loopDev := getBlockLoopDev(config, hostDir, node)
// Populate block volume with testFile containing testFileContent.
volume := setupWriteTestFile(loopDev, config, BlockLocalVolumeType, node)
volume := generateLocalTestVolume(loopDev, config, BlockLocalVolumeType, node)
volume.hostDir = loopDev
volume.loopDevDir = hostDir
return volume
}
func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestVolume {
func setupLocalVolumeBlockFsWithFormat(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
createAndMapBlockLocalVolume(config, hostDir, node)
@ -1043,13 +1052,25 @@ func setupLocalVolumeBlockFs(config *localTestConfig, node *v1.Node) *localTestV
cmd := fmt.Sprintf("sudo mkfs -t ext4 %s && sudo mount -t ext4 %s %s && sudo chmod o+rwx %s", loopDev, loopDev, hostDir, hostDir)
_, err := issueNodeCommandWithResult(config, cmd, node)
Expect(err).NotTo(HaveOccurred())
// Populate block volume with testFile containing testFileContent.
volume := setupWriteTestFile(hostDir, config, BlockFsLocalVolumeType, node)
volume := generateLocalTestVolume(hostDir, config, BlockFsWithFormatLocalVolumeType, node)
volume.hostDir = hostDir
volume.loopDevDir = loopDev
return volume
}
func setupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, node *v1.Node) *localTestVolume {
testDirName := "local-volume-test-" + string(uuid.NewUUID())
hostDir := filepath.Join(hostBase, testDirName)
createAndMapBlockLocalVolume(config, hostDir, node)
loopDev := getBlockLoopDev(config, hostDir, node)
volume := generateLocalTestVolume(loopDev, config, BlockFsWithoutFormatLocalVolumeType, node)
// we do this in order to set block device path to local PV spec path directly
// and test local volume plugin FileSystem mode on block device
volume.hostDir = loopDev
volume.loopDevDir = hostDir
return volume
}
// Determine the /dev/loopXXX device associated with this test, via its hostDir.
func getBlockLoopDev(config *localTestConfig, hostDir string, node *v1.Node) string {
loopDevCmd := fmt.Sprintf("E2E_LOOP_DEV=$(sudo losetup | grep %s/file | awk '{ print $1 }') 2>&1 > /dev/null && echo ${E2E_LOOP_DEV}", hostDir)
@ -1100,7 +1121,7 @@ func cleanupLocalVolumeDirectoryLink(config *localTestConfig, volume *localTestV
By("Removing the test directory")
hostDir := volume.hostDir
hostDirBackend := hostDir + "-backend"
removeCmd := fmt.Sprintf("rm -r %s && rm -r %s", hostDir, hostDirBackend)
removeCmd := fmt.Sprintf("sudo rm -r %s && rm -r %s", hostDir, hostDirBackend)
err := issueNodeCommand(config, removeCmd, volume.node)
Expect(err).NotTo(HaveOccurred())
}
@ -1119,7 +1140,7 @@ func cleanupLocalVolumeDirectoryLinkBindMounted(config *localTestConfig, volume
By("Removing the test directory")
hostDir := volume.hostDir
hostDirBackend := hostDir + "-backend"
removeCmd := fmt.Sprintf("rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
removeCmd := fmt.Sprintf("sudo rm %s && sudo umount %s && rm -r %s", hostDir, hostDirBackend, hostDirBackend)
err := issueNodeCommand(config, removeCmd, volume.node)
Expect(err).NotTo(HaveOccurred())
}
@ -1135,7 +1156,7 @@ func cleanupLocalVolumeBlock(config *localTestConfig, volume *localTestVolume) {
}
// Deletes the PVC/PV and removes the test directory holding the block file.
func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume) {
func cleanupLocalVolumeBlockFsWithFormat(config *localTestConfig, volume *localTestVolume) {
// umount first
By("Umount blockfs mountpoint")
umountCmd := fmt.Sprintf("sudo umount %s", volume.hostDir)
@ -1147,6 +1168,15 @@ func cleanupLocalVolumeBlockFs(config *localTestConfig, volume *localTestVolume)
Expect(err).NotTo(HaveOccurred())
}
func cleanupLocalVolumeBlockFsWithoutFormat(config *localTestConfig, volume *localTestVolume) {
volume.hostDir = volume.loopDevDir
unmapBlockLocalVolume(config, volume.hostDir, volume.node)
By("Removing the test directory")
removeCmd := fmt.Sprintf("rm -r %s", volume.hostDir)
err := issueNodeCommand(config, removeCmd, volume.node)
Expect(err).NotTo(HaveOccurred())
}
func makeLocalPVCConfig(config *localTestConfig, volumeType localVolumeType) framework.PersistentVolumeClaimConfig {
pvcConfig := framework.PersistentVolumeClaimConfig{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
@ -1337,14 +1367,6 @@ func unmapBlockLocalVolume(config *localTestConfig, dir string, node *v1.Node) {
Expect(err).NotTo(HaveOccurred())
}
// Create corresponding write and read commands
// to be executed via hostexec Pod on the node with the local PV
func createWriteAndReadCmds(testFileDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) (writeCmd string, readCmd string) {
writeCmd = createWriteCmd(testFileDir, testFile, writeTestFileContent, volumeType)
readCmd = createReadCmd(testFileDir, testFile, volumeType)
return writeCmd, readCmd
}
func createWriteCmd(testDir string, testFile string, writeTestFileContent string, volumeType localVolumeType) string {
if volumeType == BlockLocalVolumeType {
// testDir is the block device.
@ -1378,7 +1400,7 @@ func createReadCmd(testFileDir string, testFile string, volumeType localVolumeTy
// Read testFile and evaluate whether it contains the testFileContent
func testReadFileContent(testFileDir string, testFile string, testFileContent string, pod *v1.Pod, volumeType localVolumeType) {
readCmd := createReadCmd(volumeDir, testFile, volumeType)
readCmd := createReadCmd(testFileDir, testFile, volumeType)
readOut := podRWCmdExec(pod, readCmd)
Expect(readOut).To(ContainSubstring(testFileContent))
}