From f0fc0e1555c929455c3f37f9a0b05c6a6bdb2ee1 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Fri, 2 Mar 2018 16:41:40 +0100 Subject: [PATCH] Add pod cleanup timeout After client pod is stopped we need to wait for a while for kubelet to unmount and clean up the client pod before we can destroy the server pod. Killing e.g. iSCSI server too early may result in stale attached device that cannot be removed. --- test/e2e/framework/volume_util.go | 8 ++++++-- test/e2e/storage/volume_io.go | 4 ++++ test/e2e/storage/volumes.go | 2 +- test/utils/image/manifest.go | 1 - 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test/e2e/framework/volume_util.go b/test/e2e/framework/volume_util.go index 45d2c4bf34..1f0a8e019f 100644 --- a/test/e2e/framework/volume_util.go +++ b/test/e2e/framework/volume_util.go @@ -67,6 +67,10 @@ const ( // Waiting period for volume server (Ceph, ...) to initialize itself. VolumeServerPodStartupSleep = 20 * time.Second + + // Waiting period for pod to be cleaned up and unmount its volumes so we + // don't tear down containers with NFS/Ceph/Gluster server too early. + PodCleanupTimeout = 20 * time.Second ) // Configuration of one tests. The test consist of: @@ -351,8 +355,8 @@ func VolumeTestCleanup(f *Framework, config VolumeTestConfig) { } // See issue #24100. // Prevent umount errors by making sure making sure the client pod exits cleanly *before* the volume server pod exits. - By("sleeping a bit so client can stop and unmount") - time.Sleep(20 * time.Second) + By("sleeping a bit so kubelet can unmount and detach the volume") + time.Sleep(PodCleanupTimeout) err = podClient.Delete(config.Prefix+"-server", nil) if err != nil { diff --git a/test/e2e/storage/volume_io.go b/test/e2e/storage/volume_io.go index 37b5c5d2d4..ff9c61820b 100644 --- a/test/e2e/storage/volume_io.go +++ b/test/e2e/storage/volume_io.go @@ -33,6 +33,7 @@ import ( "path" "strconv" "strings" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -209,6 +210,9 @@ func testVolumeIO(f *framework.Framework, cs clientset.Interface, config framewo if err == nil { // delete err is returned if err is not set err = e } + } else { + framework.Logf("sleeping a bit so kubelet can unmount and detach the volume") + time.Sleep(framework.PodCleanupTimeout) } }() diff --git a/test/e2e/storage/volumes.go b/test/e2e/storage/volumes.go index 2ed8023d96..3be7453725 100644 --- a/test/e2e/storage/volumes.go +++ b/test/e2e/storage/volumes.go @@ -261,7 +261,7 @@ var _ = utils.SIGDescribe("Volumes", func() { config := framework.VolumeTestConfig{ Namespace: namespace.Name, Prefix: "cephfs", - ServerImage: imageutils.GetE2EImage(imageutils.VolumeCephServer), + ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer), ServerPorts: []int{6789}, } diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index d3ed428ebd..7a5e32e5cd 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -96,7 +96,6 @@ var ( VolumeNFSServer = ImageConfig{gcRegistry, "volume-nfs", "0.8", false} VolumeISCSIServer = ImageConfig{gcRegistry, "volume-iscsi", "0.2", false} VolumeGlusterServer = ImageConfig{gcRegistry, "volume-gluster", "0.2", false} - VolumeCephServer = ImageConfig{gcRegistry, "volume-ceph", "0.1", false} VolumeRBDServer = ImageConfig{gcRegistry, "volume-rbd", "0.2", false} )