add disruptive tests to verify host clean up. Issues: #31272, #37657.

test/e2e/kubelet.go

@@ -145,6 +145,26 @@ func createNfsServerPod(c clientset.Interface, config framework.VolumeTestConfig
 	return pod, ip
 }
 
+// Restart the passed-in nfs-server by issuing a `/usr/sbin/rpc.nfsd 1` command in the
+// pod's (only) container. This command changes the number of nfs server threads from
+// (presumably) zero back to 1, and therefore allows nfs to open connections again.
+func restartNfsServer(serverPod *v1.Pod) {
+
+	const startcmd = "/usr/sbin/rpc.nfsd 1"
+	ns := fmt.Sprintf("--namespace=%v", serverPod.Namespace)
+	framework.RunKubectlOrDie("exec", ns, serverPod.Name, "--", "/bin/sh", "-c", startcmd)
+}
+
+// Stop the passed-in nfs-server by issuing a `/usr/sbin/rpc.nfsd 0` command in the
+// pod's (only) container. This command changes the number of nfs server threads to 0,
+// thus closing all open nfs connections.
+func stopNfsServer(serverPod *v1.Pod) {
+
+	const stopcmd = "/usr/sbin/rpc.nfsd 0"
+	ns := fmt.Sprintf("--namespace=%v", serverPod.Namespace)
+	framework.RunKubectlOrDie("exec", ns, serverPod.Name, "--", "/bin/sh", "-c", stopcmd)
+}
+
 // Creates a pod that mounts an nfs volume that is served by the nfs-server pod. The container
 // will execute the passed in shell cmd. Waits for the pod to start.
 // Note: the nfs plugin is defined inline, no PV or PVC.
@@ -208,6 +228,23 @@ func createPodUsingNfs(f *framework.Framework, c clientset.Interface, ns, nfsIP,
 	return rtnPod
 }
 
+// move the passed-in pod's UID directory to /tmp.
+func movePodUidDir(c clientset.Interface, pod *v1.Pod) {
+
+	dest := "/tmp"
+	podDir := filepath.Join("/var/lib/kubelet/pods", string(pod.UID))
+	cmd := fmt.Sprintf("mv %v %v", podDir, dest)
+	// use ip rather than hostname in GCE
+	nodeIP, err := framework.GetHostExternalAddress(c, pod)
+	Expect(err).NotTo(HaveOccurred())
+
+	// excute cmd over ssh
+	result, _ := nodeExec(nodeIP, cmd)
+	framework.LogSSHResult(result)
+	Expect(result.Code).To(BeZero())
+	Expect(len(result.Stderr)).To(BeZero())
+}
+
 // Checks for a lingering nfs mount and/or uid directory on the pod's host. The host IP is used
 // so that this test runs in GCE, where it appears that SSH cannot resolve the hostname.
 // If expectClean is true then we expect the node to be cleaned up and thus commands like
@@ -218,42 +255,48 @@ func checkPodCleanup(c clientset.Interface, pod *v1.Pod, expectClean bool) {
 
 	timeout := 5 * time.Minute
 	poll := 20 * time.Second
-	podUID := string(pod.UID)
+	podDir := filepath.Join("/var/lib/kubelet/pods", string(pod.UID))
-	podDir := filepath.Join("/var/lib/kubelet/pods", podUID)
 	mountDir := filepath.Join(podDir, "volumes", "kubernetes.io~nfs")
 	// use ip rather than hostname in GCE
 	nodeIP, err := framework.GetHostExternalAddress(c, pod)
 	Expect(err).NotTo(HaveOccurred())
 
-	condMsg := map[bool]string{
+	condMsg := "deleted"
-		true:  "deleted",
+	if !expectClean {
-		false: "present",
+		condMsg = "present"
 	}
 
-	// table of host tests to perform
+	// table of host tests to perform (order may matter so not using a map)
-	tests := map[string]string{ //["what-to-test"] "remote-command"
+	type testT struct {
-		"pod UID directory": fmt.Sprintf("sudo ls %v", podDir),
+		feature string // feature to test
-		"pod nfs mount":     fmt.Sprintf("sudo mount | grep %v", mountDir),
+		cmd     string // remote command to execute on node
+	}
+	tests := []testT{
+		{
+			feature: "pod UID directory",
+			cmd:     fmt.Sprintf("sudo ls %v", podDir),
+		},
+		{
+			feature: "pod nfs mount",
+			cmd:     fmt.Sprintf("sudo mount | grep %v", mountDir),
+		},
 	}
 
-	for test, cmd := range tests {
+	for _, test := range tests {
-		framework.Logf("Wait up to %v for host's (%v) %q to be %v", timeout, nodeIP, test, condMsg[expectClean])
+		framework.Logf("Wait up to %v for host's (%v) %q to be %v", timeout, nodeIP, test.feature, condMsg)
 		err = wait.Poll(poll, timeout, func() (bool, error) {
-			result, _ := nodeExec(nodeIP, cmd)
+			result, _ := nodeExec(nodeIP, test.cmd)
 			framework.LogSSHResult(result)
-			sawFiles := result.Code == 0
+			ok := (result.Code == 0 && len(result.Stdout) > 0 && len(result.Stderr) == 0)
-			if expectClean && sawFiles { // keep trying
+			if expectClean && ok { // keep trying
 				return false, nil
 			}
-			if !expectClean && !sawFiles { // stop wait loop
+			if !expectClean && !ok { // stop wait loop
-				return true, fmt.Errorf("%v is gone but expected to exist", test)
+				return true, fmt.Errorf("%v is gone but expected to exist", test.feature)
 			}
 			return true, nil // done, host is as expected
 		})
-		if err != nil {
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Host (%v) cleanup error: %v. Expected %q to be %v", nodeIP, err, test.feature, condMsg))
-			framework.Logf("Host (%v) cleanup error: %v. Expected %q to be %v", nodeIP, err, test, condMsg[expectClean])
-			Expect(err).NotTo(HaveOccurred())
-		}
 	}
 
 	if expectClean {
@@ -375,14 +418,20 @@ var _ = framework.KubeDescribe("kubelet", func() {
 		}
 	})
 
-	// Delete nfs server pod after another pods accesses the mounted nfs volume.
+	// Test host cleanup when disrupting the volume environment.
-	framework.KubeDescribe("host cleanup with volume mounts [HostCleanup][Flaky]", func() {
+	framework.KubeDescribe("host cleanup with volume mounts [Disruptive][Volume][HostCleanup]", func() {
+
 		type hostCleanupTest struct {
 			itDescr string
 			podCmd  string
 		}
 
-		Context("Host cleanup after pod using NFS mount is deleted [Volume][NFS]", func() {
+		// Disrupt the nfs-server pod after a client pod accesses the nfs volume.
+		// Note: the nfs-server is stopped NOT deleted. This is done to preserve its ip addr.
+		//       If the nfs-server pod is deleted the client pod's mount can not be unmounted.
+		//       If the nfs-server pod is deleted and re-created, due to having a different ip
+		//       addr, the client pod's mount still cannot be unmounted.
+		Context("Host cleanup after disrupting NFS volume [NFS]", func() {
 			// issue #31272
 			var (
 				nfsServerPod *v1.Pod
@@ -395,21 +444,34 @@ var _ = framework.KubeDescribe("kubelet", func() {
 			testTbl := []hostCleanupTest{
 				{
 					itDescr: "after deleting the nfs-server, the host should be cleaned-up when deleting sleeping pod which mounts an NFS vol",
-					podCmd:  "sleep 6000",
+					podCmd:  "sleep 6000", // keep pod running
 				},
 				{
 					itDescr: "after deleting the nfs-server, the host should be cleaned-up when deleting a pod accessing the NFS vol",
-					podCmd:  "while true; do echo FeFieFoFum >>/mnt/SUCCESS; cat /mnt/SUCCESS; done",
+					podCmd:  "while true; do echo FeFieFoFum >>/mnt/SUCCESS; sleep 1; cat /mnt/SUCCESS; done",
 				},
 			}
 
 			BeforeEach(func() {
+<<<<<<< HEAD
 				NFSconfig = framework.VolumeTestConfig{
 					Namespace:   ns,
 					Prefix:      "nfs",
 					ServerImage: framework.NfsServerImage,
 					ServerPorts: []int{2049},
 					ServerArgs:  []string{"-G", "777", "/exports"},
+=======
+				framework.SkipUnlessProviderIs(framework.ProvidersWithSSH...)
+				// TODO: delete after pr #33447 is merged
+				framework.SkipIfProviderIs("gke")
+
+				NFSconfig = VolumeTestConfig{
+					namespace:   ns,
+					prefix:      "nfs",
+					serverImage: NfsServerImage,
+					serverPorts: []int{2049},
+					serverArgs:  []string{"-G", "777", "/exports"},
+>>>>>>> add disruptive tests to verify host clean up. Issues: #31272, #37657.
 				}
 				nfsServerPod, nfsIP = createNfsServerPod(c, NFSconfig)
 			})
@@ -420,31 +482,46 @@ var _ = framework.KubeDescribe("kubelet", func() {
 			})
 
 			// execute It blocks from above table of tests
-			for _, test := range testTbl {
+			for _, t := range testTbl {
-				t := test // local copy for closure
+				It(t.itDescr, func() {
-				It(fmt.Sprintf("%v [Serial]", t.itDescr), func() {
-					// create a pod which uses the nfs server's volume
 					pod = createPodUsingNfs(f, c, ns, nfsIP, t.podCmd)
 
-					By("Delete the NFS server pod")
+					By("Stop the NFS server")
-					framework.DeletePodWithWait(f, c, nfsServerPod)
+					stopNfsServer(nfsServerPod)
-					nfsServerPod = nil
 
 					By("Delete the pod mounted to the NFS volume")
 					framework.DeletePodWithWait(f, c, pod)
 					// pod object is now stale, but is intentionally not nil
 
-					By("Check if host running deleted pod has been cleaned up -- expect not")
+					By("Check if pod's host has been cleaned up -- expect not")
-					// expect the pod's host *not* to be cleaned up
 					checkPodCleanup(c, pod, false)
 
-					By("Recreate the nfs server pod")
+					By("Restart the nfs server")
-					nfsServerPod, nfsIP = createNfsServerPod(c, NFSconfig)
+					restartNfsServer(nfsServerPod)
+
 					By("Verify host running the deleted pod is now cleaned up")
-					// expect the pod's host to be cleaned up
 					checkPodCleanup(c, pod, true)
 				})
 			}
+
+			// Move a pod's uid dir to /tmp and delete the pod.
+			// Addresses issue #37657.
+			// Note: the pod's vol mount (as a side effect) ends up being moved to /tmp
+			//    and can be unmounted via `umount -f`.
+			It("move pod's uid dir and delete pod", func() {
+				pod = createPodUsingNfs(f, c, ns, nfsIP, "sleep 6000")
+
+				By("Move pod's uid dir to /tmp")
+				movePodUidDir(c, pod)
+
+				By("Delete the pod mounted to the NFS volume")
+				deletePodWithWait(f, c, pod)
+				// pod object is now stale, but is intentionally not nil
+				// Note: the pod's nfs mount, now in /tmp, will not be unmounted
+
+				By("Verify host running the deleted pod is cleaned up")
+				checkPodCleanup(c, pod, true)
+			})
 		})
 	})
 })