Merge pull request #50392 from dashpole/fix_inode_eviction

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>..

inode eviction tests fill a constant number of inodes

Issue: #52203

inode eviction tests pass often on some OS distributions, and almost never on others.  See [these testgrid tests](https://k8s-testgrid.appspot.com/sig-node#kubelet-flaky-gce-e2e&include-filter-by-regex=Inode)
These differences are most likely because different images have fewer or greater inode capacity, and thus percentage based rules (e.g. inodesFree<50%) make the test more stressful for some OS distributions than others.
This changes the test to require that a constant number of inodes are consumed, regardless of the number of inodes in the filesystem, by setting the new threshold to:
nodefs.inodesFree<(current_inodes_free - 200k)
so that after pods consume 200k inodes, they will be evicted.  It requires querying the summary API until we successfully determine the current number of free Inodes.

test/e2e_node/inode_eviction_test.go

@@ -91,12 +91,19 @@ var _ = framework.KubeDescribe("InodeEviction [Slow] [Serial] [Disruptive] [Flak
 			pod:              getInnocentPod(),
 		},
 	}
-	evictionTestTimeout := 30 * time.Minute
+	evictionTestTimeout := 15 * time.Minute
 	testCondition := "Disk Pressure due to Inodes"
+	inodesConsumed := uint64(200000)
 
 	Context(fmt.Sprintf("when we run containers that should cause %s", testCondition), func() {
 		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
-			initialConfig.EvictionHard = "nodefs.inodesFree<70%"
+			// Set the eviction threshold to inodesFree - inodesConsumed, so that using inodesConsumed causes an eviction.
+			inodesFree := getInodesFree()
+			if inodesFree <= inodesConsumed {
+				framework.Skipf("Too few inodes free on the host for the InodeEviction test to run")
+			}
+			initialConfig.EvictionHard = fmt.Sprintf("nodefs.inodesFree<%d", getInodesFree()-inodesConsumed)
+			initialConfig.EvictionMinimumReclaim = ""
 		})
 		// Place the remainder of the test within a context so that the kubelet config is set before and after the test.
 		Context("With kubeconfig updated", func() {
@@ -172,7 +179,8 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 				Expect(priorityPod).NotTo(BeNil())
 
 				// Check eviction ordering.
-				// Note: it is alright for a priority 1 and priority 2 pod (for example) to fail in the same round
+				// Note: it is alright for a priority 1 and priority 2 pod (for example) to fail in the same round,
+				// but never alright for a priority 1 pod to fail while the priority 2 pod is still running
 				for _, lowPriorityPodSpec := range podTestSpecs {
 					var lowPriorityPod v1.Pod
 					for _, p := range updatedPods {
@@ -249,6 +257,14 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 			}
 			return nil
 		}, postTestConditionMonitoringPeriod, evictionPollInterval).Should(BeNil())
+	})
+
+	AfterEach(func() {
+		By("deleting pods")
+		for _, spec := range podTestSpecs {
+			By(fmt.Sprintf("deleting pod: %s", spec.pod.Name))
+			f.PodClient().DeleteSync(spec.pod.Name, &metav1.DeleteOptions{}, 10*time.Minute)
+		}
 
 		By("making sure we can start a new pod after the test")
 		podName := "test-admit-pod"
@@ -266,23 +282,11 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 				},
 			},
 		})
-	})
 
-	AfterEach(func() {
-		By("deleting pods")
-		for _, spec := range podTestSpecs {
-			By(fmt.Sprintf("deleting pod: %s", spec.pod.Name))
-			f.PodClient().DeleteSync(spec.pod.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
-		}
-
-		if CurrentGinkgoTestDescription().Failed {
-			if framework.TestContext.DumpLogsOnFailure {
+		if CurrentGinkgoTestDescription().Failed && framework.TestContext.DumpLogsOnFailure {
 			logPodEvents(f)
 			logNodeEvents(f)
 		}
-			By("sleeping to allow for cleanup of test")
-			time.Sleep(postTestConditionMonitoringPeriod)
-		}
 	})
 }
 
@@ -321,6 +325,22 @@ func hasInodePressure(f *framework.Framework, testCondition string) (bool, error
 	return hasPressure, nil
 }
 
+func getInodesFree() uint64 {
+	var inodesFree uint64
+	Eventually(func() error {
+		summary, err := getNodeSummary()
+		if err != nil {
+			return err
+		}
+		if summary == nil || summary.Node.Fs == nil || summary.Node.Fs.InodesFree == nil {
+			return fmt.Errorf("some part of data is nil")
+		}
+		inodesFree = *summary.Node.Fs.InodesFree
+		return nil
+	}, time.Minute, evictionPollInterval).Should(BeNil())
+	return inodesFree
+}
+
 // returns a pod that does not use any resources
 func getInnocentPod() *v1.Pod {
 	return &v1.Pod{