Merge pull request #51039 from enisoc/deflake-sts-saturate

Automatic merge from submit-queue

StatefulSet: Deflake e2e "Saturate" phase.

This should reduce one source of flakiness found while investigating #48031.

The "Saturate" phase of StatefulSet e2e tests verifies orderly startup by controlling when each Pod is allowed to report Ready. If a Pod unexepectedly goes down during the test, the replacement Pod
created by the controller will forget if it was already allowed to report Ready.

After this change, the signal that allows each Pod to report Ready is persisted in the Pod's PVC. Thus, the replacement Pod will remember that it was already told to proceed to a Ready state.

test/e2e/apps/statefulset.go

@@ -115,7 +115,7 @@ var _ = SIGDescribe("StatefulSet", func() {
 
 			By("Restarting statefulset " + ss.Name)
 			sst.Restart(ss)
-			sst.Saturate(ss)
+			sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss)
 
 			By("Verifying statefulset mounted data directory is usable")
 			framework.ExpectNoError(sst.CheckMount(ss, "/data"))
@@ -233,13 +233,13 @@ var _ = SIGDescribe("StatefulSet", func() {
 			sst.DeleteStatefulPodAtIndex(0, ss)
 
 			By("Confirming stateful pod at index 0 is recreated.")
-			sst.WaitForRunning(2, 0, ss)
+			sst.WaitForRunning(2, 1, ss)
 
-			By("Deleting unhealthy stateful pod at index 1.")
-			sst.DeleteStatefulPodAtIndex(1, ss)
+			By("Resuming stateful pod at index 1.")
+			sst.ResumeNextPod(ss)
 
 			By("Confirming all stateful pods in statefulset are created.")
-			sst.Saturate(ss)
+			sst.WaitForRunningAndReady(*ss.Spec.Replicas, ss)
 		})
 
 		It("should perform rolling updates and roll backs of template modifications", func() {

test/e2e/framework/statefulset_utils.go

@@ -548,7 +548,7 @@ func (s *StatefulSetTester) RestorePodHttpProbe(ss *apps.StatefulSet, pod *v1.Po
 
 var pauseProbe = &v1.Probe{
 	Handler: v1.Handler{
-		Exec: &v1.ExecAction{Command: []string{"test", "-f", "/tmp/statefulset-continue"}},
+		Exec: &v1.ExecAction{Command: []string{"test", "-f", "/data/statefulset-continue"}},
 	},
 	PeriodSeconds:    1,
 	SuccessThreshold: 1,
@@ -586,7 +586,7 @@ func (s *StatefulSetTester) ResumeNextPod(ss *apps.StatefulSet) {
 		if resumedPod != "" {
 			Failf("Found multiple paused stateful pods: %v and %v", pod.Name, resumedPod)
 		}
-		_, err := RunHostCmd(pod.Namespace, pod.Name, "touch /tmp/statefulset-continue")
+		_, err := RunHostCmd(pod.Namespace, pod.Name, "touch /data/statefulset-continue")
 		ExpectNoError(err)
 		Logf("Resumed pod %v", pod.Name)
 		resumedPod = pod.Name

test/e2e/upgrades/apps/statefulset.go

@@ -110,5 +110,5 @@ func (t *StatefulSetUpgradeTest) verify() {
 func (t *StatefulSetUpgradeTest) restart() {
 	By("Restarting statefulset " + t.set.Name)
 	t.tester.Restart(t.set)
-	t.tester.Saturate(t.set)
+	t.tester.WaitForRunningAndReady(*t.set.Spec.Replicas, t.set)
 }