From 08f95aff0f81139ba9ddd7f37961a6e479ef6fe2 Mon Sep 17 00:00:00 2001 From: Kenneth Owens Date: Wed, 1 Mar 2017 15:07:48 -0800 Subject: [PATCH] Fixes e2e flake by ensuring that the StatefulSet observes mutations to Pods prior mutating the StatefulSet object to trigger sclaing. Add ObervedVersion check --- test/e2e/framework/statefulset_utils.go | 8 ++++++-- test/e2e/statefulset.go | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/e2e/framework/statefulset_utils.go b/test/e2e/framework/statefulset_utils.go index e782b5a023..1f08ba5747 100644 --- a/test/e2e/framework/statefulset_utils.go +++ b/test/e2e/framework/statefulset_utils.go @@ -355,7 +355,8 @@ func (s *StatefulSetTester) SetHealthy(ss *apps.StatefulSet) { } } -func (s *StatefulSetTester) waitForStatus(ss *apps.StatefulSet, expectedReplicas int32) { +// WaitForStatus waits for the ss.Status.Replicas to be equal to expectedReplicas +func (s *StatefulSetTester) WaitForStatus(ss *apps.StatefulSet, expectedReplicas int32) { Logf("Waiting for statefulset status.replicas updated to %d", expectedReplicas) ns, name := ss.Namespace, ss.Name @@ -365,6 +366,9 @@ func (s *StatefulSetTester) waitForStatus(ss *apps.StatefulSet, expectedReplicas if err != nil { return false, err } + if *ssGet.Status.ObservedGeneration < ss.Generation { + return false, nil + } if ssGet.Status.Replicas != expectedReplicas { Logf("Waiting for stateful set status to become %d, currently %d", expectedReplicas, ssGet.Status.Replicas) return false, nil @@ -402,7 +406,7 @@ func DeleteAllStatefulSets(c clientset.Interface, ns string) { if err := sst.Scale(&ss, 0); err != nil { errList = append(errList, fmt.Sprintf("%v", err)) } - sst.waitForStatus(&ss, 0) + sst.WaitForStatus(&ss, 0) Logf("Deleting statefulset %v", ss.Name) if err := c.Apps().StatefulSets(ss.Namespace).Delete(ss.Name, nil); err != nil { errList = append(errList, fmt.Sprintf("%v", err)) diff --git a/test/e2e/statefulset.go b/test/e2e/statefulset.go index c4debb0b08..bdb5c9d3b0 100644 --- a/test/e2e/statefulset.go +++ b/test/e2e/statefulset.go @@ -217,6 +217,7 @@ var _ = framework.KubeDescribe("StatefulSet", func() { By("Before scale up finished setting 2nd pod to be not ready by breaking readiness probe") sst.BreakProbe(ss, testProbe) + sst.WaitForStatus(ss, 0) sst.WaitForRunningAndNotReady(2, ss) By("Continue scale operation after the 2nd pod, and scaling down to 1 replica") @@ -280,6 +281,7 @@ var _ = framework.KubeDescribe("StatefulSet", func() { By("Confirming that stateful set scale up will halt with unhealthy stateful pod") sst.BreakProbe(ss, testProbe) sst.WaitForRunningAndNotReady(*ss.Spec.Replicas, ss) + sst.WaitForStatus(ss, 0) sst.UpdateReplicas(ss, 3) sst.ConfirmStatefulPodCount(1, ss, 10*time.Second) @@ -309,6 +311,7 @@ var _ = framework.KubeDescribe("StatefulSet", func() { Expect(err).NotTo(HaveOccurred()) sst.BreakProbe(ss, testProbe) + sst.WaitForStatus(ss, 0) sst.WaitForRunningAndNotReady(3, ss) sst.UpdateReplicas(ss, 0) sst.ConfirmStatefulPodCount(3, ss, 10*time.Second)