tests: fix unstable test `TestAgentAntiEntropy_Checks`. (#7594)

Example of failure: https://circleci.com/gh/hashicorp/consul/153932#tests/containers/2

agent/local/state_test.go

@@ -989,14 +989,16 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 	}
 	var checks structs.IndexedHealthChecks
 
+	retry.Run(t, func(r *retry.R) {
+
 		// Verify that we are in sync
 		if err := a.RPC("Health.NodeChecks", &req, &checks); err != nil {
-		t.Fatalf("err: %v", err)
+			r.Fatalf("err: %v", err)
 		}
 
 		// We should have 5 checks (serf included)
 		if len(checks.HealthChecks) != 5 {
-		t.Fatalf("bad: %v", checks)
+			r.Fatalf("bad: %v", checks)
 		}
 
 		// All the checks should match
@@ -1014,13 +1016,16 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 			case "serfHealth":
 				// ignore
 			default:
-			t.Fatalf("unexpected check: %v", chk)
+				r.Fatalf("unexpected check: %v", chk)
 			}
 		}
 
 		if err := checksInSync(a.State, 4, structs.DefaultEnterpriseMeta()); err != nil {
-		t.Fatal(err)
+			r.Fatal(err)
 		}
+	})
+
+	retry.Run(t, func(r *retry.R) {
 
 		// Make sure we sent along our node info addresses when we synced.
 		{
@@ -1030,7 +1035,7 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 			}
 			var services structs.IndexedNodeServices
 			if err := a.RPC("Catalog.NodeServices", &req, &services); err != nil {
-			t.Fatalf("err: %v", err)
+				r.Fatalf("err: %v", err)
 			}
 
 			id := services.NodeServices.Node.ID
@@ -1041,22 +1046,24 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 			assert.Equal(t, a.Config.TaggedAddresses, addrs)
 			assert.Equal(t, a.Config.NodeMeta, meta)
 		}
+	})
+	retry.Run(t, func(r *retry.R) {
 
 		// Remove one of the checks
 		a.State.RemoveCheck(structs.NewCheckID("redis", nil))
 
 		if err := a.State.SyncFull(); err != nil {
-		t.Fatalf("err: %v", err)
+			r.Fatalf("err: %v", err)
 		}
 
 		// Verify that we are in sync
 		if err := a.RPC("Health.NodeChecks", &req, &checks); err != nil {
-		t.Fatalf("err: %v", err)
+			r.Fatalf("err: %v", err)
 		}
 
 		// We should have 5 checks (serf included)
 		if len(checks.HealthChecks) != 4 {
-		t.Fatalf("bad: %v", checks)
+			r.Fatalf("bad: %v", checks)
 		}
 
 		// All the checks should match
@@ -1072,13 +1079,14 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 			case "serfHealth":
 				// ignore
 			default:
-			t.Fatalf("unexpected check: %v", chk)
+				r.Fatalf("unexpected check: %v", chk)
 			}
 		}
 
 		if err := checksInSync(a.State, 3, structs.DefaultEnterpriseMeta()); err != nil {
-		t.Fatal(err)
+			r.Fatal(err)
 		}
+	})
 }
 
 func TestAgentAntiEntropy_RemovingServiceAndCheck(t *testing.T) {