From 528147e5ad6fa4cea882eb2e35c8ab665b6930f8 Mon Sep 17 00:00:00 2001
From: cskh <hui.kang@hashicorp.com>
Date: Mon, 22 Jan 2024 18:35:06 -0500
Subject: [PATCH] upgrade test: run validation func at every node during
 upgrade (#20293)

* upgrade test: run validation func at every node during upgrade

* add 3 servers in each cluster
---
 test-integ/topoutil/asserter.go                  | 16 +++++++++-------
 .../upgrade/l7_traffic_management/common.go      |  6 ++++--
 testing/deployer/sprawl/consul.go                |  7 ++++++-
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/test-integ/topoutil/asserter.go b/test-integ/topoutil/asserter.go
index 894cfaf9b2..597561f9e1 100644
--- a/test-integ/topoutil/asserter.go
+++ b/test-integ/topoutil/asserter.go
@@ -367,27 +367,29 @@ func (a *Asserter) CatalogServiceExists(t *testing.T, cluster string, svc string
 	libassert.CatalogServiceExists(t, cl, svc, opts)
 }
 
-// HealthServiceEntries asserts the service has the expected number of instances
-func (a *Asserter) HealthServiceEntries(t *testing.T, cluster string, svc string, passingOnly bool, opts *api.QueryOptions, expectedInstance int) []*api.ServiceEntry {
+// HealthServiceEntries asserts the service has the expected number of instances and checks
+func (a *Asserter) HealthServiceEntries(t *testing.T, cluster string, node *topology.Node, svc string, passingOnly bool, opts *api.QueryOptions, expectedInstance int, expectedChecks int) []*api.ServiceEntry {
 	t.Helper()
-	cl := a.mustGetAPIClient(t, cluster)
+	cl, err := a.sp.APIClientForNode(cluster, node.ID(), "")
+	require.NoError(t, err)
 	health := cl.Health()
 
 	var serviceEntries []*api.ServiceEntry
-	var err error
 	retry.RunWith(&retry.Timer{Timeout: 60 * time.Second, Wait: time.Millisecond * 500}, t, func(r *retry.R) {
 		serviceEntries, _, err = health.Service(svc, "", passingOnly, opts)
 		require.NoError(r, err)
-		require.Equal(r, expectedInstance, len(serviceEntries))
+		require.Equalf(r, expectedInstance, len(serviceEntries), "dc: %s, service: %s", cluster, serviceEntries[0].Service.Service)
+		require.Equalf(r, expectedChecks, len(serviceEntries[0].Checks), "dc: %s, service: %s", cluster, serviceEntries[0].Service.Service)
 	})
 
 	return serviceEntries
 }
 
 // TokenExist asserts the token exists in the cluster and identical to the expected token
-func (a *Asserter) TokenExist(t *testing.T, cluster string, expectedToken *api.ACLToken) {
+func (a *Asserter) TokenExist(t *testing.T, cluster string, node *topology.Node, expectedToken *api.ACLToken) {
 	t.Helper()
-	cl := a.mustGetAPIClient(t, cluster)
+	cl, err := a.sp.APIClientForNode(cluster, node.ID(), "")
+	require.NoError(t, err)
 	acl := cl.ACL()
 	retry.RunWith(&retry.Timer{Timeout: 60 * time.Second, Wait: time.Millisecond * 500}, t, func(r *retry.R) {
 		retrievedToken, _, err := acl.TokenRead(expectedToken.AccessorID, &api.QueryOptions{})
diff --git a/test-integ/upgrade/l7_traffic_management/common.go b/test-integ/upgrade/l7_traffic_management/common.go
index db8459b939..ca24a75c05 100644
--- a/test-integ/upgrade/l7_traffic_management/common.go
+++ b/test-integ/upgrade/l7_traffic_management/common.go
@@ -189,19 +189,21 @@ func (ct *commonTopo) ValidateWorkloads(t *testing.T) {
 	t.Helper()
 	ct.Assert = topoutil.NewAsserter(ct.Sprawl)
 	cluster := ct.Sprawl.Topology().Clusters[dc1]
+	node := cluster.Nodes[0]
 
 	staticServerWorkload := cluster.WorkloadByID(
 		topology.NewNodeID("dc1-client1", defaultPartition),
 		ct.StaticServerSID,
 	)
 	ct.Assert.HTTPStatus(t, staticServerWorkload, staticServerWorkload.Port, 200)
-	ct.Assert.HealthServiceEntries(t, cluster.Name, ct.StaticServerSID.Name, true, &api.QueryOptions{}, 1)
+	ct.Assert.HealthServiceEntries(t, cluster.Name, node, ct.StaticServerSID.Name, true, &api.QueryOptions{}, 1, 0)
 
 	staticClientWorkload := cluster.WorkloadByID(
 		topology.NewNodeID("dc1-client2", defaultPartition),
 		ct.StaticClientSID,
 	)
-	ct.Assert.HealthServiceEntries(t, cluster.Name, ct.StaticClientSID.Name, true, &api.QueryOptions{}, 1)
+
+	ct.Assert.HealthServiceEntries(t, cluster.Name, node, ct.StaticClientSID.Name, true, &api.QueryOptions{}, 1, 0)
 
 	// check the service exists in catalog
 	svcs := cluster.WorkloadsByID(ct.StaticClientSID)
diff --git a/testing/deployer/sprawl/consul.go b/testing/deployer/sprawl/consul.go
index 205fb0235f..a816cd7480 100644
--- a/testing/deployer/sprawl/consul.go
+++ b/testing/deployer/sprawl/consul.go
@@ -74,7 +74,12 @@ func (s *Sprawl) rejoinServers(cluster *topology.Cluster) error {
 
 	servers := cluster.ServerNodes()
 
-	recoveryToken := s.secrets.ReadGeneric(cluster.Name, secrets.AgentRecovery)
+	var recoveryToken string
+	if servers[0].Images.GreaterThanVersion(topology.MinVersionAgentTokenPartition) {
+		recoveryToken = s.secrets.ReadGeneric(cluster.Name, secrets.AgentRecovery)
+	} else {
+		recoveryToken = s.secrets.ReadGeneric(cluster.Name, secrets.BootstrapToken)
+	}
 
 	node0, rest := servers[0], servers[1:]
 	client, err := util.ProxyNotPooledAPIClient(