Merge pull request #4538 from hashicorp/NET-2396 (#16516)

NET-2396: refactor test to reduce duplication

test/integration/consul-container/libs/assert/service.go

@@ -24,9 +24,9 @@ const (
 )
 
 // CatalogServiceExists verifies the service name exists in the Consul catalog
-func CatalogServiceExists(t *testing.T, c *api.Client, svc string) {
+func CatalogServiceExists(t *testing.T, c *api.Client, svc string, opts *api.QueryOptions) {
 	retry.Run(t, func(r *retry.R) {
-		services, _, err := c.Catalog().Service(svc, "", nil)
+		services, _, err := c.Catalog().Service(svc, "", opts)
 		if err != nil {
 			r.Fatal("error reading service data")
 		}

test/integration/consul-container/libs/service/helpers.go

@@ -3,10 +3,12 @@ package service
 import (
 	"context"
 	"fmt"
+	"testing"
 
 	"github.com/hashicorp/consul/api"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -171,3 +173,59 @@ func CreateAndRegisterStaticClientSidecar(
 
 	return clientConnectProxy, nil
 }
+
+func ClientsCreate(t *testing.T, numClients int, image, version string, cluster *libcluster.Cluster) {
+	opts := libcluster.BuildOptions{
+		ConsulImageName: image,
+		ConsulVersion:   version,
+	}
+	ctx := libcluster.NewBuildContext(t, opts)
+
+	conf := libcluster.NewConfigBuilder(ctx).
+		Client().
+		ToAgentConfig(t)
+	t.Logf("Cluster client config:\n%s", conf.JSON)
+
+	require.NoError(t, cluster.AddN(*conf, numClients, true))
+}
+
+func ServiceCreate(t *testing.T, client *api.Client, serviceName string) uint64 {
+	require.NoError(t, client.Agent().ServiceRegister(&api.AgentServiceRegistration{
+		Name: serviceName,
+		Port: 9999,
+		Connect: &api.AgentServiceConnect{
+			SidecarService: &api.AgentServiceRegistration{
+				Port: 22005,
+			},
+		},
+	}))
+
+	service, meta, err := client.Catalog().Service(serviceName, "", &api.QueryOptions{})
+	require.NoError(t, err)
+	require.Len(t, service, 1)
+	require.Equal(t, serviceName, service[0].ServiceName)
+	require.Equal(t, 9999, service[0].ServicePort)
+
+	return meta.LastIndex
+}
+
+func ServiceHealthBlockingQuery(client *api.Client, serviceName string, waitIndex uint64) (chan []*api.ServiceEntry, chan error) {
+	var (
+		ch    = make(chan []*api.ServiceEntry, 1)
+		errCh = make(chan error, 1)
+	)
+	go func() {
+		opts := &api.QueryOptions{WaitIndex: waitIndex}
+		service, q, err := client.Health().Service(serviceName, "", false, opts)
+		if err == nil && q.QueryBackend != api.QueryBackendStreaming {
+			err = fmt.Errorf("invalid backend for this test %s", q.QueryBackend)
+		}
+		if err != nil {
+			errCh <- err
+		} else {
+			ch <- service
+		}
+	}()
+
+	return ch, errCh
+}

test/integration/consul-container/libs/topology/peering_topology.go

@@ -69,7 +69,7 @@ func BasicPeeringTwoClustersSetup(
 			},
 		}
 		configCluster := func(cli *api.Client) error {
-			libassert.CatalogServiceExists(t, cli, "mesh")
+			libassert.CatalogServiceExists(t, cli, "mesh", nil)
 			ok, _, err := cli.ConfigEntries().Set(req, &api.WriteOptions{})
 			if !ok {
 				return fmt.Errorf("config entry is not set")
@@ -109,8 +109,8 @@ func BasicPeeringTwoClustersSetup(
 		serverService, serverSidecarService, err = libservice.CreateAndRegisterStaticServerAndSidecar(clientNode, &serviceOpts)
 		require.NoError(t, err)
 
-		libassert.CatalogServiceExists(t, acceptingClient, libservice.StaticServerServiceName)
+		libassert.CatalogServiceExists(t, acceptingClient, libservice.StaticServerServiceName, nil)
-		libassert.CatalogServiceExists(t, acceptingClient, "static-server-sidecar-proxy")
+		libassert.CatalogServiceExists(t, acceptingClient, "static-server-sidecar-proxy", nil)
 
 		require.NoError(t, serverService.Export("default", AcceptingPeerName, acceptingClient))
 	}
@@ -125,7 +125,7 @@ func BasicPeeringTwoClustersSetup(
 		clientSidecarService, err = libservice.CreateAndRegisterStaticClientSidecar(clientNode, DialingPeerName, true)
 		require.NoError(t, err)
 
-		libassert.CatalogServiceExists(t, dialingClient, "static-client-sidecar-proxy")
+		libassert.CatalogServiceExists(t, dialingClient, "static-client-sidecar-proxy", nil)
 
 	}
 
@@ -133,7 +133,7 @@ func BasicPeeringTwoClustersSetup(
 	libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", DialingPeerName), "HEALTHY", 1)
 	_, port := clientSidecarService.GetAddr()
 	libassert.HTTPServiceEchoes(t, "localhost", port, "")
-	libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+	libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 
 	return &BuiltCluster{
 			Cluster:   acceptingCluster,
@@ -198,7 +198,7 @@ func NewDialingCluster(
 	clientProxyService, err := libservice.CreateAndRegisterStaticClientSidecar(node, dialingPeerName, true)
 	require.NoError(t, err)
 
-	libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy")
+	libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil)
 
 	return cluster, client, clientProxyService
 }

test/integration/consul-container/libs/topology/service_topology.go

@@ -38,14 +38,14 @@ func CreateServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Servi
 	_, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts)
 	require.NoError(t, err)
 
-	libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticServerServiceName))
+	libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticServerServiceName), nil)
-	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName)
+	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
 
 	// Create a client proxy instance with the server as an upstream
 	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false)
 	require.NoError(t, err)
 
-	libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName))
+	libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName), nil)
 
 	return serverConnectProxy, clientConnectProxy
 }

test/integration/consul-container/test/basic/connect_service_test.go

@@ -9,7 +9,7 @@ import (
 	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
-	"github.com/hashicorp/consul/test/integration/consul-container/test"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
 )
 
 // TestBasicConnectService Summary
@@ -25,12 +25,13 @@ func TestBasicConnectService(t *testing.T) {
 	t.Parallel()
 
 	buildOptions := &libcluster.BuildOptions{
+		Datacenter:             "dc1",
 		InjectAutoEncryption:   true,
 		InjectGossipEncryption: true,
 		// TODO(rb): fix the test to not need the service/envoy stack to use :8500
 		AllowHTTPAnyway: true,
 	}
-	cluster := test.CreateCluster(t, "", nil, buildOptions, true)
+	cluster, _, _ := topology.NewPeeringCluster(t, 1, 1, buildOptions)
 
 	clientService := createServices(t, cluster)
 	_, port := clientService.GetAddr()
@@ -59,14 +60,14 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) libservice.Servic
 	_, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts)
 	require.NoError(t, err)
 
-	libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy")
+	libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil)
-	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName)
+	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
 
 	// Create a client proxy instance with the server as an upstream
 	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false)
 	require.NoError(t, err)
 
-	libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy")
+	libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil)
 
 	return clientConnectProxy
 }

test/integration/consul-container/test/common.go

@@ -24,7 +24,9 @@ func CreateCluster(
 	cmd string,
 	logConsumer *TestLogConsumer,
 	buildOptions *libcluster.BuildOptions,
-	applyDefaultProxySettings bool) *libcluster.Cluster {
+	applyDefaultProxySettings bool,
+	ports ...int,
+) *libcluster.Cluster {
 
 	// optional
 	if buildOptions == nil {
@@ -49,12 +51,12 @@ func CreateCluster(
 		conf.Cmd = append(conf.Cmd, cmd)
 	}
 
-	cluster, err := libcluster.New(t, []libcluster.Config{*conf})
+	cluster, err := libcluster.New(t, []libcluster.Config{*conf}, ports...)
 	require.NoError(t, err)
 
-	client, err := cluster.GetClient(nil, true)
+	node := cluster.Agents[0]
+	client := node.GetClient()
 
-	require.NoError(t, err)
 	libcluster.WaitForLeader(t, cluster, client)
 	libcluster.WaitForMembers(t, client, 1)
 

test/integration/consul-container/test/gateways/gateway_endpoint_test.go

@@ -3,25 +3,22 @@ package gateways
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+	"github.com/hashicorp/consul/test/integration/consul-container/test"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"io"
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-)
-
-var (
-	checkTimeout  = 1 * time.Minute
-	checkInterval = 1 * time.Second
 )
 
 // Creates a gateway service and tests to see if it is routable
@@ -32,33 +29,38 @@ func TestAPIGatewayCreate(t *testing.T) {
 	}
 
 	t.Parallel()
-
 	listenerPortOne := 6000
 
-	cluster := createCluster(t, listenerPortOne)
+	buildOpts := &libcluster.BuildOptions{
-
+		Datacenter:             "dc1",
+		InjectAutoEncryption:   true,
+		InjectGossipEncryption: true,
+		AllowHTTPAnyway:        true,
+	}
+	cluster := test.CreateCluster(t, "", nil, buildOpts, true, listenerPortOne)
 	client := cluster.APIClient(0)
 
-	//setup
+	// add api gateway config
 	apiGateway := &api.APIGatewayConfigEntry{
-		Kind: "api-gateway",
+		Kind: api.APIGateway,
 		Name: "api-gateway",
 		Listeners: []api.APIGatewayListener{
 			{
+				Name:     "listener",
 				Port:     listenerPortOne,
 				Protocol: "tcp",
 			},
 		},
 	}
-	_, _, err := client.ConfigEntries().Set(apiGateway, nil)
+
-	require.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(apiGateway))
 
 	tcpRoute := &api.TCPRouteConfigEntry{
-		Kind: "tcp-route",
+		Kind: api.TCPRoute,
 		Name: "api-gateway-route",
 		Parents: []api.ResourceReference{
 			{
-				Kind: "api-gateway",
+				Kind: api.APIGateway,
 				Name: "api-gateway",
 			},
 		},
@@ -69,8 +71,7 @@ func TestAPIGatewayCreate(t *testing.T) {
 		},
 	}
 
-	_, _, err = client.ConfigEntries().Set(tcpRoute, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(tcpRoute))
-	require.NoError(t, err)
 
 	// Create a client proxy instance with the server as an upstream
 	_, gatewayService := createServices(t, cluster, listenerPortOne)
@@ -195,8 +196,8 @@ func createService(t *testing.T, cluster *libcluster.Cluster, serviceOpts *libse
 	service, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts, containerArgs...)
 	require.NoError(t, err)
 
-	libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy")
+	libassert.CatalogServiceExists(t, client, serviceOpts.Name+"-sidecar-proxy", nil)
-	libassert.CatalogServiceExists(t, client, serviceOpts.Name)
+	libassert.CatalogServiceExists(t, client, serviceOpts.Name, nil)
 
 	return service
 
@@ -216,7 +217,7 @@ func createServices(t *testing.T, cluster *libcluster.Cluster, ports ...int) (li
 
 	gatewayService, err := libservice.NewGatewayService(context.Background(), "api-gateway", "api", cluster.Agents[0], ports...)
 	require.NoError(t, err)
-	libassert.CatalogServiceExists(t, client, "api-gateway")
+	libassert.CatalogServiceExists(t, client, "api-gateway", nil)
 
 	return clientConnectProxy, gatewayService
 }

test/integration/consul-container/test/gateways/http_route_test.go

@@ -5,13 +5,16 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"github.com/hashicorp/consul/api"
-	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
-	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"testing"
 	"time"
+
+	"github.com/hashicorp/consul/api"
+	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
+	"github.com/hashicorp/consul/test/integration/consul-container/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func getNamespace() string {
@@ -41,8 +44,15 @@ func TestHTTPRouteFlattening(t *testing.T) {
 	//infrastructure set up
 	listenerPort := 6000
 	//create cluster
-	cluster := createCluster(t, listenerPort)
+	buildOpts := &libcluster.BuildOptions{
+		Datacenter:             "dc1",
+		InjectAutoEncryption:   true,
+		InjectGossipEncryption: true,
+		AllowHTTPAnyway:        true,
+	}
+	cluster := test.CreateCluster(t, "", nil, buildOpts, true, listenerPort)
 	client := cluster.Agents[0].GetClient()
+
 	service1ResponseCode := 200
 	service2ResponseCode := 418
 	serviceOne := createService(t, cluster, &libservice.ServiceOpts{
@@ -82,8 +92,7 @@ func TestHTTPRouteFlattening(t *testing.T) {
 		},
 	}
 
-	_, _, err := client.ConfigEntries().Set(proxyDefaults, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults))
-	require.NoError(t, err)
 
 	apiGateway := &api.APIGatewayConfigEntry{
 		Kind: "api-gateway",
@@ -173,17 +182,14 @@ func TestHTTPRouteFlattening(t *testing.T) {
 		},
 	}
 
-	_, _, err = client.ConfigEntries().Set(apiGateway, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(apiGateway))
-	require.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(routeOne))
-	_, _, err = client.ConfigEntries().Set(routeOne, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(routeTwo))
-	require.NoError(t, err)
-	_, _, err = client.ConfigEntries().Set(routeTwo, nil)
-	require.NoError(t, err)
 
 	//create gateway service
 	gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort)
 	require.NoError(t, err)
-	libassert.CatalogServiceExists(t, client, gatewayName)
+	libassert.CatalogServiceExists(t, client, gatewayName, nil)
 
 	//make sure config entries have been properly created
 	checkGatewayConfigEntry(t, client, gatewayName, namespace)
@@ -284,8 +290,7 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 		},
 	}
 
-	_, _, err := client.ConfigEntries().Set(proxyDefaults, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults))
-	require.NoError(t, err)
 
 	apiGateway := createGateway(gatewayName, "http", listenerPort)
 
@@ -367,17 +372,14 @@ func TestHTTPRoutePathRewrite(t *testing.T) {
 		},
 	}
 
-	_, _, err = client.ConfigEntries().Set(apiGateway, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(apiGateway))
-	require.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(fooRoute))
-	_, _, err = client.ConfigEntries().Set(fooRoute, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(barRoute))
-	require.NoError(t, err)
-	_, _, err = client.ConfigEntries().Set(barRoute, nil)
-	require.NoError(t, err)
 
 	//create gateway service
 	gatewayService, err := libservice.NewGatewayService(context.Background(), gatewayName, "api", cluster.Agents[0], listenerPort)
 	require.NoError(t, err)
-	libassert.CatalogServiceExists(t, client, gatewayName)
+	libassert.CatalogServiceExists(t, client, gatewayName, nil)
 
 	//make sure config entries have been properly created
 	checkGatewayConfigEntry(t, client, gatewayName, namespace)
@@ -450,8 +452,8 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 			"protocol": "http",
 		},
 	}
-	_, _, err := client.ConfigEntries().Set(proxyDefaults, nil)
+
-	assert.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(proxyDefaults))
 
 	// create gateway config entry
 	gatewayOne := &api.APIGatewayConfigEntry{
@@ -466,8 +468,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 			},
 		},
 	}
-	_, _, err = client.ConfigEntries().Set(gatewayOne, nil)
+	require.NoError(t, cluster.ConfigEntryWrite(gatewayOne))
-	assert.NoError(t, err)
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayOneName, &api.QueryOptions{Namespace: namespace})
 		assert.NoError(t, err)
@@ -482,7 +483,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 	// create gateway service
 	gatewayOneService, err := libservice.NewGatewayService(context.Background(), gatewayOneName, "api", cluster.Agents[0], listenerOnePort)
 	require.NoError(t, err)
-	libassert.CatalogServiceExists(t, client, gatewayOneName)
+	libassert.CatalogServiceExists(t, client, gatewayOneName, nil)
 
 	// create gateway config entry
 	gatewayTwo := &api.APIGatewayConfigEntry{
@@ -497,8 +498,9 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 			},
 		},
 	}
-	_, _, err = client.ConfigEntries().Set(gatewayTwo, nil)
+
-	assert.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(gatewayTwo))
+
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.APIGateway, gatewayTwoName, &api.QueryOptions{Namespace: namespace})
 		assert.NoError(t, err)
@@ -513,7 +515,7 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 	// create gateway service
 	gatewayTwoService, err := libservice.NewGatewayService(context.Background(), gatewayTwoName, "api", cluster.Agents[0], listenerTwoPort)
 	require.NoError(t, err)
-	libassert.CatalogServiceExists(t, client, gatewayTwoName)
+	libassert.CatalogServiceExists(t, client, gatewayTwoName, nil)
 
 	// create route to service, targeting first gateway
 	route := &api.HTTPRouteConfigEntry{
@@ -550,8 +552,9 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 			},
 		},
 	}
-	_, _, err = client.ConfigEntries().Set(route, nil)
+
-	assert.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(route))
+
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace})
 		assert.NoError(t, err)
@@ -593,8 +596,8 @@ func TestHTTPRouteParentRefChange(t *testing.T) {
 			Namespace: namespace,
 		},
 	}
-	_, _, err = client.ConfigEntries().Set(route, nil)
+
-	assert.NoError(t, err)
+	require.NoError(t, cluster.ConfigEntryWrite(route))
 	require.Eventually(t, func() bool {
 		entry, _, err := client.ConfigEntries().Get(api.HTTPRoute, routeName, &api.QueryOptions{Namespace: namespace})
 		assert.NoError(t, err)

test/integration/consul-container/test/observability/access_logs_test.go

@@ -70,7 +70,7 @@ func TestAccessLogs(t *testing.T) {
 	// Validate Custom JSON
 	require.Eventually(t, func() bool {
 		libassert.HTTPServiceEchoes(t, "localhost", port, "banana")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 		client := libassert.ServiceLogContains(t, clientService, "\"banana_path\":\"/banana\"")
 		server := libassert.ServiceLogContains(t, serverService, "\"banana_path\":\"/banana\"")
 		return client && server
@@ -112,7 +112,7 @@ func TestAccessLogs(t *testing.T) {
 	_, port = clientService.GetAddr()
 	require.Eventually(t, func() bool {
 		libassert.HTTPServiceEchoes(t, "localhost", port, "orange")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 		client := libassert.ServiceLogContains(t, clientService, "Orange you glad I didn't say banana: /orange, -")
 		server := libassert.ServiceLogContains(t, serverService, "Orange you glad I didn't say banana: /orange, -")
 		return client && server

test/integration/consul-container/test/peering/rotate_server_and_ca_then_fail_test.go

@@ -94,7 +94,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) {
 
 		_, port := clientSidecarService.GetAddr()
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 	}
 
 	testutil.RunStep(t, "rotate exporting cluster's root CA", func(t *testing.T) {
@@ -144,7 +144,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) {
 		// Connectivity should still be contained
 		_, port := clientSidecarService.GetAddr()
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 
 		verifySidecarHasTwoRootCAs(t, clientSidecarService)
 	})
@@ -166,7 +166,7 @@ func TestPeering_RotateServerAndCAThenFail_(t *testing.T) {
 
 		_, port := clientSidecarService.GetAddr()
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 	})
 }
 

test/integration/consul-container/test/upgrade/fullstopupgrade_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
 
@@ -67,9 +68,9 @@ func TestStandardUpgradeToTarget_fromLatest(t *testing.T) {
 
 		// Create a service to be stored in the snapshot
 		const serviceName = "api"
-		index := serviceCreate(t, client, serviceName)
+		index := libservice.ServiceCreate(t, client, serviceName)
 
-		ch, errCh := serviceHealthBlockingQuery(client, serviceName, index)
+		ch, errCh := libservice.ServiceHealthBlockingQuery(client, serviceName, index)
 		require.NoError(t, client.Agent().ServiceRegister(
 			&api.AgentServiceRegistration{Name: serviceName, Port: 9998},
 		))
@@ -111,6 +112,5 @@ func TestStandardUpgradeToTarget_fromLatest(t *testing.T) {
 			func(t *testing.T) {
 				run(t, tc)
 			})
-		// time.Sleep(5 * time.Second)
 	}
 }

test/integration/consul-container/test/upgrade/healthcheck_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/consul/api"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
 	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
 )
 
@@ -22,7 +23,7 @@ func TestTargetServersWithLatestGAClients(t *testing.T) {
 
 	cluster := serversCluster(t, numServers, utils.TargetImageName, utils.TargetVersion)
 
-	clientsCreate(t, numClients, utils.LatestImageName, utils.LatestVersion, cluster)
+	libservice.ClientsCreate(t, numClients, utils.LatestImageName, utils.LatestVersion, cluster)
 
 	client := cluster.APIClient(0)
 
@@ -30,9 +31,9 @@ func TestTargetServersWithLatestGAClients(t *testing.T) {
 	libcluster.WaitForMembers(t, client, 4)
 
 	const serviceName = "api"
-	index := serviceCreate(t, client, serviceName)
+	index := libservice.ServiceCreate(t, client, serviceName)
 
-	ch, errCh := serviceHealthBlockingQuery(client, serviceName, index)
+	ch, errCh := libservice.ServiceHealthBlockingQuery(client, serviceName, index)
 	require.NoError(t, client.Agent().ServiceRegister(
 		&api.AgentServiceRegistration{Name: serviceName, Port: 9998},
 	))
@@ -120,7 +121,7 @@ func testMixedServersGAClient(t *testing.T, majorityIsTarget bool) {
 	cluster, err := libcluster.New(t, configs)
 	require.NoError(t, err)
 
-	clientsCreate(t, numClients, utils.LatestImageName, utils.LatestVersion, cluster)
+	libservice.ClientsCreate(t, numClients, utils.LatestImageName, utils.LatestVersion, cluster)
 
 	client := cluster.APIClient(0)
 
@@ -128,9 +129,9 @@ func testMixedServersGAClient(t *testing.T, majorityIsTarget bool) {
 	libcluster.WaitForMembers(t, client, 4) // TODO(rb): why 4?
 
 	const serviceName = "api"
-	index := serviceCreate(t, client, serviceName)
+	index := libservice.ServiceCreate(t, client, serviceName)
 
-	ch, errCh := serviceHealthBlockingQuery(client, serviceName, index)
+	ch, errCh := libservice.ServiceHealthBlockingQuery(client, serviceName, index)
 	require.NoError(t, client.Agent().ServiceRegister(
 		&api.AgentServiceRegistration{Name: serviceName, Port: 9998},
 	))
@@ -147,3 +148,24 @@ func testMixedServersGAClient(t *testing.T, majorityIsTarget bool) {
 		t.Fatalf("test timeout")
 	}
 }
+
+func serversCluster(t *testing.T, numServers int, image, version string) *libcluster.Cluster {
+	opts := libcluster.BuildOptions{
+		ConsulImageName: image,
+		ConsulVersion:   version,
+	}
+	ctx := libcluster.NewBuildContext(t, opts)
+
+	conf := libcluster.NewConfigBuilder(ctx).
+		Bootstrap(numServers).
+		ToAgentConfig(t)
+	t.Logf("Cluster server config:\n%s", conf.JSON)
+
+	cluster, err := libcluster.NewN(t, *conf, numServers)
+	require.NoError(t, err)
+
+	libcluster.WaitForLeader(t, cluster, nil)
+	libcluster.WaitForMembers(t, cluster.APIClient(0), numServers)
+
+	return cluster
+}

test/integration/consul-container/test/upgrade/helper_test.go

@@ -1,92 +0,0 @@
-package upgrade
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/stretchr/testify/require"
-
-	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
-)
-
-func serversCluster(t *testing.T, numServers int, image, version string) *libcluster.Cluster {
-	t.Helper()
-
-	opts := libcluster.BuildOptions{
-		ConsulImageName: image,
-		ConsulVersion:   version,
-	}
-	ctx := libcluster.NewBuildContext(t, opts)
-
-	conf := libcluster.NewConfigBuilder(ctx).
-		Bootstrap(numServers).
-		ToAgentConfig(t)
-	t.Logf("Cluster server config:\n%s", conf.JSON)
-
-	cluster, err := libcluster.NewN(t, *conf, numServers)
-	require.NoError(t, err)
-
-	libcluster.WaitForLeader(t, cluster, nil)
-	libcluster.WaitForMembers(t, cluster.APIClient(0), numServers)
-
-	return cluster
-}
-
-func clientsCreate(t *testing.T, numClients int, image, version string, cluster *libcluster.Cluster) {
-	t.Helper()
-
-	opts := libcluster.BuildOptions{
-		ConsulImageName: image,
-		ConsulVersion:   version,
-	}
-	ctx := libcluster.NewBuildContext(t, opts)
-
-	conf := libcluster.NewConfigBuilder(ctx).
-		Client().
-		ToAgentConfig(t)
-	t.Logf("Cluster client config:\n%s", conf.JSON)
-
-	require.NoError(t, cluster.AddN(*conf, numClients, true))
-}
-
-func serviceCreate(t *testing.T, client *api.Client, serviceName string) uint64 {
-	require.NoError(t, client.Agent().ServiceRegister(&api.AgentServiceRegistration{
-		Name: serviceName,
-		Port: 9999,
-		Connect: &api.AgentServiceConnect{
-			SidecarService: &api.AgentServiceRegistration{
-				Port: 22005,
-			},
-		},
-	}))
-
-	service, meta, err := client.Catalog().Service(serviceName, "", &api.QueryOptions{})
-	require.NoError(t, err)
-	require.Len(t, service, 1)
-	require.Equal(t, serviceName, service[0].ServiceName)
-	require.Equal(t, 9999, service[0].ServicePort)
-
-	return meta.LastIndex
-}
-
-func serviceHealthBlockingQuery(client *api.Client, serviceName string, waitIndex uint64) (chan []*api.ServiceEntry, chan error) {
-	var (
-		ch    = make(chan []*api.ServiceEntry, 1)
-		errCh = make(chan error, 1)
-	)
-	go func() {
-		opts := &api.QueryOptions{WaitIndex: waitIndex}
-		service, q, err := client.Health().Service(serviceName, "", false, opts)
-		if err == nil && q.QueryBackend != api.QueryBackendStreaming {
-			err = fmt.Errorf("invalid backend for this test %s", q.QueryBackend)
-		}
-		if err != nil {
-			errCh <- err
-		} else {
-			ch <- service
-		}
-	}()
-
-	return ch, errCh
-}

test/integration/consul-container/test/upgrade/ingress_gateway_test.go

@@ -170,7 +170,7 @@ func TestIngressGateway_UpgradeToTarget_fromLatest(t *testing.T) {
 			},
 		)
 		require.NoError(t, err)
-		libassert.CatalogServiceExists(t, client, nameS1)
+		libassert.CatalogServiceExists(t, client, nameS1, nil)
 
 		// create s2
 		_, _, err = libservice.CreateAndRegisterStaticServerAndSidecar(
@@ -183,7 +183,7 @@ func TestIngressGateway_UpgradeToTarget_fromLatest(t *testing.T) {
 			},
 		)
 		require.NoError(t, err)
-		libassert.CatalogServiceExists(t, client, nameS2)
+		libassert.CatalogServiceExists(t, client, nameS2, nil)
 
 		// checks
 		// TODO: other checks from verify.bats

test/integration/consul-container/test/upgrade/l7_traffic_management/resolver_default_subset_test.go

@@ -67,7 +67,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 				}
 				_, serverConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2)
 				require.NoError(t, err)
-				libassert.CatalogServiceExists(t, client, "static-server")
+				libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
 
 				// TODO: verify the number of instance of static-server is 3
 				libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 3)
@@ -104,8 +104,8 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 					libassert.AssertEnvoyRunning(t, serverAdminPortV1)
 					libassert.AssertEnvoyRunning(t, serverAdminPortV2)
 
-					libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server")
+					libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, libservice.StaticServerServiceName)
-					libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV2, "static-server")
+					libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV2, libservice.StaticServerServiceName)
 
 					libassert.AssertUpstreamEndpointStatus(t, adminPort, "v2.static-server.default", "HEALTHY", 1)
 
@@ -182,7 +182,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 					libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 1)
 
 					libassert.AssertEnvoyRunning(t, serverAdminPortV1)
-					libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, "static-server")
+					libassert.AssertEnvoyPresentsCertURI(t, serverAdminPortV1, libservice.StaticServerServiceName)
 
 					// assert static-server proxies should be healthy
 					libassert.AssertServiceHasHealthyInstances(t, node, libservice.StaticServerServiceName, true, 1)
@@ -235,7 +235,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 				}
 				_, server2ConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOpts2)
 				require.NoError(t, err)
-				libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName)
+				libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName, nil)
 
 				serviceOptsV1 := &libservice.ServiceOpts{
 					Name:     libservice.StaticServer2ServiceName,
@@ -256,7 +256,7 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 				}
 				_, server2ConnectProxyV2, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, serviceOptsV2)
 				require.NoError(t, err)
-				libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName)
+				libassert.CatalogServiceExists(t, client, libservice.StaticServer2ServiceName, nil)
 
 				// Register static-server service resolver
 				serviceResolver := &api.ServiceResolverConfigEntry{
@@ -341,8 +341,8 @@ func TestTrafficManagement_ServiceResolver(t *testing.T) {
 
 		staticClientProxy, staticServerProxy, err := createStaticClientAndServer(cluster)
 		require.NoError(t, err)
-		libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName)
+		libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
-		libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName))
+		libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName), nil)
 
 		err = cluster.ConfigEntryWrite(&api.ProxyConfigEntry{
 			Kind: api.ProxyDefaults,

test/integration/consul-container/test/upgrade/peering_control_plane_mgw_test.go

@@ -102,7 +102,7 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) {
 		require.NoError(t, clientSidecarService.Restart())
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1)
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 	}
 
 	for _, tc := range tcs {
@@ -110,6 +110,5 @@ func TestPeering_Upgrade_ControlPlane_MGW(t *testing.T) {
 			func(t *testing.T) {
 				run(t, tc)
 			})
-		// time.Sleep(3 * time.Second)
 	}
 }

test/integration/consul-container/test/upgrade/peering_http_test.go

@@ -67,7 +67,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 				if err != nil {
 					return nil, nil, nil, err
 				}
-				libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName)
+				libassert.CatalogServiceExists(t, c.Clients()[0].GetClient(), libservice.StaticServer2ServiceName, nil)
 
 				err = c.ConfigEntryWrite(&api.ProxyConfigEntry{
 					Kind: api.ProxyDefaults,
@@ -201,7 +201,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 					GRPCPort: 8078,
 				}
 				_, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(dialing.Clients()[0], serviceOpts)
-				libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName)
+				libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName, nil)
 				if err != nil {
 					return nil, nil, nil, err
 				}
@@ -246,7 +246,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 				}
 
 				clientConnectProxy, err := createAndRegisterStaticClientSidecarWith2Upstreams(dialing,
-					[]string{"static-server", "peer-static-server"},
+					[]string{libservice.StaticServerServiceName, "peer-static-server"},
 				)
 				if err != nil {
 					return nil, nil, nil, fmt.Errorf("error creating client connect proxy in cluster %s", dialing.NetworkName)
@@ -269,7 +269,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 				// make a resolver for service static-server
 				resolverConfigEntry = &api.ServiceResolverConfigEntry{
 					Kind: api.ServiceResolver,
-					Name: "static-server",
+					Name: libservice.StaticServerServiceName,
 					Failover: map[string]api.ServiceResolverFailover{
 						"*": {
 							Targets: []api.ServiceResolverFailoverTarget{
@@ -293,7 +293,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 					GRPCPort: 8078,
 				}
 				_, serverConnectProxy, err := libservice.CreateAndRegisterStaticServerAndSidecar(dialing.Clients()[0], serviceOpts)
-				libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName)
+				libassert.CatalogServiceExists(t, dialing.Clients()[0].GetClient(), libservice.StaticServerServiceName, nil)
 				if err != nil {
 					return nil, nil, nil, err
 				}
@@ -303,12 +303,12 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 					// assert traffic can fail-over to static-server in peered cluster and restor to local static-server
 					libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing", "")
 					require.NoError(t, serverConnectProxy.Stop())
-					libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server", "")
+					libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), libservice.StaticServerServiceName, "")
 					require.NoError(t, serverConnectProxy.Start())
 					libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[0]), "static-server-dialing", "")
 
 					// assert peer-static-server resolves to static-server in peered cluster
-					libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[1]), "static-server", "")
+					libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", appPorts[1]), libservice.StaticServerServiceName, "")
 				}
 				return serverConnectProxy, clientConnectProxy, assertionFn, nil
 			},
@@ -376,7 +376,7 @@ func TestPeering_UpgradeToTarget_fromLatest(t *testing.T) {
 		_, adminPort := clientSidecarService.GetAdminAddr()
 		libassert.AssertUpstreamEndpointStatus(t, adminPort, fmt.Sprintf("static-server.default.%s.external", libtopology.DialingPeerName), "HEALTHY", 1)
 		libassert.HTTPServiceEchoes(t, "localhost", port, "")
-		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), "static-server", "")
+		libassert.AssertFortioName(t, fmt.Sprintf("http://localhost:%d", port), libservice.StaticServerServiceName, "")
 
 		// TODO: restart static-server-2's sidecar
 		tc.extraAssertion(appPort)

test/integration/consul-container/test/upgrade/tenancy_ent_test.go

@@ -0,0 +1,258 @@
+//go:build consulent
+// +build consulent
+
+package upgrade
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/sdk/testutil"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
+	"github.com/stretchr/testify/require"
+
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+)
+
+// Test partition crud using Current Clients and Latest GA Servers
+func TestLatestGAServersWithCurrentClients_PartitionCRUD(t *testing.T) {
+	testLatestGAServersWithCurrentClients_TenancyCRUD(t, "Partitions",
+		func(t *testing.T, client *api.Client) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			// CRUD partitions
+			partition, _, err := client.Partitions().Read(ctx, "default", nil)
+			require.NoError(t, err)
+			fmt.Printf("%+v\n", partition)
+			require.NotNil(t, partition)
+			require.Equal(t, "default", partition.Name)
+
+			fooPartReq := api.Partition{Name: "foo-part"}
+			fooPart, _, err := client.Partitions().Create(ctx, &api.Partition{Name: "foo-part"}, nil)
+			require.NoError(t, err)
+			require.NotNil(t, fooPart)
+			require.Equal(t, "foo-part", fooPart.Name)
+
+			partition, _, err = client.Partitions().Read(ctx, "foo-part", nil)
+			require.NoError(t, err)
+			require.NotNil(t, partition)
+			require.Equal(t, "foo-part", partition.Name)
+
+			fooPartReq.Description = "foo-part part"
+			partition, _, err = client.Partitions().Update(ctx, &fooPartReq, nil)
+			require.NoError(t, err)
+			require.NotNil(t, partition)
+			require.Equal(t, "foo-part", partition.Name)
+			require.Equal(t, "foo-part part", partition.Description)
+		},
+		func(t *testing.T, client *api.Client) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			//Read partition again
+			retry.RunWith(libcluster.LongFailer(), t, func(r *retry.R) {
+				partition, _, err := client.Partitions().Read(ctx, "default", nil)
+				require.NoError(r, err)
+				require.NotNil(r, partition)
+				require.Equal(r, "default", partition.Name)
+			})
+
+			retry.RunWith(libcluster.LongFailer(), t, func(r *retry.R) {
+				partition, _, err := client.Partitions().Read(ctx, "foo-part", nil)
+				require.NoError(r, err)
+				require.NotNil(r, partition)
+				require.Equal(r, "foo-part", partition.Name)
+				require.Equal(r, "foo-part part", partition.Description)
+			})
+		},
+	)
+}
+
+// Test namespace crud using Current Clients and Latest GA Servers
+func TestLatestGAServersWithCurrentClients_NamespaceCRUD(t *testing.T) {
+	testLatestGAServersWithCurrentClients_TenancyCRUD(t, "Namespaces",
+		func(t *testing.T, client *api.Client) {
+			// CRUD namespaces
+			namespace, _, err := client.Namespaces().Read("default", nil)
+			require.NoError(t, err)
+			require.NotNil(t, namespace, "default namespace does not exist yet")
+			require.Equal(t, "default", namespace.Name)
+
+			fooNsReq := api.Namespace{Name: "foo-ns"}
+			fooNs, _, err := client.Namespaces().Create(&api.Namespace{Name: "foo-ns"}, nil)
+			require.NoError(t, err)
+			require.NotNil(t, fooNs)
+			require.Equal(t, "foo-ns", fooNs.Name)
+
+			namespace, _, err = client.Namespaces().Read("foo-ns", nil)
+			require.NoError(t, err)
+			require.NotNil(t, namespace)
+			require.Equal(t, "foo-ns", namespace.Name)
+
+			fooNsReq.Description = "foo-ns ns"
+			namespace, _, err = client.Namespaces().Update(&fooNsReq, nil)
+			require.NoError(t, err)
+			require.NotNil(t, namespace)
+			require.Equal(t, "foo-ns", namespace.Name)
+			require.Equal(t, "foo-ns ns", namespace.Description)
+		},
+		func(t *testing.T, client *api.Client) {
+			retry.RunWith(libcluster.LongFailer(), t, func(r *retry.R) {
+				namespace, _, err := client.Namespaces().Read("default", nil)
+				require.NoError(r, err)
+				require.NotNil(r, namespace)
+				require.Equal(r, "default", namespace.Name)
+			})
+			retry.RunWith(libcluster.LongFailer(), t, func(r *retry.R) {
+				namespace, _, err := client.Namespaces().Read("foo-ns", nil)
+				require.NoError(r, err)
+				require.NotNil(r, namespace)
+				require.Equal(r, "foo-ns", namespace.Name)
+				require.Equal(r, "foo-ns ns", namespace.Description)
+			})
+		},
+	)
+}
+
+func testLatestGAServersWithCurrentClients_TenancyCRUD(
+	t *testing.T,
+	tenancyName string,
+	createFn func(t *testing.T, client *api.Client),
+	readFn func(t *testing.T, client *api.Client),
+) {
+	const (
+		numServers = 3
+		numClients = 2
+	)
+
+	// Create initial cluster
+	cluster := serversCluster(t, numServers, utils.LatestImageName, utils.LatestVersion)
+	libservice.ClientsCreate(t, numClients, utils.LatestImageName, utils.LatestVersion, cluster)
+
+	client := cluster.APIClient(0)
+	libcluster.WaitForLeader(t, cluster, client)
+	libcluster.WaitForMembers(t, client, 5)
+
+	testutil.RunStep(t, "Create "+tenancyName, func(t *testing.T) {
+		fmt.Println("!!!!!!!")
+		createFn(t, client)
+		fmt.Println("!!!!DONE!!!!")
+	})
+
+	ctx := context.Background()
+
+	var snapshot io.ReadCloser
+	testutil.RunStep(t, "Save snapshot", func(t *testing.T) {
+		var err error
+		snapshot, _, err = client.Snapshot().Save(nil)
+		require.NoError(t, err)
+	})
+
+	testutil.RunStep(t, "Check "+tenancyName+" after upgrade", func(t *testing.T) {
+		// Upgrade nodes
+		leader, err := cluster.Leader()
+		require.NoError(t, err)
+
+		// upgrade things in the following order:
+		//
+		// 1. follower servers
+		// 2. leader server
+		// 3. clients
+		var upgradeOrder []libcluster.Agent
+
+		followers, err := cluster.Followers()
+		require.NoError(t, err)
+		upgradeOrder = append(upgradeOrder, followers...)
+		upgradeOrder = append(upgradeOrder, leader)
+		upgradeOrder = append(upgradeOrder, cluster.Clients()...)
+
+		for _, n := range upgradeOrder {
+			conf := n.GetConfig()
+
+			// TODO: ensure this makes sense again, it was doing an apples/orange version!=image comparison
+			if conf.Version == utils.TargetVersion {
+				return
+			}
+
+			conf.Version = utils.TargetVersion
+
+			if n.IsServer() {
+				// You only ever need bootstrap settings the FIRST time, so we do not need
+				// them again.
+				conf.ConfigBuilder.Unset("bootstrap")
+			} else {
+				// If we upgrade the clients fast enough
+				// membership might not be gossiped to all of
+				// the clients to persist into their serf
+				// snapshot, so force them to rejoin the
+				// normal way on restart.
+				conf.ConfigBuilder.Set("retry_join", []string{"agent-0"})
+			}
+
+			newJSON, err := json.MarshalIndent(conf.ConfigBuilder, "", "  ")
+			require.NoError(t, err)
+			conf.JSON = string(newJSON)
+			t.Logf("Upgraded cluster config for %q:\n%s", n.GetName(), conf.JSON)
+
+			selfBefore, err := n.GetClient().Agent().Self()
+			require.NoError(t, err)
+
+			require.NoError(t, n.Upgrade(ctx, conf))
+
+			selfAfter, err := n.GetClient().Agent().Self()
+			require.NoError(t, err)
+			require.Truef(t,
+				(selfBefore["Config"]["Version"] != selfAfter["Config"]["Version"]) || (selfBefore["Config"]["Revision"] != selfAfter["Config"]["Revision"]),
+				fmt.Sprintf("upgraded version must be different (%s, %s), (%s, %s)", selfBefore["Config"]["Version"], selfBefore["Config"]["Revision"], selfAfter["Config"]["Version"], selfAfter["Config"]["Revision"]),
+			)
+
+			client := n.GetClient()
+
+			libcluster.WaitForLeader(t, cluster, nil)
+			libcluster.WaitForMembers(t, client, 5)
+		}
+
+		//get the client again as it changed after upgrade.
+		client := cluster.APIClient(0)
+		libcluster.WaitForLeader(t, cluster, client)
+
+		// Read data again
+		readFn(t, client)
+	})
+
+	// Terminate the cluster for the snapshot test
+	testutil.RunStep(t, "Terminate the cluster", func(t *testing.T) {
+		require.NoError(t, cluster.Terminate())
+	})
+
+	{ // Clear these so they super break if you tried to use them.
+		cluster = nil
+		client = nil
+	}
+
+	// Create a fresh cluster from scratch
+	cluster2 := serversCluster(t, numServers, utils.TargetImageName, utils.TargetVersion)
+	libservice.ClientsCreate(t, numClients, utils.LatestImageName, utils.LatestVersion, cluster2)
+
+	client2 := cluster2.APIClient(0)
+
+	testutil.RunStep(t, "Restore saved snapshot", func(t *testing.T) {
+		libcluster.WaitForLeader(t, cluster2, client2)
+		libcluster.WaitForMembers(t, client2, 5)
+
+		// Restore the saved snapshot
+		require.NoError(t, client2.Snapshot().Restore(nil, snapshot))
+
+		libcluster.WaitForLeader(t, cluster2, client2)
+
+		// make sure we still have the right data
+		readFn(t, client2)
+	})
+}

test/integration/consul-container/test/wanfed/wanfed_peering_test.go

@@ -5,7 +5,6 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/sdk/testutil/retry"
 	libassert "github.com/hashicorp/consul/test/integration/consul-container/libs/assert"
 	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
 	libservice "github.com/hashicorp/consul/test/integration/consul-container/libs/service"
@@ -32,7 +31,7 @@ func TestPeering_WanFedSecondaryDC(t *testing.T) {
 
 	t.Run("secondary dc services are visible in primary dc", func(t *testing.T) {
 		createConnectService(t, c2)
-		assertCatalogService(t, c1Agent.GetClient(), "static-server", &api.QueryOptions{Datacenter: "secondary"})
+		libassert.CatalogServiceExists(t, c1Agent.GetClient(), libservice.StaticServerServiceName, &api.QueryOptions{Datacenter: "secondary"})
 	})
 
 	t.Run("secondary dc can peer to alpha dc", func(t *testing.T) {
@@ -52,7 +51,7 @@ func TestPeering_WanFedSecondaryDC(t *testing.T) {
 		// Create a testing sidecar to proxy requests through
 		clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(c2Agent, "secondary-to-alpha", false)
 		require.NoError(t, err)
-		assertCatalogService(t, c2Agent.GetClient(), "static-client-sidecar-proxy", nil)
+		libassert.CatalogServiceExists(t, c2Agent.GetClient(), "static-client-sidecar-proxy", nil)
 
 		// Ensure envoy is configured for the peer service and healthy.
 		_, adminPort := clientConnectProxy.GetAdminAddr()
@@ -68,18 +67,6 @@ func TestPeering_WanFedSecondaryDC(t *testing.T) {
 	})
 }
 
-func assertCatalogService(t *testing.T, c *api.Client, svc string, opts *api.QueryOptions) {
-	retry.Run(t, func(r *retry.R) {
-		services, _, err := c.Catalog().Service(svc, "", opts)
-		if err != nil {
-			r.Fatal("error reading catalog data", err)
-		}
-		if len(services) == 0 {
-			r.Fatal("did not find catalog entry for ", svc)
-		}
-	})
-}
-
 func createCluster(t *testing.T, dc string, f func(c *libcluster.ConfigBuilder)) (*libcluster.Cluster, libcluster.Agent) {
 	ctx := libcluster.NewBuildContext(t, libcluster.BuildOptions{Datacenter: dc})
 	conf := libcluster.NewConfigBuilder(ctx).Advanced(f)
@@ -111,8 +98,8 @@ func createConnectService(t *testing.T, cluster *libcluster.Cluster) libservice.
 	serverConnectProxy, _, err := libservice.CreateAndRegisterStaticServerAndSidecar(node, &opts)
 	require.NoError(t, err)
 
-	assertCatalogService(t, client, "static-server-sidecar-proxy", nil)
+	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
-	assertCatalogService(t, client, "static-server", nil)
+	libassert.CatalogServiceExists(t, client, "static-server-sidecar-proxy", nil)
 
 	return serverConnectProxy
 }