From 37636eab718e3873d24e11212eab059ba7ae2486 Mon Sep 17 00:00:00 2001
From: Matt Keeler <mkeeler@users.noreply.github.com>
Date: Fri, 16 Jun 2023 16:29:50 -0400
Subject: [PATCH] Catalog V2 Container Based Integration Test (#17674)

* Implement the Catalog V2 controller integration container tests

This now allows the container tests to import things from the root module. However for now we want to be very restrictive about which packages we allow importing.

* Add an upgrade test for the new catalog

Currently this should be dormant and not executed. However its put in place to detect breaking changes in the future and show an example of how to do an upgrade test with integration tests structured like catalog v2.

* Make testutil.Retry capable of performing cleanup operations

These cleanup operations are executed after each retry attempt.

* Move TestContext to taking an interface instead of a concrete testing.T

This allows this to be used on a retry.R or generally anything that meets the interface.

* Move to using TestContext instead of background contexts

Also this forces all test methods to implement the Cleanup method now instead of that being an optional interface.


Co-authored-by: Daniel Upton <daniel@floppy.co>
---
 GNUmakefile                                   |  21 ++-
 .../scripts/check-allowed-imports.sh          | 124 ++++++++++++++++++
 internal/resource/resourcetest/builder.go     |  51 ++++---
 internal/resource/resourcetest/client.go      |  40 ++++--
 internal/resource/resourcetest/testing.go     |   4 -
 sdk/testutil/context.go                       |   8 +-
 sdk/testutil/retry/counter.go                 |  23 ++++
 sdk/testutil/retry/retry.go                   |  92 ++++++-------
 sdk/testutil/retry/retry_test.go              |  63 +++++++++
 sdk/testutil/retry/timer.go                   |  37 ++++++
 test/integration/consul-container/go.mod      |  30 +++--
 test/integration/consul-container/go.sum      |  42 +++++-
 .../consul-container/libs/cluster/agent.go    |   2 +
 .../libs/cluster/container.go                 |  51 ++++++-
 .../consul-container/libs/cluster/network.go  |   1 +
 .../consul-container/libs/utils/docker.go     |  15 +++
 .../test/catalog/catalog_test.go              |  35 +++++
 .../test/upgrade/catalog/catalog_test.go      |  87 ++++++++++++
 18 files changed, 613 insertions(+), 113 deletions(-)
 create mode 100755 build-support/scripts/check-allowed-imports.sh
 create mode 100644 sdk/testutil/retry/counter.go
 create mode 100644 sdk/testutil/retry/timer.go
 create mode 100644 test/integration/consul-container/test/catalog/catalog_test.go
 create mode 100644 test/integration/consul-container/test/upgrade/catalog/catalog_test.go

diff --git a/GNUmakefile b/GNUmakefile
index 3443b71db7..fe554b3c54 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -3,6 +3,7 @@
 
 SHELL = bash
 
+
 GO_MODULES := $(shell find . -name go.mod -exec dirname {} \; | grep -v "proto-gen-rpc-glue/e2e" | sort)
 
 ###
@@ -72,6 +73,7 @@ CI_DEV_DOCKER_NAMESPACE?=hashicorpdev
 CI_DEV_DOCKER_IMAGE_NAME?=consul
 CI_DEV_DOCKER_WORKDIR?=bin/
 ################
+CONSUL_VERSION?=$(shell cat version/VERSION)
 
 TEST_MODCACHE?=1
 TEST_BUILDCACHE?=1
@@ -188,6 +190,8 @@ dev-docker: linux dev-build
 	@docker buildx use default && docker buildx build -t 'consul:local' -t '$(CONSUL_DEV_IMAGE)' \
        --platform linux/$(GOARCH) \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+		--label org.opencontainers.image.version=$(CONSUL_VERSION) \
+		--label version=$(CONSUL_VERSION) \
        --load \
        -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/
 
@@ -208,6 +212,8 @@ remote-docker: check-remote-dev-image-env
 	@docker buildx use consul-builder && docker buildx build -t '$(REMOTE_DEV_IMAGE)' \
        --platform linux/amd64,linux/arm64 \
 	   --build-arg CONSUL_IMAGE_VERSION=$(CONSUL_IMAGE_VERSION) \
+		--label org.opencontainers.image.version=$(CONSUL_VERSION) \
+		--label version=$(CONSUL_VERSION) \
        --push \
        -f $(CURDIR)/build-support/docker/Consul-Dev-Multiarch.dockerfile $(CURDIR)/pkg/bin/
 
@@ -351,16 +357,17 @@ lint/%:
 	@echo "--> Running enumcover ($*)"
 	@cd $* && GOWORK=off enumcover ./...
 
+# check that the test-container module only imports allowlisted packages
+# from the root consul module. Generally we don't want to allow these imports.
+# In a few specific instances though it is okay to import test definitions and
+# helpers from some of the packages in the root module.
 .PHONY: lint-container-test-deps
 lint-container-test-deps:
 	@echo "--> Checking container tests for bad dependencies"
-	@cd test/integration/consul-container && ( \
-		found="$$(go list -m all | grep -c '^github.com/hashicorp/consul ')" ; \
-		if [[ "$$found" != "0" ]]; then \
-			echo "test/integration/consul-container: This project should not depend on the root consul module" >&2 ; \
-			exit 1 ; \
-		fi \
-	)
+	@cd test/integration/consul-container && \
+		$(CURDIR)/build-support/scripts/check-allowed-imports.sh \
+			github.com/hashicorp/consul \
+			internal/catalog/catalogtest
 
 # Build the static web ui inside a Docker container. For local testing only; do not commit these assets.
 ui: ui-docker
diff --git a/build-support/scripts/check-allowed-imports.sh b/build-support/scripts/check-allowed-imports.sh
new file mode 100755
index 0000000000..fb0280e6ff
--- /dev/null
+++ b/build-support/scripts/check-allowed-imports.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+
+readonly SCRIPT_NAME="$(basename ${BASH_SOURCE[0]})"
+readonly SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+readonly SOURCE_DIR="$(dirname "$(dirname "${SCRIPT_DIR}")")"
+readonly FN_DIR="$(dirname "${SCRIPT_DIR}")/functions"
+
+source "${SCRIPT_DIR}/functions.sh"
+
+
+set -uo pipefail
+
+usage() {
+cat <<-EOF
+Usage: ${SCRIPT_NAME} <module root> [<allowed relative package path>...]
+
+Description:
+    Verifies that only the specified packages may be imported from the given module 
+
+Options:
+    -h | --help              Print this help text.
+EOF
+}
+
+function err_usage {
+    err "$1"
+    err ""
+    err "$(usage)"
+}
+
+function main {
+   local module_root=""
+   declare -a allowed_packages=()
+   while test $# -gt 0
+   do
+      case "$1" in
+         -h | --help )
+         usage
+         return 0
+         ;;
+      * )
+         if test -z "$module_root"
+         then
+            module_root="$1"
+         else
+            allowed_packages+="$1"
+         fi
+         shift     
+      esac
+   done
+   
+   # If we could guarantee this ran with bash 4.2+ then the final argument could
+   # be just ${allowed_packages[@]}. However that with older versions of bash
+   # in combination with set -u causes bash to emit errors about using unbound
+   # variables when no allowed packages have been specified (i.e. the module should
+   # generally be disallowed with no exceptions). This syntax is very strange
+   # but seems to be the prescribed workaround I found.
+   check_imports "$module_root" ${allowed_packages[@]+"${allowed_packages[@]}"}
+   return $?
+}
+
+function check_imports {
+   local module_root="$1"
+   shift
+   local allowed_packages="$@"
+   
+   module_imports=$( go list -test -f '{{join .TestImports "\n"}}' ./... | grep "$module_root" | sort | uniq)
+   module_test_imports=$( go list -test -f '{{join .TestImports "\n"}}' ./... | grep "$module_root" | sort | uniq)
+
+   any_error=0
+   
+   for imp in $module_imports
+   do
+      is_import_allowed "$imp" "$module_root" $allowed_packages
+      allowed=$?
+      
+      if test $any_error -ne 1
+      then
+         any_error=$allowed
+      fi
+   done
+  
+   if test $any_error -eq 1
+   then
+      echo "Only the following direct imports are allowed from module $module_root:"
+      for pkg in $allowed_packages
+      do
+         echo "   * $pkg"
+      done
+   fi
+
+   return $any_error   
+}
+
+function is_import_allowed {
+   local pkg_import=$1
+   shift
+   local module_root=$1
+   shift
+   local allowed_packages="$@"
+   
+   # check if the import path is a part of the module we are restricting imports for
+   if test "$( go list -f '{{.Module.Path}}' $pkg_import)" != "$module_root"
+   then
+      return 0
+   fi
+   
+   for pkg in $allowed_packages
+   do
+      if test "${module_root}/$pkg" == "$pkg_import"
+      then
+         return 0
+      fi
+   done
+   
+   err "Import of package $pkg_import is not allowed"
+   return 1
+}
+
+main "$@"
+exit $?
\ No newline at end of file
diff --git a/internal/resource/resourcetest/builder.go b/internal/resource/resourcetest/builder.go
index 8e66028e89..749ff4fea2 100644
--- a/internal/resource/resourcetest/builder.go
+++ b/internal/resource/resourcetest/builder.go
@@ -1,10 +1,12 @@
 package resourcetest
 
 import (
-	"context"
+	"strings"
 
-	"github.com/hashicorp/consul/internal/resource"
+	"github.com/hashicorp/consul/internal/storage"
 	"github.com/hashicorp/consul/proto-public/pbresource"
+	"github.com/hashicorp/consul/sdk/testutil"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/oklog/ulid/v2"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc/codes"
@@ -119,26 +121,37 @@ func (b *resourceBuilder) ID() *pbresource.ID {
 func (b *resourceBuilder) Write(t T, client pbresource.ResourceServiceClient) *pbresource.Resource {
 	t.Helper()
 
+	ctx := testutil.TestContext(t)
+
 	res := b.resource
 
-	rsp, err := client.Write(context.Background(), &pbresource.WriteRequest{
-		Resource: res,
-	})
+	var rsp *pbresource.WriteResponse
+	var err error
 
-	require.NoError(t, err)
+	// Retry any writes where the error is a UID mismatch and the UID was not specified. This is indicative
+	// of using a follower to rewrite an object who is not perfectly in-sync with the leader.
+	retry.Run(t, func(r *retry.R) {
+		rsp, err = client.Write(ctx, &pbresource.WriteRequest{
+			Resource: res,
+		})
+
+		if err == nil || res.Id.Uid != "" || status.Code(err) != codes.FailedPrecondition {
+			return
+		}
+
+		if strings.Contains(err.Error(), storage.ErrWrongUid.Error()) {
+			r.Fatalf("resource write failed due to uid mismatch - most likely a transient issue when talking to a non-leader")
+		} else {
+			// other errors are unexpected and should cause an immediate failure
+			r.Stop(err)
+		}
+	})
 
 	if !b.dontCleanup {
-		cleaner, ok := t.(CleanupT)
-		require.True(t, ok, "T does not implement a Cleanup method and cannot be used with automatic resource cleanup")
-		cleaner.Cleanup(func() {
-			_, err := client.Delete(context.Background(), &pbresource.DeleteRequest{
-				Id: rsp.Resource.Id,
-			})
-
-			// ignore not found errors
-			if err != nil && status.Code(err) != codes.NotFound {
-				t.Fatalf("Failed to delete resource %s of type %s: %v", rsp.Resource.Id.Name, resource.ToGVK(rsp.Resource.Id.Type), err)
-			}
+		id := proto.Clone(rsp.Resource.Id).(*pbresource.ID)
+		id.Uid = ""
+		t.Cleanup(func() {
+			NewClient(client).MustDelete(t, id)
 		})
 	}
 
@@ -151,7 +164,7 @@ func (b *resourceBuilder) Write(t T, client pbresource.ResourceServiceClient) *p
 			ObservedGeneration: rsp.Resource.Generation,
 			Conditions:         original.Conditions,
 		}
-		_, err := client.WriteStatus(context.Background(), &pbresource.WriteStatusRequest{
+		_, err := client.WriteStatus(ctx, &pbresource.WriteStatusRequest{
 			Id:     rsp.Resource.Id,
 			Key:    key,
 			Status: status,
@@ -159,7 +172,7 @@ func (b *resourceBuilder) Write(t T, client pbresource.ResourceServiceClient) *p
 		require.NoError(t, err)
 	}
 
-	readResp, err := client.Read(context.Background(), &pbresource.ReadRequest{
+	readResp, err := client.Read(ctx, &pbresource.ReadRequest{
 		Id: rsp.Resource.Id,
 	})
 
diff --git a/internal/resource/resourcetest/client.go b/internal/resource/resourcetest/client.go
index d29ccda419..5047406d05 100644
--- a/internal/resource/resourcetest/client.go
+++ b/internal/resource/resourcetest/client.go
@@ -1,12 +1,13 @@
 package resourcetest
 
 import (
-	"context"
+	"fmt"
 	"math/rand"
 	"time"
 
 	"github.com/hashicorp/consul/internal/resource"
 	"github.com/hashicorp/consul/proto-public/pbresource"
+	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/slices"
@@ -41,6 +42,8 @@ func (client *Client) retry(t T, fn func(r *retry.R)) {
 }
 
 func (client *Client) PublishResources(t T, resources []*pbresource.Resource) {
+	ctx := testutil.TestContext(t)
+
 	// Randomize the order of insertion. Generally insertion order shouldn't matter as the
 	// controllers should eventually converge on the desired state. The exception to this
 	// is that you cannot insert resources with owner refs before the resource they are
@@ -75,12 +78,17 @@ func (client *Client) PublishResources(t T, resources []*pbresource.Resource) {
 			}
 
 			t.Logf("Writing resource %s with type %s", res.Id.Name, resource.ToGVK(res.Id.Type))
-			_, err := client.Write(context.Background(), &pbresource.WriteRequest{
+			rsp, err := client.Write(ctx, &pbresource.WriteRequest{
 				Resource: res,
 			})
 			require.NoError(t, err)
 
-			// track the number o
+			id := rsp.Resource.Id
+			t.Cleanup(func() {
+				client.MustDelete(t, id)
+			})
+
+			// track the number of resources published
 			published += 1
 			written = append(written, res.Id)
 		}
@@ -102,7 +110,7 @@ func (client *Client) PublishResources(t T, resources []*pbresource.Resource) {
 func (client *Client) RequireResourceNotFound(t T, id *pbresource.ID) {
 	t.Helper()
 
-	rsp, err := client.Read(context.Background(), &pbresource.ReadRequest{Id: id})
+	rsp, err := client.Read(testutil.TestContext(t), &pbresource.ReadRequest{Id: id})
 	require.Error(t, err)
 	require.Equal(t, codes.NotFound, status.Code(err))
 	require.Nil(t, rsp)
@@ -111,7 +119,7 @@ func (client *Client) RequireResourceNotFound(t T, id *pbresource.ID) {
 func (client *Client) RequireResourceExists(t T, id *pbresource.ID) *pbresource.Resource {
 	t.Helper()
 
-	rsp, err := client.Read(context.Background(), &pbresource.ReadRequest{Id: id})
+	rsp, err := client.Read(testutil.TestContext(t), &pbresource.ReadRequest{Id: id})
 	require.NoError(t, err, "error reading %s with type %s", id.Name, resource.ToGVK(id.Type))
 	require.NotNil(t, rsp)
 	return rsp.Resource
@@ -227,10 +235,22 @@ func (client *Client) ResolveResourceID(t T, id *pbresource.ID) *pbresource.ID {
 }
 
 func (client *Client) MustDelete(t T, id *pbresource.ID) {
-	_, err := client.Delete(context.Background(), &pbresource.DeleteRequest{Id: id})
-	if status.Code(err) == codes.NotFound {
-		return
-	}
+	t.Helper()
+	ctx := testutil.TestContext(t)
 
-	require.NoError(t, err)
+	client.retry(t, func(r *retry.R) {
+		_, err := client.Delete(ctx, &pbresource.DeleteRequest{Id: id})
+		if status.Code(err) == codes.NotFound {
+			return
+		}
+
+		// codes.Aborted indicates a CAS failure and that the delete request should
+		// be retried. Anything else should be considered an unrecoverable error.
+		if err != nil && status.Code(err) != codes.Aborted {
+			r.Stop(fmt.Errorf("failed to delete the resource: %w", err))
+			return
+		}
+
+		require.NoError(r, err)
+	})
 }
diff --git a/internal/resource/resourcetest/testing.go b/internal/resource/resourcetest/testing.go
index d02b70da9d..1c774082b3 100644
--- a/internal/resource/resourcetest/testing.go
+++ b/internal/resource/resourcetest/testing.go
@@ -9,9 +9,5 @@ type T interface {
 	Errorf(format string, args ...interface{})
 	Fatalf(format string, args ...interface{})
 	FailNow()
-}
-
-type CleanupT interface {
-	T
 	Cleanup(func())
 }
diff --git a/sdk/testutil/context.go b/sdk/testutil/context.go
index 257f205aa2..47ff794c96 100644
--- a/sdk/testutil/context.go
+++ b/sdk/testutil/context.go
@@ -5,10 +5,14 @@ package testutil
 
 import (
 	"context"
-	"testing"
 )
 
-func TestContext(t *testing.T) context.Context {
+type CleanerT interface {
+	Helper()
+	Cleanup(func())
+}
+
+func TestContext(t CleanerT) context.Context {
 	t.Helper()
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)
diff --git a/sdk/testutil/retry/counter.go b/sdk/testutil/retry/counter.go
new file mode 100644
index 0000000000..96a37ab9d2
--- /dev/null
+++ b/sdk/testutil/retry/counter.go
@@ -0,0 +1,23 @@
+package retry
+
+import "time"
+
+// Counter repeats an operation a given number of
+// times and waits between subsequent operations.
+type Counter struct {
+	Count int
+	Wait  time.Duration
+
+	count int
+}
+
+func (r *Counter) Continue() bool {
+	if r.count == r.Count {
+		return false
+	}
+	if r.count > 0 {
+		time.Sleep(r.Wait)
+	}
+	r.count++
+	return true
+}
diff --git a/sdk/testutil/retry/retry.go b/sdk/testutil/retry/retry.go
index 30045f0c62..af468460d5 100644
--- a/sdk/testutil/retry/retry.go
+++ b/sdk/testutil/retry/retry.go
@@ -53,6 +53,8 @@ type R struct {
 	// and triggers t.FailNow()
 	done   bool
 	output []string
+
+	cleanups []func()
 }
 
 func (r *R) Logf(format string, args ...interface{}) {
@@ -65,6 +67,41 @@ func (r *R) Log(args ...interface{}) {
 
 func (r *R) Helper() {}
 
+// Cleanup register a function to be run to cleanup resources that
+// were allocated during the retry attempt. These functions are executed
+// after a retry attempt. If they panic, it will not stop further retry
+// attempts but will be cause for the overall test failure.
+func (r *R) Cleanup(fn func()) {
+	r.cleanups = append(r.cleanups, fn)
+}
+
+func (r *R) runCleanup() {
+
+	// Make sure that if a cleanup function panics,
+	// we still run the remaining cleanup functions.
+	defer func() {
+		err := recover()
+		if err != nil {
+			r.Stop(fmt.Errorf("error when performing test cleanup: %v", err))
+		}
+		if len(r.cleanups) > 0 {
+			r.runCleanup()
+		}
+	}()
+
+	for len(r.cleanups) > 0 {
+		var cleanup func()
+		if len(r.cleanups) > 0 {
+			last := len(r.cleanups) - 1
+			cleanup = r.cleanups[last]
+			r.cleanups = r.cleanups[:last]
+		}
+		if cleanup != nil {
+			cleanup()
+		}
+	}
+}
+
 // runFailed is a sentinel value to indicate that the func itself
 // didn't panic, rather that `FailNow` was called.
 type runFailed struct{}
@@ -190,6 +227,7 @@ func run(r Retryer, t Failer, f func(r *R)) {
 		// run f(rr), but if recover yields a runFailed value, we know
 		// FailNow was called.
 		func() {
+			defer rr.runCleanup()
 			defer func() {
 				if p := recover(); p != nil && p != (runFailed{}) {
 					panic(p)
@@ -216,16 +254,6 @@ func DefaultFailer() *Timer {
 	return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond}
 }
 
-// TwoSeconds repeats an operation for two seconds and waits 25ms in between.
-func TwoSeconds() *Timer {
-	return &Timer{Timeout: 2 * time.Second, Wait: 25 * time.Millisecond}
-}
-
-// ThreeTimes repeats an operation three times and waits 25ms in between.
-func ThreeTimes() *Counter {
-	return &Counter{Count: 3, Wait: 25 * time.Millisecond}
-}
-
 // Retryer provides an interface for repeating operations
 // until they succeed or an exit condition is met.
 type Retryer interface {
@@ -233,47 +261,3 @@ type Retryer interface {
 	// returns false to indicate retrying should stop.
 	Continue() bool
 }
-
-// Counter repeats an operation a given number of
-// times and waits between subsequent operations.
-type Counter struct {
-	Count int
-	Wait  time.Duration
-
-	count int
-}
-
-func (r *Counter) Continue() bool {
-	if r.count == r.Count {
-		return false
-	}
-	if r.count > 0 {
-		time.Sleep(r.Wait)
-	}
-	r.count++
-	return true
-}
-
-// Timer repeats an operation for a given amount
-// of time and waits between subsequent operations.
-type Timer struct {
-	Timeout time.Duration
-	Wait    time.Duration
-
-	// stop is the timeout deadline.
-	// TODO: Next()?
-	// Set on the first invocation of Next().
-	stop time.Time
-}
-
-func (r *Timer) Continue() bool {
-	if r.stop.IsZero() {
-		r.stop = time.Now().Add(r.Timeout)
-		return true
-	}
-	if time.Now().After(r.stop) {
-		return false
-	}
-	time.Sleep(r.Wait)
-	return true
-}
diff --git a/sdk/testutil/retry/retry_test.go b/sdk/testutil/retry/retry_test.go
index 1f7eda7b31..77bc2d4d9f 100644
--- a/sdk/testutil/retry/retry_test.go
+++ b/sdk/testutil/retry/retry_test.go
@@ -128,6 +128,69 @@ func TestRunWith(t *testing.T) {
 	})
 }
 
+func TestCleanup(t *testing.T) {
+	t.Run("basic", func(t *testing.T) {
+		ft := &fakeT{}
+		cleanupsExecuted := 0
+		RunWith(&Counter{Count: 2, Wait: time.Millisecond}, ft, func(r *R) {
+			r.Cleanup(func() {
+				cleanupsExecuted += 1
+			})
+		})
+
+		require.Equal(t, 0, ft.fails)
+		require.Equal(t, 1, cleanupsExecuted)
+	})
+	t.Run("cleanup-panic-recovery", func(t *testing.T) {
+		ft := &fakeT{}
+		cleanupsExecuted := 0
+		RunWith(&Counter{Count: 2, Wait: time.Millisecond}, ft, func(r *R) {
+			r.Cleanup(func() {
+				cleanupsExecuted += 1
+			})
+
+			r.Cleanup(func() {
+				cleanupsExecuted += 1
+				panic(fmt.Errorf("fake test error"))
+			})
+
+			r.Cleanup(func() {
+				cleanupsExecuted += 1
+			})
+
+			// test is successful but should fail due to the cleanup panicing
+		})
+
+		require.Equal(t, 3, cleanupsExecuted)
+		require.Equal(t, 1, ft.fails)
+		require.Contains(t, ft.out[0], "fake test error")
+	})
+
+	t.Run("cleanup-per-retry", func(t *testing.T) {
+		ft := &fakeT{}
+		iter := 0
+		cleanupsExecuted := 0
+		RunWith(&Counter{Count: 3, Wait: time.Millisecond}, ft, func(r *R) {
+			if cleanupsExecuted != iter {
+				r.Stop(fmt.Errorf("cleanups not executed between retries"))
+				return
+			}
+			iter += 1
+
+			r.Cleanup(func() {
+				cleanupsExecuted += 1
+			})
+
+			r.FailNow()
+		})
+
+		require.Equal(t, 3, cleanupsExecuted)
+		// ensure that r.Stop hadn't been called. If it was then we would
+		// have log output
+		require.Len(t, ft.out, 0)
+	})
+}
+
 type fakeT struct {
 	fails int
 	out   []string
diff --git a/sdk/testutil/retry/timer.go b/sdk/testutil/retry/timer.go
new file mode 100644
index 0000000000..a26593ddd7
--- /dev/null
+++ b/sdk/testutil/retry/timer.go
@@ -0,0 +1,37 @@
+package retry
+
+import "time"
+
+// TwoSeconds repeats an operation for two seconds and waits 25ms in between.
+func TwoSeconds() *Timer {
+	return &Timer{Timeout: 2 * time.Second, Wait: 25 * time.Millisecond}
+}
+
+// ThreeTimes repeats an operation three times and waits 25ms in between.
+func ThreeTimes() *Counter {
+	return &Counter{Count: 3, Wait: 25 * time.Millisecond}
+}
+
+// Timer repeats an operation for a given amount
+// of time and waits between subsequent operations.
+type Timer struct {
+	Timeout time.Duration
+	Wait    time.Duration
+
+	// stop is the timeout deadline.
+	// TODO: Next()?
+	// Set on the first invocation of Next().
+	stop time.Time
+}
+
+func (r *Timer) Continue() bool {
+	if r.stop.IsZero() {
+		r.stop = time.Now().Add(r.Timeout)
+		return true
+	}
+	if time.Now().After(r.stop) {
+		return false
+	}
+	time.Sleep(r.Wait)
+	return true
+}
diff --git a/test/integration/consul-container/go.mod b/test/integration/consul-container/go.mod
index 27d1357bd1..b1d092af13 100644
--- a/test/integration/consul-container/go.mod
+++ b/test/integration/consul-container/go.mod
@@ -7,9 +7,11 @@ require (
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/docker/docker v23.0.6+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/hashicorp/consul/api v1.20.0
-	github.com/hashicorp/consul/envoyextensions v0.1.2
-	github.com/hashicorp/consul/sdk v0.13.1
+	github.com/hashicorp/consul v0.0.0-00010101000000-000000000000
+	github.com/hashicorp/consul/api v1.22.0-rc1
+	github.com/hashicorp/consul/envoyextensions v0.3.0-rc1
+	github.com/hashicorp/consul/proto-public v0.4.0-rc1
+	github.com/hashicorp/consul/sdk v0.14.0-rc1
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-uuid v1.0.3
@@ -34,6 +36,7 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 // indirect
 	github.com/containerd/containerd v1.7.1 // indirect
@@ -47,6 +50,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/btree v1.0.1 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
@@ -55,6 +59,7 @@ require (
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/memberlist v0.5.0 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/itchyny/timefmt-go v0.1.4 // indirect
@@ -70,28 +75,29 @@ require (
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/oklog/ulid/v2 v2.1.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
 	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	gotest.tools/v3 v3.4.0 // indirect
 )
 
-replace github.com/hashicorp/consul/api => ../../../api
-
-replace github.com/hashicorp/consul/sdk => ../../../sdk
-
-replace github.com/hashicorp/consul => ../../..
-
-replace github.com/hashicorp/consul/envoyextensions => ../../../envoyextensions
+replace (
+	github.com/hashicorp/consul => ../../..
+	github.com/hashicorp/consul/api => ../../../api
+	github.com/hashicorp/consul/envoyextensions => ../../../envoyextensions
+	github.com/hashicorp/consul/proto-public => ../../../proto-public
+	github.com/hashicorp/consul/sdk => ../../../sdk
+)
diff --git a/test/integration/consul-container/go.sum b/test/integration/consul-container/go.sum
index 44574893cd..c4bfd79440 100644
--- a/test/integration/consul-container/go.sum
+++ b/test/integration/consul-container/go.sum
@@ -13,6 +13,7 @@ fortio.org/version v1.0.2/go.mod h1:2JQp9Ax+tm6QKiGuzR5nJY63kFeANcgrZ0osoQFDVm0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
@@ -26,18 +27,25 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/aws/aws-sdk-go v1.42.34 h1:fqGAiKmCSRY1rEa4G9VqgkKKbNmLKYq5dKmLtQkvYi8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
+github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195 h1:58f1tJ1ra+zFINPlwLWvQsR9CzAKt2e+EWV2yX9oXQ4=
@@ -95,12 +103,16 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
+github.com/hashicorp/consul-net-rpc v0.0.0-20221205195236-156cfab66a69 h1:wzWurXrxfSyG1PHskIZlfuXlTSCj1Tsyatp9DtaasuY=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-bexpr v0.1.2 h1:ijMXI4qERbzxbCnkxmfUtwMyjrrk3y+Vt0MxojNCbBs=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
@@ -109,6 +121,7 @@ github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVH
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
 github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@@ -117,11 +130,13 @@ github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
 github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
+github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -132,12 +147,18 @@ github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038 h1:n9J0rwVWXDpNd5iZnwY7w4WZyq53/rROeI7OVvLW8Ok=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
 github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=
 github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
+github.com/hashicorp/raft v1.5.0 h1:uNs9EfJ4FwiArZRxxfd/dQ5d33nV31/CdCHArH89hT8=
+github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
+github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 h1:xixZ2bWeofWV68J+x6AzmKuVM/JWCQwkWm6GW/MUR6I=
 github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
 github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/itchyny/gojq v0.12.9 h1:biKpbKwMxVYhCU1d6mR7qMr3f0Hn9F5k5YykCVb3gmM=
@@ -176,6 +197,7 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -185,7 +207,9 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 h1:hOY53G+kBFhbYFpRVxHl5eS7laP6B1+Cq+Z9Dry1iMU=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -204,6 +228,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
+github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8=
@@ -216,6 +242,7 @@ github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -228,15 +255,19 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -248,18 +279,23 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 h1:xzABM9let0HLLqFypcxvLmlvEciCHL7+Lv+4vwZqecI=
 github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569/go.mod h1:2Ly+NIftZN4de9zRmENdYbvPQeaVIYKWpLFStLFEBgI=
 github.com/testcontainers/testcontainers-go v0.20.1 h1:mK15UPJ8c5P+NsQKmkqzs/jMdJt6JMs5vlw2y4j92c0=
 github.com/testcontainers/testcontainers-go v0.20.1/go.mod h1:zb+NOlCQBkZ7RQp4QI+YMIHyO2CQ/qsXzNF5eLJ24SY=
+github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -268,6 +304,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
 golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
@@ -302,6 +339,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -318,7 +356,6 @@ golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -338,7 +375,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -348,7 +384,6 @@ golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
 golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -382,6 +417,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
-gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/test/integration/consul-container/libs/cluster/agent.go b/test/integration/consul-container/libs/cluster/agent.go
index b360ee444f..c6e4a2a002 100644
--- a/test/integration/consul-container/libs/cluster/agent.go
+++ b/test/integration/consul-container/libs/cluster/agent.go
@@ -8,6 +8,7 @@ import (
 	"io"
 
 	"github.com/testcontainers/testcontainers-go"
+	"google.golang.org/grpc"
 
 	"github.com/hashicorp/consul/api"
 
@@ -36,6 +37,7 @@ type Agent interface {
 	Upgrade(ctx context.Context, config Config) error
 	Exec(ctx context.Context, cmd []string) (string, error)
 	DataDir() string
+	GetGRPCConn() *grpc.ClientConn
 }
 
 // Config is a set of configurations required to create a Agent
diff --git a/test/integration/consul-container/libs/cluster/container.go b/test/integration/consul-container/libs/cluster/container.go
index 7ed88b0d82..a371404baf 100644
--- a/test/integration/consul-container/libs/cluster/container.go
+++ b/test/integration/consul-container/libs/cluster/container.go
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -15,11 +16,14 @@ import (
 
 	goretry "github.com/avast/retry-go"
 	dockercontainer "github.com/docker/docker/api/types/container"
+	"github.com/docker/go-connections/nat"
 	"github.com/hashicorp/go-multierror"
 	"github.com/otiai10/copy"
 	"github.com/pkg/errors"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 
 	"github.com/hashicorp/consul/api"
 
@@ -58,6 +62,8 @@ type consulContainerNode struct {
 	clientCACertFile string
 	ip               string
 
+	grpcConn *grpc.ClientConn
+
 	nextAdminPortOffset   int
 	nextConnectPortOffset int
 
@@ -172,7 +178,8 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		clientAddr       string
 		clientCACertFile string
 
-		info AgentInfo
+		info     AgentInfo
+		grpcConn *grpc.ClientConn
 	)
 	debugURI := ""
 	if utils.Debug {
@@ -236,6 +243,28 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		info.CACertFile = clientCACertFile
 	}
 
+	// TODO: Support gRPC+TLS port.
+	if pc.Ports.GRPC > 0 {
+		port, err := nat.NewPort("tcp", strconv.Itoa(pc.Ports.GRPC))
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse gRPC TLS port: %w", err)
+		}
+		endpoint, err := podContainer.PortEndpoint(ctx, port, "tcp")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get gRPC TLS endpoint: %w", err)
+		}
+		url, err := url.Parse(endpoint)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse gRPC endpoint URL: %w", err)
+		}
+		conn, err := grpc.Dial(url.Host, grpc.WithTransportCredentials(insecure.NewCredentials()))
+		if err != nil {
+			return nil, fmt.Errorf("failed to dial gRPC connection: %w", err)
+		}
+		deferClean.Add(func() { _ = conn.Close() })
+		grpcConn = conn
+	}
+
 	ip, err := podContainer.ContainerIP(ctx)
 	if err != nil {
 		return nil, err
@@ -282,6 +311,7 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		name:       name,
 		ip:         ip,
 		info:       info,
+		grpcConn:   grpcConn,
 	}
 
 	if httpPort > 0 || httpsPort > 0 {
@@ -376,6 +406,10 @@ func (c *consulContainerNode) GetClient() *api.Client {
 	return c.client
 }
 
+func (c *consulContainerNode) GetGRPCConn() *grpc.ClientConn {
+	return c.grpcConn
+}
+
 // NewClient returns an API client by making a new one based on the provided token
 // - updateDefault: if true update the default client
 func (c *consulContainerNode) NewClient(token string, updateDefault bool) (*api.Client, error) {
@@ -508,6 +542,10 @@ func (c *consulContainerNode) terminate(retainPod bool, skipFuncs bool) error {
 				continue
 			}
 		}
+
+		// if the pod is retained and therefore the IP then the grpc conn
+		// should handle reconnecting so there is no reason to close it.
+		c.closeGRPC()
 	}
 
 	var merr error
@@ -529,6 +567,16 @@ func (c *consulContainerNode) terminate(retainPod bool, skipFuncs bool) error {
 	return merr
 }
 
+func (c *consulContainerNode) closeGRPC() error {
+	if c.grpcConn != nil {
+		if err := c.grpcConn.Close(); err != nil {
+			return err
+		}
+		c.grpcConn = nil
+	}
+	return nil
+}
+
 func (c *consulContainerNode) DataDir() string {
 	return c.dataDir
 }
@@ -565,6 +613,7 @@ func newContainerRequest(config Config, opts containerOpts, ports ...int) (podRe
 		ExposedPorts: []string{
 			"8500/tcp", // Consul HTTP API
 			"8501/tcp", // Consul HTTPs API
+			"8502/tcp", // Consul gRPC API
 
 			"8443/tcp", // Envoy Gateway Listener
 
diff --git a/test/integration/consul-container/libs/cluster/network.go b/test/integration/consul-container/libs/cluster/network.go
index e0ee10f4e3..6e170b3dab 100644
--- a/test/integration/consul-container/libs/cluster/network.go
+++ b/test/integration/consul-container/libs/cluster/network.go
@@ -20,6 +20,7 @@ func createNetwork(t TestingT, name string) (testcontainers.Network, error) {
 			Name:           name,
 			Attachable:     true,
 			CheckDuplicate: true,
+			SkipReaper:     isRYUKDisabled(),
 		},
 	}
 	first := true
diff --git a/test/integration/consul-container/libs/utils/docker.go b/test/integration/consul-container/libs/utils/docker.go
index 109205855c..6be46d91ae 100644
--- a/test/integration/consul-container/libs/utils/docker.go
+++ b/test/integration/consul-container/libs/utils/docker.go
@@ -9,6 +9,9 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
+
+	"github.com/hashicorp/go-version"
 )
 
 // DockerExec simply shell out to the docker CLI binary on your host.
@@ -16,6 +19,18 @@ func DockerExec(args []string, stdout io.Writer) error {
 	return cmdExec("docker", "docker", args, stdout, "")
 }
 
+// DockerImageVersion retrieves the value of the org.opencontainers.image.version label from the specified image.
+func DockerImageVersion(imageName string) (*version.Version, error) {
+	var b strings.Builder
+	err := cmdExec("docker", "docker", []string{"image", "inspect", "--format", `{{index .Config.Labels "org.opencontainers.image.version"}}`, imageName}, &b, "")
+	if err != nil {
+		return nil, err
+	}
+	output := b.String()
+
+	return version.NewVersion(strings.TrimSpace(output))
+}
+
 func cmdExec(name, binary string, args []string, stdout io.Writer, dir string) error {
 	if binary == "" {
 		panic("binary named " + name + " was not detected")
diff --git a/test/integration/consul-container/test/catalog/catalog_test.go b/test/integration/consul-container/test/catalog/catalog_test.go
new file mode 100644
index 0000000000..8520e5a647
--- /dev/null
+++ b/test/integration/consul-container/test/catalog/catalog_test.go
@@ -0,0 +1,35 @@
+package catalog
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
+
+	"github.com/hashicorp/consul/internal/catalog/catalogtest"
+	pbresource "github.com/hashicorp/consul/proto-public/pbresource"
+)
+
+func TestCatalog(t *testing.T) {
+	t.Parallel()
+
+	cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{
+		NumServers: 3,
+		BuildOpts:  &libcluster.BuildOptions{Datacenter: "dc1"},
+		Cmd:        `-hcl=experiments=["resource-apis"]`,
+	})
+
+	followers, err := cluster.Followers()
+	require.NoError(t, err)
+	client := pbresource.NewResourceServiceClient(followers[0].GetGRPCConn())
+
+	t.Run("one-shot", func(t *testing.T) {
+		catalogtest.RunCatalogV1Alpha1IntegrationTest(t, client)
+	})
+
+	t.Run("lifecycle", func(t *testing.T) {
+		catalogtest.RunCatalogV1Alpha1LifecycleIntegrationTest(t, client)
+	})
+}
diff --git a/test/integration/consul-container/test/upgrade/catalog/catalog_test.go b/test/integration/consul-container/test/upgrade/catalog/catalog_test.go
new file mode 100644
index 0000000000..ef2de3edeb
--- /dev/null
+++ b/test/integration/consul-container/test/upgrade/catalog/catalog_test.go
@@ -0,0 +1,87 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package catalog
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/hashicorp/consul/internal/catalog/catalogtest"
+	"github.com/hashicorp/consul/proto-public/pbresource"
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+	"github.com/hashicorp/go-version"
+)
+
+var minCatalogResourceVersion = version.Must(version.NewVersion("v1.16.0"))
+
+const (
+	versionUndetermined = `
+Cannot determine the actual version the starting image represents.
+Scrutinze test failures to ensure that the starting version should
+actually be able to be used for creating the initial data set.
+	`
+)
+
+func maybeSkipUpgradeTest(t *testing.T, minVersion *version.Version) {
+	t.Helper()
+
+	image := utils.DockerImage(utils.GetLatestImageName(), utils.LatestVersion)
+	latestVersion, err := utils.DockerImageVersion(image)
+
+	if latestVersion != nil && latestVersion.LessThan(minVersion) {
+		t.Skipf("Upgrade test isn't applicable with version %q as the starting version", latestVersion.String())
+	}
+
+	if err != nil || latestVersion == nil {
+		t.Log(versionUndetermined)
+	}
+}
+
+// Test upgrade a cluster of latest version to the target version and ensure that the catalog still
+// functions properly. Note
+func TestCatalogUpgrade(t *testing.T) {
+	maybeSkipUpgradeTest(t, minCatalogResourceVersion)
+	t.Parallel()
+
+	const numServers = 1
+	buildOpts := &libcluster.BuildOptions{
+		ConsulImageName:      utils.GetLatestImageName(),
+		ConsulVersion:        utils.LatestVersion,
+		Datacenter:           "dc1",
+		InjectAutoEncryption: true,
+	}
+
+	cluster, _, _ := topology.NewCluster(t, &topology.ClusterConfig{
+		NumServers:                1,
+		BuildOpts:                 buildOpts,
+		ApplyDefaultProxySettings: true,
+		Cmd:                       `-hcl=experiments=["resource-apis"]`,
+	})
+
+	client := cluster.APIClient(0)
+
+	libcluster.WaitForLeader(t, cluster, client)
+	libcluster.WaitForMembers(t, client, numServers)
+
+	leader, err := cluster.Leader()
+	require.NoError(t, err)
+	rscClient := pbresource.NewResourceServiceClient(leader.GetGRPCConn())
+
+	// Initialize some data
+	catalogtest.PublishCatalogV1Alpha1IntegrationTestData(t, rscClient)
+
+	// upgrade the cluster to the Target version
+	t.Logf("initiating standard upgrade to version=%q", utils.TargetVersion)
+	err = cluster.StandardUpgrade(t, context.Background(), utils.GetTargetImageName(), utils.TargetVersion)
+
+	require.NoError(t, err)
+	libcluster.WaitForLeader(t, cluster, client)
+	libcluster.WaitForMembers(t, client, numServers)
+
+	catalogtest.VerifyCatalogV1Alpha1IntegrationTestResults(t, rscClient)
+}