Fix various flaky tests

agent/grpc-external/services/peerstream/stream_test.go

@@ -1252,8 +1252,8 @@ func TestStreamResources_Server_DisconnectsOnHeartbeatTimeout(t *testing.T) {
 	})
 
 	testutil.RunStep(t, "stream is disconnected due to heartbeat timeout", func(t *testing.T) {
-		disconnectTime := ptr(it.FutureNow(1))
 		retry.Run(t, func(r *retry.R) {
+			disconnectTime := ptr(it.StaticNow())
 			status, ok := srv.StreamStatus(testPeerID)
 			require.True(r, ok)
 			require.False(r, status.Connected)
@@ -1423,7 +1423,7 @@ func makeClient(t *testing.T, srv *testServer, peerID string) *MockClient {
 		},
 	}))
 
-	// Receive a services and roots subscription request pair from server
+	// Receive ExportedService, ExportedServiceList, and PeeringTrustBundle subscription requests from server
 	receivedSub1, err := client.Recv()
 	require.NoError(t, err)
 	receivedSub2, err := client.Recv()

agent/grpc-external/services/peerstream/testing.go

@@ -150,6 +150,16 @@ func (t *incrementalTime) Now() time.Time {
 	return t.base.Add(dur)
 }
 
+// StaticNow returns the current internal clock without advancing it.
+func (t *incrementalTime) StaticNow() time.Time {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	dur := time.Duration(t.next) * time.Second
+
+	return t.base.Add(dur)
+}
+
 // FutureNow will return a given future value of the Now() function.
 // The numerical argument indicates which future Now value you wanted. The
 // value must be > 0.

agent/xds/delta_test.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/armon/go-metrics"
 	envoy_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
-	"github.com/hashicorp/consul/api"
+
 	"github.com/stretchr/testify/require"
 	rpcstatus "google.golang.org/genproto/googleapis/rpc/status"
 	"google.golang.org/grpc/codes"
@@ -21,8 +21,10 @@ import (
 	"github.com/hashicorp/consul/agent/grpc-external/limiter"
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/envoyextensions/xdscommon"
 	"github.com/hashicorp/consul/sdk/testutil"
+	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/consul/version"
 )
 
@@ -1095,13 +1097,15 @@ func TestServer_DeltaAggregatedResources_v3_ACLEnforcement(t *testing.T) {
 
 			// If there is no token, check that we increment the gauge
 			if tt.token == "" {
-				data := scenario.sink.Data()
-				require.Len(t, data, 1)
-
-				item := data[0]
-				val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"]
-				require.True(t, ok)
-				require.Equal(t, float32(1), val.Value)
+				retry.Run(t, func(r *retry.R) {
+					data := scenario.sink.Data()
+					require.Len(r, data, 1)
+
+					item := data[0]
+					val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"]
+					require.True(r, ok)
+					require.Equal(r, float32(1), val.Value)
+				})
 			}
 
 			if !tt.wantDenied {
@@ -1138,13 +1142,15 @@ func TestServer_DeltaAggregatedResources_v3_ACLEnforcement(t *testing.T) {
 
 			// If there is no token, check that we decrement the gauge
 			if tt.token == "" {
-				data := scenario.sink.Data()
-				require.Len(t, data, 1)
-
-				item := data[0]
-				val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"]
-				require.True(t, ok)
-				require.Equal(t, float32(0), val.Value)
+				retry.Run(t, func(r *retry.R) {
+					data := scenario.sink.Data()
+					require.Len(r, data, 1)
+
+					item := data[0]
+					val, ok := item.Gauges["consul.xds.test.xds.server.streamsUnauthenticated"]
+					require.True(r, ok)
+					require.Equal(r, float32(0), val.Value)
+				})
 			}
 		})
 	}

command/debug/debug.go

@@ -270,7 +270,8 @@ func (c *cmd) prepare() (version string, err error) {
 	// If none are specified we will collect information from
 	// all by default
 	if len(c.capture) == 0 {
-		c.capture = defaultTargets
+		c.capture = make([]string, len(defaultTargets))
+		copy(c.capture, defaultTargets)
 	}
 
 	// If EnableDebug is not true, skip collecting pprof