Use exponential backoff instead of errorCounter in LinkWatch

internal/hcp/watch.go

@@ -6,16 +6,16 @@ package hcp
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/hashicorp/go-hclog"
 
 	"github.com/hashicorp/consul/internal/hcp/internal/types"
+	"github.com/hashicorp/consul/lib/retry"
 	pbhcp "github.com/hashicorp/consul/proto-public/pbhcp/v2"
 	"github.com/hashicorp/consul/proto-public/pbresource"
 )
 
-var linkWatchRetryCount = 10
-
 func NewLinkWatch(ctx context.Context, logger hclog.Logger, client pbresource.ResourceServiceClient) (chan *pbresource.WatchEvent, error) {
 	watchClient, err := client.WatchList(
 		ctx, &pbresource.WatchListRequest{
@@ -29,7 +29,11 @@ func NewLinkWatch(ctx context.Context, logger hclog.Logger, client pbresource.Re
 
 	eventCh := make(chan *pbresource.WatchEvent)
 	go func() {
-		errorCounter := 0
+		errorBackoff := &retry.Waiter{
+			MinFailures: 1,
+			MinWait:     1 * time.Second,
+			MaxWait:     1 * time.Minute,
+		}
 		for {
 			select {
 			case <-ctx.Done():
@@ -38,19 +42,14 @@ func NewLinkWatch(ctx context.Context, logger hclog.Logger, client pbresource.Re
 				return
 			default:
 				watchEvent, err := watchClient.Recv()
+
 				if err != nil {
 					logger.Error("error receiving link watch event", "error", err)
-
-					errorCounter++
-					if errorCounter >= linkWatchRetryCount {
-						logger.Error("received multiple consecutive errors from link watch client, will stop watching link")
-						close(eventCh)
-						return
-					}
-
+					errorBackoff.Wait(ctx)
 					continue
 				}
-				errorCounter = 0
+
+				errorBackoff.Reset()
 				eventCh <- watchEvent
 			}
 		}

internal/hcp/watch_test.go

@@ -62,24 +62,3 @@ func TestLinkWatch_ContextCanceled(t *testing.T) {
 	_, ok := <-linkWatchCh
 	require.False(t, ok)
 }
-
-// This tests ensures that when Recv returns errors repeatedly, we eventually close the channel
-// and exit the goroutine
-func TestLinkWatch_RepeatErrors(t *testing.T) {
-	mockWatchListClient := mockpbresource.NewResourceService_WatchListClient(t)
-	// Recv should be called 10 times and no more since it is repeatedly returning an error.
-	mockWatchListClient.EXPECT().Recv().Return(nil, errors.New("unexpected error")).Times(linkWatchRetryCount)
-
-	client := mockpbresource.NewResourceServiceClient(t)
-	client.EXPECT().WatchList(mock.Anything, &pbresource.WatchListRequest{
-		Type:       pbhcp.LinkType,
-		NamePrefix: types.LinkName,
-	}).Return(mockWatchListClient, nil)
-
-	linkWatchCh, err := NewLinkWatch(context.Background(), hclog.Default(), client)
-	require.NoError(t, err)
-
-	// Ensure the linkWatchCh is closed
-	_, ok := <-linkWatchCh
-	require.False(t, ok)
-}