diff --git a/internal/hcp/watch.go b/internal/hcp/watch.go index 3e3e3363d2..e963667bdd 100644 --- a/internal/hcp/watch.go +++ b/internal/hcp/watch.go @@ -6,16 +6,16 @@ package hcp import ( "context" "fmt" + "time" "github.com/hashicorp/go-hclog" "github.com/hashicorp/consul/internal/hcp/internal/types" + "github.com/hashicorp/consul/lib/retry" pbhcp "github.com/hashicorp/consul/proto-public/pbhcp/v2" "github.com/hashicorp/consul/proto-public/pbresource" ) -var linkWatchRetryCount = 10 - func NewLinkWatch(ctx context.Context, logger hclog.Logger, client pbresource.ResourceServiceClient) (chan *pbresource.WatchEvent, error) { watchClient, err := client.WatchList( ctx, &pbresource.WatchListRequest{ @@ -29,7 +29,11 @@ func NewLinkWatch(ctx context.Context, logger hclog.Logger, client pbresource.Re eventCh := make(chan *pbresource.WatchEvent) go func() { - errorCounter := 0 + errorBackoff := &retry.Waiter{ + MinFailures: 1, + MinWait: 1 * time.Second, + MaxWait: 1 * time.Minute, + } for { select { case <-ctx.Done(): @@ -38,19 +42,14 @@ func NewLinkWatch(ctx context.Context, logger hclog.Logger, client pbresource.Re return default: watchEvent, err := watchClient.Recv() + if err != nil { logger.Error("error receiving link watch event", "error", err) - - errorCounter++ - if errorCounter >= linkWatchRetryCount { - logger.Error("received multiple consecutive errors from link watch client, will stop watching link") - close(eventCh) - return - } - + errorBackoff.Wait(ctx) continue } - errorCounter = 0 + + errorBackoff.Reset() eventCh <- watchEvent } } diff --git a/internal/hcp/watch_test.go b/internal/hcp/watch_test.go index b91f6d27af..a26f870d0a 100644 --- a/internal/hcp/watch_test.go +++ b/internal/hcp/watch_test.go @@ -62,24 +62,3 @@ func TestLinkWatch_ContextCanceled(t *testing.T) { _, ok := <-linkWatchCh require.False(t, ok) } - -// This tests ensures that when Recv returns errors repeatedly, we eventually close the channel -// and exit the goroutine -func TestLinkWatch_RepeatErrors(t *testing.T) { - mockWatchListClient := mockpbresource.NewResourceService_WatchListClient(t) - // Recv should be called 10 times and no more since it is repeatedly returning an error. - mockWatchListClient.EXPECT().Recv().Return(nil, errors.New("unexpected error")).Times(linkWatchRetryCount) - - client := mockpbresource.NewResourceServiceClient(t) - client.EXPECT().WatchList(mock.Anything, &pbresource.WatchListRequest{ - Type: pbhcp.LinkType, - NamePrefix: types.LinkName, - }).Return(mockWatchListClient, nil) - - linkWatchCh, err := NewLinkWatch(context.Background(), hclog.Default(), client) - require.NoError(t, err) - - // Ensure the linkWatchCh is closed - _, ok := <-linkWatchCh - require.False(t, ok) -}