mirror of https://github.com/hashicorp/consul
Retry lint fixes (#19151)
* Add a make target to run lint-consul-retry on all the modules * Cleanup sdk/testutil/retry * Fix a bunch of retry.Run* usage to not use the outer testing.T * Fix some more recent retry lint issues and pin to v1.4.0 of lint-consul-retry * Fix codegen copywrite lint issues * Don’t perform cleanup after each retry attempt by default. * Use the common testutil.TestingTB interface in test-integ/tenancy * Fix retry tests * Update otel access logging extension test to perform requests within the retry blockpull/19840/head
Matt Keeler
12 months ago
committed by
GitHub
parent
dc02fa695f
commit
efe279f802
@ -0,0 +1,22 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// Package retry provides support for repeating operations in tests.
|
||||
//
|
||||
// A sample retry operation looks like this:
|
||||
//
|
||||
// func TestX(t *testing.T) {
|
||||
// retry.Run(t, func(r *retry.R) {
|
||||
// if err := foo(); err != nil {
|
||||
// r.Errorf("foo: %s", err)
|
||||
// return
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// Run uses the DefaultFailer, which is a Timer with a Timeout of 7s,
|
||||
// and a Wait of 25ms. To customize, use RunWith.
|
||||
//
|
||||
// WARNING: unlike *testing.T, *retry.R#Fatal and FailNow *do not*
|
||||
// fail the test function entirely, only the current run the retry func
|
||||
package retry
|
||||
@ -0,0 +1,35 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var nilInf TestingTB = nil
|
||||
|
||||
// Assertion that our TestingTB can be passed to
|
||||
var _ require.TestingT = nilInf
|
||||
var _ assert.TestingT = nilInf
|
||||
|
||||
// TestingTB is an interface that describes the implementation of the testing object.
|
||||
// Using an interface that describes testing.TB instead of the actual implementation
|
||||
// makes testutil usable in a wider variety of contexts (e.g. use with ginkgo : https://godoc.org/github.com/onsi/ginkgo#GinkgoT)
|
||||
type TestingTB interface {
|
||||
Cleanup(func())
|
||||
Error(args ...any)
|
||||
Errorf(format string, args ...any)
|
||||
Fail()
|
||||
FailNow()
|
||||
Failed() bool
|
||||
Fatal(args ...any)
|
||||
Fatalf(format string, args ...any)
|
||||
Helper()
|
||||
Log(args ...any)
|
||||
Logf(format string, args ...any)
|
||||
Name() string
|
||||
Setenv(key, value string)
|
||||
TempDir() string
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func dedup(a []string) string {
|
||||
if len(a) == 0 {
|
||||
return ""
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var b bytes.Buffer
|
||||
for _, s := range a {
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
b.WriteString(s)
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func decorate(s string) string {
|
||||
_, file, line, ok := runtime.Caller(3)
|
||||
if ok {
|
||||
n := strings.LastIndex(file, "/")
|
||||
if n >= 0 {
|
||||
file = file[n+1:]
|
||||
}
|
||||
} else {
|
||||
file = "???"
|
||||
line = 1
|
||||
}
|
||||
return fmt.Sprintf("%s:%d: %s", file, line, s)
|
||||
}
|
||||
@ -1,263 +1,232 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// Package retry provides support for repeating operations in tests.
|
||||
//
|
||||
// A sample retry operation looks like this:
|
||||
//
|
||||
// func TestX(t *testing.T) {
|
||||
// retry.Run(t, func(r *retry.R) {
|
||||
// if err := foo(); err != nil {
|
||||
// r.Errorf("foo: %s", err)
|
||||
// return
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// Run uses the DefaultFailer, which is a Timer with a Timeout of 7s,
|
||||
// and a Wait of 25ms. To customize, use RunWith.
|
||||
//
|
||||
// WARNING: unlike *testing.T, *retry.R#Fatal and FailNow *do not*
|
||||
// fail the test function entirely, only the current run the retry func
|
||||
package retry
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Failer is an interface compatible with testing.T.
|
||||
type Failer interface {
|
||||
Helper()
|
||||
var _ TestingTB = &R{}
|
||||
|
||||
// Log is called for the final test output
|
||||
Log(args ...interface{})
|
||||
type R struct {
|
||||
wrapped TestingTB
|
||||
retryer Retryer
|
||||
|
||||
// FailNow is called when the retrying is abandoned.
|
||||
FailNow()
|
||||
}
|
||||
done bool
|
||||
fullOutput bool
|
||||
immediateCleanup bool
|
||||
|
||||
// R provides context for the retryer.
|
||||
//
|
||||
// Logs from Logf, (Error|Fatal)(f) are gathered in an internal buffer
|
||||
// and printed only if the retryer fails. Printed logs are deduped and
|
||||
// prefixed with source code line numbers
|
||||
type R struct {
|
||||
// fail is set by FailNow and (Fatal|Error)(f). It indicates the pass
|
||||
// did not succeed, and should be retried
|
||||
fail bool
|
||||
// done is set by Stop. It indicates the entire run was a failure,
|
||||
// and triggers t.FailNow()
|
||||
done bool
|
||||
output []string
|
||||
attempts []*attempt
|
||||
}
|
||||
|
||||
cleanups []func()
|
||||
func (r *R) Cleanup(clean func()) {
|
||||
if r.immediateCleanup {
|
||||
a := r.getCurrentAttempt()
|
||||
a.cleanups = append(a.cleanups, clean)
|
||||
} else {
|
||||
r.wrapped.Cleanup(clean)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *R) Logf(format string, args ...interface{}) {
|
||||
r.log(fmt.Sprintf(format, args...))
|
||||
func (r *R) Error(args ...any) {
|
||||
r.Log(args...)
|
||||
r.Fail()
|
||||
}
|
||||
|
||||
func (r *R) Log(args ...interface{}) {
|
||||
r.log(fmt.Sprintln(args...))
|
||||
func (r *R) Errorf(format string, args ...any) {
|
||||
r.Logf(format, args...)
|
||||
r.Fail()
|
||||
}
|
||||
|
||||
func (r *R) Helper() {}
|
||||
func (r *R) Fail() {
|
||||
r.getCurrentAttempt().failed = true
|
||||
}
|
||||
|
||||
// Cleanup register a function to be run to cleanup resources that
|
||||
// were allocated during the retry attempt. These functions are executed
|
||||
// after a retry attempt. If they panic, it will not stop further retry
|
||||
// attempts but will be cause for the overall test failure.
|
||||
func (r *R) Cleanup(fn func()) {
|
||||
r.cleanups = append(r.cleanups, fn)
|
||||
func (r *R) FailNow() {
|
||||
r.Fail()
|
||||
panic(attemptFailed{})
|
||||
}
|
||||
|
||||
func (r *R) runCleanup() {
|
||||
func (r *R) Failed() bool {
|
||||
return r.getCurrentAttempt().failed
|
||||
}
|
||||
|
||||
// Make sure that if a cleanup function panics,
|
||||
// we still run the remaining cleanup functions.
|
||||
defer func() {
|
||||
err := recover()
|
||||
if err != nil {
|
||||
r.Stop(fmt.Errorf("error when performing test cleanup: %v", err))
|
||||
}
|
||||
if len(r.cleanups) > 0 {
|
||||
r.runCleanup()
|
||||
}
|
||||
}()
|
||||
func (r *R) Fatal(args ...any) {
|
||||
r.Log(args...)
|
||||
r.FailNow()
|
||||
}
|
||||
|
||||
for len(r.cleanups) > 0 {
|
||||
var cleanup func()
|
||||
if len(r.cleanups) > 0 {
|
||||
last := len(r.cleanups) - 1
|
||||
cleanup = r.cleanups[last]
|
||||
r.cleanups = r.cleanups[:last]
|
||||
}
|
||||
if cleanup != nil {
|
||||
cleanup()
|
||||
}
|
||||
}
|
||||
func (r *R) Fatalf(format string, args ...any) {
|
||||
r.Logf(format, args...)
|
||||
r.FailNow()
|
||||
}
|
||||
|
||||
// runFailed is a sentinel value to indicate that the func itself
|
||||
// didn't panic, rather that `FailNow` was called.
|
||||
type runFailed struct{}
|
||||
|
||||
// FailNow stops run execution. It is roughly equivalent to:
|
||||
//
|
||||
// r.Error("")
|
||||
// return
|
||||
//
|
||||
// inside the function being run.
|
||||
func (r *R) FailNow() {
|
||||
r.fail = true
|
||||
panic(runFailed{})
|
||||
func (r *R) Helper() {
|
||||
// *testing.T will just record which functions are helpers by their addresses and
|
||||
// it doesn't much matter where where we record that they are helpers
|
||||
r.wrapped.Helper()
|
||||
}
|
||||
|
||||
// Fatal is equivalent to r.Logf(args) followed by r.FailNow(), i.e. the run
|
||||
// function should be exited. Retries on the next run are allowed. Fatal is
|
||||
// equivalent to
|
||||
//
|
||||
// r.Error(args)
|
||||
// return
|
||||
//
|
||||
// inside the function being run.
|
||||
func (r *R) Fatal(args ...interface{}) {
|
||||
r.log(fmt.Sprint(args...))
|
||||
r.FailNow()
|
||||
func (r *R) Log(args ...any) {
|
||||
r.log(fmt.Sprintln(args...))
|
||||
}
|
||||
|
||||
// Fatalf is like Fatal but allows a format string
|
||||
func (r *R) Fatalf(format string, args ...interface{}) {
|
||||
func (r *R) Logf(format string, args ...any) {
|
||||
r.log(fmt.Sprintf(format, args...))
|
||||
r.FailNow()
|
||||
}
|
||||
|
||||
// Error indicates the current run encountered an error and should be retried.
|
||||
// It *does not* stop execution of the rest of the run function.
|
||||
func (r *R) Error(args ...interface{}) {
|
||||
r.log(fmt.Sprint(args...))
|
||||
r.fail = true
|
||||
// Name will return the name of the underlying TestingT.
|
||||
func (r *R) Name() string {
|
||||
return r.wrapped.Name()
|
||||
}
|
||||
|
||||
// Errorf is like Error but allows a format string
|
||||
func (r *R) Errorf(format string, args ...interface{}) {
|
||||
r.log(fmt.Sprintf(format, args...))
|
||||
r.fail = true
|
||||
// Setenv will save the current value of the specified env var, set it to the
|
||||
// specified value and then restore it to the original value in a cleanup function
|
||||
// once the retry attempt has finished.
|
||||
func (r *R) Setenv(key, value string) {
|
||||
prevValue, ok := os.LookupEnv(key)
|
||||
|
||||
if err := os.Setenv(key, value); err != nil {
|
||||
r.wrapped.Fatalf("cannot set environment variable: %v", err)
|
||||
}
|
||||
|
||||
if ok {
|
||||
r.Cleanup(func() {
|
||||
os.Setenv(key, prevValue)
|
||||
})
|
||||
} else {
|
||||
r.Cleanup(func() {
|
||||
os.Unsetenv(key)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// If err is non-nil, equivalent to r.Fatal(err.Error()) followed by
|
||||
// r.FailNow(). Otherwise a no-op.
|
||||
// TempDir will use the wrapped TestingT to create a temporary directory
|
||||
// that will be cleaned up when ALL RETRYING has finished.
|
||||
func (r *R) TempDir() string {
|
||||
return r.wrapped.TempDir()
|
||||
}
|
||||
|
||||
// Check will call r.Fatal(err) if err is not nil
|
||||
func (r *R) Check(err error) {
|
||||
if err != nil {
|
||||
r.log(err.Error())
|
||||
r.FailNow()
|
||||
r.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *R) log(s string) {
|
||||
r.output = append(r.output, decorate(s))
|
||||
}
|
||||
|
||||
// Stop retrying, and fail the test, logging the specified error.
|
||||
// Does not stop execution, so return should be called after.
|
||||
func (r *R) Stop(err error) {
|
||||
r.log(err.Error())
|
||||
r.done = true
|
||||
}
|
||||
|
||||
func decorate(s string) string {
|
||||
_, file, line, ok := runtime.Caller(3)
|
||||
if ok {
|
||||
n := strings.LastIndex(file, "/")
|
||||
if n >= 0 {
|
||||
file = file[n+1:]
|
||||
}
|
||||
} else {
|
||||
file = "???"
|
||||
line = 1
|
||||
}
|
||||
return fmt.Sprintf("%s:%d: %s", file, line, s)
|
||||
func (r *R) failCurrentAttempt() {
|
||||
r.getCurrentAttempt().failed = true
|
||||
}
|
||||
|
||||
func Run(t Failer, f func(r *R)) {
|
||||
t.Helper()
|
||||
run(DefaultFailer(), t, f)
|
||||
func (r *R) log(s string) {
|
||||
a := r.getCurrentAttempt()
|
||||
a.output = append(a.output, decorate(s))
|
||||
}
|
||||
|
||||
func RunWith(r Retryer, t Failer, f func(r *R)) {
|
||||
t.Helper()
|
||||
run(r, t, f)
|
||||
func (r *R) getCurrentAttempt() *attempt {
|
||||
if len(r.attempts) == 0 {
|
||||
panic("no retry attempts have been started yet")
|
||||
}
|
||||
|
||||
return r.attempts[len(r.attempts)-1]
|
||||
}
|
||||
|
||||
func dedup(a []string) string {
|
||||
if len(a) == 0 {
|
||||
return ""
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var b bytes.Buffer
|
||||
for _, s := range a {
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
// cleanupAttempt will perform all the register cleanup operations recorded
|
||||
// during execution of the single round of the test function.
|
||||
func (r *R) cleanupAttempt(a *attempt) {
|
||||
// Make sure that if a cleanup function panics,
|
||||
// we still run the remaining cleanup functions.
|
||||
defer func() {
|
||||
err := recover()
|
||||
if err != nil {
|
||||
r.Stop(fmt.Errorf("error when performing test cleanup: %v", err))
|
||||
}
|
||||
if len(a.cleanups) > 0 {
|
||||
r.cleanupAttempt(a)
|
||||
}
|
||||
}()
|
||||
|
||||
for len(a.cleanups) > 0 {
|
||||
var cleanup func()
|
||||
if len(a.cleanups) > 0 {
|
||||
last := len(a.cleanups) - 1
|
||||
cleanup = a.cleanups[last]
|
||||
a.cleanups = a.cleanups[:last]
|
||||
}
|
||||
if cleanup != nil {
|
||||
cleanup()
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
b.WriteString(s)
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func run(r Retryer, t Failer, f func(r *R)) {
|
||||
t.Helper()
|
||||
rr := &R{}
|
||||
// runAttempt will execute one round of the test function and handle cleanups and panic recovery
|
||||
// of a failed attempt that should not stop retrying.
|
||||
func (r *R) runAttempt(f func(r *R)) {
|
||||
r.Helper()
|
||||
|
||||
a := &attempt{}
|
||||
r.attempts = append(r.attempts, a)
|
||||
|
||||
fail := func() {
|
||||
t.Helper()
|
||||
out := dedup(rr.output)
|
||||
if out != "" {
|
||||
t.Log(out)
|
||||
defer r.cleanupAttempt(a)
|
||||
defer func() {
|
||||
if p := recover(); p != nil && p != (attemptFailed{}) {
|
||||
panic(p)
|
||||
}
|
||||
t.FailNow()
|
||||
}
|
||||
}()
|
||||
f(r)
|
||||
}
|
||||
|
||||
func (r *R) run(f func(r *R)) {
|
||||
r.Helper()
|
||||
|
||||
for r.Continue() {
|
||||
// run f(rr), but if recover yields a runFailed value, we know
|
||||
// FailNow was called.
|
||||
func() {
|
||||
defer rr.runCleanup()
|
||||
defer func() {
|
||||
if p := recover(); p != nil && p != (runFailed{}) {
|
||||
panic(p)
|
||||
}
|
||||
}()
|
||||
f(rr)
|
||||
}()
|
||||
for r.retryer.Continue() {
|
||||
r.runAttempt(f)
|
||||
|
||||
switch {
|
||||
case rr.done:
|
||||
fail()
|
||||
case r.done:
|
||||
r.recordRetryFailure()
|
||||
return
|
||||
case !rr.fail:
|
||||
case !r.Failed():
|
||||
// the current attempt did not fail so we can go ahead and return
|
||||
return
|
||||
}
|
||||
rr.fail = false
|
||||
}
|
||||
fail()
|
||||
|
||||
// We cannot retry any more and no attempt has succeeded yet.
|
||||
r.recordRetryFailure()
|
||||
}
|
||||
|
||||
// DefaultFailer provides default retry.Run() behavior for unit tests, namely
|
||||
// 7s timeout with a wait of 25ms
|
||||
func DefaultFailer() *Timer {
|
||||
return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond}
|
||||
func (r *R) recordRetryFailure() {
|
||||
r.Helper()
|
||||
output := r.getCurrentAttempt().output
|
||||
if r.fullOutput {
|
||||
var combined []string
|
||||
for _, attempt := range r.attempts {
|
||||
combined = append(combined, attempt.output...)
|
||||
}
|
||||
output = combined
|
||||
}
|
||||
|
||||
out := dedup(output)
|
||||
if out != "" {
|
||||
r.wrapped.Log(out)
|
||||
}
|
||||
r.wrapped.FailNow()
|
||||
}
|
||||
|
||||
// Retryer provides an interface for repeating operations
|
||||
// until they succeed or an exit condition is met.
|
||||
type Retryer interface {
|
||||
// Continue returns true if the operation should be repeated, otherwise it
|
||||
// returns false to indicate retrying should stop.
|
||||
Continue() bool
|
||||
type attempt struct {
|
||||
failed bool
|
||||
output []string
|
||||
cleanups []func()
|
||||
}
|
||||
|
||||
// attemptFailed is a sentinel value to indicate that the func itself
|
||||
// didn't panic, rather that `FailNow` was called.
|
||||
type attemptFailed struct{}
|
||||
|
||||
@ -0,0 +1,36 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry
|
||||
|
||||
import "time"
|
||||
|
||||
// Retryer provides an interface for repeating operations
|
||||
// until they succeed or an exit condition is met.
|
||||
type Retryer interface {
|
||||
// Continue returns true if the operation should be repeated, otherwise it
|
||||
// returns false to indicate retrying should stop.
|
||||
Continue() bool
|
||||
}
|
||||
|
||||
// DefaultRetryer provides default retry.Run() behavior for unit tests, namely
|
||||
// 7s timeout with a wait of 25ms
|
||||
func DefaultRetryer() Retryer {
|
||||
return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond}
|
||||
}
|
||||
|
||||
// ThirtySeconds repeats an operation for thirty seconds and waits 500ms in between.
|
||||
// Best for known slower operations like waiting on eventually consistent state.
|
||||
func ThirtySeconds() *Timer {
|
||||
return &Timer{Timeout: 30 * time.Second, Wait: 500 * time.Millisecond}
|
||||
}
|
||||
|
||||
// TwoSeconds repeats an operation for two seconds and waits 25ms in between.
|
||||
func TwoSeconds() *Timer {
|
||||
return &Timer{Timeout: 2 * time.Second, Wait: 25 * time.Millisecond}
|
||||
}
|
||||
|
||||
// ThreeTimes repeats an operation three times and waits 25ms in between.
|
||||
func ThreeTimes() *Counter {
|
||||
return &Counter{Count: 3, Wait: 25 * time.Millisecond}
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry
|
||||
|
||||
type Option func(r *R)
|
||||
|
||||
func WithRetryer(retryer Retryer) Option {
|
||||
return func(r *R) {
|
||||
r.retryer = retryer
|
||||
}
|
||||
}
|
||||
|
||||
func WithFullOutput() Option {
|
||||
return func(r *R) {
|
||||
r.fullOutput = true
|
||||
}
|
||||
}
|
||||
|
||||
// WithImmediateCleanup will cause all cleanup operations added
|
||||
// by calling the Cleanup method on *R to be performed after
|
||||
// the retry attempt completes (regardless of pass/fail status)
|
||||
// Use this only if all resources created during the retry loop should
|
||||
// not persist after the retry has finished.
|
||||
func WithImmediateCleanup() Option {
|
||||
return func(r *R) {
|
||||
r.immediateCleanup = true
|
||||
}
|
||||
}
|
||||
|
||||
func Run(t TestingTB, f func(r *R), opts ...Option) {
|
||||
t.Helper()
|
||||
r := &R{
|
||||
wrapped: t,
|
||||
retryer: DefaultRetryer(),
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(r)
|
||||
}
|
||||
|
||||
r.run(f)
|
||||
}
|
||||
|
||||
func RunWith(r Retryer, t TestingTB, f func(r *R)) {
|
||||
t.Helper()
|
||||
Run(t, f, WithRetryer(r))
|
||||
}
|
||||
Loading…
Reference in new issue