mirror of https://github.com/hashicorp/consul
Browse Source
* Add a make target to run lint-consul-retry on all the modules * Cleanup sdk/testutil/retry * Fix a bunch of retry.Run* usage to not use the outer testing.T * Fix some more recent retry lint issues and pin to v1.4.0 of lint-consul-retry * Fix codegen copywrite lint issues * Don’t perform cleanup after each retry attempt by default. * Use the common testutil.TestingTB interface in test-integ/tenancy * Fix retry tests * Update otel access logging extension test to perform requests within the retry blockpull/19840/head
Matt Keeler
12 months ago
committed by
GitHub
45 changed files with 585 additions and 373 deletions
@ -0,0 +1,22 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// Package retry provides support for repeating operations in tests.
|
||||
//
|
||||
// A sample retry operation looks like this:
|
||||
//
|
||||
// func TestX(t *testing.T) {
|
||||
// retry.Run(t, func(r *retry.R) {
|
||||
// if err := foo(); err != nil {
|
||||
// r.Errorf("foo: %s", err)
|
||||
// return
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// Run uses the DefaultFailer, which is a Timer with a Timeout of 7s,
|
||||
// and a Wait of 25ms. To customize, use RunWith.
|
||||
//
|
||||
// WARNING: unlike *testing.T, *retry.R#Fatal and FailNow *do not*
|
||||
// fail the test function entirely, only the current run the retry func
|
||||
package retry |
||||
@ -0,0 +1,35 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry |
||||
|
||||
import ( |
||||
"github.com/stretchr/testify/assert" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
var nilInf TestingTB = nil |
||||
|
||||
// Assertion that our TestingTB can be passed to
|
||||
var _ require.TestingT = nilInf |
||||
var _ assert.TestingT = nilInf |
||||
|
||||
// TestingTB is an interface that describes the implementation of the testing object.
|
||||
// Using an interface that describes testing.TB instead of the actual implementation
|
||||
// makes testutil usable in a wider variety of contexts (e.g. use with ginkgo : https://godoc.org/github.com/onsi/ginkgo#GinkgoT)
|
||||
type TestingTB interface { |
||||
Cleanup(func()) |
||||
Error(args ...any) |
||||
Errorf(format string, args ...any) |
||||
Fail() |
||||
FailNow() |
||||
Failed() bool |
||||
Fatal(args ...any) |
||||
Fatalf(format string, args ...any) |
||||
Helper() |
||||
Log(args ...any) |
||||
Logf(format string, args ...any) |
||||
Name() string |
||||
Setenv(key, value string) |
||||
TempDir() string |
||||
} |
||||
@ -0,0 +1,42 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"runtime" |
||||
"strings" |
||||
) |
||||
|
||||
func dedup(a []string) string { |
||||
if len(a) == 0 { |
||||
return "" |
||||
} |
||||
seen := map[string]struct{}{} |
||||
var b bytes.Buffer |
||||
for _, s := range a { |
||||
if _, ok := seen[s]; ok { |
||||
continue |
||||
} |
||||
seen[s] = struct{}{} |
||||
b.WriteString(s) |
||||
b.WriteRune('\n') |
||||
} |
||||
return b.String() |
||||
} |
||||
|
||||
func decorate(s string) string { |
||||
_, file, line, ok := runtime.Caller(3) |
||||
if ok { |
||||
n := strings.LastIndex(file, "/") |
||||
if n >= 0 { |
||||
file = file[n+1:] |
||||
} |
||||
} else { |
||||
file = "???" |
||||
line = 1 |
||||
} |
||||
return fmt.Sprintf("%s:%d: %s", file, line, s) |
||||
} |
||||
@ -1,263 +1,232 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// Package retry provides support for repeating operations in tests.
|
||||
//
|
||||
// A sample retry operation looks like this:
|
||||
//
|
||||
// func TestX(t *testing.T) {
|
||||
// retry.Run(t, func(r *retry.R) {
|
||||
// if err := foo(); err != nil {
|
||||
// r.Errorf("foo: %s", err)
|
||||
// return
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// Run uses the DefaultFailer, which is a Timer with a Timeout of 7s,
|
||||
// and a Wait of 25ms. To customize, use RunWith.
|
||||
//
|
||||
// WARNING: unlike *testing.T, *retry.R#Fatal and FailNow *do not*
|
||||
// fail the test function entirely, only the current run the retry func
|
||||
package retry |
||||
|
||||
import ( |
||||
"bytes" |
||||
"fmt" |
||||
"runtime" |
||||
"strings" |
||||
"time" |
||||
"os" |
||||
) |
||||
|
||||
// Failer is an interface compatible with testing.T.
|
||||
type Failer interface { |
||||
Helper() |
||||
var _ TestingTB = &R{} |
||||
|
||||
// Log is called for the final test output
|
||||
Log(args ...interface{}) |
||||
|
||||
// FailNow is called when the retrying is abandoned.
|
||||
FailNow() |
||||
} |
||||
|
||||
// R provides context for the retryer.
|
||||
//
|
||||
// Logs from Logf, (Error|Fatal)(f) are gathered in an internal buffer
|
||||
// and printed only if the retryer fails. Printed logs are deduped and
|
||||
// prefixed with source code line numbers
|
||||
type R struct { |
||||
// fail is set by FailNow and (Fatal|Error)(f). It indicates the pass
|
||||
// did not succeed, and should be retried
|
||||
fail bool |
||||
// done is set by Stop. It indicates the entire run was a failure,
|
||||
// and triggers t.FailNow()
|
||||
wrapped TestingTB |
||||
retryer Retryer |
||||
|
||||
done bool |
||||
output []string |
||||
fullOutput bool |
||||
immediateCleanup bool |
||||
|
||||
cleanups []func() |
||||
attempts []*attempt |
||||
} |
||||
|
||||
func (r *R) Logf(format string, args ...interface{}) { |
||||
r.log(fmt.Sprintf(format, args...)) |
||||
func (r *R) Cleanup(clean func()) { |
||||
if r.immediateCleanup { |
||||
a := r.getCurrentAttempt() |
||||
a.cleanups = append(a.cleanups, clean) |
||||
} else { |
||||
r.wrapped.Cleanup(clean) |
||||
} |
||||
} |
||||
|
||||
func (r *R) Log(args ...interface{}) { |
||||
r.log(fmt.Sprintln(args...)) |
||||
func (r *R) Error(args ...any) { |
||||
r.Log(args...) |
||||
r.Fail() |
||||
} |
||||
|
||||
func (r *R) Helper() {} |
||||
func (r *R) Errorf(format string, args ...any) { |
||||
r.Logf(format, args...) |
||||
r.Fail() |
||||
} |
||||
|
||||
// Cleanup register a function to be run to cleanup resources that
|
||||
// were allocated during the retry attempt. These functions are executed
|
||||
// after a retry attempt. If they panic, it will not stop further retry
|
||||
// attempts but will be cause for the overall test failure.
|
||||
func (r *R) Cleanup(fn func()) { |
||||
r.cleanups = append(r.cleanups, fn) |
||||
func (r *R) Fail() { |
||||
r.getCurrentAttempt().failed = true |
||||
} |
||||
|
||||
func (r *R) runCleanup() { |
||||
func (r *R) FailNow() { |
||||
r.Fail() |
||||
panic(attemptFailed{}) |
||||
} |
||||
|
||||
// Make sure that if a cleanup function panics,
|
||||
// we still run the remaining cleanup functions.
|
||||
defer func() { |
||||
err := recover() |
||||
if err != nil { |
||||
r.Stop(fmt.Errorf("error when performing test cleanup: %v", err)) |
||||
func (r *R) Failed() bool { |
||||
return r.getCurrentAttempt().failed |
||||
} |
||||
if len(r.cleanups) > 0 { |
||||
r.runCleanup() |
||||
|
||||
func (r *R) Fatal(args ...any) { |
||||
r.Log(args...) |
||||
r.FailNow() |
||||
} |
||||
}() |
||||
|
||||
for len(r.cleanups) > 0 { |
||||
var cleanup func() |
||||
if len(r.cleanups) > 0 { |
||||
last := len(r.cleanups) - 1 |
||||
cleanup = r.cleanups[last] |
||||
r.cleanups = r.cleanups[:last] |
||||
func (r *R) Fatalf(format string, args ...any) { |
||||
r.Logf(format, args...) |
||||
r.FailNow() |
||||
} |
||||
if cleanup != nil { |
||||
cleanup() |
||||
|
||||
func (r *R) Helper() { |
||||
// *testing.T will just record which functions are helpers by their addresses and
|
||||
// it doesn't much matter where where we record that they are helpers
|
||||
r.wrapped.Helper() |
||||
} |
||||
|
||||
func (r *R) Log(args ...any) { |
||||
r.log(fmt.Sprintln(args...)) |
||||
} |
||||
|
||||
func (r *R) Logf(format string, args ...any) { |
||||
r.log(fmt.Sprintf(format, args...)) |
||||
} |
||||
|
||||
// runFailed is a sentinel value to indicate that the func itself
|
||||
// didn't panic, rather that `FailNow` was called.
|
||||
type runFailed struct{} |
||||
|
||||
// FailNow stops run execution. It is roughly equivalent to:
|
||||
//
|
||||
// r.Error("")
|
||||
// return
|
||||
//
|
||||
// inside the function being run.
|
||||
func (r *R) FailNow() { |
||||
r.fail = true |
||||
panic(runFailed{}) |
||||
} |
||||
|
||||
// Fatal is equivalent to r.Logf(args) followed by r.FailNow(), i.e. the run
|
||||
// function should be exited. Retries on the next run are allowed. Fatal is
|
||||
// equivalent to
|
||||
//
|
||||
// r.Error(args)
|
||||
// return
|
||||
//
|
||||
// inside the function being run.
|
||||
func (r *R) Fatal(args ...interface{}) { |
||||
r.log(fmt.Sprint(args...)) |
||||
r.FailNow() |
||||
// Name will return the name of the underlying TestingT.
|
||||
func (r *R) Name() string { |
||||
return r.wrapped.Name() |
||||
} |
||||
|
||||
// Fatalf is like Fatal but allows a format string
|
||||
func (r *R) Fatalf(format string, args ...interface{}) { |
||||
r.log(fmt.Sprintf(format, args...)) |
||||
r.FailNow() |
||||
// Setenv will save the current value of the specified env var, set it to the
|
||||
// specified value and then restore it to the original value in a cleanup function
|
||||
// once the retry attempt has finished.
|
||||
func (r *R) Setenv(key, value string) { |
||||
prevValue, ok := os.LookupEnv(key) |
||||
|
||||
if err := os.Setenv(key, value); err != nil { |
||||
r.wrapped.Fatalf("cannot set environment variable: %v", err) |
||||
} |
||||
|
||||
// Error indicates the current run encountered an error and should be retried.
|
||||
// It *does not* stop execution of the rest of the run function.
|
||||
func (r *R) Error(args ...interface{}) { |
||||
r.log(fmt.Sprint(args...)) |
||||
r.fail = true |
||||
if ok { |
||||
r.Cleanup(func() { |
||||
os.Setenv(key, prevValue) |
||||
}) |
||||
} else { |
||||
r.Cleanup(func() { |
||||
os.Unsetenv(key) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
// Errorf is like Error but allows a format string
|
||||
func (r *R) Errorf(format string, args ...interface{}) { |
||||
r.log(fmt.Sprintf(format, args...)) |
||||
r.fail = true |
||||
// TempDir will use the wrapped TestingT to create a temporary directory
|
||||
// that will be cleaned up when ALL RETRYING has finished.
|
||||
func (r *R) TempDir() string { |
||||
return r.wrapped.TempDir() |
||||
} |
||||
|
||||
// If err is non-nil, equivalent to r.Fatal(err.Error()) followed by
|
||||
// r.FailNow(). Otherwise a no-op.
|
||||
// Check will call r.Fatal(err) if err is not nil
|
||||
func (r *R) Check(err error) { |
||||
if err != nil { |
||||
r.log(err.Error()) |
||||
r.FailNow() |
||||
r.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
func (r *R) log(s string) { |
||||
r.output = append(r.output, decorate(s)) |
||||
} |
||||
|
||||
// Stop retrying, and fail the test, logging the specified error.
|
||||
// Does not stop execution, so return should be called after.
|
||||
func (r *R) Stop(err error) { |
||||
r.log(err.Error()) |
||||
r.done = true |
||||
} |
||||
|
||||
func decorate(s string) string { |
||||
_, file, line, ok := runtime.Caller(3) |
||||
if ok { |
||||
n := strings.LastIndex(file, "/") |
||||
if n >= 0 { |
||||
file = file[n+1:] |
||||
func (r *R) failCurrentAttempt() { |
||||
r.getCurrentAttempt().failed = true |
||||
} |
||||
} else { |
||||
file = "???" |
||||
line = 1 |
||||
|
||||
func (r *R) log(s string) { |
||||
a := r.getCurrentAttempt() |
||||
a.output = append(a.output, decorate(s)) |
||||
} |
||||
return fmt.Sprintf("%s:%d: %s", file, line, s) |
||||
|
||||
func (r *R) getCurrentAttempt() *attempt { |
||||
if len(r.attempts) == 0 { |
||||
panic("no retry attempts have been started yet") |
||||
} |
||||
|
||||
func Run(t Failer, f func(r *R)) { |
||||
t.Helper() |
||||
run(DefaultFailer(), t, f) |
||||
return r.attempts[len(r.attempts)-1] |
||||
} |
||||
|
||||
func RunWith(r Retryer, t Failer, f func(r *R)) { |
||||
t.Helper() |
||||
run(r, t, f) |
||||
// cleanupAttempt will perform all the register cleanup operations recorded
|
||||
// during execution of the single round of the test function.
|
||||
func (r *R) cleanupAttempt(a *attempt) { |
||||
// Make sure that if a cleanup function panics,
|
||||
// we still run the remaining cleanup functions.
|
||||
defer func() { |
||||
err := recover() |
||||
if err != nil { |
||||
r.Stop(fmt.Errorf("error when performing test cleanup: %v", err)) |
||||
} |
||||
if len(a.cleanups) > 0 { |
||||
r.cleanupAttempt(a) |
||||
} |
||||
}() |
||||
|
||||
func dedup(a []string) string { |
||||
if len(a) == 0 { |
||||
return "" |
||||
for len(a.cleanups) > 0 { |
||||
var cleanup func() |
||||
if len(a.cleanups) > 0 { |
||||
last := len(a.cleanups) - 1 |
||||
cleanup = a.cleanups[last] |
||||
a.cleanups = a.cleanups[:last] |
||||
} |
||||
seen := map[string]struct{}{} |
||||
var b bytes.Buffer |
||||
for _, s := range a { |
||||
if _, ok := seen[s]; ok { |
||||
continue |
||||
if cleanup != nil { |
||||
cleanup() |
||||
} |
||||
seen[s] = struct{}{} |
||||
b.WriteString(s) |
||||
b.WriteRune('\n') |
||||
} |
||||
return b.String() |
||||
} |
||||
|
||||
func run(r Retryer, t Failer, f func(r *R)) { |
||||
t.Helper() |
||||
rr := &R{} |
||||
// runAttempt will execute one round of the test function and handle cleanups and panic recovery
|
||||
// of a failed attempt that should not stop retrying.
|
||||
func (r *R) runAttempt(f func(r *R)) { |
||||
r.Helper() |
||||
|
||||
fail := func() { |
||||
t.Helper() |
||||
out := dedup(rr.output) |
||||
if out != "" { |
||||
t.Log(out) |
||||
} |
||||
t.FailNow() |
||||
} |
||||
a := &attempt{} |
||||
r.attempts = append(r.attempts, a) |
||||
|
||||
for r.Continue() { |
||||
// run f(rr), but if recover yields a runFailed value, we know
|
||||
// FailNow was called.
|
||||
func() { |
||||
defer rr.runCleanup() |
||||
defer r.cleanupAttempt(a) |
||||
defer func() { |
||||
if p := recover(); p != nil && p != (runFailed{}) { |
||||
if p := recover(); p != nil && p != (attemptFailed{}) { |
||||
panic(p) |
||||
} |
||||
}() |
||||
f(rr) |
||||
}() |
||||
f(r) |
||||
} |
||||
|
||||
func (r *R) run(f func(r *R)) { |
||||
r.Helper() |
||||
|
||||
for r.retryer.Continue() { |
||||
r.runAttempt(f) |
||||
|
||||
switch { |
||||
case rr.done: |
||||
fail() |
||||
case r.done: |
||||
r.recordRetryFailure() |
||||
return |
||||
case !rr.fail: |
||||
case !r.Failed(): |
||||
// the current attempt did not fail so we can go ahead and return
|
||||
return |
||||
} |
||||
rr.fail = false |
||||
} |
||||
fail() |
||||
|
||||
// We cannot retry any more and no attempt has succeeded yet.
|
||||
r.recordRetryFailure() |
||||
} |
||||
|
||||
// DefaultFailer provides default retry.Run() behavior for unit tests, namely
|
||||
// 7s timeout with a wait of 25ms
|
||||
func DefaultFailer() *Timer { |
||||
return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond} |
||||
func (r *R) recordRetryFailure() { |
||||
r.Helper() |
||||
output := r.getCurrentAttempt().output |
||||
if r.fullOutput { |
||||
var combined []string |
||||
for _, attempt := range r.attempts { |
||||
combined = append(combined, attempt.output...) |
||||
} |
||||
output = combined |
||||
} |
||||
|
||||
// Retryer provides an interface for repeating operations
|
||||
// until they succeed or an exit condition is met.
|
||||
type Retryer interface { |
||||
// Continue returns true if the operation should be repeated, otherwise it
|
||||
// returns false to indicate retrying should stop.
|
||||
Continue() bool |
||||
out := dedup(output) |
||||
if out != "" { |
||||
r.wrapped.Log(out) |
||||
} |
||||
r.wrapped.FailNow() |
||||
} |
||||
|
||||
type attempt struct { |
||||
failed bool |
||||
output []string |
||||
cleanups []func() |
||||
} |
||||
|
||||
// attemptFailed is a sentinel value to indicate that the func itself
|
||||
// didn't panic, rather that `FailNow` was called.
|
||||
type attemptFailed struct{} |
||||
|
||||
@ -0,0 +1,36 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry |
||||
|
||||
import "time" |
||||
|
||||
// Retryer provides an interface for repeating operations
|
||||
// until they succeed or an exit condition is met.
|
||||
type Retryer interface { |
||||
// Continue returns true if the operation should be repeated, otherwise it
|
||||
// returns false to indicate retrying should stop.
|
||||
Continue() bool |
||||
} |
||||
|
||||
// DefaultRetryer provides default retry.Run() behavior for unit tests, namely
|
||||
// 7s timeout with a wait of 25ms
|
||||
func DefaultRetryer() Retryer { |
||||
return &Timer{Timeout: 7 * time.Second, Wait: 25 * time.Millisecond} |
||||
} |
||||
|
||||
// ThirtySeconds repeats an operation for thirty seconds and waits 500ms in between.
|
||||
// Best for known slower operations like waiting on eventually consistent state.
|
||||
func ThirtySeconds() *Timer { |
||||
return &Timer{Timeout: 30 * time.Second, Wait: 500 * time.Millisecond} |
||||
} |
||||
|
||||
// TwoSeconds repeats an operation for two seconds and waits 25ms in between.
|
||||
func TwoSeconds() *Timer { |
||||
return &Timer{Timeout: 2 * time.Second, Wait: 25 * time.Millisecond} |
||||
} |
||||
|
||||
// ThreeTimes repeats an operation three times and waits 25ms in between.
|
||||
func ThreeTimes() *Counter { |
||||
return &Counter{Count: 3, Wait: 25 * time.Millisecond} |
||||
} |
||||
@ -0,0 +1,48 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package retry |
||||
|
||||
type Option func(r *R) |
||||
|
||||
func WithRetryer(retryer Retryer) Option { |
||||
return func(r *R) { |
||||
r.retryer = retryer |
||||
} |
||||
} |
||||
|
||||
func WithFullOutput() Option { |
||||
return func(r *R) { |
||||
r.fullOutput = true |
||||
} |
||||
} |
||||
|
||||
// WithImmediateCleanup will cause all cleanup operations added
|
||||
// by calling the Cleanup method on *R to be performed after
|
||||
// the retry attempt completes (regardless of pass/fail status)
|
||||
// Use this only if all resources created during the retry loop should
|
||||
// not persist after the retry has finished.
|
||||
func WithImmediateCleanup() Option { |
||||
return func(r *R) { |
||||
r.immediateCleanup = true |
||||
} |
||||
} |
||||
|
||||
func Run(t TestingTB, f func(r *R), opts ...Option) { |
||||
t.Helper() |
||||
r := &R{ |
||||
wrapped: t, |
||||
retryer: DefaultRetryer(), |
||||
} |
||||
|
||||
for _, opt := range opts { |
||||
opt(r) |
||||
} |
||||
|
||||
r.run(f) |
||||
} |
||||
|
||||
func RunWith(r Retryer, t TestingTB, f func(r *R)) { |
||||
t.Helper() |
||||
Run(t, f, WithRetryer(r)) |
||||
} |
||||
Loading…
Reference in new issue