Switch the client event recorder from exponential backoff to one random

sleep on the first failed request followed by a constant amount on all
subsequent consecutive failed requests.

pkg/client/record/event.go

@@ -18,7 +18,7 @@ package record
 
 import (
 	"fmt"
-	"math"
+	"math/rand"
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
@@ -26,19 +26,14 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
-	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"
 
 	"github.com/golang/glog"
 )
 
-const maxTriesPerEvent = 10
+const maxTriesPerEvent = 12
 
-var (
-	minSleep   = float64(1 * time.Second)
-	maxSleep   = float64(15 * time.Second)
-	backoffExp = 1.5
-)
+var sleepDuration = time.Duration(10 * time.Second)
 
 // EventRecorder knows how to store events (client.Client implements it.)
 // EventRecorder must respect the namespace that will be embedded in 'event'.
@@ -70,9 +65,13 @@ func StartRecording(recorder EventRecorder, source api.EventSource) watch.Interf
 				glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event)
 				break
 			}
-			sleepDuration := time.Duration(
-				math.Min(maxSleep, minSleep*math.Pow(backoffExp, float64(tries-1))))
-			time.Sleep(wait.Jitter(sleepDuration, 0.5))
+			// Randomize the first sleep so that various clients won't all be
+			// synced up if the master goes down.
+			if tries == 1 {
+				time.Sleep(time.Duration(float64(sleepDuration) * rand.Float64()))
+			} else {
+				time.Sleep(sleepDuration)
+			}
 		}
 	})
 }

pkg/client/record/event_test.go

@@ -22,7 +22,6 @@ import (
 	"strconv"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
@@ -33,8 +32,7 @@ import (
 
 func init() {
 	// Don't bother sleeping between retries.
-	minSleep = 0
-	maxSleep = 0
+	sleepDuration = 0
 }
 
 type testEventRecorder struct {
@@ -195,12 +193,12 @@ func TestWriteEventError(t *testing.T) {
 		},
 		"retry1": {
 			timesToSendError: 1000,
-			attemptsWanted:   10,
+			attemptsWanted:   12,
 			err:              &errors.UnexpectedObjectError{},
 		},
 		"retry2": {
 			timesToSendError: 1000,
-			attemptsWanted:   10,
+			attemptsWanted:   12,
 			err:              fmt.Errorf("A weird error"),
 		},
 		"succeedEventually": {