From feaa3aba82520437503a4602c3badd2b84e5d15f Mon Sep 17 00:00:00 2001 From: Phillip Wittrock Date: Thu, 26 May 2016 13:01:49 -0700 Subject: [PATCH] Retry concurrency exceptions in e2e/kubectl.go. Fixes #26138 --- pkg/registry/generic/registry/store.go | 4 +++- test/e2e/kubectl.go | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pkg/registry/generic/registry/store.go b/pkg/registry/generic/registry/store.go index 2a4979e7b7..776ba5a5b1 100644 --- a/pkg/registry/generic/registry/store.go +++ b/pkg/registry/generic/registry/store.go @@ -117,6 +117,8 @@ type Store struct { Storage storage.Interface } +const OptimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again" + // NamespaceKeyRootFunc is the default function for constructing storage paths to resource directories enforcing namespace rules. func NamespaceKeyRootFunc(ctx api.Context, prefix string) string { key := prefix @@ -315,7 +317,7 @@ func (e *Store) Update(ctx api.Context, name string, objInfo rest.UpdatedObjectI return nil, nil, kubeerr.NewInvalid(qualifiedKind, name, fieldErrList) } if newVersion != version { - return nil, nil, kubeerr.NewConflict(e.QualifiedResource, name, fmt.Errorf("the object has been modified; please apply your changes to the latest version and try again")) + return nil, nil, kubeerr.NewConflict(e.QualifiedResource, name, fmt.Errorf(OptimisticLockErrorMsg)) } } if err := rest.BeforeUpdate(e.UpdateStrategy, ctx, obj, existing); err != nil { diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 7c85f8202d..1e1116e3a2 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -49,6 +49,7 @@ import ( "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/registry/generic/registry" pkgutil "k8s.io/kubernetes/pkg/util" utilnet "k8s.io/kubernetes/pkg/util/net" "k8s.io/kubernetes/pkg/util/wait" @@ -138,6 +139,23 @@ func readTestFileOrDie(file string) []byte { return framework.ReadOrDie(path.Join(kubeCtlManifestPath, file)) } +func runKubectlRetryOrDie(args ...string) string { + var err error + var output string + for i := 0; i < 3; i++ { + output, err = framework.RunKubectl(args...) + if err == nil || !strings.Contains(err.Error(), registry.OptimisticLockErrorMsg) { + break + } + time.Sleep(time.Second) + } + // Expect no errors to be present after retries are finished + // Copied from framework #ExecOrDie + framework.Logf("stdout: %q", output) + Expect(err).NotTo(HaveOccurred()) + return output +} + var _ = framework.KubeDescribe("Kubectl client", func() { defer GinkgoRecover() f := framework.NewDefaultFramework("kubectl") @@ -907,7 +925,8 @@ var _ = framework.KubeDescribe("Kubectl client", func() { framework.WaitForRCToStabilize(c, ns, rcName, framework.PodStartTimeout) By("rolling-update to same image controller") - framework.RunKubectlOrDie("rolling-update", rcName, "--update-period=1s", "--image="+nginxImage, "--image-pull-policy="+string(api.PullIfNotPresent), nsFlag) + + runKubectlRetryOrDie("rolling-update", rcName, "--update-period=1s", "--image="+nginxImage, "--image-pull-policy="+string(api.PullIfNotPresent), nsFlag) framework.ValidateController(c, nginxImage, 1, rcName, "run="+rcName, noOpValidatorFn, ns) }) })