diff --git a/test/e2e_node/gcloud/gcloud.go b/test/e2e_node/gcloud/gcloud.go index 6d5ed940c5..40a0b6dedf 100644 --- a/test/e2e_node/gcloud/gcloud.go +++ b/test/e2e_node/gcloud/gcloud.go @@ -136,6 +136,16 @@ func (gc *gCloudClientImpl) Run( go func() { // Create the tmp directory out, err := gc.Command("mkdir", "-p", tDir) + + // Work around for gcloud flakiness - TODO: debug why gcloud sometimes cannot find credentials for some hosts + // If there was an error about credentials, retry making the directory 6 times to see if it can be resolved + // This is to help debug if the credential issues are persistent for a given host on a given run, or transient + // And if downstream gcloud commands are also impacted + for i := 0; i < 6 && err != nil && strings.Contains(string(out), "does not have any valid credentials"); i++ { + glog.Errorf("mkdir failed on host %s due to credential issues, retrying in 5 seconds %v %s", gc.host, err, out) + time.Sleep(5 * time.Second) + out, err = gc.Command("mkdir", "-p", tDir) + } if err != nil { glog.Errorf("mkdir failed %v %s", err, out) h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}