github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

node e2e tests - retry fkaky failures caused by gcloud authentication

pull/6/head
Phillip Wittrock 2016-01-07 11:26:08 -08:00
parent 9ac13f665d
commit aebc1bcc92
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test/e2e_node/gcloud/gcloud.go
View File

@ -136,6 +136,16 @@ func (gc *gCloudClientImpl) Run(
go func() { go func() {
// Create the tmp directory // Create the tmp directory
out, err := gc.Command("mkdir", "-p", tDir) out, err := gc.Command("mkdir", "-p", tDir)
// Work around for gcloud flakiness - TODO: debug why gcloud sometimes cannot find credentials for some hosts
// If there was an error about credentials, retry making the directory 6 times to see if it can be resolved
// This is to help debug if the credential issues are persistent for a given host on a given run, or transient
// And if downstream gcloud commands are also impacted
for i := 0; i < 6 && err != nil && strings.Contains(string(out), "does not have any valid credentials"); i++ {
glog.Errorf("mkdir failed on host %s due to credential issues, retrying in 5 seconds %v %s", gc.host, err, out)
time.Sleep(5 * time.Second)
out, err = gc.Command("mkdir", "-p", tDir)
}
if err != nil { if err != nil {
glog.Errorf("mkdir failed %v %s", err, out) glog.Errorf("mkdir failed %v %s", err, out)
h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)} h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}