From 1e7384c99753d830b3886d1386a6ae3d9c3ac08f Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 27 Jun 2017 12:10:43 -0700 Subject: [PATCH 1/2] Adding a retry to the master version checking --- test/e2e/framework/BUILD | 1 + test/e2e/framework/upgrade_util.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/e2e/framework/BUILD b/test/e2e/framework/BUILD index 4eddbf5995..e1b7222a20 100644 --- a/test/e2e/framework/BUILD +++ b/test/e2e/framework/BUILD @@ -128,6 +128,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/version:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library", diff --git a/test/e2e/framework/upgrade_util.go b/test/e2e/framework/upgrade_util.go index 8e5f4609e0..ae8a3c98d3 100644 --- a/test/e2e/framework/upgrade_util.go +++ b/test/e2e/framework/upgrade_util.go @@ -20,7 +20,10 @@ import ( "fmt" "path" "strings" + "time" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/version" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" ) @@ -38,8 +41,13 @@ func RealVersion(s string) (string, error) { func CheckMasterVersion(c clientset.Interface, want string) error { Logf("Checking master version") - v, err := c.Discovery().ServerVersion() - if err != nil { + var err error + var v *version.Info + waitErr := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { + v, err = c.Discovery().ServerVersion() + return err != nil, nil + }) + if waitErr != nil { return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err) } // We do prefix trimming and then matching because: From faf7dff7630156ae3f5af6ba973c37917b205049 Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 28 Jun 2017 12:25:20 -0700 Subject: [PATCH 2/2] Add traceroute logging on connection failure --- test/e2e/framework/upgrade_util.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/e2e/framework/upgrade_util.go b/test/e2e/framework/upgrade_util.go index ae8a3c98d3..fa12401289 100644 --- a/test/e2e/framework/upgrade_util.go +++ b/test/e2e/framework/upgrade_util.go @@ -18,6 +18,7 @@ package framework import ( "fmt" + "os/exec" "path" "strings" "time" @@ -39,13 +40,34 @@ func RealVersion(s string) (string, error) { return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil } +func traceRouteToMaster() { + path, err := exec.LookPath("traceroute") + if err != nil { + Logf("Could not find traceroute program") + return + } + + cmd := exec.Command(path, "-I", GetMasterHost()) + out, err := cmd.Output() + if len(out) != 0 { + Logf(string(out)) + } + if exiterr, ok := err.(*exec.ExitError); err != nil && ok { + Logf("error while running traceroute: %s", exiterr.Stderr) + } +} + func CheckMasterVersion(c clientset.Interface, want string) error { Logf("Checking master version") var err error var v *version.Info waitErr := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { v, err = c.Discovery().ServerVersion() - return err != nil, nil + if err != nil { + traceRouteToMaster() + return false, nil + } + return true, nil }) if waitErr != nil { return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err)