Merge pull request #48154 from krousey/upgrades

Automatic merge from submit-queue (batch tested with PRs 48214, 48154)

Adding a retry and traceroute to the master version checking

This is hitting a lot of connection refused errors in the e2e upgrade tests. We should make this more robust in case this is intermittent network errors. In the event of an error, attempt to log a traceroute to the master.

cc @kubernetes/sig-cluster-lifecycle-bugs @dchen1107 

#47379

test/e2e/framework/BUILD

@@ -128,6 +128,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",

test/e2e/framework/upgrade_util.go

@@ -18,9 +18,13 @@ package framework
 
 import (
 	"fmt"
+	"os/exec"
 	"path"
 	"strings"
+	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 )
 
@@ -36,10 +40,36 @@ func RealVersion(s string) (string, error) {
 	return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil
 }
 
+func traceRouteToMaster() {
+	path, err := exec.LookPath("traceroute")
+	if err != nil {
+		Logf("Could not find traceroute program")
+		return
+	}
+
+	cmd := exec.Command(path, "-I", GetMasterHost())
+	out, err := cmd.Output()
+	if len(out) != 0 {
+		Logf(string(out))
+	}
+	if exiterr, ok := err.(*exec.ExitError); err != nil && ok {
+		Logf("error while running traceroute: %s", exiterr.Stderr)
+	}
+}
+
 func CheckMasterVersion(c clientset.Interface, want string) error {
 	Logf("Checking master version")
-	v, err := c.Discovery().ServerVersion()
-	if err != nil {
+	var err error
+	var v *version.Info
+	waitErr := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
+		v, err = c.Discovery().ServerVersion()
+		if err != nil {
+			traceRouteToMaster()
+			return false, nil
+		}
+		return true, nil
+	})
+	if waitErr != nil {
 		return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err)
 	}
 	// We do prefix trimming and then matching because: