Merge pull request #48154 from krousey/upgrades

Automatic merge from submit-queue (batch tested with PRs 48214, 48154) Adding a retry and traceroute to the master version checking This is hitting a lot of connection refused errors in the e2e upgrade tests. We should make this more robust in case this is intermittent network errors. In the event of an error, attempt to log a traceroute to the master. cc @kubernetes/sig-cluster-lifecycle-bugs @dchen1107 #47379

Merge pull request #48154 from krousey/upgrades
b353792f · Kubernetes Submit Queue · GitHub · bcb44655 · faf7dff7 · b353792f
Commit b353792f authored Jun 28, 2017 by Kubernetes Submit Queue Committed by GitHub Jun 28, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 33 additions and 2 deletions

BUILD test/e2e/framework/BUILD +1 -0

upgrade_util.go test/e2e/framework/upgrade_util.go +32 -2

No files found.
--- a/test/e2e/framework/BUILD
+++ b/test/e2e/framework/BUILD
@@ -128,6 +128,7 @@ go_library(
        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
        "//vendor/k8s.io/client-go/discovery:go_default_library",
        "//vendor/k8s.io/client-go/dynamic:go_default_library",

--- a/test/e2e/framework/upgrade_util.go
+++ b/test/e2e/framework/upgrade_util.go
@@ -18,9 +18,13 @@ package framework

 import (
 	"fmt"
+	"os/exec"
 	"path"
 	"strings"
+	"time"

+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 )

@@ -36,10 +40,36 @@ func RealVersion(s string) (string, error) {
 	return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil
 }

+func traceRouteToMaster() {
+	path, err := exec.LookPath("traceroute")
+	if err != nil {
+		Logf("Could not find traceroute program")
+		return
+	}
+
+	cmd := exec.Command(path, "-I", GetMasterHost())
+	out, err := cmd.Output()
+	if len(out) != 0 {
+		Logf(string(out))
+	}
+	if exiterr, ok := err.(*exec.ExitError); err != nil && ok {
+		Logf("error while running traceroute: %s", exiterr.Stderr)
+	}
+}
+
 func CheckMasterVersion(c clientset.Interface, want string) error {
 	Logf("Checking master version")
-	v, err := c.Discovery().ServerVersion()
-	if err != nil {
+	var err error
+	var v *version.Info
+	waitErr := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
+		v, err = c.Discovery().ServerVersion()
+		if err != nil {
+			traceRouteToMaster()
+			return false, nil
+		}
+		return true, nil
+	})
+	if waitErr != nil {
 		return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err)
 	}
 	// We do prefix trimming and then matching because: