Merge pull request #26188 from brendandburns/test_main

hack/test-integration.sh

@@ -52,7 +52,7 @@ runTests() {
   # KUBE_RACE="-race"
   KUBE_GOFLAGS="-tags 'integration no-docker' " \
     KUBE_RACE="" \
-    KUBE_TIMEOUT="${KUBE_TIMEOUT}" \
+    KUBE_TEST_ARGS="--integration-timeout=${KUBE_TIMEOUT}"
     KUBE_TEST_API_VERSIONS="$1" \
     "${KUBE_ROOT}/hack/test-go.sh" test/integration
 

hack/verify-flags/known-flags.txt

@@ -206,6 +206,7 @@ insecure-bind-address
 insecure-port
 insecure-skip-tls-verify
 instance-name-prefix
+integration-timeout
 iptables-masquerade-bit
 iptables-sync-period
 ir-data-source
@@ -448,6 +449,7 @@ target-port
 tcp-services
 terminated-pod-gc-threshold
 test-timeout
+timeout-full-stack
 tls-cert-file
 tls-private-key-file
 to-version

test/integration/main_test.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+)
+
+var integrationTimeout = flag.Duration("integration-timeout", 10*time.Minute, "If non-zero timeout all tests after this amount of time.")
+var fullStack = flag.Bool("timeout-full-stack", false, "If true, don't filter stacks on timeout")
+var timer *time.Timer
+
+func printInterestingStacks() {
+	buf := make([]byte, 2<<20)
+	buf = buf[:runtime.Stack(buf, true)]
+	stacks := [][]string{}
+	for _, stack := range strings.Split(string(buf), "\n\n") {
+		lines := strings.Split(stack, "\n")
+		for _, line := range lines {
+			if strings.Contains(line, "src/k8s.io") || *fullStack {
+				stacks = append(stacks, lines)
+				break
+			}
+		}
+	}
+	for _, stack := range stacks {
+		for _, line := range stack {
+			fmt.Printf("%s\n", line)
+		}
+		fmt.Printf("\n")
+	}
+	return
+}
+
+func startAlarm() {
+	if *integrationTimeout > 0 {
+		timer = time.AfterFunc(*integrationTimeout, func() {
+			printInterestingStacks()
+			fmt.Printf("Integration tests timed out after %v\n", integrationTimeout.String())
+			os.Exit(1)
+		})
+	}
+}
+
+func stopAlarm() {
+	if timer != nil {
+		timer.Stop()
+	}
+}
+
+/**
+ * Adds a custom test main handler (see: TestMain in https://golang.org/pkg/testing/ for details)
+ * This custom does the standard timeout, but strips non-kubernetes stacks out of the stack trace.
+ * It filters things like:
+
+goroutine 466 [IO wait, 7 minutes]:
+net.runtime_pollWait(0x7fd74c4672c0, 0x72, 0xc821614000)
+    /usr/local/go/src/runtime/netpoll.go:160 +0x60
+net.(*pollDesc).Wait(0xc8215c21b0, 0x72, 0x0, 0x0)
+    /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
+net.(*pollDesc).WaitRead(0xc8215c21b0, 0x0, 0x0)
+    /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
+net.(*netFD).Read(0xc8215c2150, 0xc821614000, 0x1000, 0x1000, 0x0, 0x7fd74c491050, 0xc820014058)
+    /usr/local/go/src/net/fd_unix.go:250 +0x23a
+net.(*conn).Read(0xc820a5a090, 0xc821614000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
+    /usr/local/go/src/net/net.go:172 +0xe4
+net/http.noteEOFReader.Read(0x7fd74c465258, 0xc820a5a090, 0xc8215f0068, 0xc821614000, 0x1000, 0x1000, 0x405773, 0x0, 0x0)
+    /usr/local/go/src/net/http/transport.go:1687 +0x67
+net/http.(*noteEOFReader).Read(0xc8215ae1a0, 0xc821614000, 0x1000, 0x1000, 0xc82159ad1d, 0x0, 0x0)
+    <autogenerated>:284 +0xd0
+bufio.(*Reader).fill(0xc8202a2b40)
+    /usr/local/go/src/bufio/bufio.go:97 +0x1e9
+bufio.(*Reader).Peek(0xc8202a2b40, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
+    /usr/local/go/src/bufio/bufio.go:132 +0xcc
+net/http.(*persistConn).readLoop(0xc8215f0000)
+    /usr/local/go/src/net/http/transport.go:1073 +0x177
+created by net/http.(*Transport).dialConn
+    /usr/local/go/src/net/http/transport.go:857 +0x10a6
+
+  * See printInterestingStacks above for details.
+  * We may want to get even more aggressive in the future.
+*/
+// TestMain replaces the built-in golang Main for running tests. See https://golang.org/pkg/testing/
+func TestMain(m *testing.M) {
+	flag.Parse()
+	startAlarm()
+	v := m.Run()
+	stopAlarm()
+	os.Exit(v)
+}