Merge pull request #22404 from yujuhong/dump_mem

test/e2e/kubelet_perf.go

@@ -99,7 +99,7 @@ func runResourceTrackingTest(framework *Framework, podsPerNode int, nodeNames se
 	usageSummary, err := rm.GetLatest()
 	Expect(err).NotTo(HaveOccurred())
 	Logf("%s", rm.FormatResourceUsage(usageSummary))
-	verifyMemoryLimits(expectedMemory, usageSummary)
+	verifyMemoryLimits(framework.Client, expectedMemory, usageSummary)
 
 	cpuSummary := rm.GetCPUSummary()
 	Logf("%s", rm.FormatCPUSummary(cpuSummary))
@@ -109,7 +109,7 @@ func runResourceTrackingTest(framework *Framework, podsPerNode int, nodeNames se
 	DeleteRC(framework.Client, framework.Namespace.Name, rcName)
 }
 
-func verifyMemoryLimits(expected resourceUsagePerContainer, actual resourceUsagePerNode) {
+func verifyMemoryLimits(c *client.Client, expected resourceUsagePerContainer, actual resourceUsagePerNode) {
 	if expected == nil {
 		return
 	}
@@ -132,6 +132,12 @@ func verifyMemoryLimits(expected resourceUsagePerContainer, actual resourceUsage
 		}
 		if len(nodeErrs) > 0 {
 			errList = append(errList, fmt.Sprintf("node %v:\n %s", nodeName, strings.Join(nodeErrs, ", ")))
+			heapStats, err := getKubeletHeapStats(c, nodeName)
+			if err != nil {
+				Logf("Unable to get heap stats from %q", nodeName)
+			} else {
+				Logf("Heap stats on %q\n:%v", nodeName, heapStats)
+			}
 		}
 	}
 	if len(errList) > 0 {

test/e2e/kubelet_stats.go

@@ -407,6 +407,22 @@ func getKubeletMetricsThroughNode(nodeName string) (string, error) {
 	return string(body), nil
 }
 
+func getKubeletHeapStats(c *client.Client, nodeName string) (string, error) {
+	client, err := nodeProxyRequest(c, nodeName, "debug/pprof/heap")
+	if err != nil {
+		return "", err
+	}
+	raw, errRaw := client.Raw()
+	if errRaw != nil {
+		return "", err
+	}
+	stats := string(raw)
+	// Only dumping the runtime.MemStats numbers to avoid polluting the log.
+	numLines := 23
+	lines := strings.Split(stats, "\n")
+	return strings.Join(lines[len(lines)-numLines:], "\n"), nil
+}
+
 // GetKubeletPods retrieves the list of running pods on the kubelet. The pods
 // includes necessary information (e.g., UID, name, namespace for
 // pods/containers), but do not contain the full spec.