Merge pull request #33278 from Crassirostris/gcl-e2e-test

cluster/gce/config-test.sh

@@ -122,7 +122,7 @@ KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 
 # Optional: Enable node logging.
 ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}"
-LOGGING_DESTINATION="${KUBE_LOGGING_DESTINATION:-elasticsearch}" # options: elasticsearch, gcp
+LOGGING_DESTINATION="${KUBE_LOGGING_DESTINATION:-gcp}" # options: elasticsearch, gcp
 
 # Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
 ENABLE_CLUSTER_LOGGING="${KUBE_ENABLE_CLUSTER_LOGGING:-true}"

hack/verify-flags/exceptions.txt

@@ -119,9 +119,9 @@ test/e2e/common/host_path.go:			fmt.Sprintf("--file_content_in_loop=%v", filePat
 test/e2e/common/host_path.go:			fmt.Sprintf("--file_content_in_loop=%v", filePathInReader),
 test/e2e/common/host_path.go:			fmt.Sprintf("--retry_time=%d", retryDuration),
 test/e2e/common/host_path.go:			fmt.Sprintf("--retry_time=%d", retryDuration),
-test/e2e/es_cluster_logging.go:		framework.Failf("No cluster_name field in Elasticsearch response: %v", esResponse)
-test/e2e/es_cluster_logging.go:	// Check to see if have a cluster_name field.
-test/e2e/es_cluster_logging.go:	clusterName, ok := esResponse["cluster_name"]
+test/e2e/cluster_logging_es.go:		return fmt.Errorf("No cluster_name field in Elasticsearch response: %v", esResponse)
+test/e2e/cluster_logging_es.go:	// Check to see if have a cluster_name field.
+test/e2e/cluster_logging_es.go:	clusterName, ok := esResponse["cluster_name"]
 test/e2e_node/container_manager_test.go:		return fmt.Errorf("expected pid %d's oom_score_adj to be %d; found %d", pid, expectedOOMScoreAdj, oomScore)
 test/e2e_node/container_manager_test.go:		return fmt.Errorf("expected pid %d's oom_score_adj to be < %d; found %d", pid, expectedMaxOOMScoreAdj, oomScore)
 test/e2e_node/container_manager_test.go:		return fmt.Errorf("expected pid %d's oom_score_adj to be >= %d; found %d", pid, expectedMinOOMScoreAdj, oomScore)

test/e2e/cluster_logging_gcl.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/util/json"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = framework.KubeDescribe("Cluster level logging using GCL", func() {
+	f := framework.NewDefaultFramework("gcl-logging")
+
+	BeforeEach(func() {
+		// TODO (crassirostris): Expand to GKE once the test is stable
+		framework.SkipUnlessProviderIs("gce")
+	})
+
+	It("should check that logs from containers are ingested in GCL", func() {
+		By("Running synthetic logger")
+		createSynthLogger(f, expectedLinesCount)
+		defer f.PodClient().Delete(synthLoggerPodName, &api.DeleteOptions{})
+		err := framework.WaitForPodSuccessInNamespace(f.Client, synthLoggerPodName, synthLoggerPodName, f.Namespace.Name)
+		framework.ExpectNoError(err, fmt.Sprintf("Should've successfully waited for pod %s to succeed", synthLoggerPodName))
+
+		By("Waiting for logs to ingest")
+		totalMissing := expectedLinesCount
+		for start := time.Now(); time.Since(start) < ingestionTimeout; time.Sleep(ingestionRetryDelay) {
+			var err error
+			totalMissing, err = getMissingLinesCountGcl(f, synthLoggerPodName, expectedLinesCount)
+			if err != nil {
+				framework.Logf("Failed to get missing lines count due to %v", err)
+				totalMissing = expectedLinesCount
+			} else if totalMissing > 0 {
+				framework.Logf("Still missing %d lines", totalMissing)
+			}
+
+			if totalMissing == 0 {
+				break
+			}
+		}
+
+		Expect(totalMissing).To(Equal(0), "Some log lines are still missing")
+	})
+})
+
+func getMissingLinesCountGcl(f *framework.Framework, podName string, expectedCount int) (int, error) {
+	gclFilter := fmt.Sprintf("resource.labels.pod_id:%s AND resource.labels.namespace_id:%s", podName, f.Namespace.Name)
+	entries, err := readFilteredEntriesFromGcl(gclFilter)
+	if err != nil {
+		return 0, err
+	}
+
+	occurrences := make(map[int]int)
+	for _, entry := range entries {
+		lineNumber, err := strconv.Atoi(strings.TrimSpace(entry))
+		if err != nil {
+			continue
+		}
+		if lineNumber < 0 || lineNumber >= expectedCount {
+			framework.Logf("Unexpected line number: %d", lineNumber)
+		} else {
+			// Duplicates are possible and fine, fluentd has at-least-once delivery
+			occurrences[lineNumber]++
+		}
+	}
+
+	return expectedCount - len(occurrences), nil
+}
+
+type LogEntry struct {
+	TextPayload string
+}
+
+// Since GCL API is not easily available from the outside of cluster
+// we use gcloud command to perform search with filter
+func readFilteredEntriesFromGcl(filter string) ([]string, error) {
+	framework.Logf("Reading entries from GCL with filter '%v'", filter)
+	argList := []string{"beta",
+		"logging",
+		"read",
+		filter,
+		"--format",
+		"json",
+		"--project",
+		framework.TestContext.CloudConfig.ProjectID,
+	}
+	output, err := exec.Command("gcloud", argList...).CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+
+	var entries []*LogEntry
+	if err = json.Unmarshal(output, &entries); err != nil {
+		return nil, err
+	}
+	framework.Logf("Read %d entries from GCL", len(entries))
+
+	var result []string
+	for _, entry := range entries {
+		if entry.TextPayload != "" {
+			result = append(result, entry.TextPayload)
+		}
+	}
+
+	return result, nil
+}

test/e2e/cluster_logging_utils.go

+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	// ingestionTimeout is how long to keep retrying to wait for all the
+	// logs to be ingested.
+	ingestionTimeout = 10 * time.Minute
+	// ingestionRetryDelay is how long test should wait between
+	// two attempts to check for ingestion
+	ingestionRetryDelay = 25 * time.Second
+
+	synthLoggerPodName = "synthlogger"
+
+	// expectedLinesCount is the number of log lines emitted (and checked) for each synthetic logging pod.
+	expectedLinesCount = 100
+)
+
+func createSynthLogger(f *framework.Framework, linesCount int) {
+	f.PodClient().Create(&api.Pod{
+		ObjectMeta: api.ObjectMeta{
+			Name:      synthLoggerPodName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: api.PodSpec{
+			Containers: []api.Container{
+				{
+					Name:  synthLoggerPodName,
+					Image: "gcr.io/google_containers/busybox:1.24",
+					// notice: the subshell syntax is escaped with `$$`
+					Command: []string{"/bin/sh", "-c", fmt.Sprintf("i=0; while [ $i -lt %d ]; do echo $i; i=`expr $i + 1`; done", linesCount)},
+				},
+			},
+		},
+	})
+}

test/e2e/kibana_logging.go

@@ -27,7 +27,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = framework.KubeDescribe("Kibana Logging Instances Is Alive", func() {
+var _ = framework.KubeDescribe("Kibana Logging Instances Is Alive [Feature:Elasticsearch]", func() {
 	f := framework.NewDefaultFramework("kibana-logging")
 
 	BeforeEach(func() {