Launch Elasticsearch and Kibana automatically

contrib/logging/fluentd-ek/es-pod.yml → cluster/addons/fluentd-elasticsearch/es-controller.yaml.in

 apiVersion: v1beta1
-kind: Pod
-id: elasticsearch-pod
+kind: ReplicationController
+id: elasticsearch-logging-controller
 desiredState:
-  manifest:
-    version: v1beta1
-    id: es
-    containers:
-      - name: elasticsearch
-        image: dockerfile/elasticsearch
-        ports:
-          - name: es-port
-            containerPort: 9200
-          - name: es-transport-port
-            containerPort: 9300
-        volumeMounts:
+  replicas: {ELASTICSEARCH_LOGGING_REPLICAS}
+  replicaSelector:
+    name: elasticsearch-logging
+  podTemplate:
+    desiredState:
+      manifest:
+        version: v1beta1
+        id: es-log-ingestion
+        containers:
+          - name: elasticsearch-logging
+            image: dockerfile/elasticsearch
+            ports:
+              - name: es-port
+                containerPort: 9200
+              - name: es-transport-port
+                containerPort: 9300
+            volumeMounts:
+              - name: es-persistent-storage
+                mountPath: /data
+        volumes:
           - name: es-persistent-storage
-            mountPath: /data
-    volumes:
-      - name: es-persistent-storage
-        source:
-          emptyDir: {}
+            source:
+              emptyDir: {}
+    labels:
+      name: elasticsearch-logging
 labels:
-  app: elasticsearch
- 
+  name: elasticsearch-logging
+ 
\ No newline at end of file

contrib/logging/fluentd-ek/es-service.yml → cluster/addons/fluentd-elasticsearch/es-service.yaml

 apiVersion: v1beta1
 kind: Service
-id: elasticsearch
+id: elasticsearch-logging
 containerPort: es-port
 port: 9200
 selector:
-  app: elasticsearch
+  name: elasticsearch-logging
 createExternalLoadBalancer: true

contrib/logging/fluentd-es-image/Makefile → cluster/addons/fluentd-elasticsearch/fluentd-es-image/Makefile

 .PHONY:	build push
 
-TAG = latest
+TAG = 1.0
 
 build:	
 	sudo docker build -t kubernetes/fluentd-elasticsearch:$(TAG) .

contrib/logging/fluentd-es-image/td-agent.conf → cluster/addons/fluentd-elasticsearch/fluentd-es-image/td-agent.conf

@@ -47,7 +47,7 @@
    type elasticsearch
    log_level info
    include_tag_key true
-   host elasticsearch.default
+   host elasticsearch-logging.default
    port 9200
    logstash_format true
    flush_interval 5s
@@ -69,7 +69,7 @@
    type elasticsearch
    log_level info
    include_tag_key true
-   host elasticsearch.default
+   host elasticsearch-logging.default
    port 9200
    logstash_format true
    flush_interval 5s

cluster/addons/fluentd-elasticsearch/kibana-controller.yaml

+apiVersion: v1beta1
+kind:  ReplicationController
+id: kibana-logging-controller
+desiredState:
+  replicas: 1
+  replicaSelector:
+    name: kibana-logging
+  podTemplate:
+    desiredState:
+      manifest:
+        version: v1beta1
+        id: kibana-viewer
+        containers:
+          - name: kibana-logging
+            image: kubernetes/kibana:1.0
+            ports:
+              - name: kibana-port
+                containerPort: 80
+    labels:
+      name: kibana-logging
+labels:
+  name: kibana-logging

cluster/addons/fluentd-elasticsearch/kibana-image/Makefile

+.PHONY:	build push
+
+TAG = 1.0
+
+build:	
+	docker build -t kubernetes/kibana:$(TAG) .
+
+push:	
+	docker push kubernetes/kibana:$(TAG)

contrib/logging/kibana-image/run_kibana_nginx.sh → cluster/addons/fluentd-elasticsearch/kibana-image/run_kibana_nginx.sh

@@ -26,16 +26,16 @@
 #    PROXY_PORT is set to 9200 because Elasticsearch is running on the
 #    same name as Kibana. If KIBANA_IP is the external IP address of
 #    the Kubernetes Kibna service then all requests to:
-#       KIBANA_SERVICE:$ES_PORT/elasticsearch/XXX
+#       KIBANA_LOGGING_SERVICE:$ES_PORT/elasticsearch/XXX
 #    are proxied to:
 #       http://127.0.0.1:9200/XXX
 # 2. Elasticsearch and Kibana are run in separate pods and Elasticsearch
 #    has an IP and port exposed via a Kubernetes service. In this case
 #    the Elasticsearch service *must* be called 'elasticsearch' and then
 #    all requests sent to:
-#       KIBANA_SERVICE:$ES_PORT/elasticsearch/XXX
+#       KIBANA_LOGGING_SERVICE:$ES_PORT/elasticsearch/XXX
 #    are proxied to:
-#       http://$ELASTICSEARCH_SERVICE_HOST:$ELASTICSEARCH_SERVICE_PORT:9200/XXX
+#       http://$ELASTICSEARCH_LOGGING_SERVICE_HOST:$ELASTICSEARCH_LOGGING_SERVICE_PORT:9200/XXX
 # The proxy configuration occurs in a location block of the nginx configuration
 # file /etc/nginx/sites-available/default.
 
@@ -57,9 +57,9 @@ echo ES_PORT=$ES_PORT
 # code in the configuration for nginx. If a Kubernetes Elasticsearch
 # service called 'elasticsearch' is defined, use that. Otherwise, use
 # a local instance of Elasticsearch on port 9200.
-PROXY_HOST=${ELASTICSEARCH_SERVICE_HOST:-127.0.0.1}
+PROXY_HOST=${ELASTICSEARCH_LOGGING_SERVICE_HOST:-127.0.0.1}
 echo PROXY_HOST=${PROXY_HOST}
-PROXY_PORT=${ELASTICSEARCH_SERVICE_PORT:-9200}
+PROXY_PORT=${ELASTICSEARCH_SERVICE_LOGGING_PORT:-9200}
 echo PROXY_PORT=${PROXY_PORT}
 # Test the connection to Elasticsearch
 echo "Running curl http://${PROXY_HOST}:${PROXY_PORT}"

contrib/logging/fluentd-ek/kibana-service.yml → cluster/addons/fluentd-elasticsearch/kibana-service.yaml

 apiVersion: v1beta1
 kind: Service
-id: kibana
+id: kibana-logging
 containerPort: kibana-port
 port: 5601
 selector:
-  app: kibana-viewer
+  name: kibana-logging
 createExternalLoadBalancer: true
- 
\ No newline at end of file
+ 

cluster/addons/fluentd-elasticsearch/logging-demo/Makefile

+# Makefile for launching syntheitc logging sources (any platform)
+# and for reporting the forwarding rules for the
+# Elasticsearch and Kibana pods for the GCE platform.
+
+
+.PHONY: up down logger-up logger-down logger10-up logger10-downget net
+
+KUBECTL=../../../kubectl.sh
+
+up:	logger-up logger10-up
+
+down:	logger-down logger10-down
+
+
+logger-up:
+	-${KUBECTL} create -f synthetic_0_25lps.yaml
+
+logger-down:
+	-${KUBECTL} delete pods synthetic-logger-0.25lps-pod
+
+logger10-up:
+	-${KUBECTL} create -f synthetic_10lps.yaml
+
+logger10-down:
+	-${KUBECTL} delete pods synthetic-logger-10lps-pod
+
+get:	
+	${KUBECTL} get pods
+	${KUBECTL} get replicationControllers
+	${KUBECTL} get services
+
+net:	
+	gcloud compute forwarding-rules describe elasticsearch-logging
+	gcloud compute forwarding-rules describe kibana-logging

cluster/addons/fluentd-elasticsearch/logging-demo/README.md

+# Elasticsearch/Kibana Logging Demonstration
+This directory contains two pod specifications which can be used as synthetic
+loggig sources. The pod specification in [synthetic_0_25lps.yaml](synthetic_0_25lps.yaml)
+describes a pod that just emits a log message once every 4 seconds:
+```
+# This pod specification creates an instance of a synthetic logger. The logger
+# is simply a program that writes out the hostname of the pod, a count which increments
+# by one on each iteration (to help notice missing log enteries) and the date using
+# a long format (RFC-3339) to nano-second precision. This program logs at a frequency
+# of 0.25 lines per second. The shellscript program is given directly to bash as -c argument
+# and could have been written out as:
+#   i="0"
+#   while true
+#   do
+#     echo -n "`hostname`: $i: "
+#     date --rfc-3339 ns
+#     sleep 4
+#     i=$[$i+1]
+#   done
+
+apiVersion: v1beta1
+kind: Pod
+id: synthetic-logger-0.25lps-pod
+desiredState:
+  manifest:
+    version: v1beta1
+    id: synth-logger-0.25lps
+    containers:
+      - name: synth-lgr
+        image: ubuntu:14.04
+        command: ["bash", "-c", "i=\"0\"; while true; do echo -n \"`hostname`: $i: \"; date --rfc-3339 ns; sleep 4; i=$[$i+1]; done"]
+labels:
+  name: synth-logging-source
+```
+
+The other YAML file [synthetic_10lps.yaml](synthetic_10lps.yaml) specifies a similar synthetic logger that emits 10 log messages every second. To run both synthetic loggers:
+```
+$ make up
+../../../kubectl.sh create -f synthetic_0_25lps.yaml
+Running: ../../../cluster/../cluster/gce/../../_output/dockerized/bin/linux/amd64/kubectl create -f synthetic_0_25lps.yaml
+synthetic-logger-0.25lps-pod
+../../../kubectl.sh create -f synthetic_10lps.yaml
+Running: ../../../cluster/../cluster/gce/../../_output/dockerized/bin/linux/amd64/kubectl create -f synthetic_10lps.yaml
+synthetic-logger-10lps-pod
+
+```
+
+Visiting the Kibana dashboard should make it clear that logs are being collected from the two synthetic loggers:
+![Synthetic loggers](synth-logger.png)
+
+You can report the running pods, replication controllers and services with another Makefile rule:
+```
+$ make get
+../../../kubectl.sh get pods
+Running: ../../../../cluster/gce/../../_output/dockerized/bin/linux/amd64/kubectl get pods
+POD                                    CONTAINER(S)            IMAGE(S)                           HOST                                                            LABELS                       STATUS
+7e1c7ce6-9764-11e4-898c-42010af03582   kibana-logging          kubernetes/kibana                  kubernetes-minion-3.c.kubernetes-elk.internal/130.211.129.169   name=kibana-logging          Running
+synthetic-logger-0.25lps-pod           synth-lgr               ubuntu:14.04                       kubernetes-minion-2.c.kubernetes-elk.internal/146.148.41.87     name=synth-logging-source    Running
+synthetic-logger-10lps-pod             synth-lgr               ubuntu:14.04                       kubernetes-minion-1.c.kubernetes-elk.internal/146.148.42.44     name=synth-logging-source    Running
+influx-grafana                         influxdb                kubernetes/heapster_influxdb       kubernetes-minion-3.c.kubernetes-elk.internal/130.211.129.169   name=influxdb                Running
+                                       grafana                 kubernetes/heapster_grafana                                                                                                     
+                                       elasticsearch           dockerfile/elasticsearch                                                                                                        
+heapster                               heapster                kubernetes/heapster                kubernetes-minion-2.c.kubernetes-elk.internal/146.148.41.87     name=heapster                Running
+67cfcb1f-9764-11e4-898c-42010af03582   etcd                    quay.io/coreos/etcd:latest         kubernetes-minion-3.c.kubernetes-elk.internal/130.211.129.169   k8s-app=skydns               Running
+                                       kube2sky                kubernetes/kube2sky:1.0                                                                                                         
+                                       skydns                  kubernetes/skydns:2014-12-23-001                                                                                                
+6ba20338-9764-11e4-898c-42010af03582   elasticsearch-logging   dockerfile/elasticsearch           kubernetes-minion-3.c.kubernetes-elk.internal/130.211.129.169   name=elasticsearch-logging   Running
+../../../cluster/kubectl.sh get replicationControllers
+Running: ../../../cluster/../cluster/gce/../../_output/dockerized/bin/linux/amd64/kubectl get replicationControllers
+CONTROLLER                         CONTAINER(S)            IMAGE(S)                           SELECTOR                     REPLICAS
+skydns                             etcd                    quay.io/coreos/etcd:latest         k8s-app=skydns               1
+                                   kube2sky                kubernetes/kube2sky:1.0                                         
+                                   skydns                  kubernetes/skydns:2014-12-23-001                                
+elasticsearch-logging-controller   elasticsearch-logging   dockerfile/elasticsearch           name=elasticsearch-logging   1
+kibana-logging-controller          kibana-logging          kubernetes/kibana                  name=kibana-logging          1
+../../.../kubectl.sh get services
+Running: ../../../cluster/../cluster/gce/../../_output/dockerized/bin/linux/amd64/kubectl get services
+NAME                    LABELS                                    SELECTOR                     IP                  PORT
+kubernetes-ro           component=apiserver,provider=kubernetes   <none>                       10.0.83.3           80
+kubernetes              component=apiserver,provider=kubernetes   <none>                       10.0.79.4           443
+influx-master           <none>                                    name=influxdb                10.0.232.223        8085
+skydns                  k8s-app=skydns                            k8s-app=skydns               10.0.0.10           53
+elasticsearch-logging   <none>                                    name=elasticsearch-logging   10.0.25.103         9200
+kibana-logging          <none>                                    name=kibana-logging          10.0.208.114        5601
+
+```
+On the GCE provider you can also obtain the external IP addresses of the Elasticsearch and Kibana services:
+```
+$ make net
+IPAddress: 130.211.120.118
+IPProtocol: TCP
+creationTimestamp: '2015-01-08T10:30:34.210-08:00'
+id: '12815488049392139704'
+kind: compute#forwardingRule
+name: elasticsearch-logging
+portRange: 9200-9200
+region: https://www.googleapis.com/compute/v1/projects/kubernetes-elk/regions/us-central1
+selfLink: https://www.googleapis.com/compute/v1/projects/kubernetes-elk/regions/us-central1/forwardingRules/elasticsearch-logging
+target: https://www.googleapis.com/compute/v1/projects/kubernetes-elk/regions/us-central1/targetPools/elasticsearch-logging
+gcloud compute forwarding-rules describe kibana-logging
+IPAddress: 146.148.40.158
+IPProtocol: TCP
+creationTimestamp: '2015-01-08T10:31:05.715-08:00'
+id: '2755171906970792849'
+kind: compute#forwardingRule
+name: kibana-logging
+portRange: 5601-5601
+region: https://www.googleapis.com/compute/v1/projects/kubernetes-elk/regions/us-central1
+selfLink: https://www.googleapis.com/compute/v1/projects/kubernetes-elk/regions/us-central1/forwardingRules/kibana-logging
+target: https://www.googleapis.com/compute/v1/projects/kubernetes-elk/regions/us-central1/targetPools/kibana-logging
+```

cluster/aws/config-default.sh

@@ -36,12 +36,14 @@ MINION_SCOPES=""
 POLL_SLEEP_INTERVAL=3
 PORTAL_NET="10.0.0.0/16"
 
-# Optional: Install node monitoring.
-ENABLE_NODE_MONITORING=true
-
 # Optional: Install node logging
-ENABLE_NODE_LOGGING=true
+ENABLE_NODE_LOGGING=false
 LOGGING_DESTINATION=elasticsearch # options: elasticsearch, gcp
+
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1
+
 IAM_PROFILE="kubernetes"
 LOG="/dev/null"
 

cluster/aws/util.sh

@@ -497,3 +497,11 @@ function kube-down {
   $AWS_CMD delete-route --route-table-id $route_table_id --destination-cidr-block 0.0.0.0/0 > $LOG
   $AWS_CMD delete-vpc --vpc-id $vpc_id > $LOG
 }
+
+function setup-logging {
+  echo "TODO: setup logging"
+}
+
+function teardown-logging {
+  echo "TODO: teardown logging"
+}
\ No newline at end of file

cluster/azure/config-default.sh

@@ -36,3 +36,11 @@ MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24"))
 MINION_SCOPES=""
 
 PORTAL_NET="10.250.0.0/16"
+
+# Optional: Install node logging
+ENABLE_NODE_LOGGING=false
+LOGGING_DESTINATION=elasticsearch # options: elasticsearch, gcp
+
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1

cluster/azure/util.sh

@@ -565,3 +565,11 @@ function setup-monitoring {
 function teardown-monitoring {
     echo "not implemented"  >/dev/null
 }
+
+function setup-logging {
+  echo "TODO: setup logging"
+}
+
+function teardown-logging {
+  echo "TODO: teardown logging"
+}
\ No newline at end of file

cluster/gce/config-default.sh

@@ -54,6 +54,10 @@ ENABLE_DOCKER_REGISTRY_CACHE=true
 ENABLE_NODE_LOGGING=true
 LOGGING_DESTINATION=elasticsearch # options: elasticsearch, gcp
 
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=true
+ELASTICSEARCH_LOGGING_REPLICAS=1
+
 # Don't require https for registries in our local RFC1918 network
 EXTRA_DOCKER_OPTS="--insecure-registry 10.0.0.0/8"
 

cluster/gce/util.sh

@@ -348,14 +348,14 @@ function kube-up {
   local htpasswd
   htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
 
-  if ! gcloud compute networks describe "${NETWORK}" &>/dev/null; then
+  if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then
     echo "Creating new network: ${NETWORK}"
     # The network needs to be created synchronously or we have a race. The
     # firewalls can be added concurrent with instance creation.
-    gcloud compute networks create "${NETWORK}" --range "10.240.0.0/16"
+    gcloud compute networks create --project "${PROJECT}" "${NETWORK}" --range "10.240.0.0/16"
   fi
 
-  if ! gcloud compute firewall-rules describe "${NETWORK}-default-internal" &>/dev/null; then
+  if ! gcloud compute firewall-rules --project "${PROJECT}" describe "${NETWORK}-default-internal" &>/dev/null; then
     gcloud compute firewall-rules create "${NETWORK}-default-internal" \
       --project "${PROJECT}" \
       --network "${NETWORK}" \
@@ -363,7 +363,7 @@ function kube-up {
       --allow "tcp:1-65535" "udp:1-65535" "icmp" &
   fi
 
-  if ! gcloud compute firewall-rules describe "${NETWORK}-default-ssh" &>/dev/null; then
+  if ! gcloud compute firewall-rules describe --project "${PROJECT}" "${NETWORK}-default-ssh" &>/dev/null; then
     gcloud compute firewall-rules create "${NETWORK}-default-ssh" \
       --project "${PROJECT}" \
       --network "${NETWORK}" \
@@ -718,7 +718,7 @@ function test-teardown {
 function ssh-to-node {
   local node="$1"
   local cmd="$2"
-  gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --zone="${ZONE}" "${node}" --command "${cmd}"
+  gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}"
 }
 
 # Restart the kube-proxy on a node ($1)
@@ -732,7 +732,7 @@ function setup-monitoring {
     echo "Setting up cluster monitoring using Heapster."
 
     detect-project
-    if ! gcloud compute firewall-rules describe monitoring-heapster &>/dev/null; then
+    if ! gcloud compute firewall-rules --project "{$PROJECT}" describe monitoring-heapster &>/dev/null; then
       if ! gcloud compute firewall-rules create monitoring-heapster \
           --project "${PROJECT}" \
           --target-tags="${MINION_TAG}" \
@@ -774,7 +774,7 @@ function teardown-monitoring {
     "${kubectl}" delete pods heapster &> /dev/null || true
     "${kubectl}" delete pods influx-grafana &> /dev/null || true
     "${kubectl}" delete services influx-master &> /dev/null || true
-    if gcloud compute firewall-rules describe monitoring-heapster &> /dev/null; then
+    if gcloud compute firewall-rules describe --project "${PROJECT}" monitoring-heapster &> /dev/null; then
       gcloud compute firewall-rules delete \
           --project "${PROJECT}" \
           --quiet \
@@ -783,6 +783,48 @@ function teardown-monitoring {
   fi
 }
 
+function setup-logging {
+  # If logging with Fluentd to Elasticsearch is enabled then create pods
+  # and services for Elasticsearch (for ingesting logs) and Kibana (for
+  # viewing logs).
+  if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]] && \
+     [[ "${LOGGING_DESTINATION-}" == "elasticsearch" ]] && \
+     [[ "${ENABLE_CLUSTER_LOGGING-}" == "true" ]]; then
+    local -r kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+    if sed -e "s/{ELASTICSEARCH_LOGGING_REPLICAS}/${ELASTICSEARCH_LOGGING_REPLICAS}/g" \
+              "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in | \
+              "${kubectl}" create -f - &> /dev/null && \
+       "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/es-service.yaml &> /dev/null && \
+       "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml &> /dev/null && \
+       "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/kibana-service.yaml &> /dev/null; then
+      gcloud compute firewall-rules create fluentd-elasticsearch-logging --project "${PROJECT}" \
+             --allow tcp:5601 tcp:9200 tcp:9300 --target-tags "${INSTANCE_PREFIX}"-minion || true
+      local -r region="${ZONE::-2}"
+      local -r es_ip=$(gcloud compute forwarding-rules --project "${PROJECT}" describe --region "${region}" elasticsearch-logging | grep IPAddress | awk '{print $2}')
+      local -r kibana_ip=$(gcloud compute forwarding-rules --project "${PROJECT}" describe --region "${region}" kibana-logging | grep IPAddress | awk '{print $2}')
+      echo
+      echo -e "${color_green}Cluster logs are ingested into Elasticsearch running at ${color_yellow}http://${es_ip}:9200"
+      echo -e "${color_green}Kibana logging dashboard will be available at ${color_yellow}http://${kibana_ip}:5601${color_norm}"
+      echo
+    else
+      echo -e "${color_red}Failed to launch Elasticsearch and Kibana pods and services for logging.${color_norm}"
+    fi
+  fi
+}
+
+function teardown-logging {
+  if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]] && \
+     [[ "${LOGGING_DESTINATION-}" == "elasticsearch" ]] && \
+     [[ "${ENABLE_CLUSTER_LOGGING-}" == "true" ]]; then
+    local -r kubectl="${KUBE_ROOT}/cluster/kubectl.sh"
+    "${kubectl}" delete replicationController elasticsearch-logging-controller &> /dev/null || true
+    "${kubectl}" delete service elasticsearch-logging &> /dev/null || true
+    "${kubectl}" delete replicationController kibana-logging-controller &> /dev/null || true
+    "${kubectl}" delete service kibana-logging &> /dev/null || true
+    gcloud compute firewall-rules delete -q fluentd-elasticsearch-logging --project "${PROJECT}" || true
+  fi
+}
+
 # Perform preparations required to run e2e tests
 function prepare-e2e() {
   detect-project

cluster/gke/config-default.sh

@@ -21,3 +21,11 @@ NETWORK=${KUBE_GKE_NETWORK:-default}
 # For ease of maintenance, extract any pieces that do not vary between default
 # and test in a common config.
 source $(dirname "${BASH_SOURCE}")/config-common.sh
+
+# Optional: Install node logging
+ENABLE_NODE_LOGGING=false
+LOGGING_DESTINATION=gcp # options: elasticsearch, gcp
+
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1

cluster/gke/util.sh

@@ -261,3 +261,11 @@ function kube-down() {
   "${GCLOUD}" preview container clusters delete --project="${PROJECT}" \
     --zone="${ZONE}" "${CLUSTER_NAME}"
 }
+
+function setup-logging {
+  echo "TODO: setup logging"
+}
+
+function teardown-logging {
+  echo "TODO: teardown logging"
+}

cluster/kube-down.sh

@@ -28,6 +28,8 @@ echo "Bringing down cluster using provider: $KUBERNETES_PROVIDER"
 
 verify-prereqs
 teardown-monitoring
+teardown-logging
+
 kube-down
 
 echo "Done"

cluster/kube-up.sh

@@ -43,7 +43,7 @@ echo "... calling setup-monitoring" >&2
 setup-monitoring
 
 if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then
-  echo ".. setting up cluster DNS"
+  echo "... setting up cluster DNS"
   sed -e "s/{DNS_DOMAIN}/$DNS_DOMAIN/g" \
       -e "s/{DNS_REPLICAS}/$DNS_REPLICAS/g" \
       "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.in" \
@@ -54,4 +54,7 @@ if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then
       | "${KUBE_ROOT}/cluster/kubectl.sh" create -f -
 fi
 
+echo "... calling setup-logging" >&2
+setup-logging
+
 echo "Done" >&2

cluster/rackspace/config-default.sh

@@ -42,9 +42,13 @@ PORTAL_NET="10.0.0.0/16"
 ENABLE_NODE_MONITORING=true
 
 # Optional: Enable node logging.
-ENABLE_NODE_LOGGING=true
+ENABLE_NODE_LOGGING=false
 LOGGING_DESTINATION=elasticsearch
 
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1
+
 # Optional: Install cluster DNS.
 ENABLE_CLUSTER_DNS=true
 DNS_SERVER_IP="10.0.0.10"

cluster/rackspace/util.sh

@@ -350,6 +350,14 @@ function teardown-monitoring {
   echo "TODO"
 }
 
+function setup-logging {
+  echo "TODO: setup logging"
+}
+
+function teardown-logging {
+  echo "TODO: teardown logging"
+}
+
 # Perform preparations required to run e2e tests
 function prepare-e2e() {
   echo "Rackspace doesn't need special preparations for e2e tests"

cluster/saltbase/salt/fluentd-es/fluentd-es.manifest

@@ -2,7 +2,7 @@ version: v1beta2
 id: fluentd-to-elasticsearch
 containers:
   - name: fluentd-es
-    image: kubernetes/fluentd-elasticsearch
+    image: kubernetes/fluentd-elasticsearch:1.0
     volumeMounts:
       - name: containers
         mountPath: /var/lib/docker/containers

cluster/vagrant/config-default.sh

@@ -50,9 +50,13 @@ MASTER_PASSWD=vagrant
 ENABLE_NODE_MONITORING=true
 
 # Optional: Enable node logging.
-ENABLE_NODE_LOGGING=true
+ENABLE_NODE_LOGGING=false
 LOGGING_DESTINATION=elasticsearch
 
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1
+
 # Extra options to set on the Docker command line.  This is useful for setting
 # --insecure-registry for local registries.
 DOCKER_OPTS=""

cluster/vagrant/util.sh

@@ -273,3 +273,11 @@ function teardown-monitoring {
 function prepare-e2e() {
   echo "Vagrant doesn't need special preparations for e2e tests"
 }
+
+function setup-logging {
+  echo "TODO: setup logging"
+}
+
+function teardown-logging {
+  echo "TODO: teardown logging"
+}
\ No newline at end of file

cluster/vsphere/config-default.sh

@@ -37,9 +37,13 @@ PORTAL_NET="10.244.240.0/20"
 ENABLE_NODE_MONITORING=true
 
 # Optional: Enable node logging.
-ENABLE_NODE_LOGGING=true
+ENABLE_NODE_LOGGING=false
 LOGGING_DESTINATION=elasticsearch
 
+# Optional: When set to true, Elasticsearch and Kibana will be setup as part of the cluster bring up.
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1
+
 # Optional: Install cluster DNS.
 ENABLE_CLUSTER_DNS=true
 DNS_SERVER_IP="10.244.240.240"

cluster/vsphere/util.sh

@@ -485,3 +485,11 @@ function setup-monitoring {
 function teardown-monitoring {
   echo "TODO"
 }
+
+function setup-logging {
+  echo "TODO: setup logging"
+}
+
+function teardown-logging {
+  echo "TODO: teardown logging"
+}
\ No newline at end of file

contrib/logging/fluentd-ek/Makefile

-# Makefile for Fluentd to Elastiscsearch and Kibana configured
-# in separate pods.
-
-
-.PHONY: up dow es-up kibana-up es-down kibana-down update \
-        logger-up logger-down get net firewall rmfirewall
-
-KUBECTL=kubectl.sh
-
-up:	logger-up es-up kibana-up
-
-down:	logger-down es-down kibana-down
-
-
-es-up:	
-	-${KUBECTL} create -f es-pod.yml
-	-${KUBECTL} create -f es-service.yml
-
-kibana-up:
-	-${KUBECTL} create -f kibana-pod.yml
-	-${KUBECTL} create -f kibana-service.yml
-
-es-down:
-	-${KUBECTL} delete pods elasticsearch-pod
-	-${KUBECTL} delete service elasticsearch
-
-kibana-down:
-	-${KUBECTL} delete pods kibana-pod
-	-${KUBECTL} delete service kibana
-
-
-update:	
-	-${KUBECTL} delete pods kibana-pod
-	-${KUBECTL} create -f kibana-pod.yml
-
-logger-up:
-	-${KUBECTL} create -f synthetic_0_25lps.yml
-
-logger-down:
-	-${KUBECTL} delete pods synthetic-logger-0.25lps-pod
-
-logger10-up:
-	-${KUBECTL} create -f synthetic_10lps.yml
-
-logger10-down:
-	-${KUBECTL} delete pods synthetic-logger-10lps-pod
-
-get:	
-	${KUBECTL} get pods
-	${KUBECTL} get services
-
-net:	
-	gcutil getforwardingrule elasticsearch
-	gcutil getforwardingrule kibana
-
-firewall:	
-	gcutil addfirewall --allowed=tcp:5601,tcp:9200,tcp:9300 --target_tags=kubernetes-minion kubernetes-elk-example
- 
-
-rmfirewall:
-	gcutil deletefirewall -f kubernetes-elk-example
-

contrib/logging/fluentd-ek/kibana-pod.yml

-apiVersion: v1beta1
-kind: Pod
-id: kibana-pod
-desiredState:
-  manifest:
-    version: v1beta1
-    id: kibana-server
-    containers:
-      - name: kibana-image
-        image: kubernetes/kibana:latest
-        ports:
-          - name: kibana-port
-            containerPort: 80
-labels:
-  app: kibana-viewer

contrib/logging/kibana-image/Makefile

-build:	
-	docker build -t kubernetes/kibana .
-
-push:	
-	docker push kubernetes/kibana

docs/logging.md

@@ -4,9 +4,45 @@
 Kubernetes components, such as kubelet and apiserver, use the [glog](https://godoc.org/github.com/golang/glog) logging library.  Developer conventions for logging severity are described in [devel/logging.md](devel/logging.md).
 
 ## Logging in Containers
-There are no Kubernetes-specific requirements for logging from within containers.  A
-[search](https://www.google.com/?q=docker+container+logging) will turn up any number of articles about logging and
+There are no Kubernetes-specific requirements for logging from within containers. [search](https://www.google.com/?q=docker+container+logging) will turn up any number of articles about logging and
 Docker containers.  However, we do provide an example of how to collect, index, and view pod logs [using Fluentd, Elasticsearch, and Kibana](./getting-started-guides/logging.md)
 
 
+## Logging to Elasticsearch on the GCE platform
+Currently the collection of container logs using the [Fluentd](http://www.fluentd.org/) log collector is 
+enabled by default for clusters created for the GCE platform. Each node uses Fluentd to collect
+the container logs which are submitted in [Logstash](http://logstash.net/docs/1.4.2/tutorials/getting-started-with-logstash)
+format (in JSON) to an [Elasticsearch](http://www.elasticsearch.org/) cluster which runs as a Kubernetes service.
+When you create a cluster the console output reports the URL of both the Elasticsearch cluster as well as
+a URL for a [Kibana](http://www.elasticsearch.org/overview/kibana/) dashboard viewer for the logs that have been ingested
+into Elasticsearch.
+```
+Cluster logs are ingested into Elasticsearch running at http://130.211.121.21:9200
+Kibana logging dashboard will be available at http://130.211.137.206:5601
+```
+Visiting the Kibana dashboard URL in a browser should give a display like this:
+![Kibana](kibana.png)
 
+To learn how to query, fitler etc. using Kibana you might like to look at this [tutorial](http://www.elasticsearch.org/guide/en/kibana/current/working-with-queries-and-filters.html).
+
+You can check to see if any logs are being ingested into Elasticsearch by curling against its URL:
+```
+$ curl http://130.211.120.118:9200/_search?size=10
+{"took":7,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":3705,"max_score":1.0,"hits":[{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8Durpdh","_score":1.0,"_source":{"message":"I0108 18:30:47.694725    4927 server.go:313] GET /healthz: (9.249us) 200","tag":"kubelet","@timestamp":"2015-01-08T18:30:47+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8Durpdm","_score":1.0,"_source":{"message":"E0108 18:30:52.299372    4927 metadata.go:109] while reading 'google-dockercfg' metadata: http status code: 404 while fetching url http://metadata.google.internal./computeMetadata/v1/instance/attributes/google-dockercfg","tag":"kubelet","@timestamp":"2015-01-08T18:30:52+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8Durpdr","_score":1.0,"_source":{"message":"I0108 18:30:52.317636    4927 docker.go:214] Pulling image kubernetes/kube2sky without credentials","tag":"kubelet","@timestamp":"2015-01-08T18:30:52+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8Durpdw","_score":1.0,"_source":{"message":"I0108 18:30:54.500174    4927 event.go:92] Event(api.ObjectReference{Kind:\"BoundPod\", Namespace:\"default\", Name:\"67cfcb1f-9764-11e4-898c-42010af03582\", UID:\"67cfcb1f-9764-11e4-898c-42010af03582\", APIVersion:\"v1beta1\", ResourceVersion:\"\", FieldPath:\"spec.containers{kube2sky}\"}): status: 'waiting', reason: 'created' Created with docker id ff24ec6eb3b10d1163a2bcb7c63ccef78e6e3e7a1185eba3fe430f6b3d871eb5","tag":"kubelet","@timestamp":"2015-01-08T18:30:54+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8Durpd1","_score":1.0,"_source":{"message":"goroutine 114 [running]:","tag":"kubelet","@timestamp":"2015-01-08T18:30:56+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8Durpd6","_score":1.0,"_source":{"message":"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet.(*Server).error(0xc2080e0060, 0x7fe0ba496840, 0xc208278840, 0x7fe0ba4881b0, 0xc20822daa0)","tag":"kubelet","@timestamp":"2015-01-08T18:30:56+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRRanI4L8DurpeB","_score":1.0,"_source":{"message":"\t/go/src/github.com/GoogleCloudPlatform/kubernetes/_output/dockerized/go/src/github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/server.go:94 +0x44","tag":"kubelet","@timestamp":"2015-01-08T18:30:56+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRSanI4L8DurpeJ","_score":1.0,"_source":{"message":"goroutine 114 [running]:","tag":"kubelet","@timestamp":"2015-01-08T18:30:56+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRSanI4L8DurpeO","_score":1.0,"_source":{"message":"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet.(*Server).error(0xc2080e0060, 0x7fe0ba496840, 0xc208278a80, 0x7fe0ba4881b0, 0xc20822df00)","tag":"kubelet","@timestamp":"2015-01-08T18:30:56+00:00"}},{"_index":"logstash-2015.01.08","_type":"fluentd","_id":"AUrK0hRSanI4L8DurpeT","_score":1.0,"_source":{"message":"\t/go/src/github.com/GoogleCloudPlatform/kubernetes/_output/dockerized/go/src/github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/server.go:240 +0x45","tag":"kubelet","@timestamp":"2015-01-08T18:30:56+00:00"}}]}}
+```
+
+A [demonstration](../cluster/addons/fluentd-elasticsearch/logging-demo/README.md) of two synthetic logging sources can be used
+to check that logging is working correctly.
+
+Cluster logging can be turned on or off using the environment variable `ENABLE_NODE_LOGGING` which is defined in the
+`config-default.sh` file for each provider. For the GCE provider this is set by default to `true`. Set this
+to `false` to disable cluster logging.
+
+The type of logging is used is specified by the environment variable `LOGGING_DESTINATION` which for the
+GCE provider has the default value `elasticsearch`. If this is set to `gcp` for the GCE provder then
+logs will be sent to the Google Cloud Logging system instead.
+
+When using Elasticsearch the number of Elasticsearch instances can be controlled by setting the
+variable `ELASTICSEARCH_LOGGING_REPLICAS` which has the default value of `1`. For large clusters
+or clusters that are generating log information at a high rate you may wish to use more
+Elasticsearch instances.