Unify fluentd-gcp configurations

cluster/addons/fluentd-gcp/fluentd-gcp-image/Dockerfile

@@ -50,11 +50,11 @@ RUN apt-get -qq update && \
         /opt/td-agent/embedded/share/postgresql \
         /etc/td-agent/td-agent.conf
 
-# Copy the Fluentd configuration files for logging Docker container logs.
-# Either configuration file can be used by specifying `-c <file>` as a command
-# line argument.
-COPY google-fluentd.conf /etc/td-agent/td-agent.conf
-COPY google-fluentd-journal.conf /etc/td-agent/td-agent-journal.conf
+# Copy the Fluentd configuration file for logging Docker container logs.
+COPY fluent.conf /etc/td-agent/td-agent.conf
+
+# Copy the entrypoint for the container
+COPY run.sh /run.sh
 
 # Start Fluentd to pick up our config that watches Docker container logs.
-CMD /usr/sbin/td-agent $FLUENTD_ARGS
+CMD /run.sh $FLUENTD_ARGS

cluster/addons/fluentd-gcp/fluentd-gcp-image/Makefile

@@ -13,23 +13,20 @@
 # limitations under the License.
 
 # The build rule builds a Docker image that logs all Docker contains logs to
-# Google Compute Platform using the Cloud Logging API. The push rule pushes
-# the image to DockerHub.
+# Google Compute Platform using the Cloud Logging API.
 
 # Procedure for change:
 # 1. Bump the tag number.
-# 2. Create an image for the kubernetes user on Dockerhub and push there: make kbuild kpush
-# 3. Temporarily adjust any config files etc. that reference gcr.io/google_containers/fluentd-gcp:$(TAG)
-#    to use kubernetes/fluentd-gcp:$(TAG) instead.
-# 4. Test etc.
-# 5. Issue PR.
-# 6. When PR is approved make the gcr.io version of the image: make build push
-# 7. Revert the referendes to kubernetes/fluentd-gcp:$(TAG) to gcr.io/google_containers/fluentd-gcp:$(TAG)
+# 2. Push to the private repo and test using newer version
+# 3. Issue PR.
+# 4. Assuming permissions to do so, when PR is approved
+#    make the gcr.io version of the image: make build push
+# 5. Issue PR with config files changes
 
 .PHONY:	build push
 
 PREFIX=gcr.io/google_containers
-TAG = 1.28
+TAG = 1.29
 
 build:
 	docker build -t $(PREFIX)/fluentd-gcp:$(TAG) .

cluster/addons/fluentd-gcp/fluentd-gcp-image/google-fluentd.conf → cluster/addons/fluentd-gcp/fluentd-gcp-image/fluent.conf

@@ -5,10 +5,6 @@
 # pod name and container name. These logs are then submitted to
 # Google Cloud Logging which assumes the installation of the cloud-logging plug-in.
 #
-# This configuration is almost identical to google-fluentd-journal.conf, with
-# the one difference being that this doesn't try to collect systemd journal
-# logs.
-#
 # Example
 # =======
 # A line in the Docker log file might like like this JSON:
@@ -43,11 +39,6 @@
 # the name of the Kubernetes container regardless of how many times the
 # Kubernetes pod has been restarted (resulting in a several Docker container IDs).
 
-# Do not directly collect fluentd's own logs to avoid infinite loops.
-<match fluent.**>
-  type null
-</match>
-
 # Example:
 # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
 <source>
@@ -227,6 +218,23 @@
   tag cluster-autoscaler
 </source>
 
+# Logs from systemd-journal for interesting services.
+<source>
+  type systemd
+  filters [{ "_SYSTEMD_UNIT": "docker.service" }]
+  pos_file /var/log/gcp-journald-docker.pos
+  read_from_head true
+  tag docker
+</source>
+
+<source>
+  type systemd
+  filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+  pos_file /var/log/gcp-journald-kubelet.pos
+  read_from_head true
+  tag kubelet
+</source>
+
 # We use 2 output stanzas - one to handle the container logs and one to handle
 # the node daemon logs, the latter of which explicitly sends its logs to the
 # compute.googleapis.com service rather than container.googleapis.com to keep

cluster/addons/fluentd-gcp/fluentd-gcp-image/run.sh

+#!/bin/sh
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# For systems without journald
+mkdir -p /var/log/journal
+
+if [ -e /host/lib/libsystemd* ]
+then
+  rm /lib/x86_64-linux-gnu/libsystemd*
+  cp /host/lib/libsystemd* /lib/x86_64-linux-gnu/
+fi
+
+LD_PRELOAD=/opt/td-agent/embedded/lib/libjemalloc.so
+RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0.9
+
+/usr/sbin/td-agent "$@"