Create a Docker image for use as a sidecar log collector that sends logs

to Google Cloud Logging.

contrib/logging/fluentd-sidecar-gcp/Dockerfile

@@ -0,0 +1,35 @@
+# This Dockerfile will build an image that is configured to use Fluentd to
+# collect container log files from the specified paths and send them to the
+# Google Cloud Logging API.
+# The environment variable that controls which log files are collected is
+# FILES_TO_COLLECT. Files specified in the environment variable should be
+# separated by whitespace, as in "/var/log/syslog /var/log/nginx/access.log".
+# This configuration assumes that the host performning the collection is a VM
+# that has been created with a logging.write scope and that the Logging API
+# has been enabled for the project in the Google Developer Console. 
+
+FROM ubuntu:14.04
+MAINTAINER Alex Robinson "arob@google.com"
+
+# Disable prompts from apt.
+ENV DEBIAN_FRONTEND noninteractive
+ENV OPTS_APT -y --force-yes --no-install-recommends
+
+# Install the Fluentd agent that knows how to send logs to Google Cloud Logging.
+RUN apt-get -q update && \
+    apt-get -y install curl && \
+    apt-get clean && \
+    curl -s https://storage.googleapis.com/signals-agents/logging/google-fluentd-install.sh | sudo bash
+
+# Copy the configuration file generator for creating input configurations for
+# each file specified in the FILES_TO_COLLECT environment variable.
+COPY config_generator.sh /usr/local/sbin/config_generator.sh
+
+# Copy the Fluentd configuration file for collecting from all the inputs
+# generated by the config generator and sending them to Google Cloud Logging.
+COPY google-fluentd.conf /etc/google-fluentd/google-fluentd.conf
+
+# Run the config generator to get the config files in place and start Fluentd.
+# We have to run the config generator at runtime rather than now so that it can
+# incorporate the files provided in the environment variable in its config.
+CMD /usr/local/sbin/config_generator.sh && /usr/sbin/google-fluentd -qq --use-v1-config --suppress-repeated-stacktrace > /var/log/google-fluentd/google-fluentd.log

contrib/logging/fluentd-sidecar-gcp/Makefile

@@ -0,0 +1,9 @@
+.PHONY:	build push
+
+TAG = 1.0
+
+build:
+	docker build -t gcr.io/google_containers/fluentd-sidecar-gcp:$(TAG) .
+
+push:
+	gcloud preview docker push gcr.io/google_containers/fluentd-sidecar-gcp:$(TAG)

contrib/logging/fluentd-sidecar-gcp/README.md

@@ -0,0 +1,8 @@
+# Collecting log files from within containers with Fluentd and sending to the Google Cloud Logging service.
+This directory contains the source files needed to make a Docker image that collects log files from arbitrary files within a container using [Fluentd](http://www.fluentd.org/) and sends them to GCP.
+This image is designed to be used as a sidecar container as part of a [Kubernetes](https://github.com/GoogleCloudPlatform/kubernetes) pod.
+The image resides at DockerHub under the name
+[kubernetes/fluentd-sidecar-gcp](https://registry.hub.docker.com/u/kubernetes/fluentd-sidecar-gcp/).
+
+# TODO: Add example pod config.
+# TODO: say that it resides at gcr.io instead?

contrib/logging/fluentd-sidecar-gcp/config_generator.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+mkdir -p /etc/google-fluentd/files
+if [ -z "$FILES_TO_COLLECT" ]; then
+  exit 0
+fi
+
+for filepath in $FILES_TO_COLLECT
+do
+  filename=$(basename $filepath)
+  cat > "/etc/google-fluentd/files/${filename}" << EndOfMessage
+<source>
+  type tail
+  format none
+  time_key time
+  path ${filepath}
+  pos_file /etc/google-fluentd/fluentd-gcp.log.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag file.${filename}
+  read_from_head true
+</source>
+EndOfMessage
+done

contrib/logging/fluentd-sidecar-gcp/google-fluentd.conf

@@ -0,0 +1,26 @@
+# This Fluentd configuration file enables the collection of log files
+# that can be specified at the time of its creation in an environment
+# variable, assuming that the config_generator.sh script runs to generate
+# a configuration file for each log file to collect.
+# Logs collected will be sent to the Google Cloud Logging API.
+#
+# Currently the collector uses a text format rather than allowing the user
+# to specify how to parse each file.
+#
+# This configuration assumes the correct installation of the the Google
+# Fluentd plug-in, and that the VM host running this configuration is on
+# Google Compute Engine and has been created with the logging.write scope.
+
+# Pick up all the auto-generated input config files, one for each file
+# specified in the FILES_TO_COLLECT environment variable.
+@include files/*
+
+# All the auto-generated files should use the tag "file.<filename>".
+<match file.**>
+  type google_cloud
+  flush_interval 5s
+  # Never wait longer than 5 minutes between retries.
+  max_retry_wait 300
+  # Disable the limit on the number of retries (retry forever).
+  disable_retry_limit
+</match>