Add script to upload test artifacts and log to GCS using gsutil

This new script is designed to be source'd by Jenkins jobs. It'll
automatically upload all test artifacts and the console log to GCS on
exit.

hack/jenkins/upload-to-gcs.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Source this script in the Jenkins "Execute shell" build action to have all
+# test artifacts and the console log uploaded at the end of the test run.
+
+# For example, you might use the following line as the first command:
+# mkdir -p _tmp/ && curl -fsS --retry 3 -o _tmp/upload-to-gcs.sh \
+#   "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh" \
+#   && source _tmp/upload-to-gcs.sh
+
+# Note that this script requires the Jenkins shell binary to be set to bash, not
+# the system default (which may be dash on Debian-based systems).
+# Also, for magicfile support to work correctly, the "file" utility must be
+# installed.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+if [[ ${JOB_NAME} =~ -pull- ]]; then
+  : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/${ghprbActualCommit:-unknown}"}
+else
+  : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
+fi
+
+: ${JENKINS_UPLOAD_TO_GCS:="y"}
+
+# Upload the build log and all test artifacts (under _artifacts) to GCS when
+# JENKINS_UPLOAD_TO_GCS is set to y.
+# We intentionally ignore gsutil errors since we don't want failed uploads to
+# fail the entire test run (#13548).
+function upload_logs_to_gcs() {
+  if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then
+      return
+  fi
+  local -r artifacts_path="${WORKSPACE}/_artifacts"
+  local -r gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}"
+  local -r gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}"
+  local -r gcs_acl="public-read"
+  for upload_attempt in $(seq 3); do
+    echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})"
+    if [[ -d "${artifacts_path}" ]]; then
+      gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c \
+        -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue
+    fi
+    # Remove ANSI escape sequences from the console log before uploading.
+    local -r filtered_console_log="${WORKSPACE}/console-log.txt"
+    sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" \
+      "${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}/log" \
+      > "${filtered_console_log}"
+    gsutil -q cp -a "${gcs_acl}" -z txt "${filtered_console_log}" "${gcs_build_path}/" || continue
+    # Mark this build as the latest completed.
+    {
+      echo "BUILD_NUMBER=${BUILD_NUMBER}"
+      echo "GIT_COMMIT=${GIT_COMMIT:-}"
+    } | gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
+      cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
+    break  # all uploads succeeded if we hit this point
+  done
+}
+
+
+# Automatically upload logs to GCS on exit or timeout.
+trap upload_logs_to_gcs EXIT SIGTERM SIGINT