add junit recording in e2e runner

hack/jenkins/e2e-runner.sh

@@ -21,9 +21,48 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
+# include shell2junit library
+source <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/third_party/forked/shell2junit/sh2ju.sh')
+
+# Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
+ARTIFACTS=${WORKSPACE}/_artifacts
+mkdir -p ${ARTIFACTS}
+
+# E2E runner stages
+STAGE_PRE="PRE-SETUP"
+STAGE_SETUP="SETUP"
+STAGE_CLEANUP="CLEANUP"
+STAGE_KUBEMARK="KUBEMARK"
+
 : ${KUBE_GCS_RELEASE_BUCKET:="kubernetes-release"}
 : ${KUBE_GCS_DEV_RELEASE_BUCKET:="kubernetes-release-dev"}
 
+# record_command runs the command and records its output/error messages in junit format
+# it expects the first argument to be the class and the second to be the name of the command
+# Example:
+# record_command PRESETUP curltest curl google.com
+# record_command CLEANUP check false
+#
+# WARNING: Variable changes in the command will NOT be effective after record_command returns.
+#          This is because the command runs in subshell.
+function record_command() {
+    set +o xtrace
+    set +o nounset
+    set +o errexit
+
+    local class=$1
+    shift
+    local name=$1
+    shift
+    echo "Recording: ${class} ${name}"
+    echo "Running command: $@"
+    juLog -output="${ARTIFACTS}" -class="${class}" -name="${name}" "$@"
+
+    set -o nounset
+    set -o errexit
+    set -o xtrace
+}
+
 function running_in_docker() {
     grep -q docker /proc/self/cgroup
 }
@@ -114,7 +153,7 @@ function install_google_cloud_sdk_tarball() {
     tar xzf "${tarball}" -C "${install_dir}"
 
     export CLOUDSDK_CORE_DISABLE_PROMPTS=1
-    "${install_dir}/google-cloud-sdk/install.sh" --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
+    record_command "${STAGE_PRE}" "install_gcloud" "${install_dir}/google-cloud-sdk/install.sh" --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
     export PATH=${install_dir}/google-cloud-sdk/bin:${PATH}
 }
 
@@ -144,7 +183,7 @@ function dump_cluster_logs() {
 
 ### Pre Set Up ###
 if running_in_docker; then
-    curl -fsSL --retry 3 --keepalive-time 2 -o "${WORKSPACE}/google-cloud-sdk.tar.gz" 'https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz'
+    record_command "${STAGE_PRE}" "download_gcloud" curl -fsSL --retry 3 --keepalive-time 2 -o "${WORKSPACE}/google-cloud-sdk.tar.gz" 'https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz'
     install_google_cloud_sdk_tarball "${WORKSPACE}/google-cloud-sdk.tar.gz" /
 fi
 
@@ -280,9 +319,6 @@ if [[ ! "${JOB_NAME}" =~ -pull- ]]; then
     JENKINS_BUILD_STARTED=true bash <(curl -fsS --retry 3 --keepalive-time 2 "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh")
 fi
 
-# Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
-ARTIFACTS=${WORKSPACE}/_artifacts
-mkdir -p ${ARTIFACTS}
 # When run inside Docker, we need to make sure all files are world-readable
 # (since they will be owned by root on the host).
 trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
@@ -401,11 +437,16 @@ fi
 # * neither started nor destroyed (soak test)
 if [[ "${E2E_UP:-}" == "${E2E_DOWN:-}" && -f "${gcp_resources_before}" && -f "${gcp_resources_after}" ]]; then
   difference=$(diff -sw -U0 -F'^\[.*\]$' "${gcp_resources_before}" "${gcp_resources_after}") || true
+  noleak=true
   if [[ -n $(echo "${difference}" | tail -n +3 | grep -E "^\+") ]] && [[ "${FAIL_ON_GCP_RESOURCE_LEAK:-}" == "true" ]]; then
+    noleak=false
+  fi
+  if ! ${noleak} ; then
     echo "${difference}"
     echo "!!! FAIL: Google Cloud Platform resources leaked while running tests!"
     EXIT_CODE=1
   fi
+  record_command "${STAGE_CLEANUP}" "gcp_resource_leak_check" ${noleak}
 fi
 
 exit ${EXIT_CODE}