Upload logs to gcloud

.drone.yml

@@ -57,6 +57,10 @@ steps:
 
 - name: sonobuoy-e2e-tests
   image: rancher/dapper:v0.4.2
+  secrets: [ gcloud_auth ]
+  environment:
+    GCLOUD_AUTH:
+      from_secret: gcloud_auth
   commands:
   - dapper -f Dockerfile.sonobuoy.dapper
   volumes:
@@ -144,6 +148,10 @@ steps:
 
 - name: sonobuoy-e2e-tests
   image: rancher/dapper:v0.4.2
+  secrets: [ gcloud_auth ]
+  environment:
+    GCLOUD_AUTH:
+      from_secret: gcloud_auth
   commands:
   - dapper -f Dockerfile.sonobuoy.dapper
   volumes:
@@ -214,6 +222,10 @@ steps:
 
 - name: sonobuoy-e2e-tests
   image: rancher/dapper:v0.4.2
+  secrets: [ gcloud_auth ]
+  environment:
+    GCLOUD_AUTH:
+      from_secret: gcloud_auth
   commands:
   - dapper -f Dockerfile.sonobuoy.dapper
   volumes:

Dockerfile.sonobuoy.dapper

@@ -1,6 +1,6 @@
 FROM golang:1.13.3-alpine3.10
 
-RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils
+RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2
 RUN SONOBUOY_VER=v0.16.2 && \
     SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \
     go get -d ${SONOBUOY_PKG} && \
@@ -18,7 +18,7 @@ RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
     chmod a+x /usr/local/bin/kubectl
 
 ENV DAPPER_RUN_ARGS --privileged --network host
-ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME
+ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH
 ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
 ENV DAPPER_OUTPUT ./dist
 ENV DAPPER_DOCKER_SOCKET true

scripts/log-upload

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+[ -d "$1" ] || {
+  echo "First argument should be a directory" >&2
+  exit 1
+}
+
+umask 077
+
+TMPDIR=$(mktemp -d)
+cleanup() {
+  exit_code=$?
+  trap - EXIT INT
+  rm -rf ${TMPDIR}
+  exit ${exit_code}
+}
+trap cleanup EXIT INT
+
+GCLOUD_JSON=${TMPDIR}/.gcloud.json
+[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
+[ -s "${GCLOUD_JSON}" ] || {
+  echo "gcloud auth not defined" >&2
+  exit 1
+}
+
+BOTO_CONF=${TMPDIR}/.boto
+[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
+[Credentials]
+gs_service_key_file = ${GCLOUD_JSON}
+[Boto]
+https_validate_certificates = True
+[GSUtil]
+content_language = en
+default_api_version = 2
+default_project_id = rancher-dev
+END
+
+[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
+
+HOME=${TMPDIR}
+PATH=$PATH:${HOME}/gsutil
+
+LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz
+
+tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
+gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1
+echo "Logs uploaded" >&2
+echo "https://storage.googleapis.com/k3s-ci-logs/${LOG_TGZ}"

scripts/sonobuoy

@@ -1,9 +1,10 @@
 #!/bin/bash
-
 set -xe
 
 cd $(dirname $0)/..
 
+. ./scripts/test-helpers
+
 if [ -z "${K3S_IMAGE}" ]; then
     echo 'K3S_IMAGE environment variable should be defined'
     exit 1
@@ -11,150 +12,6 @@ fi
 
 # ---
 
-port-used() {
-    (cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
-}
-export -f port-used
-
-# ---
-
-get-port() {
-    while
-        PORT=$((10000 + RANDOM % 50000))
-        port-used ${PORT}
-    do continue; done
-    echo ${PORT}
-}
-export -f get-port
-
-# ---
-
-fetch-kubeconfig() {
-    docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
-}
-export -f fetch-kubeconfig
-
-# ---
-
-wait-for-kubeconfig() {
-    while ! fetch-kubeconfig; do
-        echo 'Waiting for kubeconfig to become available...'
-        sleep 5
-    done
-}
-export -f wait-for-kubeconfig
-
-# ---
-
-count-ready-nodes() {
-    kubectl get nodes -o json \
-        | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
-        | wc -l \
-        | tr -d '[:space:]'
-}
-export -f count-ready-nodes
-
-# ---
-
-wait-for-nodes() {
-    while [[ $(count-ready-nodes) -ne $1 ]]; do
-        echo 'Waiting for nodes to be ready...'
-        sleep 5
-    done
-}
-export -f wait-for-nodes
-
-# ---
-
-pod-ready() {
-    kubectl get pods -n kube-system -o json \
-        | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
-}
-export -f pod-ready
-
-# ---
-
-wait-for-services() {
-    for service in $@; do
-        while [[ "$(pod-ready ${service})" != 'true' ]]; do
-            echo "Waiting for service ${service} to be ready..."
-            sleep 5
-        done
-        echo "Service ${service} is ready"
-    done
-}
-export -f wait-for-services
-
-# ---
-
-verify-valid-version() {
-    if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
-        return 1
-    fi
-}
-export -f verify-valid-version
-
-# ---
-
-verify-valid-versions() {
-    verify-valid-version $1 kubectl version
-    verify-valid-version $1 ctr version
-    verify-valid-version $1 crictl version
-}
-export -f verify-valid-versions
-
-# ---
-
-dump-container-logs() {
-    mkdir -p ${LOGS}
-    for container in ${CONTAINERS}; do
-        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log
-        docker logs ${container} >${LOGS}/${container}.log 2>&1
-    done
-}
-export -f dump-container-logs
-
-# ---
-
-retrieve-sonobuoy-logs() {
-    if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
-        status=passed
-        exit_code=0
-    else
-        status=failed
-        exit_code=1
-    fi
-
-    mkdir -p ${E2E}
-    sonobuoy retrieve ${E2E}
-    tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
-    if [ ! -s ${RESULTS} ]; then
-        return 1
-    fi
-    if [ -n "${E2E_LOG_OUTPUT}" ]; then
-        cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
-    fi
-    awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
-    return ${exit_code}
-}
-export -f retrieve-sonobuoy-logs
-
-# ---
-
-sonobuoy-test() {
-    time sonobuoy run \
-        --config=scripts/sonobuoy-config.json \
-        --plugin-env=e2e.E2E_USE_GO_RUNNER=true \
-        --wait=30 \
-        "${@}" &
-    SONOBUOY_PID=$!
-    wait $SONOBUOY_PID
-    retrieve-sonobuoy-logs
-}
-export -f sonobuoy-test
-
-# ---
-
 cleanup() {
     exit_status=$?
     set +e
@@ -162,7 +19,7 @@ cleanup() {
     trap - EXIT
     [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
     if [ "${exit_status}" -ne "0" ]; then
-        dump-container-logs
+        dump-logs
     fi
     docker rm -f ${CONTAINERS} 2>/dev/null
     rm ${KUBECONFIG}
@@ -180,8 +37,8 @@ E2E_LOG='plugins/e2e/results/global/e2e.log'
 RESULTS="${E2E}/${E2E_LOG}"
 
 SECRET=random-$((100000 + RANDOM % 999999))
-export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
+export K3S_AGENT=k3s-agent-${K3S_PORT}
-export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
+export K3S_SERVER=k3s-server-${K3S_PORT}
 export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
 export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
 

scripts/sonobuoy-e2e-tests

@@ -25,12 +25,12 @@ show-logs() {
         fi
         echo
         echo "#- Begin: logs for sonobuoy run pid ${pid}"
-        for log in $(pwd)/logs/${pid}/*; do
+        for log in $(pwd)/logs/${pid}/*.log; do
             if [ -f ${log} ]; then
                 echo
-                echo "#- Start: ${log}"
+                echo "#- Tail: ${log}"
-                cat ${log}
+                tail -10 ${log}
-                echo "#- End: ${log}"
+                echo "#- Done: ${log}"
                 echo
             fi
         done
@@ -57,18 +57,23 @@ cleanup() {
 }
 trap cleanup EXIT INT
 
+
+# ---
+
 run-sonobuoy() {
     output+=(${log_output})
-    E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} &
+    E2E_LOG_OUTPUT=${log_output} \
+        ./scripts/sonobuoy ${@} \
+            > >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 &
     pids+=($!)
 }
 
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log \
+log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \
     run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
 
 sleep 60
 
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \
+log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \
     run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
 
 exit_code=0

scripts/test-helpers

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# ---
+
+port-used() {
+    (cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
+}
+export -f port-used
+
+# ---
+
+get-port() {
+    while
+        PORT=$((10000 + RANDOM % 50000))
+        port-used ${PORT}
+    do continue; done
+    echo ${PORT}
+}
+export -f get-port
+
+# ---
+
+fetch-kubeconfig() {
+    docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
+}
+export -f fetch-kubeconfig
+
+# ---
+
+wait-for-kubeconfig() {
+    while ! fetch-kubeconfig; do
+        echo 'Waiting for kubeconfig to become available...'
+        sleep 5
+    done
+}
+export -f wait-for-kubeconfig
+
+# ---
+
+count-ready-nodes() {
+    kubectl get nodes -o json \
+        | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
+        | wc -l \
+        | tr -d '[:space:]'
+}
+export -f count-ready-nodes
+
+# ---
+
+wait-for-nodes() {
+    while [[ $(count-ready-nodes) -ne $1 ]]; do
+        echo 'Waiting for nodes to be ready...'
+        sleep 5
+    done
+}
+export -f wait-for-nodes
+
+# ---
+
+pod-ready() {
+    kubectl get pods -n kube-system -o json \
+        | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
+}
+export -f pod-ready
+
+# ---
+
+wait-for-services() {
+    for service in $@; do
+        while [[ "$(pod-ready ${service})" != 'true' ]]; do
+            echo "Waiting for service ${service} to be ready..."
+            sleep 5
+        done
+        echo "Service ${service} is ready"
+    done
+}
+export -f wait-for-services
+
+# ---
+
+verify-valid-version() {
+    if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
+        return 1
+    fi
+}
+export -f verify-valid-version
+
+# ---
+
+verify-valid-versions() {
+    verify-valid-version $1 kubectl version
+    verify-valid-version $1 ctr version
+    verify-valid-version $1 crictl version
+}
+export -f verify-valid-versions
+
+# ---
+
+dump-logs() {
+    mkdir -p ${LOGS}/sonobuoy
+    tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy
+    for container in ${CONTAINERS}; do
+        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log
+        docker logs ${container} >${LOGS}/${container}-system.log 2>&1
+    done
+    ./scripts/log-upload ${LOGS}
+}
+export -f dump-logs
+
+# ---
+
+retrieve-sonobuoy-logs() {
+    sonobuoy status || true
+    if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
+        status=passed
+        exit_code=0
+    else
+        status=failed
+        exit_code=1
+    fi
+
+    mkdir -p ${E2E}
+    sonobuoy retrieve ${E2E}
+    tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
+    if [ ! -s ${RESULTS} ]; then
+        return 1
+    fi
+    if [ -n "${E2E_LOG_OUTPUT}" ]; then
+        cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
+    fi
+    awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
+    return ${exit_code}
+}
+export -f retrieve-sonobuoy-logs
+
+# ---
+
+sonobuoy-test() {
+    time sonobuoy run \
+        --config=scripts/sonobuoy-config.json \
+        --plugin-env=e2e.E2E_USE_GO_RUNNER=true \
+        --wait=30 \
+        "${@}" &
+    SONOBUOY_PID=$!
+    wait $SONOBUOY_PID
+    retrieve-sonobuoy-logs
+}
+export -f sonobuoy-test
+
+# ---