diff --git a/.drone.yml b/.drone.yml index 232ae9bd5d..4c5d08f3dc 100644 --- a/.drone.yml +++ b/.drone.yml @@ -57,6 +57,10 @@ steps: - name: sonobuoy-e2e-tests image: rancher/dapper:v0.4.2 + secrets: [ gcloud_auth ] + environment: + GCLOUD_AUTH: + from_secret: gcloud_auth commands: - dapper -f Dockerfile.sonobuoy.dapper volumes: @@ -144,6 +148,10 @@ steps: - name: sonobuoy-e2e-tests image: rancher/dapper:v0.4.2 + secrets: [ gcloud_auth ] + environment: + GCLOUD_AUTH: + from_secret: gcloud_auth commands: - dapper -f Dockerfile.sonobuoy.dapper volumes: @@ -214,6 +222,10 @@ steps: - name: sonobuoy-e2e-tests image: rancher/dapper:v0.4.2 + secrets: [ gcloud_auth ] + environment: + GCLOUD_AUTH: + from_secret: gcloud_auth commands: - dapper -f Dockerfile.sonobuoy.dapper volumes: diff --git a/Dockerfile.sonobuoy.dapper b/Dockerfile.sonobuoy.dapper index 5a15c1d6b3..0563598e31 100644 --- a/Dockerfile.sonobuoy.dapper +++ b/Dockerfile.sonobuoy.dapper @@ -1,6 +1,6 @@ FROM golang:1.13.3-alpine3.10 -RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils +RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 RUN SONOBUOY_VER=v0.16.2 && \ SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \ go get -d ${SONOBUOY_PKG} && \ @@ -18,7 +18,7 @@ RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \ chmod a+x /usr/local/bin/kubectl ENV DAPPER_RUN_ARGS --privileged --network host -ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME +ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/ ENV DAPPER_OUTPUT ./dist ENV DAPPER_DOCKER_SOCKET true diff --git a/scripts/log-upload b/scripts/log-upload new file mode 100755 index 0000000000..1cb9762867 --- /dev/null +++ b/scripts/log-upload @@ -0,0 +1,48 @@ +#!/bin/bash + +[ -d "$1" ] || { + echo "First argument should be a directory" >&2 + exit 1 +} + +umask 077 + +TMPDIR=$(mktemp -d) +cleanup() { + exit_code=$? + trap - EXIT INT + rm -rf ${TMPDIR} + exit ${exit_code} +} +trap cleanup EXIT INT + +GCLOUD_JSON=${TMPDIR}/.gcloud.json +[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON} +[ -s "${GCLOUD_JSON}" ] || { + echo "gcloud auth not defined" >&2 + exit 1 +} + +BOTO_CONF=${TMPDIR}/.boto +[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <&2 +echo "https://storage.googleapis.com/k3s-ci-logs/${LOG_TGZ}" diff --git a/scripts/sonobuoy b/scripts/sonobuoy index 25f6c98080..448a6f90ee 100755 --- a/scripts/sonobuoy +++ b/scripts/sonobuoy @@ -1,9 +1,10 @@ #!/bin/bash - set -xe cd $(dirname $0)/.. +. ./scripts/test-helpers + if [ -z "${K3S_IMAGE}" ]; then echo 'K3S_IMAGE environment variable should be defined' exit 1 @@ -11,150 +12,6 @@ fi # --- -port-used() { - (cat /dev/tcp/127.0.0.1/$1) 2>/dev/null -} -export -f port-used - -# --- - -get-port() { - while - PORT=$((10000 + RANDOM % 50000)) - port-used ${PORT} - do continue; done - echo ${PORT} -} -export -f get-port - -# --- - -fetch-kubeconfig() { - docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null -} -export -f fetch-kubeconfig - -# --- - -wait-for-kubeconfig() { - while ! fetch-kubeconfig; do - echo 'Waiting for kubeconfig to become available...' - sleep 5 - done -} -export -f wait-for-kubeconfig - -# --- - -count-ready-nodes() { - kubectl get nodes -o json \ - | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \ - | wc -l \ - | tr -d '[:space:]' -} -export -f count-ready-nodes - -# --- - -wait-for-nodes() { - while [[ $(count-ready-nodes) -ne $1 ]]; do - echo 'Waiting for nodes to be ready...' - sleep 5 - done -} -export -f wait-for-nodes - -# --- - -pod-ready() { - kubectl get pods -n kube-system -o json \ - | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null -} -export -f pod-ready - -# --- - -wait-for-services() { - for service in $@; do - while [[ "$(pod-ready ${service})" != 'true' ]]; do - echo "Waiting for service ${service} to be ready..." - sleep 5 - done - echo "Service ${service} is ready" - done -} -export -f wait-for-services - -# --- - -verify-valid-version() { - if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then - return 1 - fi -} -export -f verify-valid-version - -# --- - -verify-valid-versions() { - verify-valid-version $1 kubectl version - verify-valid-version $1 ctr version - verify-valid-version $1 crictl version -} -export -f verify-valid-versions - -# --- - -dump-container-logs() { - mkdir -p ${LOGS} - for container in ${CONTAINERS}; do - docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log - docker logs ${container} >${LOGS}/${container}.log 2>&1 - done -} -export -f dump-container-logs - -# --- - -retrieve-sonobuoy-logs() { - if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then - status=passed - exit_code=0 - else - status=failed - exit_code=1 - fi - - mkdir -p ${E2E} - sonobuoy retrieve ${E2E} - tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} - if [ ! -s ${RESULTS} ]; then - return 1 - fi - if [ -n "${E2E_LOG_OUTPUT}" ]; then - cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}") - fi - awk '/^Summarizing .* Failures:$/,0' ${RESULTS} - return ${exit_code} -} -export -f retrieve-sonobuoy-logs - -# --- - -sonobuoy-test() { - time sonobuoy run \ - --config=scripts/sonobuoy-config.json \ - --plugin-env=e2e.E2E_USE_GO_RUNNER=true \ - --wait=30 \ - "${@}" & - SONOBUOY_PID=$! - wait $SONOBUOY_PID - retrieve-sonobuoy-logs -} -export -f sonobuoy-test - -# --- - cleanup() { exit_status=$? set +e @@ -162,7 +19,7 @@ cleanup() { trap - EXIT [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null if [ "${exit_status}" -ne "0" ]; then - dump-container-logs + dump-logs fi docker rm -f ${CONTAINERS} 2>/dev/null rm ${KUBECONFIG} @@ -180,8 +37,8 @@ E2E_LOG='plugins/e2e/results/global/e2e.log' RESULTS="${E2E}/${E2E_LOG}" SECRET=random-$((100000 + RANDOM % 999999)) -export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT} -export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT} +export K3S_AGENT=k3s-agent-${K3S_PORT} +export K3S_SERVER=k3s-server-${K3S_PORT} export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}" export KUBECONFIG=${OUTPUT}/kubeconfig.yaml diff --git a/scripts/sonobuoy-e2e-tests b/scripts/sonobuoy-e2e-tests index 9f57691119..d6a4fafc5f 100755 --- a/scripts/sonobuoy-e2e-tests +++ b/scripts/sonobuoy-e2e-tests @@ -25,12 +25,12 @@ show-logs() { fi echo echo "#- Begin: logs for sonobuoy run pid ${pid}" - for log in $(pwd)/logs/${pid}/*; do + for log in $(pwd)/logs/${pid}/*.log; do if [ -f ${log} ]; then echo - echo "#- Start: ${log}" - cat ${log} - echo "#- End: ${log}" + echo "#- Tail: ${log}" + tail -10 ${log} + echo "#- Done: ${log}" echo fi done @@ -57,18 +57,23 @@ cleanup() { } trap cleanup EXIT INT + +# --- + run-sonobuoy() { output+=(${log_output}) - E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} & + E2E_LOG_OUTPUT=${log_output} \ + ./scripts/sonobuoy ${@} \ + > >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 & pids+=($!) } -log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log \ +log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \ run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y sleep 60 -log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \ +log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \ run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]' exit_code=0 diff --git a/scripts/test-helpers b/scripts/test-helpers new file mode 100755 index 0000000000..cf931487fb --- /dev/null +++ b/scripts/test-helpers @@ -0,0 +1,150 @@ +#!/bin/bash + +# --- + +port-used() { + (cat /dev/tcp/127.0.0.1/$1) 2>/dev/null +} +export -f port-used + +# --- + +get-port() { + while + PORT=$((10000 + RANDOM % 50000)) + port-used ${PORT} + do continue; done + echo ${PORT} +} +export -f get-port + +# --- + +fetch-kubeconfig() { + docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null +} +export -f fetch-kubeconfig + +# --- + +wait-for-kubeconfig() { + while ! fetch-kubeconfig; do + echo 'Waiting for kubeconfig to become available...' + sleep 5 + done +} +export -f wait-for-kubeconfig + +# --- + +count-ready-nodes() { + kubectl get nodes -o json \ + | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \ + | wc -l \ + | tr -d '[:space:]' +} +export -f count-ready-nodes + +# --- + +wait-for-nodes() { + while [[ $(count-ready-nodes) -ne $1 ]]; do + echo 'Waiting for nodes to be ready...' + sleep 5 + done +} +export -f wait-for-nodes + +# --- + +pod-ready() { + kubectl get pods -n kube-system -o json \ + | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null +} +export -f pod-ready + +# --- + +wait-for-services() { + for service in $@; do + while [[ "$(pod-ready ${service})" != 'true' ]]; do + echo "Waiting for service ${service} to be ready..." + sleep 5 + done + echo "Service ${service} is ready" + done +} +export -f wait-for-services + +# --- + +verify-valid-version() { + if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then + return 1 + fi +} +export -f verify-valid-version + +# --- + +verify-valid-versions() { + verify-valid-version $1 kubectl version + verify-valid-version $1 ctr version + verify-valid-version $1 crictl version +} +export -f verify-valid-versions + +# --- + +dump-logs() { + mkdir -p ${LOGS}/sonobuoy + tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy + for container in ${CONTAINERS}; do + docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log + docker logs ${container} >${LOGS}/${container}-system.log 2>&1 + done + ./scripts/log-upload ${LOGS} +} +export -f dump-logs + +# --- + +retrieve-sonobuoy-logs() { + sonobuoy status || true + if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then + status=passed + exit_code=0 + else + status=failed + exit_code=1 + fi + + mkdir -p ${E2E} + sonobuoy retrieve ${E2E} + tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} + if [ ! -s ${RESULTS} ]; then + return 1 + fi + if [ -n "${E2E_LOG_OUTPUT}" ]; then + cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}") + fi + awk '/^Summarizing .* Failures:$/,0' ${RESULTS} + return ${exit_code} +} +export -f retrieve-sonobuoy-logs + +# --- + +sonobuoy-test() { + time sonobuoy run \ + --config=scripts/sonobuoy-config.json \ + --plugin-env=e2e.E2E_USE_GO_RUNNER=true \ + --wait=30 \ + "${@}" & + SONOBUOY_PID=$! + wait $SONOBUOY_PID + retrieve-sonobuoy-logs +} +export -f sonobuoy-test + +# ---