#!/bin/bash set -xe cd $(dirname $0)/.. if [ -z "${K3S_IMAGE}" ]; then echo 'K3S_IMAGE environment variable should be defined' exit 1 fi # --- port-used() { (cat /dev/tcp/127.0.0.1/$1) 2>/dev/null } export -f port-used # --- get-port() { while PORT=$((10000 + RANDOM % 50000)) port-used ${PORT} do continue; done echo ${PORT} } export -f get-port # --- fetch-kubeconfig() { docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null } export -f fetch-kubeconfig # --- wait-for-kubeconfig() { while ! fetch-kubeconfig; do echo 'Waiting for kubeconfig to become available...' sleep 5 done } export -f wait-for-kubeconfig # --- count-ready-nodes() { kubectl get nodes -o json \ | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \ | wc -l \ | tr -d '[:space:]' } export -f count-ready-nodes # --- wait-for-nodes() { while [[ $(count-ready-nodes) -ne $1 ]]; do echo 'Waiting for nodes to be ready...' sleep 5 done } export -f wait-for-nodes # --- pod-ready() { kubectl get pods -n kube-system -o json \ | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null } export -f pod-ready # --- wait-for-services() { for service in $@; do while [[ "$(pod-ready ${service})" != 'true' ]]; do echo "Waiting for service ${service} to be ready..." sleep 5 done echo "Service ${service} is ready" done } export -f wait-for-services # --- verify-valid-version() { if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then return 1 fi } export -f verify-valid-version # --- verify-valid-versions() { verify-valid-version $1 kubectl version verify-valid-version $1 ctr version verify-valid-version $1 crictl version } export -f verify-valid-versions # --- dump-container-logs() { mkdir -p ${LOGS} for container in ${CONTAINERS}; do docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log docker logs ${container} >${LOGS}/${container}.log 2>&1 done } export -f dump-container-logs # --- retrieve-sonobuoy-logs() { if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then status=passed exit_code=0 else status=failed exit_code=1 fi mkdir -p ${E2E} sonobuoy retrieve ${E2E} tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG} if [ ! -s ${RESULTS} ]; then return 1 fi if [ -n "${E2E_LOG_OUTPUT}" ]; then cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}") fi awk '/^Summarizing .* Failures:$/,0' ${RESULTS} return ${exit_code} } export -f retrieve-sonobuoy-logs # --- sonobuoy-test() { time sonobuoy run \ --config=scripts/sonobuoy-config.json \ --plugin-env=e2e.E2E_USE_GO_RUNNER=true \ --wait=30 \ "${@}" & SONOBUOY_PID=$! wait $SONOBUOY_PID retrieve-sonobuoy-logs } export -f sonobuoy-test # --- cleanup() { exit_status=$? set +e echo 'Cleaning up' trap - EXIT [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null if [ "${exit_status}" -ne "0" ]; then dump-container-logs fi docker rm -f ${CONTAINERS} 2>/dev/null rm ${KUBECONFIG} exit ${exit_status} } trap cleanup EXIT # --- K3S_PORT=$(timeout --foreground 5s bash -c get-port) OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT} LOGS=$(pwd)/logs/$$ E2E="${OUTPUT}/e2e" E2E_LOG='plugins/e2e/results/global/e2e.log' RESULTS="${E2E}/${E2E_LOG}" SECRET=random-$((100000 + RANDOM % 999999)) export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT} export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT} export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}" export KUBECONFIG=${OUTPUT}/kubeconfig.yaml mkdir -p ${OUTPUT} # --- docker run -d --name ${K3S_SERVER} --privileged \ -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \ -e K3S_CLUSTER_SECRET=${SECRET} \ ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT} K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER}) echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}" timeout --foreground 1m bash -c wait-for-kubeconfig verify-valid-versions ${K3S_SERVER} # --- docker run -d --name ${K3S_AGENT} --privileged \ -e K3S_CLUSTER_SECRET=${SECRET} \ -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \ ${K3S_IMAGE} agent echo "Started ${K3S_AGENT}" # --- timeout --foreground 1m bash -c 'wait-for-nodes 2' timeout --foreground 1m bash -c 'wait-for-services coredns local-path-provisioner metrics-server' if [ "$ARCH" = 'arm' ]; then echo "Aborting sonobuoy tests, images not available for $ARCH" exit 0 fi echo 'Starting sonobuoy tests' sonobuoy-test "${@}"