k3s/scripts/sonobuoy

#!/bin/bash

set -xe

cd $(dirname $0)/..

if [ -z "${K3S_IMAGE}" ]; then
    echo "K3S_IMAGE environment variable should be defined"
    exit 1
fi

# ---

port-used() {
    (cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
}
export -f port-used

# ---

get-port() {
    while
        PORT=$((10000 + RANDOM % 50000))
        port-used ${PORT}
    do continue; done
    echo ${PORT}
}
export -f get-port

# ---

fetch-kubeconfig() {
    docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
}
export -f fetch-kubeconfig

# ---

wait-for-kubeconfig() {
    while ! fetch-kubeconfig; do
        echo "Waiting for kubeconfig to become available..."
        sleep 5
    done
}
export -f wait-for-kubeconfig

# ---

count-ready-nodes() {
    kubectl get nodes -o json \
        | jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
        | wc -l \
        | tr -d '[:space:]'
}
export -f count-ready-nodes

# ---

wait-for-nodes() {
    while [[ "$(count-ready-nodes)" != "2" ]]; do
        echo "Waiting for nodes to be ready..."
        sleep 5
    done
}
export -f wait-for-nodes

# ---

pod-ready() {
    kubectl get pods -n kube-system -o json \
        | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready

# ---

wait-for-services() {
    for service in coredns; do
        while [[ "$(pod-ready ${service})" != "true" ]]; do
            echo "Waiting for service ${service} to be ready..."
            sleep 5
        done
        echo "Service ${service} is ready"
    done
}
export -f wait-for-services

# ---

dump-container-logs() {
    for container in ${CONTAINERS}; do
        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${OUTPUT}/${container}-containerd.log
        docker logs ${container} >${OUTPUT}/${container}.log 2>&1
        cat ${OUTPUT}/${container}-containerd.log
        cat ${OUTPUT}/${container}.log
    done
}
export -f dump-container-logs

# ---

sonobuoy-destruct() {
    sonobuoy logs
    sonobuoy delete --wait
}
export -f sonobuoy-destruct

# ---

sonobuoy-test() {
    timeout --foreground 30m sonobuoy run \
        --config scripts/sonobuoy-config.json \
        --wait \
        "${@}"
    if sonobuoy status | grep "failed"; then
        sonobuoy-destruct
        return 1
    fi

    rm -rf ${E2E}
    mkdir -p ${E2E}
    sonobuoy retrieve ${E2E}

    tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
    if [ ! -s ${RESULTS} ]; then
        sonobuoy-destruct
        return 1
    fi

    tail -20 ${RESULTS}
}
export -f sonobuoy-test

# ---

sonobuoy-retry-test() {
    SECONDS=0
    LIMIT=300
    for i in $(seq 1 3); do
        sonobuoy-test "${@}" && return
        echo "*** Failed Sonobuoy try #${i} on port ${K3S_PORT} at ${SECONDS} seconds ***"
        if [ "$SECONDS" -gt "$LIMIT" ]; then
            echo "Stopping sonobuoy re-try after ${SECONDS} seconds (limit ${LIMIT}s)"
            break
        fi
    done
    dump-container-logs
    return 1
}
export -f sonobuoy-retry-test

# ---

cleanup() {
    exit_status=$?
    set +e
    echo "Cleaning up"
    docker rm -f ${CONTAINERS} 2>/dev/null
    rm ${KUBECONFIG}
    exit ${exit_status}
}
trap cleanup EXIT

# ---

K3S_PORT=$(timeout --foreground 5s bash -c get-port)
OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
E2E="${OUTPUT}/e2e"
E2E_LOG="plugins/e2e/results/e2e.log"
RESULTS="${E2E}/${E2E_LOG}"
mkdir -p ${OUTPUT}

SECRET=random-$((100000 + RANDOM % 999999))
export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
export KUBECONFIG=${OUTPUT}/kubeconfig.yaml

# ---

docker run -d --name ${K3S_SERVER} --privileged \
    -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
    -e K3S_CLUSTER_SECRET=${SECRET} \
    ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}

K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"

# ---

timeout --foreground 1m bash -c wait-for-kubeconfig

docker run -d --name ${K3S_AGENT} --privileged \
    -e K3S_CLUSTER_SECRET=${SECRET} \
    -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
    ${K3S_IMAGE} agent

echo "Started ${K3S_AGENT}"

# ---

timeout --foreground 1m bash -c wait-for-nodes
timeout --foreground 1m bash -c wait-for-services

echo "Starting sonobuoy tests"
sonobuoy-retry-test "${@}"

# ---

exit_code=0
status=$(tail -5 ${RESULTS} | grep '^SUCCESS!.*| 0 Failed |' >/dev/null && echo passed || echo failed)
[ "${status}" = "failed" ] && exit_code=1

if [ -n "${E2E_LOG_OUTPUT}" ]; then
    cp ${RESULTS} $(echo ${E2E_LOG_OUTPUT} | sed -e "s/-STATUS-/-${status}-/g")
fi

exit ${exit_code}