Merge pull request #1138 from erikwilson/test-cleanup

Test cleanup

.drone.yml

@@ -59,14 +59,14 @@ steps:
     event:
     - tag
 
-- name: sonobuoy-e2e-tests
+- name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
   environment:
     GCLOUD_AUTH:
       from_secret: gcloud_auth
   commands:
-  - dapper -f Dockerfile.sonobuoy.dapper
+  - dapper -f Dockerfile.test.dapper
   volumes:
   - name: docker
     path: /var/run/docker.sock
@@ -154,14 +154,14 @@ steps:
     event:
     - tag
 
-- name: sonobuoy-e2e-tests
+- name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
   environment:
     GCLOUD_AUTH:
       from_secret: gcloud_auth
   commands:
-  - dapper -f Dockerfile.sonobuoy.dapper
+  - dapper -f Dockerfile.test.dapper
   volumes:
   - name: docker
     path: /var/run/docker.sock
@@ -232,14 +232,14 @@ steps:
     event:
     - tag
 
-- name: sonobuoy-e2e-tests
+- name: test
   image: rancher/dapper:v0.4.2
   secrets: [ gcloud_auth ]
   environment:
     GCLOUD_AUTH:
       from_secret: gcloud_auth
   commands:
-  - dapper -f Dockerfile.sonobuoy.dapper
+  - dapper -f Dockerfile.test.dapper
   volumes:
   - name: docker
     path: /var/run/docker.sock

Dockerfile.test.dapper

@@ -1,22 +1,23 @@
 FROM golang:1.13.5-alpine3.10
 
-RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2
+RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 openssl
-RUN SONOBUOY_VER=v0.16.2 && \
-    SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \
-    go get -d ${SONOBUOY_PKG} && \
-    cd /go/src/${SONOBUOY_PKG} && \
-    git checkout -b current ${SONOBUOY_VER} && \
-    go build -o /usr/local/bin/sonobuoy
-RUN rm -rf /go/src /go/pkg
 
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
 
+RUN if [ "${ARCH}" == "amd64" ] || [ "${ARCH}" == "arm64" ]; then \
+        VERSION=0.17.0 OS=linux && \
+        curl -sL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_${ARCH}.tar.gz" | \
+        tar -xzf - -C /usr/local/bin; \
+    fi
+
 RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
             curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt \
         )/bin/linux/${ARCH}/kubectl -o /usr/local/bin/kubectl && \
     chmod a+x /usr/local/bin/kubectl
 
+ENV TEST_CLEANUP true
+
 ENV DAPPER_RUN_ARGS --privileged --network host
 ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH
 ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
@@ -26,4 +27,4 @@ ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}
 
 ENTRYPOINT ["./scripts/entry.sh"]
-CMD ["sonobuoy-e2e-tests"]
+CMD ["test"]

scripts/log-upload

@@ -40,7 +40,7 @@ END
 HOME=${TMPDIR}
 PATH=$PATH:${HOME}/gsutil
 
-LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz
+LOG_TGZ=k3s-log-$(date +%s)-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(basename $1).tgz
 
 tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
 gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1

scripts/sonobuoy

@@ -1,82 +0,0 @@
-#!/bin/bash
-set -xe
-
-cd $(dirname $0)/..
-
-. ./scripts/test-helpers
-
-if [ -z "${K3S_IMAGE}" ]; then
-    echo 'K3S_IMAGE environment variable should be defined'
-    exit 1
-fi
-
-# ---
-
-cleanup() {
-    exit_status=$?
-    set +e
-    echo 'Cleaning up'
-    trap - EXIT
-    [ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
-    if [ "${exit_status}" -ne "0" ]; then
-        dump-logs
-    fi
-    docker rm -f ${CONTAINERS} 2>/dev/null
-    rm ${KUBECONFIG}
-    exit ${exit_status}
-}
-trap cleanup EXIT
-
-# ---
-
-K3S_PORT=$(timeout --foreground 5s bash -c get-port)
-OUTPUT=$(pwd)/sonobuoy-output/${K3S_PORT}
-LOGS=$(pwd)/logs/$$
-E2E="${OUTPUT}/e2e"
-E2E_LOG='plugins/e2e/results/global/e2e.log'
-RESULTS="${E2E}/${E2E_LOG}"
-
-SECRET=random-$((100000 + RANDOM % 999999))
-export K3S_AGENT=k3s-agent-${K3S_PORT}
-export K3S_SERVER=k3s-server-${K3S_PORT}
-export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
-export KUBECONFIG=${OUTPUT}/kubeconfig.yaml
-
-mkdir -p ${OUTPUT}
-
-# ---
-
-docker run -d --name ${K3S_SERVER} --privileged \
-    -p 127.0.0.1:${K3S_PORT}:${K3S_PORT} \
-    -e K3S_CLUSTER_SECRET=${SECRET} \
-    -e K3S_DEBUG=true \
-    ${K3S_IMAGE} server --no-deploy=traefik --https-listen-port=${K3S_PORT}
-
-K3S_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${K3S_SERVER})
-echo "Started ${K3S_SERVER} @ ${K3S_IP}:${K3S_PORT}"
-
-docker exec ${K3S_SERVER} check-config || true
-timeout --foreground 1m bash -c wait-for-kubeconfig
-verify-valid-versions ${K3S_SERVER}
-
-# ---
-
-docker run -d --name ${K3S_AGENT} --privileged \
-    -e K3S_CLUSTER_SECRET=${SECRET} \
-    -e K3S_URL=https://${K3S_IP}:${K3S_PORT} \
-    ${K3S_IMAGE} agent
-
-echo "Started ${K3S_AGENT}"
-
-# ---
-
-timeout --foreground 1m bash -c 'wait-for-nodes 2'
-timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server'
-
-if [ "$ARCH" = 'arm' ]; then
-    echo "Aborting sonobuoy tests, images not available for $ARCH"
-    exit 0
-fi
-
-echo 'Starting sonobuoy tests'
-sonobuoy-test "${@}"

scripts/sonobuoy-e2e-tests

@@ -1,83 +0,0 @@
-#!/bin/bash
-set -e -x
-
-cd $(dirname $0)/..
-
-if [ -z "$K3S_IMAGE" ]; then
-    . ./scripts/version.sh
-    TAG=${TAG:-${VERSION_TAG}${SUFFIX}}
-    REPO=${REPO:-rancher}
-    IMAGE_NAME=${IMAGE_NAME:-k3s}
-    export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG}
-fi
-
-OUTPUT=$(pwd)/dist/artifacts
-mkdir -p ${OUTPUT}
-
-pids=()
-output=()
-
-show-logs() {
-    for pid in "${pids[@]}"; do
-        logdir=$(pwd)/logs/${pid}
-        if [ ! -d $logdir ]; then
-            continue
-        fi
-        echo
-        echo "#- Begin: logs for sonobuoy run pid ${pid}"
-        for log in $(pwd)/logs/${pid}/*.log; do
-            if [ -f ${log} ]; then
-                echo
-                echo "#- Tail: ${log}"
-                tail -10 ${log}
-                echo "#- Done: ${log}"
-                echo
-            fi
-        done
-        echo "#- Finish: logs for sonobuoy run pid ${pid}"
-        echo
-    done
-}
-
-cleanup() {
-    exit_status=$?
-    set +e
-    trap - EXIT INT
-    kill ${pids[@]} 2>/dev/null
-    wait
-    set +x
-    echo -n "Tests "
-    if [ "${exit_status}" -eq "0" ]; then
-        echo "passed"
-    else
-        echo "failed"
-        show-logs
-    fi
-    exit ${exit_status}
-}
-trap cleanup EXIT INT
-
-
-# ---
-
-run-sonobuoy() {
-    output+=(${log_output})
-    E2E_LOG_OUTPUT=${log_output} \
-        ./scripts/sonobuoy ${@} \
-            > >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 &
-    pids+=($!)
-}
-
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \
-    run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
-
-sleep 60
-
-log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \
-    run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
-
-exit_code=0
-for pid in "${pids[@]}"; do
-    wait $pid || exit_code=$?
-done
-exit ${exit_code}

scripts/test

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e -x
+cd $(dirname $0)/..
+
+. ./scripts/test-helpers
+
+artifacts=$(pwd)/dist/artifacts
+mkdir -p $artifacts
+
+# ---
+
+. ./scripts/test-run-basics
+
+# # ---
+
+E2E_OUTPUT=$artifacts test-run-sonobuoy
+
+# # ---
+
+# if [ "$ARCH" != 'amd64' ]; then
+#     printf "\033[33mSkipping remaining tests, images not available for $ARCH.\033[m\n"
+#     exit 0
+# fi
+
+# # ---
+
+# test-run-sonobuoy mysql
+# test-run-sonobuoy postgres
+# test-run-sonobuoy dqlite

scripts/test-certs-openssl.cnf

@@ -0,0 +1,72 @@
+# db OpenSSL configuration file.
+SAN = "IP:127.0.0.1, IP:172.17.0.1, DNS:host.docker.internal"
+dir = .
+
+[ ca ]
+default_ca = db_ca
+
+[ db_ca ]
+certs            = $dir/certs
+certificate      = $dir/certs/ca.crt
+crl              = $dir/crl.pem
+crl_dir          = $dir/crl
+crlnumber        = $dir/crlnumber
+database         = $dir/index.txt
+email_in_dn      = no
+new_certs_dir    = $dir/newcerts
+private_key      = $dir/private/ca.key
+serial           = $dir/serial
+RANDFILE         = $dir/private/.rand
+name_opt         = ca_default
+cert_opt         = ca_default
+default_days     = 3650
+default_crl_days = 30
+default_md       = sha512
+preserve         = no
+policy           = policy_db
+
+[ policy_db ]
+organizationName = optional
+commonName       = supplied
+
+[ req ]
+default_bits       = 1024
+default_keyfile    = privkey.pem
+distinguished_name = req_distinguished_name
+attributes         = req_attributes
+x509_extensions    = v3_ca
+string_mask        = utf8only
+req_extensions     = db_client
+
+[ req_distinguished_name ]
+countryName                = Country Name (2 letter code)
+countryName_default        = US
+countryName_min            = 2
+countryName_max            = 2
+commonName                 = Common Name (FQDN)
+0.organizationName         = Organization Name (eg, company)
+0.organizationName_default = db-ca
+
+[ req_attributes ]
+
+[ v3_ca ]
+basicConstraints       = CA:true
+keyUsage               = keyCertSign,cRLSign
+subjectKeyIdentifier   = hash
+
+[ db_client ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth
+keyUsage               = digitalSignature, keyEncipherment
+
+[ db_peer ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth, serverAuth
+keyUsage               = digitalSignature, keyEncipherment
+subjectAltName         = ${ENV::SAN}
+
+[ db_server ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth, serverAuth
+keyUsage               = digitalSignature, keyEncipherment
+subjectAltName         = ${ENV::SAN}

scripts/test-helpers

@@ -10,26 +10,31 @@ export -f port-used
 # ---
 
 get-port() {
+    local port=0
     while
-        PORT=$((10000 + RANDOM % 50000))
+        port=$((10000 + RANDOM % 50000))
-        port-used ${PORT}
+        port-used $port
     do continue; done
-    echo ${PORT}
+    echo $port
 }
 export -f get-port
 
 # ---
 
-fetch-kubeconfig() {
+fetch-kubeconfig() {(
-    docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
+    set -e -o pipefail
-}
+    local num=${1:-1}
+    local name=$(cat $TEST_DIR/servers/$num/metadata/name)
+    local port=$(cat $TEST_DIR/servers/$num/metadata/port)
+    docker cp $name:/etc/rancher/k3s/k3s.yaml - 2>/dev/null | tar -xO 2>/dev/null | sed -e "s/:6443/:$port/g" >$TEST_DIR/servers/$num/kubeconfig.yaml
+)}
 export -f fetch-kubeconfig
 
 # ---
 
 wait-for-kubeconfig() {
-    while ! fetch-kubeconfig; do
+    while ! fetch-kubeconfig $1; do
-        echo 'Waiting for kubeconfig to become available...'
+        echo 'Waiting for kubeconfig to become available...' >&2
         sleep 5
     done
 }
@@ -49,7 +54,7 @@ export -f count-ready-nodes
 
 wait-for-nodes() {
     while [[ $(count-ready-nodes) -ne $1 ]]; do
-        echo 'Waiting for nodes to be ready...'
+        echo 'Waiting for nodes to be ready...' >&2
         sleep 5
     done
 }
@@ -67,17 +72,31 @@ export -f pod-ready
 
 wait-for-services() {
     for service in $@; do
-        while [[ "$(pod-ready ${service})" != 'true' ]]; do
+        while [[ "$(pod-ready $service)" != 'true' ]]; do
-            echo "Waiting for service ${service} to be ready..."
+            echo "Waiting for service $service to be ready..." >&2
             sleep 5
         done
-        echo "Service ${service} is ready"
+        echo "Service $service is ready"
     done
 }
 export -f wait-for-services
 
 # ---
 
+wait-for-db-connection() {
+    if [ -z "$DB_CONNECTION_TEST" ]; then
+      echo 'DB_CONNECTION_TEST is not defined' >&2
+      return 1
+    fi
+    while ! $DB_CONNECTION_TEST 2>/dev/null; do
+        echo 'Waiting for database to become available...' >&2
+        sleep 5
+    done
+}
+export -f wait-for-db-connection
+
+# ---
+
 verify-valid-version() {
     if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
         return 1
@@ -97,13 +116,30 @@ export -f verify-valid-versions
 # ---
 
 dump-logs() {
-    mkdir -p ${LOGS}/sonobuoy
+    local testID=$(basename $TEST_DIR)
-    tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy
+    echo "#---------------------------------"
-    for container in ${CONTAINERS}; do
+    echo "#- Begin: logs for run ($testID)"
-        docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log
+    echo
-        docker logs ${container} >${LOGS}/${container}-system.log 2>&1
+    for node in $TEST_DIR/*/*; do
+        [ -d "$node" ] || continue
+        local name=$(cat $node/metadata/name 2>/dev/null)
+        [ "$name" ] || continue
+        mkdir -p $node/logs
+        docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null
+        docker logs $name >$node/logs/system.log 2>&1
+        for log in $node/logs/*.log; do
+            echo
+            echo "#- Tail: $log"
+            tail -5 $log
+            echo "#- Done: $log"
+            echo
+        done
     done
-    ./scripts/log-upload ${LOGS}
+    echo
+    echo "#- Finish: logs for run ($testID)"
+    echo "#---------------------------------"
+    echo
+    ./scripts/log-upload $TEST_DIR
 }
 export -f dump-logs
 
@@ -111,40 +147,389 @@ export -f dump-logs
 
 retrieve-sonobuoy-logs() {
     sonobuoy status || true
-    if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
+
-        status=passed
+    local status=passed
-        exit_code=0
+    local code=0
-    else
+
+    if ! sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
         status=failed
-        exit_code=1
+        code=1
     fi
 
-    mkdir -p ${E2E}
+    mkdir -p $TEST_DIR/sonobuoy
-    sonobuoy retrieve ${E2E}
+    sonobuoy retrieve $TEST_DIR/sonobuoy 2>/dev/null || true
-    tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
+    local logTarball=$TEST_DIR/sonobuoy/*_sonobuoy_*.tar.gz
-    if [ ! -s ${RESULTS} ]; then
+
+    if [ -f $logTarball ]; then
+        tar -xz -f $logTarball -C $TEST_DIR/sonobuoy
+        rm $logTarball
+    else
+        rm -rf $TEST_DIR/sonobuoy
+    fi
+
+    local e2eLog=$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log
+    if [ ! -s $e2eLog ]; then
         return 1
     fi
-    if [ -n "${E2E_LOG_OUTPUT}" ]; then
+    if [ -n "$LOG_OUTPUT" ]; then
-        cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
+        cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT")
     fi
-    awk '/^Summarizing .* Failures?:$/,0' ${RESULTS}
+    awk '/^Summarizing .* Failures?:$/,0' $e2eLog
-    return ${exit_code}
+    return $code
 }
 export -f retrieve-sonobuoy-logs
 
 # ---
 
 sonobuoy-test() {
-    time sonobuoy run \
+    if [ "$ARCH" = 'arm' ]; then
-        --config=scripts/sonobuoy-config.json \
+        echo "Aborting sonobuoy tests, images not available for $ARCH"
-        --plugin-env=e2e.E2E_USE_GO_RUNNER=true \
+        return 0
-        --wait=30 \
+    fi
-        "${@}" &
+    echo 'Starting sonobuoy tests'
-    SONOBUOY_PID=$!
+
-    wait $SONOBUOY_PID
+    sonobuoy run --config=scripts/sonobuoy-config.json --plugin-env=e2e.E2E_USE_GO_RUNNER=true --wait=30 $@ &
+    local sonobuoyPID=$!
+    local code=0
+    time wait $sonobuoyPID || code=$?
+
     retrieve-sonobuoy-logs
+    return $code
 }
 export -f sonobuoy-test
 
 # ---
+
+test-cleanup() {
+    local code=$?
+    set +e +x
+    echo 'Cleaning up...'
+    trap - EXIT INT TERM
+    if [[ $code -ne 0 ]]; then
+        dump-logs
+    fi
+    for name in $TEST_DIR/*/*/metadata/name; do
+        [ -f "$name" ] || continue
+        local container=$(cat $name)
+        docker rm -f -v $container >/dev/null 2>&1 &
+    done
+    if [ "$TEST_CLEANUP" = true ]; then
+        echo "Removing $TEST_DIR"
+        rm -rf $TEST_DIR >/dev/null 2>&1 &
+    fi
+    [ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK >/dev/null 2>&1 &
+    echo
+    echo -n "Test $(basename $TEST_DIR) "
+    if [ $code -eq 0 ]; then
+        echo "passed."
+    else
+        echo "failed."
+    fi
+    echo
+    exit $code
+}
+export -f test-cleanup
+
+# ---
+
+test-setup() {
+    export TEST_DIR=$(mktemp -d '/tmp/XXXXXX')
+    trap test-cleanup EXIT INT TERM
+
+    mkdir -p $TEST_DIR/metadata
+    if [ "$LABEL" ]; then
+        exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \
+            2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2)
+        echo "$LABEL" >$TEST_DIR/metadata/label
+    fi
+
+    mkdir -p $TEST_DIR/logs
+    exec > >(tee -a $TEST_DIR/logs/test.log) \
+        2> >(tee -a $TEST_DIR/logs/test.log >&2)
+
+    if [ -z "$K3S_IMAGE" ]; then
+        . ./scripts/version.sh
+        TAG=${TAG:-${VERSION_TAG}${SUFFIX}}
+        REPO=${REPO:-rancher}
+        IMAGE_NAME=${IMAGE_NAME:-k3s}
+        export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG}
+    fi
+
+    if [ -z "$K3S_IMAGE" ]; then
+        echo 'K3S_IMAGE environment variable should be defined'
+        return 1
+    fi
+
+    local setupFile=./scripts/test-setup-${TEST_TYPE}
+    [ -f $setupFile ] && source $setupFile
+
+    echo ${RANDOM}${RANDOM}${RANDOM} >$TEST_DIR/metadata/secret
+}
+export -f test-setup
+
+# ---
+
+gen-certs() {(
+    set -e -x
+    #umask 077
+
+    local opensslConfig=$(pwd)/scripts/test-certs-openssl.cnf
+    local subject="/C=US/ST=AZ/L=Tempe/O=Rancher/OU=DevOps/DC=com/DC=rancher"
+    local caDir=$TEST_DIR/db-ca
+    [ -d $caDir ] && rm -rf $caDir
+    mkdir -p $caDir
+
+    cd $caDir
+    mkdir -p private certs newcerts crl
+    touch index.txt
+    echo '01' > serial
+
+    openssl req \
+        -config $opensslConfig \
+        -new -x509 \
+        -nodes \
+        -subj $subject \
+        -extensions v3_ca \
+        -keyout private/ca.key \
+        -out certs/ca.crt
+
+    openssl req \
+        -config $opensslConfig \
+        -new \
+        -nodes \
+        -subj $subject"/CN=server" \
+        -keyout private/db.key \
+        -out db.csr
+
+    openssl ca \
+        -config $opensslConfig \
+        -batch \
+        -extensions db_server \
+        -keyfile private/ca.key \
+        -cert certs/ca.crt \
+        -out certs/db.crt \
+        -infiles db.csr
+
+    openssl req \
+        -config $opensslConfig \
+        -new \
+        -nodes \
+        -subj $subject"/CN=client" \
+        -keyout private/client.key \
+        -out client.csr
+
+    openssl ca \
+        -config $opensslConfig \
+        -batch \
+        -extensions db_client \
+        -keyfile private/ca.key \
+        -cert certs/ca.crt \
+        -out certs/client.crt \
+        -infiles client.csr
+)}
+export -f gen-certs
+
+# ---
+
+inc-count() {(
+    shopt -s extglob
+    local count=$(exec 2>/dev/null; ls -1d $TEST_DIR/$1/+([0-9]) | xargs -n1 basename | sort -n -r | head -1)
+    count=$((count+1))
+    mkdir -p $TEST_DIR/$1/$count/metadata
+    echo $count
+)}
+export -f inc-count
+
+# ---
+
+run-function() {
+    declare -f $1 >/dev/null 2>&1 || return 0
+    $@
+}
+export -f run-function
+
+# ---
+
+provision-server() {
+    local count=$(inc-count servers)
+    local testID=$(basename $TEST_DIR)
+    local name=$(echo "k3s-server-$count-$testID" | tee $TEST_DIR/servers/$count/metadata/name)
+    #local args=$(cat $TEST_DIR/args $TEST_DIR/servers/args $TEST_DIR/servers/$count/args 2>/dev/null)
+    local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/servers/$count/metadata/port)
+    local SERVER_INSTANCE_ARGS="SERVER_${count}_ARGS"
+
+    run-function server-pre-hook $count
+
+    docker run \
+        -d --name $name \
+        --privileged \
+        -v $TEST_DIR/db-ca/:/db-ca \
+        -p 127.0.0.1:$port:6443 \
+        -p 6443 \
+        -e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \
+        -e K3S_DEBUG=true \
+        $K3S_IMAGE server $ARGS $SERVER_ARGS ${!SERVER_INSTANCE_ARGS}
+
+    local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/servers/$count/metadata/ip)
+    local url=$(echo "https://$ip:6443" | tee $TEST_DIR/servers/$count/metadata/url)
+
+    echo "Started $name @ $url"
+    run-function server-post-hook $count
+}
+export -f provision-server
+
+# ---
+
+provision-agent() {
+    local K3S_URL=${K3S_URL:-"$(cat $TEST_DIR/servers/1/metadata/url)"}
+    local count=$(inc-count agents)
+    local testID=$(basename $TEST_DIR)
+    local name=$(echo "k3s-agent-$count-$testID" | tee $TEST_DIR/agents/$count/metadata/name)
+    #local args=$(cat $TEST_DIR/args $TEST_DIR/agents/args $TEST_DIR/agents/$count/args 2>/dev/null)
+    local AGENT_INSTANCE_ARGS="AGENT_${count}_ARGS"
+
+    run-function agent-pre-hook $count
+    docker run \
+        -d --name $name \
+        --privileged \
+        -e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \
+        -e K3S_URL=$K3S_URL \
+        $K3S_IMAGE agent $ARGS $AGENT_ARGS ${!AGENT_INSTANCE_ARGS}
+
+    echo "Started $name"
+    run-function agent-post-hook $count
+}
+export -f provision-agent
+
+# ---
+
+provision-cluster() {
+    run-function cluster-pre-hook
+
+    for i in $(seq 1 $NUM_SERVERS); do
+        provision-server
+        timeout --foreground 30s bash -c "wait-for-kubeconfig $i"
+    done
+    export KUBECONFIG=$TEST_DIR/servers/1/kubeconfig.yaml
+
+    if [ $NUM_AGENTS -gt 0 ]; then
+        for _ in $(seq 1 $NUM_AGENTS); do
+            provision-agent
+        done
+    fi
+
+    timeout --foreground 1m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))"
+    timeout --foreground 3m bash -c "wait-for-services $WAIT_SERVICES"
+
+    run-function cluster-post-hook
+
+    [ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK
+}
+export -f provision-cluster
+
+# ---
+
+run-test() {
+    export PROVISION_LOCK=$(mktemp)
+    ./scripts/test-runner $@ &
+    pids+=($!)
+    (
+        set +x
+        while [ -f "$PROVISION_LOCK" ]; do
+            sleep 1
+        done
+        sleep 5
+    )
+}
+export -f run-test
+
+# ---
+
+e2e-test() {
+  local label=$label
+  if [ -n "$LABEL_SUFFIX" ]; then
+    label="$label-$LABEL_SUFFIX"
+  fi
+  local logOutput=
+  if [ -n "$E2E_OUTPUT" ]; then
+    logOutput=$E2E_OUTPUT/$logname
+  fi
+  LABEL=$label LOG_OUTPUT=$logOutput run-test $@
+}
+
+# ---
+
+run-e2e-tests() {
+  label=PARALLEL \
+      logName=e2e-STATUS-${ARCH}-parallel.log \
+      e2e-test ${sonobuoyParallelArgs[@]}
+
+  label=SERIAL \
+      logName=e2e-STATUS-${ARCH}-serial.log \
+      e2e-test ${sonobuoySerialArgs[@]}
+}
+export -f run-e2e-tests
+
+# ---
+
+test-run-sonobuoy() {
+    local suffix
+    if [ "$1" ]; then
+        suffix="-$1"
+        export LABEL_SUFFIX=$1
+    fi
+
+    . ./scripts/test-setup-sonobuoy$suffix
+    run-e2e-tests
+}
+export -f test-run-sonobuoy
+
+# ---
+
+pid-cleanup() {
+    local code=$?
+    set +e
+    local failCount=0
+    if [ $code -ne 0 ]; then
+        for pid in ${pids[@]}; do
+            pkill -P $pid
+            wait $pid || failCount=$((failCount+1))
+        done
+    fi
+    wait
+    trap - EXIT INT TERM
+    set +x
+    echo
+    if [ $failCount -eq 0 ]; then
+        printf '\033[32mAll tests passed.\033[m\n'
+        if [ $code -ne 0 ]; then
+            printf "\033[31mExit code is $code.\033[m\n"
+        fi
+    else
+        printf "\033[31m$failCount tests failed.\033[m\n"
+    fi
+    echo
+    exit $code
+}
+export -f pid-cleanup
+
+# ---
+
+wait-pids() {
+    trap - EXIT
+    set +e
+    local code=0
+    for pid in "${pids[@]}"; do
+        wait $pid || code=$?
+    done
+    [ $code -eq 0 ]
+    pid-cleanup
+    exit $code
+}
+export -f wait-pids
+
+# ---
+
+pids=()
+trap pid-cleanup INT TERM
+trap wait-pids EXIT

scripts/test-run-basics

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export NUM_SERVERS=1
+export NUM_AGENTS=1
+export SERVER_ARGS='--no-deploy=traefik,coredns,local-storage,metrics-server'
+
+start-test() {
+    docker exec $(cat $TEST_DIR/servers/1/metadata/name) check-config || true
+    verify-valid-versions $(cat $TEST_DIR/servers/1/metadata/name)
+}
+export -f start-test
+
+# --- create a basic cluster and check for valid versions
+LABEL=BASICS run-test

scripts/test-runner

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -x -e
+cd $(dirname $0)/..
+
+# ---
+
+for include in $TEST_INCLUDES; do
+  . $include
+done
+
+test-setup
+provision-cluster
+start-test $@

scripts/test-setup-sonobuoy

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export NUM_SERVERS=1
+export NUM_AGENTS=1
+export SERVER_ARGS='--no-deploy=traefik'
+export WAIT_SERVICES='coredns local-path-provisioner metrics-server'
+
+export sonobuoyParallelArgs=(--e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y)
+export sonobuoySerialArgs=(--e2e-focus='\[Serial\].*\[Conformance\]')
+
+start-test() {
+  sonobuoy-test $@
+}
+export -f start-test

scripts/test-setup-sonobuoy-dqlite

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+. ./scripts/test-setup-sonobuoy
+
+export NUM_SERVERS=2
+export NUM_AGENTS=0
+
+export SERVER_1_ARGS=--cluster-init
+
+# ---
+
+server-post-hook() {
+  if [ $1 -eq 1 ]; then
+    local url=$(cat $TEST_DIR/servers/1/metadata/url)
+    export SERVER_ARGS="--server $url"
+  fi
+}
+export -f server-post-hook

scripts/test-setup-sonobuoy-mysql

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+. ./scripts/test-setup-sonobuoy
+
+# ---
+
+cluster-pre-hook() {
+  # gen-certs
+
+  mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata
+  local testID=$(basename $TEST_DIR)
+  local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name)
+  local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port)
+  local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret)
+
+  docker run --name $name \
+    --privileged \
+    -p 0.0.0.0:$port:3306 \
+    -v $TEST_DIR/db-ca/:/db-ca \
+    -e MYSQL_ROOT_PASSWORD=$secret \
+    -e MYSQL_ROOT_HOST=% \
+    -d mysql:latest \
+    >/dev/null
+      #  --require-secure-transport=ON \
+      #  --ssl-ca /db-ca/certs/ca.crt \
+      #  --ssl-cert /db-ca/certs/db.crt \
+      #  --ssl-key /db-ca/private/db.key \
+
+  local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip)
+  # local host=host.docker.internal
+  local host=172.17.0.1
+
+  DB_CONNECTION_TEST="
+    docker run
+      -v $TEST_DIR/db-ca/:/db-ca
+      --rm mysql
+      mysql
+        -h$host
+        -P$port
+        -uroot
+        -p$secret
+        -e status" \
+    timeout --foreground 1m bash -c "wait-for-db-connection"
+        # --ssl-ca /db-ca/certs/ca.crt
+        # --ssl-cert /db-ca/certs/client.crt
+        # --ssl-key /db-ca/private/client.key
+
+  echo "Started $LABEL_SUFFIX db @ $host"
+  export SERVER_ARGS="${SERVER_ARGS}
+    --datastore-endpoint=mysql://root:$secret@tcp($host:$port)/testdb
+  "
+    # --datastore-cafile /db-ca/certs/ca.crt
+    # --datastore-certfile /db-ca/certs/client.crt
+    # --datastore-keyfile /db-ca/private/client.key
+}
+export -f cluster-pre-hook

scripts/test-setup-sonobuoy-postgres

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+. ./scripts/test-setup-sonobuoy
+
+# ---
+
+cluster-pre-hook() {
+  # gen-certs
+
+  mkdir -p $TEST_DIR/db/$LABEL_SUFFIX/metadata
+  local testID=$(basename $TEST_DIR)
+  local name=$(echo $LABEL_SUFFIX-$testID | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/name)
+  local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/port)
+  local secret=$(echo "${RANDOM}${RANDOM}${RANDOM}" | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/secret)
+
+  docker run --name $name \
+    --privileged \
+    -p 0.0.0.0:$port:5432 \
+    -v $TEST_DIR/db-ca/:/db-ca \
+    -e POSTGRES_USER=root \
+    -e POSTGRES_PASSWORD=$secret \
+    -d postgres:latest \
+    >/dev/null
+      # -c ssl=on \
+      # -c ssl_ca_file=/db-ca/certs/ca.crt \
+      # -c ssl_cert_file=/db-ca/certs/db.crt \
+      # -c ssl_key_file=/db-ca/private/db.key \
+
+  local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/db/$LABEL_SUFFIX/metadata/ip)
+  # local host=host.docker.internal
+  local host=172.17.0.1
+
+  DB_CONNECTION_TEST="
+    docker run
+      -v $TEST_DIR/db-ca/:/db-ca
+      -e PGPASSWORD=$secret
+      --rm postgres
+      psql
+        -h $host
+        -p $port
+        -U root
+        -c \conninfo" \
+    timeout --foreground 1m bash -c "wait-for-db-connection"
+        # --set sslrootcert=/db-ca/certs/ca.crt
+        # --set sslcert=/db-ca/certs/client.crt
+        # --set sslkey=/db-ca/private/client.key
+
+  echo "Started $LABEL_SUFFIX db @ $host"
+  export SERVER_ARGS="${SERVER_ARGS}
+    --datastore-endpoint=postgres://root:$secret@$host:$port/testdb?sslmode=disable
+  "
+    # --datastore-cafile /db-ca/certs/ca.crt
+    # --datastore-certfile /db-ca/certs/client.crt
+    # --datastore-keyfile /db-ca/private/client.key
+}
+export -f cluster-pre-hook