From cccba681a9cb6679742e5888ccd2fe71761273ad Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Tue, 8 Dec 2020 10:43:31 -0700 Subject: [PATCH] Test script cleanup backport from RKE2 (#2650) * Test script cleanup backport from RKE2 Signed-off-by: Erik Wilson * Update scripts/test Co-authored-by: Brian Downs Co-authored-by: Brian Downs --- scripts/test | 9 +++++- scripts/test-helpers | 70 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/scripts/test b/scripts/test index 2a35b9d2e6..7cffdd3439 100755 --- a/scripts/test +++ b/scripts/test @@ -10,7 +10,12 @@ mkdir -p $artifacts # --- +docker ps + +# --- + . ./scripts/test-run-basics +echo "Did test-run-basics $?" # --- @@ -18,7 +23,7 @@ mkdir -p $artifacts early-exit "Skipping remaining tests, images not available for $ARCH." E2E_OUTPUT=$artifacts test-run-sonobuoy - +echo "Did test-run-sonobuoy $?" # --- [ "$DRONE_BUILD_EVENT" = 'tag' ] && \ @@ -28,3 +33,5 @@ E2E_OUTPUT=$artifacts test-run-sonobuoy test-run-sonobuoy mysql test-run-sonobuoy postgres + +exit 0 diff --git a/scripts/test-helpers b/scripts/test-helpers index 2f4187991f..dbbe8c35db 100755 --- a/scripts/test-helpers +++ b/scripts/test-helpers @@ -64,7 +64,7 @@ export -f wait-for-nodes pod-ready() { kubectl get pods -n kube-system -o json \ - | jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null + | jq ".items[].status | select(.containerStatuses != null) | .containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null } export -f pod-ready @@ -72,7 +72,7 @@ export -f pod-ready wait-for-services() { for service in $@; do - while [[ "$(pod-ready $service)" != 'true' ]]; do + while [[ "$(pod-ready $service | sort -u)" != 'true' ]]; do echo "Waiting for service $service to be ready..." >&2 sleep 5 done @@ -122,13 +122,27 @@ dump-logs() { echo "#---------------------------------" echo "#- Begin: logs for run ($testID)" echo + local server=$(cat $TEST_DIR/servers/1/metadata/name) + docker exec $server kubectl get pods -A -o wide >$TEST_DIR/logs/kubectl-get-pods.txt + docker exec $server kubectl get nodes -o wide >$TEST_DIR/logs/kubectl-get-nodes.txt + docker exec $server kubectl describe pods -A >$TEST_DIR/logs/kubectl-describe-pods.txt + for node in $TEST_DIR/*/*; do [ -d "$node" ] || continue local name=$(cat $node/metadata/name 2>/dev/null) [ "$name" ] || continue mkdir -p $node/logs + local hostname=$(docker exec $name hostname) + docker exec $server kubectl describe node/$hostname >$node/logs/kubectl-describe-node.txt docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null docker logs $name >$node/logs/system.log 2>&1 + docker exec $name crictl pods >$node/logs/crictl-pods.txt + docker exec $name crictl ps -a >$node/logs/crictl-ps.txt + docker exec $name crictl ps -a -o json >$node/metadata/crictl-ps.json + for container in $(jq -r '.containers[].id' <$node/metadata/crictl-ps.json); do + local cname=$(jq -r '.containers[] | select(.id == "'$container'") | .metadata.name' <$node/metadata/crictl-ps.json) + docker exec $name crictl logs $container >$node/logs/$cname-$container.log 2>&1 + done for log in $node/logs/*.log; do echo echo "#- Tail: $log" @@ -137,6 +151,13 @@ dump-logs() { echo done done + for txt in $TEST_DIR/logs/*.txt $TEST_DIR/*/*/logs/*.txt; do + echo + echo "#- Cat: $txt" + cat $txt + echo "#- Done: $txt" + echo + done echo echo "#- Finish: logs for run ($testID)" echo "#---------------------------------" @@ -176,6 +197,7 @@ retrieve-sonobuoy-logs() { if [ -n "$LOG_OUTPUT" ]; then cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT") fi + tail -11 $e2eLog awk '/^Summarizing .* Failures?:$/,0' $e2eLog return $code } @@ -183,6 +205,23 @@ export -f retrieve-sonobuoy-logs # --- +test-wait() { + wait $1 || echo "test wait exit code $?" + local delay=15 + sleep $delay + while sonobuoy status | grep -q -E ' +e2e +running +'; do + sleep $delay + done + sleep $delay + if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then + return 0 + fi + return 1 +} +export -f test-wait + +# --- + sonobuoy-test() { if [ "$ARCH" = 'arm' ]; then echo "Aborting sonobuoy tests, images not available for $ARCH" @@ -200,8 +239,8 @@ sonobuoy-test() { local sonobuoyPID=$! local code=0 - time wait $sonobuoyPID || code=$? - + time timeout --foreground 30m bash -c test-wait $sonobuoyPID || code=$? + echo "Sonobuoy finished with code $code" retrieve-sonobuoy-logs return $code } @@ -211,7 +250,7 @@ export -f sonobuoy-test test-cleanup() { local code=$? - set +e +x + set +e -x echo 'Cleaning up...' trap - EXIT INT TERM if [[ $code -ne 0 ]]; then @@ -220,13 +259,14 @@ test-cleanup() { for name in $TEST_DIR/*/*/metadata/name; do [ -f "$name" ] || continue local container=$(cat $name) - docker rm -f -v $container >/dev/null 2>&1 & + echo "Removing container $container" + docker rm -f -v $container done if [ "$TEST_CLEANUP" = true ]; then - echo "Removing $TEST_DIR" - rm -rf $TEST_DIR >/dev/null 2>&1 & + echo "Removing test directory $TEST_DIR" + rm -rf $TEST_DIR fi - [ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK >/dev/null 2>&1 & + [ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK echo echo -n "Test $(basename $TEST_DIR) " if [ $code -eq 0 ]; then @@ -247,8 +287,8 @@ test-setup() { mkdir -p $TEST_DIR/metadata if [ "$LABEL" ]; then - exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \ - 2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2) + exec > >(awk -W interactive "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \ + 2> >(awk -W interactive "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2) echo "$LABEL" >$TEST_DIR/metadata/label fi @@ -417,7 +457,7 @@ provision-cluster() { for i in $(seq 1 $NUM_SERVERS); do provision-server - timeout --foreground 30s bash -c "wait-for-kubeconfig $i" + timeout --foreground 120s bash -c "wait-for-kubeconfig $i" done export KUBECONFIG=$TEST_DIR/servers/1/kubeconfig.yaml @@ -427,8 +467,8 @@ provision-cluster() { done fi - timeout --foreground 1m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))" - timeout --foreground 3m bash -c "wait-for-services $WAIT_SERVICES" + timeout --foreground 4m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))" + timeout --foreground 8m bash -c "wait-for-services $WAIT_SERVICES" run-function cluster-post-hook @@ -480,10 +520,12 @@ run-e2e-tests() { label=PARALLEL \ logName=e2e-STATUS-${ARCH}-parallel.log \ e2e-test ${sonobuoyParallelArgs[@]} + echo "Exit code $? for parallel start" label=SERIAL \ logName=e2e-STATUS-${ARCH}-serial.log \ e2e-test ${sonobuoySerialArgs[@]} + echo "Exit code $? for serial start" } export -f run-e2e-tests