github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Test script cleanup backport from RKE2 (#2650) 

* Test script cleanup backport from RKE2

Signed-off-by: Erik Wilson <Erik.E.Wilson@gmail.com>

* Update scripts/test

Co-authored-by: Brian Downs <brian.downs@gmail.com>

Co-authored-by: Brian Downs <brian.downs@gmail.com>
pull/2654/head
Erik Wilson 4 years ago committed by GitHub
parent
fadc5a8057
commit
cccba681a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 64 additions and 15 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 9
      scripts/test
  2. 70
      scripts/test-helpers

9
scripts/test
Unescape View File

@ -10,7 +10,12 @@ mkdir -p $artifacts
# ---
docker ps
# ---
. ./scripts/test-run-basics
echo "Did test-run-basics $?"
# ---
@ -18,7 +23,7 @@ mkdir -p $artifacts
early-exit "Skipping remaining tests, images not available for $ARCH."
E2E_OUTPUT=$artifacts test-run-sonobuoy
echo "Did test-run-sonobuoy $?"
# ---
[ "$DRONE_BUILD_EVENT" = 'tag' ] && \
@ -28,3 +33,5 @@ E2E_OUTPUT=$artifacts test-run-sonobuoy
test-run-sonobuoy mysql
test-run-sonobuoy postgres
exit 0

70
scripts/test-helpers
Unescape View File

@ -64,7 +64,7 @@ export -f wait-for-nodes
pod-ready() {
kubectl get pods -n kube-system -o json \
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
| jq ".items[].status | select(.containerStatuses != null) | .containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready
@ -72,7 +72,7 @@ export -f pod-ready
wait-for-services() {
for service in $@; do
while [[ "$(pod-ready $service)" != 'true' ]]; do
while [[ "$(pod-ready $service | sort -u)" != 'true' ]]; do
echo "Waiting for service $service to be ready..." >&2
sleep 5
done
@ -122,13 +122,27 @@ dump-logs() {
echo "#---------------------------------"
echo "#- Begin: logs for run ($testID)"
echo
local server=$(cat $TEST_DIR/servers/1/metadata/name)
docker exec $server kubectl get pods -A -o wide >$TEST_DIR/logs/kubectl-get-pods.txt
docker exec $server kubectl get nodes -o wide >$TEST_DIR/logs/kubectl-get-nodes.txt
docker exec $server kubectl describe pods -A >$TEST_DIR/logs/kubectl-describe-pods.txt
for node in $TEST_DIR/*/*; do
[ -d "$node" ] || continue
local name=$(cat $node/metadata/name 2>/dev/null)
[ "$name" ] || continue
mkdir -p $node/logs
local hostname=$(docker exec $name hostname)
docker exec $server kubectl describe node/$hostname >$node/logs/kubectl-describe-node.txt
docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null
docker logs $name >$node/logs/system.log 2>&1
docker exec $name crictl pods >$node/logs/crictl-pods.txt
docker exec $name crictl ps -a >$node/logs/crictl-ps.txt
docker exec $name crictl ps -a -o json >$node/metadata/crictl-ps.json
for container in $(jq -r '.containers[].id' <$node/metadata/crictl-ps.json); do
local cname=$(jq -r '.containers[] | select(.id == "'$container'") | .metadata.name' <$node/metadata/crictl-ps.json)
docker exec $name crictl logs $container >$node/logs/$cname-$container.log 2>&1
done
for log in $node/logs/*.log; do
echo
echo "#- Tail: $log"
@ -137,6 +151,13 @@ dump-logs() {
echo
done
done
for txt in $TEST_DIR/logs/*.txt $TEST_DIR/*/*/logs/*.txt; do
echo
echo "#- Cat: $txt"
cat $txt
echo "#- Done: $txt"
echo
done
echo
echo "#- Finish: logs for run ($testID)"
echo "#---------------------------------"
@ -176,6 +197,7 @@ retrieve-sonobuoy-logs() {
if [ -n "$LOG_OUTPUT" ]; then
cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT")
fi
tail -11 $e2eLog
awk '/^Summarizing .* Failures?:$/,0' $e2eLog
return $code
}
@ -183,6 +205,23 @@ export -f retrieve-sonobuoy-logs
# ---
test-wait() {
wait $1 || echo "test wait exit code $?"
local delay=15
sleep $delay
while sonobuoy status | grep -q -E ' +e2e +running +'; do
sleep $delay
done
sleep $delay
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
return 0
fi
return 1
}
export -f test-wait
# ---
sonobuoy-test() {
if [ "$ARCH" = 'arm' ]; then
echo "Aborting sonobuoy tests, images not available for $ARCH"
@ -200,8 +239,8 @@ sonobuoy-test() {
local sonobuoyPID=$!
local code=0
time wait $sonobuoyPID || code=$?
time timeout --foreground 30m bash -c test-wait $sonobuoyPID || code=$?
echo "Sonobuoy finished with code $code"
retrieve-sonobuoy-logs
return $code
}
@ -211,7 +250,7 @@ export -f sonobuoy-test
test-cleanup() {
local code=$?
set +e +x
set +e -x
echo 'Cleaning up...'
trap - EXIT INT TERM
if [[ $code -ne 0 ]]; then
@ -220,13 +259,14 @@ test-cleanup() {
for name in $TEST_DIR/*/*/metadata/name; do
[ -f "$name" ] || continue
local container=$(cat $name)
docker rm -f -v $container >/dev/null 2>&1 &
echo "Removing container $container"
docker rm -f -v $container
done
if [ "$TEST_CLEANUP" = true ]; then
echo "Removing $TEST_DIR"
rm -rf $TEST_DIR >/dev/null 2>&1 &
echo "Removing test directory $TEST_DIR"
rm -rf $TEST_DIR
fi
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK >/dev/null 2>&1 &
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK
echo
echo -n "Test $(basename $TEST_DIR) "
if [ $code -eq 0 ]; then
@ -247,8 +287,8 @@ test-setup() {
mkdir -p $TEST_DIR/metadata
if [ "$LABEL" ]; then
exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \
2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2)
exec > >(awk -W interactive "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0 }") \
2> >(awk -W interactive "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0 }" >&2)
echo "$LABEL" >$TEST_DIR/metadata/label
fi
@ -417,7 +457,7 @@ provision-cluster() {
for i in $(seq 1 $NUM_SERVERS); do
provision-server
timeout --foreground 30s bash -c "wait-for-kubeconfig $i"
timeout --foreground 120s bash -c "wait-for-kubeconfig $i"
done
export KUBECONFIG=$TEST_DIR/servers/1/kubeconfig.yaml
@ -427,8 +467,8 @@ provision-cluster() {
done
fi
timeout --foreground 1m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))"
timeout --foreground 3m bash -c "wait-for-services $WAIT_SERVICES"
timeout --foreground 4m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))"
timeout --foreground 8m bash -c "wait-for-services $WAIT_SERVICES"
run-function cluster-post-hook
@ -480,10 +520,12 @@ run-e2e-tests() {
label=PARALLEL \
logName=e2e-STATUS-${ARCH}-parallel.log \
e2e-test ${sonobuoyParallelArgs[@]}
echo "Exit code $? for parallel start"
label=SERIAL \
logName=e2e-STATUS-${ARCH}-serial.log \
e2e-test ${sonobuoySerialArgs[@]}
echo "Exit code $? for serial start"
}
export -f run-e2e-tests

Write Preview
Loading…
Cancel
Save