github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fixes to Drone CI Stability (#4897)

Browse Source 
* Do serial only first

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Add test-post-hook for extremely flaky tests, enable parallel

Signed-off-by: Derek Nola <derek.nola@suse.com>

* Add flaky test post hook to etcd

* Add concurrent parallel, fix flaky tests checks

Signed-off-by: Derek Nola <derek.nola@suse.com>
pull/5050/head^2
Derek Nola 3 years ago committed by GitHub
parent d583a99f62
commit 444a7cb0ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 130 additions and 19 deletions
Show Stats Download Patch File Download Diff File

40
scripts/test
Unescape View File

@ -32,17 +32,43 @@ echo "Did test-run-lazypull $?"
[ "$ARCH" != 'amd64' ] && \
early-exit "Skipping remaining tests, images not available for $ARCH."
E2E_OUTPUT=$artifacts test-run-sonobuoy
echo "Did test-run-sonobuoy $?"
E2E_OUTPUT=$artifacts test-run-sonobuoy serial
echo "Did test-run-sonobuoy serial $?"
# ---
[ "$DRONE_BUILD_EVENT" = 'tag' ] && \
if [ "$DRONE_BUILD_EVENT" = 'tag' ]; then
E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
echo "Did test-run-sonobuoy parallel $?"
early-exit 'Skipping remaining tests on tag.'
fi
# ---
test-run-sonobuoy mysql
test-run-sonobuoy postgres
test-run-sonobuoy etcd skip-parallel
test-run-sonobuoy etcd serial
echo "Did test-run-sonobuoy-etcd serial $?"
test-run-sonobuoy mysql serial
echo "Did test-run-sonobuoy-mysqk serial $?"
test-run-sonobuoy postgres serial
echo "Did test-run-sonobuoy-postgres serial $?"
# Wait until all serial tests have finished
delay=15
(
set +x
while [ $(count-running-tests) -ge 1 ]; do
sleep $delay
done
)
E2E_OUTPUT=$artifacts test-run-sonobuoy parallel
echo "Did test-run-sonobuoy parallel $?"
test-run-sonobuoy etcd parallel
echo "Did test-run-sonobuoy-etcd parallel $?"
test-run-sonobuoy mysql parallel
echo "Did test-run-sonobuoy-mysql parallel $?"
test-run-sonobuoy postgres parallel
echo "Did test-run-sonobuoy-postgres parallel $?"
exit 0

42
scripts/test-helpers
Unescape View File

@ -145,6 +145,9 @@ dump-logs() {
docker exec $name crictl logs $container >$node/logs/$cname-$container.log 2>&1
done
fi
if [ "$1" == "skip-output" ]; then
continue
fi
for log in $node/logs/*.log; do
echo
echo "#- Tail: $log"
@ -153,6 +156,11 @@ dump-logs() {
echo
done
done
if [ "$1" == "skip-output" ]; then
return
fi
for txt in $TEST_DIR/logs/*.txt $TEST_DIR/*/*/logs/*.txt; do
echo
echo "#- Cat: $txt"
@ -254,6 +262,11 @@ test-cleanup() {
set +e -x
echo 'Cleaning up...'
trap - EXIT INT TERM
if has-function test-post-hook; then
echo 'Running post-hook'
test-post-hook $code
code=$?
fi
if [[ $code -ne 0 ]]; then
dump-logs
fi
@ -264,10 +277,6 @@ test-cleanup() {
docker rm -f -v $container
done
echo
if has-function test-post-hook; then
test-post-hook $code
code=$?
fi
if [ "$TEST_CLEANUP" = true ]; then
echo "Removing test directory $TEST_DIR"
rm -rf $TEST_DIR
@ -554,7 +563,7 @@ run-test() {
local delay=15
(
set +x
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-3} ]; do
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-4} ]; do
sleep $delay
done
)
@ -614,32 +623,41 @@ e2e-test() {
if [ -n "$E2E_OUTPUT" ]; then
logOutput=$E2E_OUTPUT/$logName
fi
LABEL=$label LOG_OUTPUT=$logOutput run-test $@
if [[ $label =~ ^PARALLEL.* ]]; then
LABEL=$label LOG_OUTPUT=$logOutput MAX_CONCURRENT_TESTS=3 run-test $@
else
LABEL=$label LOG_OUTPUT=$logOutput run-test $@
fi
}
# ---
test-run-sonobuoy() {
local suffix
if [ "$1" ]; then
if [ "$1" ] && [[ ! $1 =~ ^(serial|parallel)$ ]] ; then
suffix="-$1"
export LABEL_SUFFIX=$1
else
unset LABEL_SUFFIX
fi
cleanup-test-env
. ./scripts/test-setup-sonobuoy$suffix
if [ "$2" != "skip-parallel" ]; then
if [ "$1" = "parallel" ] || [ "$2" = "parallel" ]; then
label=PARALLEL \
logName=e2e-STATUS-${ARCH}-parallel.log \
e2e-test ${sonobuoyParallelArgs[@]}
echo "Exit code $? for parallel start"
fi
label=SERIAL \
logName=e2e-STATUS-${ARCH}-serial.log \
e2e-test ${sonobuoySerialArgs[@]}
echo "Exit code $? for serial start"
if [ "$1" = "serial" ] || [ "$2" = "serial" ]; then
label=SERIAL \
logName=e2e-STATUS-${ARCH}-serial.log \
e2e-test ${sonobuoySerialArgs[@]}
echo "Exit code $? for serial start"
fi
}
export -f test-run-sonobuoy

1
scripts/test-run-compat
Unescape View File

@ -22,6 +22,7 @@ test-post-hook() {
if [[ $1 -eq 0 ]]; then
return
fi
dump-logs skip-output
grep -sqF 'incompatible down-level server detected' $TEST_DIR/agents/*/logs/system.log
}
export -f test-post-hook

1
scripts/test-run-etcd
Unescape View File

@ -37,6 +37,7 @@ test-post-hook() {
if [[ $1 -eq 0 ]]; then
return
fi
dump-logs skip-output
grep -sqF 'critical configuration value mismatch' $TEST_DIR/servers/2/logs/system.log
}
export -f test-post-hook

16
scripts/test-setup-sonobuoy
Unescape View File

@ -12,3 +12,19 @@ start-test() {
sonobuoy-test $@
}
export -f start-test
test-post-hook() {
if [[ $1 -eq 0 ]]; then
return
fi
local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log)
# Ignore sonobuoy failures if only these flaky tests have failed
local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]'
local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]'
local flakyTest3='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]'
local flakyTest4='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]'
flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" )
totalFails=$( grep -scF -e "[Fail]" <<< "$failures" )
[ "$totalFails" -le "$flakyFails" ]
}
export -f test-post-hook

17
scripts/test-setup-sonobuoy-etcd
Unescape View File

@ -13,3 +13,20 @@ server-post-hook() {
fi
}
export -f server-post-hook
test-post-hook() {
if [[ $1 -eq 0 ]]; then
return
fi
local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log)
# Ignore sonobuoy failures if only these flaky tests have failed
local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]'
local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]'
local flakyTest3='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]'
local flakyTest4='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]'
flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" )
totalFails=$( grep -scF -e "[Fail]" <<< "$failures" )
[ "$totalFails" -le "$flakyFails" ]
}
export -f test-post-hook

16
scripts/test-setup-sonobuoy-mysql
Unescape View File

@ -54,3 +54,19 @@ cluster-pre-hook() {
# --datastore-keyfile /db-ca/private/client.key
}
export -f cluster-pre-hook
test-post-hook() {
if [[ $1 -eq 0 ]]; then
return
fi
local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log)
# Ignore sonobuoy failures if only these flaky tests have failed
local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]'
local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]'
local flakyTest3='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]'
local flakyTest4='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]'
flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" )
totalFails=$( grep -scF -e "[Fail]" <<< "$failures" )
[ "$totalFails" -le "$flakyFails" ]
}
export -f test-post-hook

16
scripts/test-setup-sonobuoy-postgres
Unescape View File

@ -54,3 +54,19 @@ cluster-pre-hook() {
# --datastore-keyfile /db-ca/private/client.key
}
export -f cluster-pre-hook
test-post-hook() {
if [[ $1 -eq 0 ]]; then
return
fi
local failures=$(awk '/^Summarizing .* Failures?:$/,0' "$TEST_DIR"/sonobuoy/plugins/e2e/results/global/e2e.log)
# Ignore sonobuoy failures if only these flaky tests have failed
local flakyTest1='[Fail] [sig-node] Probing container [It] should have monotonically increasing restart count [NodeConformance] [Conformance]'
local flakyTest2='[Fail] [sig-node] Pods [It] should delete a collection of pods [Conformance]'
local flakyTest3='[Fail] [sig-node] Pods [It] should run through the lifecycle of Pods and PodStatus [Conformance]'
local flakyTest4='[Fail] [sig-network] Proxy version v1 [It] A set of valid responses are returned for both pod and service ProxyWithPath [Conformance]'
flakyFails=$( grep -scF -e "$flakyTest1" -e "$flakyTest2" -e "$flakyTest3" -e "$flakyTest4" <<< "$failures" )
totalFails=$( grep -scF -e "[Fail]" <<< "$failures" )
[ "$totalFails" -le "$flakyFails" ]
}
export -f test-post-hook
Write Preview
Loading…
Cancel
Save