mirror of https://github.com/k3s-io/k3s
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
645 lines
17 KiB
645 lines
17 KiB
#!/bin/bash |
|
|
|
# --- |
|
|
|
port-used() { |
|
netstat -tuna | grep -q ":$1 " |
|
} |
|
export -f port-used |
|
|
|
# --- |
|
|
|
get-port() { |
|
local port=0 |
|
while |
|
port=$((10000 + RANDOM % 50000)) |
|
port-used $port |
|
do continue; done |
|
echo $port |
|
} |
|
export -f get-port |
|
|
|
# --- |
|
|
|
fetch-kubeconfig() {( |
|
set -e -o pipefail |
|
local num=${1:-1} |
|
local name=$(cat $TEST_DIR/servers/$num/metadata/name) |
|
local port=$(cat $TEST_DIR/servers/$num/metadata/port) |
|
docker cp $name:/etc/rancher/k3s/k3s.yaml - 2>/dev/null | tar -xO 2>/dev/null | sed -e "s/:6443/:$port/g" >$TEST_DIR/servers/$num/kubeconfig.yaml |
|
)} |
|
export -f fetch-kubeconfig |
|
|
|
# --- |
|
|
|
wait-for-kubeconfig() { |
|
while ! fetch-kubeconfig $1; do |
|
echo 'Waiting for kubeconfig to become available...' >&2 |
|
sleep 5 |
|
done |
|
} |
|
export -f wait-for-kubeconfig |
|
|
|
# --- |
|
|
|
count-ready-nodes() { |
|
kubectl get nodes -o json \ |
|
| jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \ |
|
| wc -l \ |
|
| tr -d '[:space:]' |
|
} |
|
export -f count-ready-nodes |
|
|
|
# --- |
|
|
|
wait-for-nodes() { |
|
while [[ $(count-ready-nodes) -ne $1 ]]; do |
|
echo 'Waiting for nodes to be ready...' >&2 |
|
sleep 5 |
|
done |
|
} |
|
export -f wait-for-nodes |
|
|
|
# --- |
|
|
|
pod-ready() { |
|
kubectl get pods -n kube-system -o json \ |
|
| jq ".items[].status | select(.containerStatuses != null) | .containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null |
|
} |
|
export -f pod-ready |
|
|
|
# --- |
|
|
|
wait-for-services() { |
|
for service in "$@"; do |
|
while [[ "$(pod-ready $service | sort -u)" != 'true' ]]; do |
|
echo "Waiting for service $service to be ready..." >&2 |
|
sleep 5 |
|
done |
|
echo "Service $service is ready" |
|
done |
|
} |
|
export -f wait-for-services |
|
|
|
# --- |
|
|
|
wait-for-db-connection() { |
|
if [ -z "$DB_CONNECTION_TEST" ]; then |
|
echo 'DB_CONNECTION_TEST is not defined' >&2 |
|
return 1 |
|
fi |
|
while ! $DB_CONNECTION_TEST 2>/dev/null; do |
|
echo 'Waiting for database to become available...' >&2 |
|
sleep 5 |
|
done |
|
} |
|
export -f wait-for-db-connection |
|
|
|
# --- |
|
|
|
verify-valid-version() { |
|
docker exec "$@" 2>&1 | tee .version.tmp |
|
# check for bad strings in the version output, including '.' in the build metadata |
|
if grep -oiE '.*(dev|head|unknown|fail|refuse|\+[^"]*\.).*' .version.tmp; then |
|
return 1 |
|
fi |
|
} |
|
export -f verify-valid-version |
|
|
|
# --- |
|
|
|
verify-valid-versions() { |
|
verify-valid-version $1 kubectl version |
|
verify-valid-version $1 ctr version |
|
verify-valid-version $1 crictl version |
|
} |
|
export -f verify-valid-versions |
|
|
|
# --- |
|
|
|
dump-logs() { |
|
local testID=$(basename $TEST_DIR) |
|
echo "#---------------------------------" |
|
echo "#- Begin: logs for run ($testID)" |
|
echo |
|
local server=$(cat $TEST_DIR/servers/1/metadata/name) |
|
docker exec $server kubectl get pods -A -o wide >$TEST_DIR/logs/kubectl-get-pods.txt |
|
docker exec $server kubectl get nodes -o wide >$TEST_DIR/logs/kubectl-get-nodes.txt |
|
docker exec $server kubectl describe pods -A >$TEST_DIR/logs/kubectl-describe-pods.txt |
|
|
|
for node in $TEST_DIR/*/*; do |
|
[ -d "$node" ] || continue |
|
local name=$(cat $node/metadata/name 2>/dev/null) |
|
[ "$name" ] || continue |
|
mkdir -p $node/logs |
|
local hostname=$(docker exec $name hostname) |
|
docker logs $name >$node/logs/system.log 2>&1 |
|
if [[ ! -z "$hostname" && $name == k3s-* ]]; then |
|
docker exec $server kubectl describe node/$hostname >$node/logs/kubectl-describe-node.txt |
|
docker cp $name:/var/lib/rancher/k3s/agent/containerd/containerd.log $node/logs/containerd.log 2>/dev/null |
|
docker exec $name crictl pods >$node/logs/crictl-pods.txt |
|
docker exec $name crictl ps -a >$node/logs/crictl-ps.txt |
|
docker exec $name crictl ps -a -o json >$node/metadata/crictl-ps.json |
|
for container in $(jq -r '.containers[].id' <$node/metadata/crictl-ps.json); do |
|
local cname=$(jq -r '.containers[] | select(.id == "'$container'") | .metadata.name' <$node/metadata/crictl-ps.json) |
|
docker exec $name crictl logs $container >$node/logs/$cname-$container.log 2>&1 |
|
done |
|
fi |
|
if [ "$1" == "skip-output" ]; then |
|
continue |
|
fi |
|
for log in $node/logs/*.log; do |
|
echo |
|
echo "#- Tail: $log" |
|
tail -5 $log |
|
echo "#- Done: $log" |
|
echo |
|
done |
|
done |
|
|
|
if [ "$1" == "skip-output" ]; then |
|
return |
|
fi |
|
|
|
for txt in $TEST_DIR/logs/*.txt $TEST_DIR/*/*/logs/*.txt; do |
|
echo |
|
echo "#- Cat: $txt" |
|
cat $txt |
|
echo "#- Done: $txt" |
|
echo |
|
done |
|
echo |
|
echo "#- Finish: logs for run ($testID)" |
|
echo "#---------------------------------" |
|
echo |
|
./scripts/log-upload $TEST_DIR |
|
} |
|
export -f dump-logs |
|
|
|
# --- |
|
|
|
retrieve-sonobuoy-logs() { |
|
local status=passed |
|
local code=0 |
|
local testStatus=$(sonobuoy status 2>&1) |
|
cat <<< $testStatus |
|
|
|
if ! grep -q -E '\s+e2e\s+complete\s+passed\s+' <<< $testStatus; then |
|
status=failed |
|
code=1 |
|
fi |
|
|
|
mkdir -p $TEST_DIR/sonobuoy |
|
local logTarball="$(sonobuoy retrieve $TEST_DIR/sonobuoy)" |
|
if [ -f "$logTarball" ]; then |
|
tar -xz -f "$logTarball" -C $TEST_DIR/sonobuoy |
|
rm "$logTarball" |
|
else |
|
rm -rf $TEST_DIR/sonobuoy |
|
fi |
|
|
|
local e2eLog=$TEST_DIR/sonobuoy/plugins/e2e/results/global/e2e.log |
|
if [ ! -s $e2eLog ]; then |
|
return 1 |
|
fi |
|
if [ -n "$LOG_OUTPUT" ]; then |
|
cp $e2eLog $(sed -e "s/-STATUS-/-$status-/g" <<< "$LOG_OUTPUT") |
|
fi |
|
tail -11 $e2eLog |
|
awk '/^Summarizing .* Failures?:$/,0' $e2eLog |
|
return $code |
|
} |
|
export -f retrieve-sonobuoy-logs |
|
|
|
# --- |
|
|
|
test-wait() { |
|
wait $1 || echo "test wait exit code $?" |
|
local delay=15 |
|
sleep $delay |
|
while sonobuoy status | grep -q -E ' +e2e +running +'; do |
|
sleep $delay |
|
done |
|
sleep $delay |
|
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then |
|
return 0 |
|
fi |
|
return 1 |
|
} |
|
export -f test-wait |
|
|
|
# --- |
|
|
|
sonobuoy-test() { |
|
if [ "$ARCH" = 'arm' ]; then |
|
echo "Aborting sonobuoy tests, images not available for $ARCH" |
|
return 0 |
|
fi |
|
echo 'Starting sonobuoy tests' |
|
|
|
sonobuoy run \ |
|
--config=scripts/sonobuoy-config.json \ |
|
--plugin-env=e2e.E2E_USE_GO_RUNNER=true \ |
|
--kubernetes-version=${VERSION_K8S} \ |
|
--wait=90 \ |
|
"$@" & |
|
|
|
local sonobuoyPID=$! |
|
local code=0 |
|
time timeout --foreground 60m bash -c test-wait $sonobuoyPID || code=$? |
|
echo "Sonobuoy finished with code $code" |
|
retrieve-sonobuoy-logs |
|
return $code |
|
} |
|
export -f sonobuoy-test |
|
|
|
# --- |
|
|
|
test-cleanup() { |
|
local code=$? |
|
set +e -x |
|
echo 'Cleaning up...' |
|
trap - EXIT INT TERM |
|
if has-function test-post-hook; then |
|
test-post-hook $code |
|
code=$? |
|
fi |
|
if [[ $code -ne 0 ]]; then |
|
dump-logs |
|
fi |
|
for name in $TEST_DIR/*/*/metadata/name; do |
|
[ -f "$name" ] || continue |
|
local container=$(cat $name) |
|
echo "Removing container $container" |
|
docker rm -f -v $container |
|
done |
|
if has-function test-cleanup-hook; then |
|
test-cleanup-hook |
|
fi |
|
echo |
|
if [ "$TEST_CLEANUP" = true ]; then |
|
echo "Removing test directory $TEST_DIR" |
|
rm -rf $TEST_DIR |
|
fi |
|
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK |
|
echo -n "Test $(basename $TEST_DIR) " |
|
if [ $code -eq 0 ]; then |
|
echo "passed." |
|
else |
|
echo "failed." |
|
fi |
|
echo |
|
exit $code |
|
} |
|
export -f test-cleanup |
|
|
|
# --- |
|
|
|
test-setup() { |
|
export TEST_DIR=$(mktemp -d '/tmp/XXXXXX') |
|
trap test-cleanup EXIT INT TERM |
|
|
|
mkdir -p $TEST_DIR/metadata |
|
if [ "$LABEL" ]; then |
|
exec > >(awk "{ printf \"[\033[36m${LABEL}\033[m] %s\n\", \$0; fflush() }") \ |
|
2> >(awk "{ printf \"[\033[35m${LABEL}\033[m] %s\n\", \$0; fflush() }" >&2) |
|
echo "$LABEL" >$TEST_DIR/metadata/label |
|
fi |
|
|
|
mkdir -p $TEST_DIR/logs |
|
exec > >(tee -a $TEST_DIR/logs/test.log) \ |
|
2> >(tee -a $TEST_DIR/logs/test.log >&2) |
|
|
|
if [ -z "$K3S_IMAGE" ]; then |
|
. ./scripts/version.sh |
|
TAG=${TAG:-${VERSION_TAG}${SUFFIX}} |
|
REPO=${REPO:-rancher} |
|
IMAGE_NAME=${IMAGE_NAME:-k3s} |
|
export K3S_IMAGE=${REPO}/${IMAGE_NAME}:${TAG} |
|
fi |
|
|
|
if [ -z "$K3S_IMAGE" ]; then |
|
echo 'K3S_IMAGE environment variable should be defined' |
|
return 1 |
|
fi |
|
|
|
SERVER_MINOR=$(awk -F. '{print $2}' <<<${K3S_IMAGE_SERVER:-$K3S_IMAGE}) |
|
AGENT_MINOR=$(awk -F. '{print $2}' <<<${K3S_IMAGE_AGENT:-$K3S_IMAGE}) |
|
if [ $NUM_AGENTS -gt 0 ] && [ $AGENT_MINOR -gt $SERVER_MINOR ]; then |
|
echo "Agent minor version cannot be higher than server - not supported by Kubernetes version skew policy" |
|
exit 0 |
|
fi |
|
|
|
echo ${RANDOM}${RANDOM}${RANDOM} >$TEST_DIR/metadata/secret |
|
} |
|
export -f test-setup |
|
|
|
# --- |
|
|
|
inc-count() { |
|
local count=$(find $TEST_DIR -type d -mindepth 2 -maxdepth 2 -regex ".*/$1/[0-9]+" -printf '%f\n' | sort -nr | head -1) |
|
count=$((count+1)) |
|
mkdir -p $TEST_DIR/$1/$count/metadata |
|
echo $count |
|
} |
|
export -f inc-count |
|
|
|
# --- |
|
|
|
has-function() { |
|
[[ -n "$1" && $(type -t $1) == "function" ]] |
|
} 2> /dev/null |
|
export -f has-function |
|
|
|
# --- |
|
|
|
run-function() { |
|
has-function $1 || return 0 |
|
"$@" |
|
} |
|
export -f run-function |
|
|
|
# --- |
|
|
|
provision-server() { |
|
local count=$(inc-count servers) |
|
local testID=$(basename $TEST_DIR) |
|
local name=$(echo "k3s-server-${count}-${testID,,}" | tee $TEST_DIR/servers/$count/metadata/name) |
|
local port=$(timeout --foreground 5s bash -c get-port | tee $TEST_DIR/servers/$count/metadata/port) |
|
local SERVER_INSTANCE_ARGS="SERVER_${count}_ARGS" |
|
|
|
run-function server-pre-hook $count |
|
|
|
docker run \ |
|
-d --name $name \ |
|
--hostname $name \ |
|
--privileged \ |
|
-p 127.0.0.1:$port:6443 \ |
|
-p 6443 \ |
|
-e K3S_TOKEN="$(cat $TEST_DIR/metadata/secret)" \ |
|
-e K3S_DEBUG=true \ |
|
${SERVER_DOCKER_ARGS:-} \ |
|
${REGISTRY_CLUSTER_ARGS:-} \ |
|
${K3S_IMAGE_SERVER:-$K3S_IMAGE} server ${ARGS} ${SERVER_ARGS} ${!SERVER_INSTANCE_ARGS} |
|
|
|
local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/servers/$count/metadata/ip) |
|
local url=$(echo "https://$ip:6443" | tee $TEST_DIR/servers/$count/metadata/url) |
|
|
|
echo "Started $name @ $url" |
|
run-function server-post-hook $count |
|
} |
|
export -f provision-server |
|
|
|
# --- |
|
|
|
provision-agent() { |
|
local K3S_URL=${K3S_URL:-"$(cat $TEST_DIR/servers/1/metadata/url)"} |
|
local count=$(inc-count agents) |
|
local testID=$(basename $TEST_DIR) |
|
local name=$(echo "k3s-agent-${count}-${testID,,}" | tee $TEST_DIR/agents/$count/metadata/name) |
|
local AGENT_INSTANCE_ARGS="AGENT_${count}_ARGS" |
|
|
|
run-function agent-pre-hook $count |
|
docker run \ |
|
-d --name $name \ |
|
--hostname $name \ |
|
--privileged \ |
|
-e K3S_TOKEN=$(cat $TEST_DIR/metadata/secret) \ |
|
-e K3S_URL=$K3S_URL \ |
|
${AGENT_DOCKER_ARGS:-} \ |
|
${REGISTRY_CLUSTER_ARGS:-} \ |
|
${K3S_IMAGE_AGENT:-$K3S_IMAGE} agent ${ARGS} ${AGENT_ARGS} ${!AGENT_INSTANCE_ARGS} |
|
|
|
local ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $name | tee $TEST_DIR/agents/$count/metadata/ip) |
|
|
|
echo "Started $name" |
|
run-function agent-post-hook $count |
|
} |
|
export -f provision-agent |
|
|
|
# --- |
|
|
|
provision-cluster() { |
|
run-function cluster-pre-hook |
|
|
|
if [ "${ENABLE_REGISTRY}" == 'true' ]; then |
|
provision-registry-proxy |
|
fi |
|
|
|
for i in $(seq 1 $NUM_SERVERS); do |
|
provision-server |
|
timeout --foreground 120s bash -c "wait-for-kubeconfig $i" |
|
done |
|
export KUBECONFIG=$TEST_DIR/servers/${KUBECONFIG_SERVER:-1}/kubeconfig.yaml |
|
|
|
if [ $NUM_AGENTS -gt 0 ]; then |
|
for _ in $(seq 1 $NUM_AGENTS); do |
|
provision-agent |
|
done |
|
fi |
|
|
|
[ -f "$PROVISION_LOCK" ] && rm $PROVISION_LOCK |
|
|
|
timeout --foreground 2m bash -c "wait-for-nodes $(( NUM_SERVERS + NUM_AGENTS ))" |
|
timeout --foreground 4m bash -c "wait-for-services $WAIT_SERVICES" |
|
|
|
run-function cluster-post-hook |
|
} |
|
export -f provision-cluster |
|
|
|
# --- |
|
|
|
provision-registry-proxy() { |
|
set -e -o pipefail |
|
local image="docker.io/library/registry:2.8.1" |
|
local prefix="docker-registry-" |
|
local registries="docker.io:registry-1.docker.io registry.k8s.io gcr.io quay.io ghcr.io" |
|
local registries_yaml="$TEST_DIR/registries.yaml" |
|
|
|
echo "mirrors:" > $registries_yaml |
|
|
|
for registry in $registries; do |
|
IFS=: read registry_name registry_endpoint <<< $registry |
|
if [ -z "$registry_endpoint" ]; then |
|
registry_endpoint=$registry_name |
|
fi |
|
|
|
local name="registry_${registry_name//./_}" |
|
local status=$(docker inspect $name --format '{{ .State.Status }} {{ .Config.Image }} {{ (index .HostConfig.PortBindings "5000/tcp" 0).HostPort }}' 2>/dev/null || true) |
|
|
|
read state_status config_image hostport <<< $status |
|
if [ "$state_status" != "running" ] || [ "$config_image" != "$image" ]; then |
|
hostport=$(timeout --foreground 5s bash -c get-port) |
|
docker rm --force $name 2>/dev/null || true |
|
docker run \ |
|
-d --name $name \ |
|
-p 0.0.0.0:$hostport:5000 \ |
|
-v "registry-cache:/var/lib/registry" \ |
|
-e "REGISTRY_HTTP_SECRET=shared-secret" \ |
|
-e "REGISTRY_PROXY_REMOTEURL=https://$registry_endpoint" \ |
|
-e "REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR=inmemory" \ |
|
-e "REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/var/lib/registry/$registry_name" \ |
|
$image |
|
fi |
|
echo -e " $registry_name:\n endpoint:\n - http://172.17.0.1:$hostport" >> $registries_yaml |
|
done |
|
|
|
echo "Using registry mirror with cluster registries.yaml:" |
|
cat $registries_yaml |
|
|
|
export REGISTRY_CLUSTER_ARGS="-v $registries_yaml:/etc/rancher/k3s/registries.yaml" |
|
} |
|
export -f provision-registry-proxy |
|
|
|
# --- |
|
|
|
early-exit() { |
|
printf "\033[33m%s\033[m\n" "$1" |
|
exit $2 |
|
} |
|
export -f early-exit |
|
|
|
# --- |
|
|
|
run-test() { |
|
local delay=15 |
|
( |
|
set +x |
|
while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-4} ]; do |
|
sleep $delay |
|
done |
|
) |
|
|
|
export PROVISION_LOCK=$(mktemp) |
|
./tests/docker/test-runner "$@" & |
|
pids+=($!) |
|
|
|
( |
|
set +x |
|
# busy-wait on the provisioning lock before imposing a final inter-test delay |
|
while [ -f "$PROVISION_LOCK" ]; do |
|
sleep 1 |
|
done |
|
sleep $delay |
|
) |
|
} |
|
export -f run-test |
|
|
|
# --- |
|
|
|
cleanup-test-env(){ |
|
export NUM_SERVERS=1 |
|
export NUM_AGENTS=1 |
|
export AGENT_ARGS='' |
|
export SERVER_ARGS='' |
|
export WAIT_SERVICES="${all_services[*]}" |
|
|
|
unset AGENT_1_ARGS AGENT_2_ARGS AGENT_3_ARGS AGENT_DOCKER_ARGS K3S_IMAGE_AGENT |
|
unset SERVER_1_ARGS SERVER_2_ARGS SERVER_3_ARGS SERVER_DOCKER_ARGS K3S_IMAGE_SERVER |
|
|
|
unset -f server-pre-hook server-post-hook agent-pre-hook agent-post-hook cluster-pre-hook cluster-post-hook test-post-hook test-cleanup-hook |
|
} |
|
|
|
# --- |
|
|
|
count-running-tests(){ |
|
local count=0 |
|
for pid in "${pids[@]}"; do |
|
if [ $(pgrep -c -P $pid) -gt 0 ]; then |
|
((count++)) |
|
fi |
|
done |
|
echo "Currently running ${count} tests" 1>&2 |
|
echo ${count} |
|
} |
|
export -f count-running-tests |
|
|
|
# --- |
|
|
|
e2e-test() { |
|
local label=$label |
|
if [ -n "$LABEL_SUFFIX" ]; then |
|
label="$label-$LABEL_SUFFIX" |
|
fi |
|
local logOutput= |
|
if [ -n "$E2E_OUTPUT" ]; then |
|
logOutput=$E2E_OUTPUT/$logName |
|
fi |
|
if [[ $label =~ ^PARALLEL.* ]]; then |
|
LABEL=$label LOG_OUTPUT=$logOutput MAX_CONCURRENT_TESTS=3 run-test "$@" |
|
else |
|
LABEL=$label LOG_OUTPUT=$logOutput run-test "$@" |
|
fi |
|
|
|
} |
|
|
|
# --- |
|
|
|
test-run-sonobuoy() { |
|
local suffix |
|
if [ "$1" ] && [[ ! $1 =~ ^(serial|parallel)$ ]] ; then |
|
suffix="-$1" |
|
export LABEL_SUFFIX=$1 |
|
else |
|
unset LABEL_SUFFIX |
|
fi |
|
|
|
cleanup-test-env |
|
. ./tests/docker/test-setup-sonobuoy$suffix |
|
|
|
if [ "$1" = "parallel" ] || [ "$2" = "parallel" ]; then |
|
label=PARALLEL \ |
|
logName=e2e-STATUS-${ARCH}-parallel.log \ |
|
e2e-test "${sonobuoyParallelArgs[@]}" |
|
echo "Exit code $? for parallel start" |
|
fi |
|
|
|
if [ "$1" = "serial" ] || [ "$2" = "serial" ]; then |
|
label=SERIAL \ |
|
logName=e2e-STATUS-${ARCH}-serial.log \ |
|
e2e-test "${sonobuoySerialArgs[@]}" |
|
echo "Exit code $? for serial start" |
|
fi |
|
} |
|
export -f test-run-sonobuoy |
|
|
|
# --- |
|
|
|
pid-cleanup() { |
|
local code=$? |
|
local failCount=0 |
|
set +e |
|
if [ $code -eq 0 ]; then |
|
for pid in "${pids[@]}"; do |
|
wait $pid || code=$? |
|
done |
|
fi |
|
if [ $code -ne 0 ]; then |
|
for pid in "${pids[@]}"; do |
|
pkill -P $pid |
|
wait $pid || failCount=$((failCount+1)) |
|
done |
|
fi |
|
trap - EXIT INT TERM |
|
set +x |
|
echo |
|
if [ $failCount -eq 0 ]; then |
|
printf '\033[32mAll tests passed.\033[m\n' |
|
else |
|
printf "\033[31m%s tests failed.\033[m\n" "$failCount" |
|
if [ "$DRONE_BUILD_EVENT" = 'tag' ]; then |
|
printf "\033[31mIgnoring test failures on tag.\033[m\n" |
|
code=0 |
|
else |
|
code=1 |
|
fi |
|
fi |
|
echo |
|
exit $code |
|
} |
|
export -f pid-cleanup |
|
|
|
# --- |
|
|
|
pids=() |
|
trap pid-cleanup EXIT INT TERM
|
|
|