Merge pull request #1044 from erikwilson/upload-logs

Upload logs to gcloud
pull/1054/head
Erik Wilson 5 years ago committed by GitHub
commit 16ec08d425
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -57,6 +57,10 @@ steps:
- name: sonobuoy-e2e-tests
image: rancher/dapper:v0.4.2
secrets: [ gcloud_auth ]
environment:
GCLOUD_AUTH:
from_secret: gcloud_auth
commands:
- dapper -f Dockerfile.sonobuoy.dapper
volumes:
@ -144,6 +148,10 @@ steps:
- name: sonobuoy-e2e-tests
image: rancher/dapper:v0.4.2
secrets: [ gcloud_auth ]
environment:
GCLOUD_AUTH:
from_secret: gcloud_auth
commands:
- dapper -f Dockerfile.sonobuoy.dapper
volumes:
@ -214,6 +222,10 @@ steps:
- name: sonobuoy-e2e-tests
image: rancher/dapper:v0.4.2
secrets: [ gcloud_auth ]
environment:
GCLOUD_AUTH:
from_secret: gcloud_auth
commands:
- dapper -f Dockerfile.sonobuoy.dapper
volumes:

@ -1,6 +1,6 @@
FROM golang:1.13.3-alpine3.10
RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils
RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2
RUN SONOBUOY_VER=v0.16.2 && \
SONOBUOY_PKG=github.com/vmware-tanzu/sonobuoy && \
go get -d ${SONOBUOY_PKG} && \
@ -18,7 +18,7 @@ RUN curl -sL https://storage.googleapis.com/kubernetes-release/release/$( \
chmod a+x /usr/local/bin/kubectl
ENV DAPPER_RUN_ARGS --privileged --network host
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME GCLOUD_AUTH
ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
ENV DAPPER_OUTPUT ./dist
ENV DAPPER_DOCKER_SOCKET true

@ -0,0 +1,48 @@
#!/bin/bash
[ -d "$1" ] || {
echo "First argument should be a directory" >&2
exit 1
}
umask 077
TMPDIR=$(mktemp -d)
cleanup() {
exit_code=$?
trap - EXIT INT
rm -rf ${TMPDIR}
exit ${exit_code}
}
trap cleanup EXIT INT
GCLOUD_JSON=${TMPDIR}/.gcloud.json
[ -z "${GCLOUD_AUTH}" ] || echo "${GCLOUD_AUTH}" >${GCLOUD_JSON}
[ -s "${GCLOUD_JSON}" ] || {
echo "gcloud auth not defined" >&2
exit 1
}
BOTO_CONF=${TMPDIR}/.boto
[ -s "${BOTO_CONF}" ] || cat >${BOTO_CONF} <<END
[Credentials]
gs_service_key_file = ${GCLOUD_JSON}
[Boto]
https_validate_certificates = True
[GSUtil]
content_language = en
default_api_version = 2
default_project_id = rancher-dev
END
[ -d "${TMPDIR}/gsutil" ] || curl -sfL https://storage.googleapis.com/pub/gsutil.tar.gz | tar xz -C ${TMPDIR}
HOME=${TMPDIR}
PATH=$PATH:${HOME}/gsutil
LOG_TGZ=k3s-log-$(uname -s | tr '[:upper:]' '[:lower:]' )-$(go env GOARCH)-$(git rev-parse --short HEAD)-$(date +%s)-$(basename $1).tgz
tar -cz -f ${TMPDIR}/${LOG_TGZ} -C $(dirname $1) $(basename $1)
gsutil cp ${TMPDIR}/${LOG_TGZ} gs://k3s-ci-logs || exit 1
echo "Logs uploaded" >&2
echo "https://storage.googleapis.com/k3s-ci-logs/${LOG_TGZ}"

@ -1,9 +1,10 @@
#!/bin/bash
set -xe
cd $(dirname $0)/..
. ./scripts/test-helpers
if [ -z "${K3S_IMAGE}" ]; then
echo 'K3S_IMAGE environment variable should be defined'
exit 1
@ -11,150 +12,6 @@ fi
# ---
port-used() {
(cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
}
export -f port-used
# ---
get-port() {
while
PORT=$((10000 + RANDOM % 50000))
port-used ${PORT}
do continue; done
echo ${PORT}
}
export -f get-port
# ---
fetch-kubeconfig() {
docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
}
export -f fetch-kubeconfig
# ---
wait-for-kubeconfig() {
while ! fetch-kubeconfig; do
echo 'Waiting for kubeconfig to become available...'
sleep 5
done
}
export -f wait-for-kubeconfig
# ---
count-ready-nodes() {
kubectl get nodes -o json \
| jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
| wc -l \
| tr -d '[:space:]'
}
export -f count-ready-nodes
# ---
wait-for-nodes() {
while [[ $(count-ready-nodes) -ne $1 ]]; do
echo 'Waiting for nodes to be ready...'
sleep 5
done
}
export -f wait-for-nodes
# ---
pod-ready() {
kubectl get pods -n kube-system -o json \
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready
# ---
wait-for-services() {
for service in $@; do
while [[ "$(pod-ready ${service})" != 'true' ]]; do
echo "Waiting for service ${service} to be ready..."
sleep 5
done
echo "Service ${service} is ready"
done
}
export -f wait-for-services
# ---
verify-valid-version() {
if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
return 1
fi
}
export -f verify-valid-version
# ---
verify-valid-versions() {
verify-valid-version $1 kubectl version
verify-valid-version $1 ctr version
verify-valid-version $1 crictl version
}
export -f verify-valid-versions
# ---
dump-container-logs() {
mkdir -p ${LOGS}
for container in ${CONTAINERS}; do
docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/containerd-${container}.log
docker logs ${container} >${LOGS}/${container}.log 2>&1
done
}
export -f dump-container-logs
# ---
retrieve-sonobuoy-logs() {
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
status=passed
exit_code=0
else
status=failed
exit_code=1
fi
mkdir -p ${E2E}
sonobuoy retrieve ${E2E}
tar x -z -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
if [ ! -s ${RESULTS} ]; then
return 1
fi
if [ -n "${E2E_LOG_OUTPUT}" ]; then
cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
fi
awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
return ${exit_code}
}
export -f retrieve-sonobuoy-logs
# ---
sonobuoy-test() {
time sonobuoy run \
--config=scripts/sonobuoy-config.json \
--plugin-env=e2e.E2E_USE_GO_RUNNER=true \
--wait=30 \
"${@}" &
SONOBUOY_PID=$!
wait $SONOBUOY_PID
retrieve-sonobuoy-logs
}
export -f sonobuoy-test
# ---
cleanup() {
exit_status=$?
set +e
@ -162,7 +19,7 @@ cleanup() {
trap - EXIT
[ -n "$SONOBUOY_PID" ] && kill $SONOBUOY_PID 2>/dev/null
if [ "${exit_status}" -ne "0" ]; then
dump-container-logs
dump-logs
fi
docker rm -f ${CONTAINERS} 2>/dev/null
rm ${KUBECONFIG}
@ -180,8 +37,8 @@ E2E_LOG='plugins/e2e/results/global/e2e.log'
RESULTS="${E2E}/${E2E_LOG}"
SECRET=random-$((100000 + RANDOM % 999999))
export K3S_AGENT=sonobuoy-k3s-agent-${K3S_PORT}
export K3S_SERVER=sonobuoy-k3s-server-${K3S_PORT}
export K3S_AGENT=k3s-agent-${K3S_PORT}
export K3S_SERVER=k3s-server-${K3S_PORT}
export CONTAINERS="${K3S_SERVER} ${K3S_AGENT}"
export KUBECONFIG=${OUTPUT}/kubeconfig.yaml

@ -25,12 +25,12 @@ show-logs() {
fi
echo
echo "#- Begin: logs for sonobuoy run pid ${pid}"
for log in $(pwd)/logs/${pid}/*; do
for log in $(pwd)/logs/${pid}/*.log; do
if [ -f ${log} ]; then
echo
echo "#- Start: ${log}"
cat ${log}
echo "#- End: ${log}"
echo "#- Tail: ${log}"
tail -10 ${log}
echo "#- Done: ${log}"
echo
fi
done
@ -57,18 +57,23 @@ cleanup() {
}
trap cleanup EXIT INT
# ---
run-sonobuoy() {
output+=(${log_output})
E2E_LOG_OUTPUT=${log_output} ./scripts/sonobuoy ${@} &
E2E_LOG_OUTPUT=${log_output} \
./scripts/sonobuoy ${@} \
> >(stdbuf -oL sed "s/^/[${label}] /") 2>&1 &
pids+=($!)
}
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log \
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-parallel.log label=PARALLEL \
run-sonobuoy --e2e-focus='\[Conformance\]' --e2e-skip='\[Serial\]' --e2e-parallel=y
sleep 60
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log \
log_output=${OUTPUT}/e2e-STATUS-${ARCH}-serial.log label=SERIAL \
run-sonobuoy --e2e-focus='\[Serial\].*\[Conformance\]'
exit_code=0

@ -0,0 +1,150 @@
#!/bin/bash
# ---
port-used() {
(cat </dev/null >/dev/tcp/127.0.0.1/$1) 2>/dev/null
}
export -f port-used
# ---
get-port() {
while
PORT=$((10000 + RANDOM % 50000))
port-used ${PORT}
do continue; done
echo ${PORT}
}
export -f get-port
# ---
fetch-kubeconfig() {
docker cp ${K3S_SERVER}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} 2>/dev/null
}
export -f fetch-kubeconfig
# ---
wait-for-kubeconfig() {
while ! fetch-kubeconfig; do
echo 'Waiting for kubeconfig to become available...'
sleep 5
done
}
export -f wait-for-kubeconfig
# ---
count-ready-nodes() {
kubectl get nodes -o json \
| jq '.items[].status.conditions[] | select(.type == "Ready" and .status == "True") | .type' \
| wc -l \
| tr -d '[:space:]'
}
export -f count-ready-nodes
# ---
wait-for-nodes() {
while [[ $(count-ready-nodes) -ne $1 ]]; do
echo 'Waiting for nodes to be ready...'
sleep 5
done
}
export -f wait-for-nodes
# ---
pod-ready() {
kubectl get pods -n kube-system -o json \
| jq ".items[].status.containerStatuses[] | select(.name == \"$1\") | .ready" 2>/dev/null
}
export -f pod-ready
# ---
wait-for-services() {
for service in $@; do
while [[ "$(pod-ready ${service})" != 'true' ]]; do
echo "Waiting for service ${service} to be ready..."
sleep 5
done
echo "Service ${service} is ready"
done
}
export -f wait-for-services
# ---
verify-valid-version() {
if docker exec $@ 2>&1 | grep -iE '(dev|head|unknown|fail|refuse)'; then
return 1
fi
}
export -f verify-valid-version
# ---
verify-valid-versions() {
verify-valid-version $1 kubectl version
verify-valid-version $1 ctr version
verify-valid-version $1 crictl version
}
export -f verify-valid-versions
# ---
dump-logs() {
mkdir -p ${LOGS}/sonobuoy
tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${LOGS}/sonobuoy
for container in ${CONTAINERS}; do
docker cp ${container}:/var/lib/rancher/k3s/agent/containerd/containerd.log ${LOGS}/${container}-containerd.log
docker logs ${container} >${LOGS}/${container}-system.log 2>&1
done
./scripts/log-upload ${LOGS}
}
export -f dump-logs
# ---
retrieve-sonobuoy-logs() {
sonobuoy status || true
if sonobuoy status | grep -q -E ' +e2e +complete +passed +'; then
status=passed
exit_code=0
else
status=failed
exit_code=1
fi
mkdir -p ${E2E}
sonobuoy retrieve ${E2E}
tar -xz -f ${E2E}/*_sonobuoy_*.tar.gz -C ${E2E} ${E2E_LOG}
if [ ! -s ${RESULTS} ]; then
return 1
fi
if [ -n "${E2E_LOG_OUTPUT}" ]; then
cp ${RESULTS} $(sed -e "s/-STATUS-/-${status}-/g" <<< "${E2E_LOG_OUTPUT}")
fi
awk '/^Summarizing .* Failures:$/,0' ${RESULTS}
return ${exit_code}
}
export -f retrieve-sonobuoy-logs
# ---
sonobuoy-test() {
time sonobuoy run \
--config=scripts/sonobuoy-config.json \
--plugin-env=e2e.E2E_USE_GO_RUNNER=true \
--wait=30 \
"${@}" &
SONOBUOY_PID=$!
wait $SONOBUOY_PID
retrieve-sonobuoy-logs
}
export -f sonobuoy-test
# ---
Loading…
Cancel
Save