Merge pull request #65139 from wojtek-t/fix_logexporter

Automatic merge from submit-queue (batch tested with PRs 65123, 65176, 65139, 65084, 65056). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Periodically fetch logexported nodes instead of sleeping
pull/8/head
Kubernetes Submit Queue 2018-06-21 16:56:13 -07:00 committed by GitHub
commit 624dec20c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 38 additions and 12 deletions

View File

@ -281,6 +281,24 @@ function dump_nodes() {
fi
}
# Collect names of nodes which didn't run logexporter successfully.
# Note: This step is O(#nodes^2) as we check if each node is present in the list of succeeded nodes.
# Making it linear would add code complexity without much benefit (as it just takes ~1s for 5k nodes).
# Assumes:
# NODE_NAMES
# Sets:
# NON_LOGEXPORTED_NODES
function find_non_logexported_nodes() {
succeeded_nodes=$(gsutil ls ${gcs_artifacts_dir}/logexported-nodes-registry) || return 1
echo "Successfully listed marker files for successful nodes"
NON_LOGEXPORTED_NODES=()
for node in "${NODE_NAMES[@]}"; do
if [[ ! "${succeeded_nodes}" =~ "${node}" ]]; then
NON_LOGEXPORTED_NODES+=("${node}")
fi
done
}
function dump_nodes_with_logexporter() {
echo "Detecting nodes in the cluster"
detect-node-names &> /dev/null
@ -312,14 +330,27 @@ function dump_nodes_with_logexporter() {
return
fi
# Give some time for the pods to finish uploading logs.
sleep "${logexport_sleep_seconds}"
# Periodically fetch list of already logexported nodes to verify
# if we aren't already done.
start="$(date +%s)"
while true; do
now="$(date +%s)"
if [[ $((now - start)) -gt ${logexport_sleep_seconds} ]]; then
echo "Waiting for all nodes to be logexported timed out."
break
fi
if find_non_logexported_nodes; then
if [[ -z "${NON_LOGEXPORTED_NODES:-}" ]]; then
break
fi
fi
sleep 15
done
# List registry of marker files (of nodes whose logexporter succeeded) from GCS.
local nodes_succeeded
for retry in {1..10}; do
if nodes_succeeded=$(gsutil ls ${gcs_artifacts_dir}/logexported-nodes-registry); then
echo "Successfully listed marker files for successful nodes"
if find_non_logexported_nodes; then
break
else
echo "Attempt ${retry} failed to list marker files for succeessful nodes"
@ -333,15 +364,10 @@ function dump_nodes_with_logexporter() {
fi
done
# Collect names of nodes which didn't run logexporter successfully.
# Note: This step is O(#nodes^2) as we check if each node is present in the list of succeeded nodes.
# Making it linear would add code complexity without much benefit (as it just takes ~1s for 5k nodes).
failed_nodes=()
for node in "${NODE_NAMES[@]}"; do
if [[ ! "${nodes_succeeded}" =~ "${node}" ]]; then
echo "Logexporter didn't succeed on node ${node}. Queuing it for logdump through SSH."
failed_nodes+=("${node}")
fi
for node in "${NON_LOGEXPORTED_NODES[@]:-}"; do
echo "Logexporter didn't succeed on node ${node}. Queuing it for logdump through SSH."
failed_nodes+=("${node}")
done
# Delete the logexporter resources and dump logs for the failed nodes (if any) through SSH.