diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index 60d962dc38..6609fd0129 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -429,6 +429,30 @@ function dump_nodes_with_logexporter() { fi } +function detect_node_failures() { + if ! [[ "${gcloud_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then + return + fi + + detect-node-names + for group in "${INSTANCE_GROUPS[@]}"; do + local creation_timestamp=$(gcloud compute instance-groups managed describe \ + "${group}" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --format='value(creationTimestamp)') + echo "Failures for ${group}" + gcloud logging read --order=asc \ + --format='table(timestamp,jsonPayload.resource.name,jsonPayload.event_subtype)' \ + --project "${PROJECT}" \ + "resource.type=\"gce_instance\" + logName=\"projects/${PROJECT}/logs/compute.googleapis.com%2Factivity_log\" + (jsonPayload.event_subtype=\"compute.instances.hostError\" OR jsonPayload.event_subtype=\"compute.instances.automaticRestart\") + jsonPayload.resource.name:\"${group}\" + timestamp >= \"${creation_timestamp}\"" + done +} + function main() { setup # Copy master logs to artifacts dir locally (through SSH). @@ -447,6 +471,8 @@ function main() { echo "Dumping logs from nodes locally to '${report_dir}'" dump_nodes fi + + detect_node_failures } main