k3s/contrib/util/diagnostics.sh

#!/usr/bin/env bash

# update for posix shell?

set -e

[ $(id -u) -eq 0 ] || exec sudo -E $0 $@

DIAGCMD=${1:-gather-upload-confirm}

DIAGPROG=${DIAGPROG:-k3s}
BUCKET_NAME=${BUCKET_NAME:-"$DIAGPROG-diagnostic-logs"}

bin=/var/lib/rancher/$DIAGPROG/data/current/bin/
if [ -d $bin ]; then
  export PATH=$PATH:$bin:$bin/aux
else
  for bin in /var/lib/rancher/k3s/data/**/bin/; do
    [ -d $bin ] && export PATH=$PATH:$bin:$bin/aux
  done
fi

PUBKEY=${PUBKEY:-'-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1SlmOKCafhG5EzqJHWnT
cEupADJ/2WgbU2PgvTG9TlbaoVyiB5AX6pGFy9hasEJtscmngLvpgY+65te0cJBo
WJ+CMa3nTFXmiX+PGbrBhWMGT5bdM9Lhx5pKvkoaHzL1nNvN/DMeusGqyIdJr3gk
1wlNHr0bZYjlUOvJ3c+X0uIyjX5y0JTjaF5AcbBMlz//zdf7beToPlPuKIlz8FZd
ff4h6dKBYpOnqJW2NBxwICD8ZVokPRRMSZvSY3Mr7HZL1gDoCkOvCsWml27xB0S6
Z6Ib8zB8PFCVWtMZxzcj7ae4tI79OHmaFkEEBAqkBNNU/9S+J0F5tz0caVVnZ+j1
fy13JKIp75vwuDxGgfaru8012QM9zLwXQOcYcHLkLbaTJJ4HpMLC/v0R7TahlLVw
3F1OtQrhQH5PFNtCecpk8SNMgFhYyuCAuWGoai3BtYMNiKFbvuakFSq/XMLFUZS9
T89FaJF2S9liz3VFfCUapBFoD4rZkFCbNufhypwnSVq6MRe1k9V5EaYIsUpfJs33
mpKDVuU/yWwYM+bnlJYo9Sn1QcnjqxRVhUePIActoQ0s9b1CA9NpbqTRiSn7Qxx5
dcnKK+f2NUEdQroCDeUxe2dBLfAvKTCM+c4VCEt2o2d9poSwPytd4K9VdDfiUor+
6u2c2QnLeIcdfRM4j7SmxM0CAwEAAQ==
-----END PUBLIC KEY-----'}

gen-uuid() {
  uuid=$(uuidgen 2>/dev/null)
  [ -z "$uuid" ] && uuid=$(cat /proc/sys/kernel/random/uuid 2>/dev/null)
  [ -z "$uuid" ] && uuid=$(od -x /dev/urandom | head -1 | awk '{OFS="-"; srand($6); sub(/./,"4",$5); sub(/./,substr("89ab",rand()*4,1),$6); print $2$3,$4,$5,$6,$7$8$9}')
  if [ -z "$uuid" ]; then
    echo "Unable to generate UUID" >&2
    return 1
  fi
  tr '[:lower:]' '[:upper:]' <<< "$uuid"
}
echo setup $DIAGUUID
UUID=${UUID:-$(gen-uuid)}
if [ -z "$UUID" ]; then
  echo "UUID is not set and could not be created" >&2
  exit 1
fi

no-cleanup() {
  echo
  echo "Skipping cleanup for $DIAGDIR"
}

cleanup() {
  exit_code=$?
  set +e +x +v
  trap - EXIT INT
  if [ -n "$DIAGDIR" ] && [ -d "$DIAGDIR" ]; then
    rm -rf $DIAGDIR $DIAGDIR.*
  fi
  exit $exit_code
}

setup_diagdir() {
  if [ -z "$DIAGDIR" ]; then
    DIAGDIR=$(readlink -m $(mktemp -d ${TMPDIR:-/tmp}/$DIAGPROG-diagnostics-$UUID-XXXXXXXX))
    trap cleanup INT
  fi
  trap no-cleanup EXIT

  set +e +x +v
  echo "Diagnostics location: $DIAGDIR"
}

remove_empty() {
  if [ -f "$1" ] && [ ! -s "$1" ]; then
    rm "$1"
  fi
}

run_cmd() {
  cmd="$@"
  cmd=${cmd//-/}
  cmd=${cmd// /-}
  cmd=${cmd//\//_}
  logCmdFile="$LOGDIR/$cmd.cmd.txt"
  logOutFile="$LOGDIR/$cmd.txt"
  logErrFile="$LOGDIR/$cmd.err.txt"

  if [ -f "$logCmdFile" ]; then
    echo "Error already ran: $@" >&2
    return 1
  fi

  echo "Gathering command: $@"
  echo "$@" >"$logCmdFile"
  $@ >"$logOutFile"  2>"$logErrFile"
  remove_empty "$logOutFile"
  remove_empty "$logErrFile"
  return 0
}

copy() {
  from=$1
  to=${2:-$1}
  to=${to//\//_}
  to="$LOGDIR/$to"

  if [ ! -e "$from" ]; then
    echo "Skipping copy, does not exist: $from"
    return 1
  fi
  echo "Copying: $from"
  cp --recursive --dereference $from $to
}

setup_logs() {
  export LOGDIR=$DIAGDIR/$1
  mkdir -p $LOGDIR
  echo
  echo "Using subdirectory: $1"
}

log_system()  {
  setup_logs system

  copy /etc/os-release
  run_cmd sysctl -a
  run_cmd uname -a
  run_cmd ps uax
  run_cmd dmesg
  run_cmd id
  run_cmd mount
  run_cmd df -h
  run_cmd ifconfig -a
  run_cmd netstat -ln
  run_cmd netstat -nr
  run_cmd lsof -n -P -p $(pgrep -o $DIAGPROG)
  run_cmd iptables -L
  run_cmd iptables -S
  run_cmd hostname -f
}

log_prog() {
  setup_logs $DIAGPROG

  run_cmd $DIAGPROG --version
  run_cmd $DIAGPROG check-config
  for log in /var/log/$DIAGPROG*.log; do
    copy $log
  done
  if command -v journalctl >/dev/null 2>&1; then
    for unit in $(journalctl --field _SYSTEMD_UNIT | grep "$DIAGPROG"); do
      run_cmd journalctl --unit "$unit" --no-pager
    done
  fi
  copy "/var/lib/rancher/$DIAGPROG/agent/containerd/containerd.log"

  # log cert openssl data?
}

log_kube() {
  setup_logs kube

  copy /var/log/pods # copies all pod logs
  run_cmd command -v kubectl
  run_cmd kubectl version 
  run_cmd kubectl config get-contexts
  run_cmd kubectl config current-context
  run_cmd kubectl cluster-info dump
  run_cmd kubectl get namespaces
  run_cmd kubectl get nodes
  run_cmd kubectl describe nodes
  run_cmd kubectl describe pods --all-namespaces
  run_cmd kubectl describe services --all-namespaces
  run_cmd kubectl describe daemonset --all-namespaces
  run_cmd kubectl describe deployments --all-namespaces
  run_cmd kubectl describe replicaset --all-namespaces
  run_cmd kubectl describe storageclass,pv,pvc
}

contains() {
  [ -z "${1##*$2*}" ]
}

gather() {
  log_system
  log_prog
  log_kube
}

confirm() {
  local def=${2:-'N'}
  local prompt='(y/N)'
  if [ "$def" = 'Y' ]; then
    prompt='(Y/n)'
  fi
  echo
  read -p "$1 $prompt: " -n 1 input
  echo
  if [ "$(tr '[:lower:]' '[:upper:]' <<<"$input")" = 'Y' ]; then
    return 0
  fi
  if [ -z "$input" ] && [ "$def" = 'Y' ]; then
    return 0
  fi
  return 1
}

upload() {
  echo
  echo "Prepare upload of $DIAGDIR"

  if contains "$DIAGCMD" confirm && ! confirm "Perform upload?"; then
    return 1
  fi
  trap cleanup EXIT

  local salt=${salt:-$(openssl rand -hex 8)}
  local key=${key:-$(openssl rand -hex 32)}
  local iv=${iv:-$(openssl rand -hex 16)}
  local base=$(basename $DIAGDIR)
  local dir=$(dirname $DIAGDIR)
  local tar=${TMPDIR:-/tmp}/$base.tar.gz

  echo
  echo "Creating $tar"

  tar -c -z -C $dir $base | \
    openssl enc -aes-256-cbc -S $salt -K $key -iv $iv -in /dev/stdin -out $DIAGDIR.logs.tar.gz.enc

  cat >$DIAGDIR.meta <<EOF
salt=$salt
key=$key
iv=$iv
EOF

  if contains "$DIAGCMD" nometa || ( contains "$DIAGCMD" confirm && ! confirm "Include encrypted metadata in upload?" Y ); then
    echo
    echo "Save secret metadata for log decryption:"
    cat $DIAGDIR.meta
  else
    echo "$PUBKEY" | openssl rsautl -encrypt -inkey /dev/stdin -pubin -in $DIAGDIR.meta -out $DIAGDIR.meta.enc
  fi
  (cd $dir; tar -c -z -f $tar $base.*.enc)

  echo
  echo "Uploading $tar"
  if curl --upload-file $tar "https://storage.googleapis.com/$BUCKET_NAME/"; then
    echo
    echo "Saved diagnostics log in cloud storage as: $base"
    echo
  fi
}

{
  setup_diagdir
  for cmd in gather upload; do
    if contains "$DIAGCMD" $cmd; then
      $cmd
    else
      echo
      echo "Skipping $cmd..."
    fi
  done
}

# record log of above about

exit 0