From dbcf0faa00332319c491630a7993b1215fcb01bb Mon Sep 17 00:00:00 2001 From: Marcin Owsiany Date: Tue, 7 Nov 2017 15:36:14 +0100 Subject: [PATCH] Add some error handling in place of ilusory one. TL;DR: "set -e" is ignored inside function foo when it's called like "foo || something". See https://github.com/kubernetes/kubernetes/issues/55229 for details. This is a short-term hack that will hopefully let us at least see the error messages whenever we hit intermittent certificate setup errors next time. Once we know what fails there, we can start working on an actual fix, which may very well involve rewriting this in a language other than shell, with better error handling. --- cluster/common.sh | 61 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 579bbfa7a8..1c4e57f580 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -1002,7 +1002,6 @@ function create-certs { PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs - CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" # By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces. # Note 'base64 -w0' doesn't work on Mac OS X, which has different flags. CA_KEY_BASE64=$(cat "${CERT_DIR}/pki/private/ca.key" | base64 | tr -d '\r\n') @@ -1019,13 +1018,20 @@ function create-certs { # Setting up an addition directory (beyond pki) as it is the simplest way to # ensure we get a different CA pair to sign the proxy-client certs and which # we can send CA public key to the user-apiserver to validate communication. - AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" AGGREGATOR_CA_KEY_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" | base64 | tr -d '\r\n') REQUESTHEADER_CA_CERT_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | base64 | tr -d '\r\n') PROXY_CLIENT_CERT_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | base64 | tr -d '\r\n') PROXY_CLIENT_KEY_BASE64=$(cat "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | base64 | tr -d '\r\n') } +# Set up easy-rsa directory structure. +# +# Assumed vars +# KUBE_TEMP +# +# Vars set: +# CERT_DIR +# AGGREGATOR_CERT_DIR function setup-easyrsa { local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") # Note: This was heavily cribbed from make-ca-cert.sh @@ -1036,21 +1042,25 @@ function setup-easyrsa { mkdir easy-rsa-master/kubelet cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet mkdir easy-rsa-master/aggregator - cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator) &>${cert_create_debug_output} || { - # If there was an error in the subshell, just die. - # TODO(roberthbailey): add better error handling here + cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator) &>${cert_create_debug_output} || true + CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" + AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" + if [ ! -x "${CERT_DIR}/easyrsa" -o ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 cat "${cert_create_debug_output}" >&2 echo "=== Failed to setup easy-rsa: Aborting ===" >&2 exit 2 - } + fi } # Runs the easy RSA commands to generate certificate files. -# The generated files are at ${KUBE_TEMP}/easy-rsa-master/easyrsa3 +# The generated files are IN ${CERT_DIR} # # Assumed vars # KUBE_TEMP # MASTER_NAME +# CERT_DIR # PRIMARY_CN: Primary canonical name # SANS: Subject alternate names # @@ -1059,7 +1069,8 @@ function generate-certs { local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") # Note: This was heavily cribbed from make-ca-cert.sh (set -x - cd "${KUBE_TEMP}/easy-rsa-master/easyrsa3" + set -o errexit + cd "${CERT_DIR}" ./easyrsa init-pki # this puts the cert into pki/ca.crt and the key into pki/private/ca.key ./easyrsa --batch "--req-cn=${PRIMARY_CN}@$(date +%s)" build-ca nopass @@ -1080,21 +1091,32 @@ function generate-certs { ./easyrsa --dn-mode=org \ --req-cn=kubecfg --req-org=system:masters \ --req-c= --req-st= --req-city= --req-email= --req-ou= \ - build-client-full kubecfg nopass) &>${cert_create_debug_output} || { - # If there was an error in the subshell, just die. - # TODO(roberthbailey): add better error handling here + build-client-full kubecfg nopass) &>${cert_create_debug_output} || true + if [ ! -s "${CERT_DIR}/pki/private/ca.key" \ + -o ! -s "${CERT_DIR}/pki/ca.crt" \ + -o ! -s "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" \ + -o ! -s "${CERT_DIR}/pki/private/${MASTER_NAME}.key" \ + -o ! -s "${CERT_DIR}/pki/issued/kubelet.crt" \ + -o ! -s "${CERT_DIR}/pki/private/kubelet.key" \ + -o ! -s "${CERT_DIR}/pki/issued/kubecfg.crt" \ + -o ! -s "${CERT_DIR}/pki/private/kubecfg.key" \ + -o ! -s "${CERT_DIR}/pki/issued/kube-apiserver.crt" \ + -o ! -s "${CERT_DIR}/pki/private/kube-apiserver.key" ]; then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 cat "${cert_create_debug_output}" >&2 echo "=== Failed to generate master certificates: Aborting ===" >&2 exit 2 - } + fi } # Runs the easy RSA commands to generate aggregator certificate files. -# The generated files are at ${KUBE_TEMP}/easy-rsa-master/aggregator +# The generated files are in ${AGGREGATOR_CERT_DIR} # # Assumed vars # KUBE_TEMP # AGGREGATOR_MASTER_NAME +# AGGREGATOR_CERT_DIR # AGGREGATOR_PRIMARY_CN: Primary canonical name # AGGREGATOR_SANS: Subject alternate names # @@ -1124,13 +1146,18 @@ function generate-aggregator-certs { ./easyrsa --dn-mode=org \ --req-cn=proxy-clientcfg --req-org=system:aggregator \ --req-c= --req-st= --req-city= --req-email= --req-ou= \ - build-client-full proxy-clientcfg nopass) &>${cert_create_debug_output} || { - # If there was an error in the subshell, just die. - # TODO(roberthbailey): add better error handling here + build-client-full proxy-clientcfg nopass) &>${cert_create_debug_output} || true + + if [ ! -s "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" \ + -o ! -s "${AGGREGATOR_CERT_DIR}/pki/ca.crt" \ + -o ! -s "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" \ + -o ! -s "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" ]; then + # TODO(roberthbailey,porridge): add better error handling here, + # see https://github.com/kubernetes/kubernetes/issues/55229 cat "${cert_create_debug_output}" >&2 echo "=== Failed to generate aggregator certificates: Aborting ===" >&2 exit 2 - } + fi } # Run the cfssl command to generates certificate files for etcd service, the