From d8da39ecd0a253ddb03670c5ba7a67408318e6a9 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Thu, 18 Jun 2015 11:31:21 -0700 Subject: [PATCH] Validate binaries downloaded from GCS: * Set SHA1 for Kubernetes server binary and Salt tar in kube-env. * Check SHA1 in configure-vm.sh. If the env variable isn't available, download the SHA1 from GCS and double check that. * Fixes a bug in the devel path where we were actually uploading the wrong sha1 to the bucket. Fixes #10021 --- cluster/common.sh | 14 ++++++- cluster/gce/configure-vm.sh | 77 ++++++++++++++++++++++++------------ cluster/gce/coreos/helper.sh | 2 + cluster/gce/debian/helper.sh | 2 + cluster/gce/util.sh | 19 +++++---- 5 files changed, 80 insertions(+), 34 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 8026430052..38184f175c 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -199,7 +199,9 @@ function set_binary_version() { # PROJECT # Vars set: # SERVER_BINARY_TAR_URL +# SERVER_BINARY_TAR_HASH # SALT_TAR_URL +# SALT_TAR_HASH function tars_from_version() { if [[ -z "${KUBE_VERSION-}" ]]; then find-release-tars @@ -214,9 +216,19 @@ function tars_from_version() { echo "Version doesn't match regexp" >&2 exit 1 fi + until SERVER_BINARY_TAR_HASH=$(curl --fail --silent "${SERVER_BINARY_TAR_URL}.sha1"); do + echo "Failure trying to curl release .sha1" + done + until SALT_TAR_HASH=$(curl --fail --silent "${SALT_TAR_URL}.sha1"); do + echo "Failure trying to curl Salt tar .sha1" + done - if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then + if ! curl -Ss --range 0-1 "${SERVER_BINARY_TAR_URL}" >&/dev/null; then echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2 exit 1 fi + if ! curl -Ss --range 0-1 "${SALT_TAR_URL}" >&/dev/null; then + echo "Can't find Salt tar at ${SALT_TAR_URL}" >&2 + exit 1 + fi } diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 9ac82ace2d..da0f565afd 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -107,11 +107,23 @@ download-or-bust() { local -r url="$1" local -r file="${url##*/}" rm -f "$file" - until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"; do - echo "Failed to download file ($1). Retrying." + until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; do + echo "Failed to download file (${url}). Retrying." done } +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + # Install salt from GCS. See README.md for instructions on how to update these # debs. install-salt() { @@ -430,31 +442,46 @@ EOF fi } +function try-download-release() { + # TODO(zmerlynn): Now we REALLy have no excuse not to do the reboot + # optimization. + + # TODO(zmerlynn): This may not be set yet by everyone (GKE). + if [[ -z "${SERVER_BINARY_TAR_HASH:-}" ]]; then + echo "Downloading binary release sha1 (not found in env)" + download-or-bust "${SERVER_BINARY_TAR_URL}.sha1" + SERVER_BINARY_TAR_HASH=$(cat "${SERVER_BINARY_TAR_URL##*/}.sha1") + fi + + echo "Downloading binary release tar (${SERVER_BINARY_TAR_URL})" + download-or-bust "${SERVER_BINARY_TAR_URL}" + + validate-hash "${SERVER_BINARY_TAR_URL##*/}" "${SERVER_BINARY_TAR_HASH}" + echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" + + # TODO(zmerlynn): This may not be set yet by everyone (GKE). + if [[ -z "${SALT_TAR_HASH:-}" ]]; then + echo "Downloading Salt tar sha1 (not found in env)" + download-or-bust "${SALT_TAR_URL}.sha1" + SALT_TAR_HASH=$(cat "${SALT_TAR_URL##*/}.sha1") + fi + + echo "Downloading Salt tar ($SALT_TAR_URL)" + download-or-bust "$SALT_TAR_URL" + + validate-hash "${SALT_TAR_URL##*/}" "${SALT_TAR_HASH}" + echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" + + echo "Unpacking Salt tree and checking integrity of binary release tar" + rm -rf kubernetes + tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null +} + function download-release() { - # TODO(zmerlynn): We should optimize for the reboot case here, but - # unlike the .debs, we don't have version information in the - # filenames here, nor do the URLs even provide useful information in - # the dev environment case (because they're just a project - # bucket). We should probably push a hash into the kube-env, and - # store it when we download, and then when it's different infer that - # a push occurred (otherwise it's a simple reboot). - - # In case of failure of unpacking Salt tree or checking integrity of - # binary release tar (the last command in the "until" block) retry - # downloading both release and Salt tars. - until - echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" - download-or-bust "$SERVER_BINARY_TAR_URL" - - echo "Downloading Salt tar ($SALT_TAR_URL)" - download-or-bust "$SALT_TAR_URL" - - echo "Unpacking Salt tree and checking integrity of binary release tar" - rm -rf kubernetes - tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null - do + # In case of failure checking integrity of release, retry. + until try-download-release; do sleep 15 - echo "Couldn't unpack Salt tree. Retrying..." + echo "Couldn't download release. Retrying..." done echo "Running release install script" diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index d50fb58f09..4b6c7f8de5 100644 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -31,7 +31,9 @@ INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX}) NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX}) CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16}) SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL}) +SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH}) SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL}) +SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH}) SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE}) ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false}) ENABLE_CLUSTER_MONITORING: $(yaml-quote ${ENABLE_CLUSTER_MONITORING:-none}) diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 2b29eb26d7..3bed23afd9 100644 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -28,7 +28,9 @@ INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX}) NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX}) CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16}) SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL}) +SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH}) SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL}) +SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH}) SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE}) KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME}) ALLOCATE_NODE_CIDRS: $(yaml-quote ${ALLOCATE_NODE_CIDRS:-false}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index d50c6a831b..1b6ae674da 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -178,10 +178,11 @@ function copy-if-not-staged() { if already-staged "${tar}" "${hash}"; then echo "+++ $(basename ${tar}) already staged ('rm ${tar}.sha1' to force)" else - echo "${server_hash}" > "${tar}.sha1" + echo "${hash}" > "${tar}.sha1" gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}" gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1 - echo "${server_hash}" > "${tar}.uploaded.sha1" + echo "${hash}" > "${tar}.uploaded.sha1" + echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})" fi } @@ -194,10 +195,14 @@ function copy-if-not-staged() { # SALT_TAR # Vars set: # SERVER_BINARY_TAR_URL +# SERVER_BINARY_TAR_HASH # SALT_TAR_URL +# SALT_TAR_HASH function upload-server-tars() { SERVER_BINARY_TAR_URL= + SERVER_BINARY_TAR_HASH= SALT_TAR_URL= + SALT_TAR_HASH= local project_hash if which md5 > /dev/null 2>&1; then @@ -220,16 +225,14 @@ function upload-server-tars() { local -r staging_path="${staging_bucket}/devel${KUBE_GCS_STAGING_PATH_SUFFIX}" - local server_hash - local salt_hash - server_hash=$(sha1sum-file "${SERVER_BINARY_TAR}") - salt_hash=$(sha1sum-file "${SALT_TAR}") + SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}") + SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}") echo "+++ Staging server tars to Google Storage: ${staging_path}" local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" local salt_gs_url="${staging_path}/${SALT_TAR##*/}" - copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${server_hash}" - copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${salt_hash}" + copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}" + copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}" # Convert from gs:// URL to an https:// URL SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"