diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 1477784aa5..a4574e0028 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -18,6 +18,7 @@ # gcloud multiplexing for shared GCE/GKE tests. GCLOUD=gcloud ZONE=${KUBE_GCE_ZONE:-us-central1-b} +RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false} MASTER_SIZE=${MASTER_SIZE:-n1-standard-2} NODE_SIZE=${NODE_SIZE:-n1-standard-2} NUM_NODES=${NUM_NODES:-3} diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index d5328abefa..3a8d42b55b 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -18,6 +18,7 @@ # gcloud multiplexing for shared GCE/GKE tests. GCLOUD=gcloud ZONE=${KUBE_GCE_ZONE:-us-central1-b} +RELEASE_REGION_FALLBACK=${RELEASE_REGION_FALLBACK:-false} MASTER_SIZE=${MASTER_SIZE:-n1-standard-2} NODE_SIZE=${NODE_SIZE:-n1-standard-2} NUM_NODES=${NUM_NODES:-3} diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 7f05f18bec..c5bc4509f9 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -140,15 +140,32 @@ function remove-docker-artifacts() { echo "== Finished deleting docker0 ==" } -# Retry a download until we get it. +# Retry a download until we get it. Takes a hash and a set of URLs. # -# $1 is the URL to download +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. download-or-bust() { - local -r url="$1" - local -r file="${url##*/}" - rm -f "$file" - until curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; do - echo "Failed to download file (${url}). Retrying." + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done done } @@ -603,39 +620,43 @@ EOF fi } +function split-commas() { + echo $1 | tr "," "\n" +} + function try-download-release() { # TODO(zmerlynn): Now we REALLy have no excuse not to do the reboot # optimization. - # TODO(zmerlynn): This may not be set yet by everyone (GKE). - if [[ -z "${SERVER_BINARY_TAR_HASH:-}" ]]; then + local -r server_binary_tar_urls=( $(split-commas "${SERVER_BINARY_TAR_URL}") ) + local -r server_binary_tar="${server_binary_tar_urls[0]##*/}" + if [[ -n "${SERVER_BINARY_TAR_HASH:-}" ]]; then + local -r server_binary_tar_hash="${SERVER_BINARY_TAR_HASH}" + else echo "Downloading binary release sha1 (not found in env)" - download-or-bust "${SERVER_BINARY_TAR_URL}.sha1" - SERVER_BINARY_TAR_HASH=$(cat "${SERVER_BINARY_TAR_URL##*/}.sha1") + download-or-bust "" "${server_binary_tar_urls[@]/.tar.gz/.tar.gz.sha1}" + local -r server_binary_tar_hash=$(cat "${server_binary_tar}.sha1") fi - echo "Downloading binary release tar (${SERVER_BINARY_TAR_URL})" - download-or-bust "${SERVER_BINARY_TAR_URL}" + echo "Downloading binary release tar (${server_binary_tar_urls[@]})" + download-or-bust "${server_binary_tar_hash}" "${server_binary_tar_urls[@]}" - validate-hash "${SERVER_BINARY_TAR_URL##*/}" "${SERVER_BINARY_TAR_HASH}" - echo "Validated ${SERVER_BINARY_TAR_URL} SHA1 = ${SERVER_BINARY_TAR_HASH}" - - # TODO(zmerlynn): This may not be set yet by everyone (GKE). - if [[ -z "${SALT_TAR_HASH:-}" ]]; then + local -r salt_tar_urls=( $(split-commas "${SALT_TAR_URL}") ) + local -r salt_tar="${salt_tar_urls[0]##*/}" + if [[ -n "${SALT_TAR_HASH:-}" ]]; then + local -r salt_tar_hash="${SALT_TAR_HASH}" + else echo "Downloading Salt tar sha1 (not found in env)" - download-or-bust "${SALT_TAR_URL}.sha1" - SALT_TAR_HASH=$(cat "${SALT_TAR_URL##*/}.sha1") + download-or-bust "" "${salt_tar_urls[@]/.tar.gz/.tar.gz.sha1}" + local -r salt_tar_hash=$(cat "${salt_tar}.sha1") fi - echo "Downloading Salt tar ($SALT_TAR_URL)" - download-or-bust "$SALT_TAR_URL" - - validate-hash "${SALT_TAR_URL##*/}" "${SALT_TAR_HASH}" - echo "Validated ${SALT_TAR_URL} SHA1 = ${SALT_TAR_HASH}" + echo "Downloading Salt tar (${salt_tar_urls[@]})" + download-or-bust "${salt_tar_hash}" "${salt_tar_urls[@]}" echo "Unpacking Salt tree and checking integrity of binary release tar" rm -rf kubernetes - tar xzf "${SALT_TAR_URL##*/}" && tar tzf "${SERVER_BINARY_TAR_URL##*/}" > /dev/null + tar xzf "${salt_tar}" && tar tzf "${server_binary_tar}" > /dev/null } function download-release() { diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 0a9f52b217..fd1ea94c42 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -115,33 +115,43 @@ function detect-project () { fi } -function already-staged() { - local -r file=$1 - local -r newsum=$2 - - [[ -e "${file}.uploaded.sha1" ]] || return 1 - - local oldsum - oldsum=$(cat "${file}.uploaded.sha1") - - [[ "${oldsum}" == "${newsum}" ]] -} - -# Copy a release tar, if we don't already think it's staged in GCS -function copy-if-not-staged() { +# Copy a release tar and its accompanying hash. +function copy-to-staging() { local -r staging_path=$1 local -r gs_url=$2 local -r tar=$3 local -r hash=$4 - if already-staged "${tar}" "${hash}"; then - echo "+++ $(basename ${tar}) already staged ('rm ${tar}.uploaded.sha1' to force)" - else - echo "${hash}" > "${tar}.sha1" - gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}" - gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1 - echo "${hash}" > "${tar}.uploaded.sha1" - echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})" + echo "${hash}" > "${tar}.sha1" + gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha1" "${staging_path}" + gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha1" >/dev/null 2>&1 + echo "+++ $(basename ${tar}) uploaded (sha1 = ${hash})" +} + +# Given the cluster zone, return the list of regional GCS release +# bucket suffixes for the release in preference order. GCS doesn't +# give us an API for this, so we hardcode it. +# +# Assumed vars: +# REGIONAL_RELEASE +# ZONE +# Vars set: +# PREFERRED_REGION +function set-preferred-region() { + case ${ZONE} in + asia-*) + PREFERRED_REGION=("asia" "us" "eu") + ;; + europe-*) + PREFERRED_REGION=("eu" "us" "asia") + ;; + *) + PREFERRED_REGION=("us" "eu" "asia") + ;; + esac + + if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then + PREFERRED_REGION=( "${PREFERRED_REGION[0]}" ) fi } @@ -155,6 +165,7 @@ function copy-if-not-staged() { # SERVER_BINARY_TAR # SALT_TAR # KUBE_MANIFESTS_TAR +# ZONE # Vars set: # SERVER_BINARY_TAR_URL # SERVER_BINARY_TAR_HASH @@ -181,35 +192,59 @@ function upload-server-tars() { # that's probably good enough for now :P project_hash=${project_hash:0:10} - local -r staging_bucket="gs://kubernetes-staging-${project_hash}" - - # Ensure the bucket is created - if ! gsutil ls "$staging_bucket" > /dev/null 2>&1 ; then - echo "Creating $staging_bucket" - gsutil mb "${staging_bucket}" - fi - - local -r staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel" + set-preferred-region SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}") SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}") + if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then + KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}") + fi - echo "+++ Staging server tars to Google Storage: ${staging_path}" - local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" - local salt_gs_url="${staging_path}/${SALT_TAR##*/}" - copy-if-not-staged "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}" - copy-if-not-staged "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}" + local server_binary_tar_urls=() + local salt_tar_urls=() + local kube_manifest_tar_urls=() - # Convert from gs:// URL to an https:// URL - SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" - SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" + for region in "${PREFERRED_REGION[@]}"; do + suffix="-${region}" + if [[ "${suffix}" == "-us" ]]; then + suffix="" + fi + local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}" + + # Ensure the buckets are created + if ! gsutil ls "${staging_bucket}" > /dev/null 2>&1 ; then + echo "Creating ${staging_bucket}" + gsutil mb -l "${region}" "${staging_bucket}" + fi + + local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel" + + echo "+++ Staging server tars to Google Storage: ${staging_path}" + local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" + local salt_gs_url="${staging_path}/${SALT_TAR##*/}" + copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}" + copy-to-staging "${staging_path}" "${salt_gs_url}" "${SALT_TAR}" "${SALT_TAR_HASH}" + + # Convert from gs:// URL to an https:// URL + server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}") + salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}") + + if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then + local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}" + copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}" + # Convert from gs:// URL to an https:// URL + kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}") + fi + done if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "coreos" ]]; then - local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}" - KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}") - copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}" - # Convert from gs:// URL to an https:// URL - KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}" + # TODO: Support fallback .tar.gz settings on CoreOS/Trusty + SERVER_BINARY_TAR_URL="${server_binary_tar_urls[0]}" + SALT_TAR_URL="${salt_tar_urls[0]}" + KUBE_MANIFESTS_TAR_URL="${kube_manifests_tar_urls[0]}" + else + SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}") + SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}") fi }