From 0142e4c9c24da55908eb3fb9191d363529bf5727 Mon Sep 17 00:00:00 2001 From: Piotr Szczesniak Date: Mon, 1 Jun 2015 17:59:12 +0200 Subject: [PATCH] Refactored kube-push.sh script The script allows also to push binaries only to the master or specified node. Added support for released tars. Introduced new push methods and implemented them for GCE. --- cluster/common.sh | 52 +++++++++++++++++ cluster/gce/upgrade.sh | 73 ++++-------------------- cluster/gce/util.sh | 123 ++++++++++++++++++++++++++--------------- cluster/kube-push.sh | 55 +++++++++++++++++- cluster/kube-util.sh | 17 +++++- 5 files changed, 210 insertions(+), 110 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index fc605413b9..ad0b697aaf 100644 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -168,3 +168,55 @@ function get-kubeconfig-bearertoken() { KUBE_BEARER_TOKEN='' fi } + +# Sets binary_version variable to the version passed in as an argument, or if argument is +# latest_stable, latest_release, or latest_ci fetches and sets the correponding version number +# +# Args: +# $1 version string from command line +# Vars set: +# KUBE_VERSION +function set_binary_version() { + if [[ "${1}" == "latest_stable" ]]; then + KUBE_VERSION=$(gsutil cat gs://kubernetes-release/release/stable.txt) + echo "Using latest stable version: ${binary_version}" + elif [[ "${1}" == "latest_release" ]]; then + KUBE_VERSION=$(gsutil cat gs://kubernetes-release/release/latest.txt) + echo "Using latest release version: ${binary_version}" + elif [[ "${1}" == "latest_ci" ]]; then + KUBE_VERSION=$(gsutil cat gs://kubernetes-release/ci/latest.txt) + echo "Using latest ci version: ${binary_version}" + else + KUBE_VERSION=${1} + fi +} + +# Figure out which binary use on the server and assure it is available. +# If KUBE_VERSION is specified use binaries specified by it, otherwise +# use local dev binaries. +# +# Assumed vars: +# PROJECT +# Vars set: +# SERVER_BINARY_TAR_URL +# SALT_TAR_URL +function tars_from_version() { + if [[ -z "${KUBE_VERSION-}" ]]; then + find-release-tars + upload-server-tars + elif [[ ${KUBE_VERSION} =~ ${KUBE_VERSION_REGEX} ]]; then + SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" + SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-salt.tar.gz" + elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then + SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" + SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${KUBE_VERSION}/kubernetes-salt.tar.gz" + else + echo "Version doesn't match regexp" >&2 + exit 1 + fi + + if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then + echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2 + exit 1 + fi +} diff --git a/cluster/gce/upgrade.sh b/cluster/gce/upgrade.sh index e75ffd6c80..3d5741b0be 100755 --- a/cluster/gce/upgrade.sh +++ b/cluster/gce/upgrade.sh @@ -22,12 +22,6 @@ set -o errexit set -o nounset set -o pipefail -# VERSION_REGEX matches things like "v0.13.1" -readonly VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" - -# CI_VERSION_REGEX matches things like "v0.14.1-341-ge0c9d9e" -readonly CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(.*)$" - if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then echo "!!! ${1} only works on GCE" >&2 exit 1 @@ -95,41 +89,14 @@ function wait-for-master() { echo "== Done ==" } -# Sets binary_version variable to the version passed in as an argument, or if argument is -# latest_stable, latest_release, or latest_ci fetches and sets the correponding version number -# -# Args: -# $1 version string from command line -function set_binary_version() { - if [[ "${1}" == "latest_stable" ]]; then - binary_version=$(gsutil cat gs://kubernetes-release/release/stable.txt) - echo "Using latest stable version: ${binary_version}" - elif [[ "${1}" == "latest_release" ]]; then - binary_version=$(gsutil cat gs://kubernetes-release/release/latest.txt) - echo "Using latest release version: ${binary_version}" - elif [[ "${1}" == "latest_ci" ]]; then - binary_version=$(gsutil cat gs://kubernetes-release/ci/latest.txt) - echo "Using latest ci version: ${binary_version}" - else - binary_version=${1} - fi -} - # Perform common upgrade setup tasks # # Assumed vars -# local_binaries -# binary_version +# KUBE_VERSION function prepare-upgrade() { ensure-temp-dir detect-project - - if [[ "${local_binaries}" == "true" ]]; then - find-release-tars - upload-server-tars - else - tars_from_version ${binary_version} - fi + tars_from_version } # Reads kube-env metadata from master and extracts value from provided key. @@ -148,11 +115,15 @@ function get-env-val() { | grep ${1} | cut -d : -f 2 | cut -d \' -f 2 } -# $1 veresion +# Assumed vars: +# KUBE_VERSION +# MINION_SCOPES +# NODE_INSTANCE_PREFIX +# PROJECT +# ZONE function upgrade-nodes() { - local version=${1} - local sanitized_version=$(echo ${version} | sed s/"\."/-/g) - echo "== Upgrading nodes to ${version}. ==" + local sanitized_version=$(echo ${KUBE_VERSION} | sed s/"\."/-/g) + echo "== Upgrading nodes to ${KUBE_VERSION}. ==" detect-minion-names @@ -187,28 +158,6 @@ function upgrade-nodes() { echo "== Done ==" } -function tars_from_version() { - version=${1-} - - if [[ ${version} =~ ${VERSION_REGEX} ]]; then - SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-server-linux-amd64.tar.gz" - SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/release/${version}/kubernetes-salt.tar.gz" - elif [[ ${version} =~ ${CI_VERSION_REGEX} ]]; then - SERVER_BINARY_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${version}/kubernetes-server-linux-amd64.tar.gz" - SALT_TAR_URL="https://storage.googleapis.com/kubernetes-release/ci/${version}/kubernetes-salt.tar.gz" - else - echo "!!! Version not provided or version doesn't match regexp" >&2 - exit 1 - fi - - if ! curl -Ss --range 0-1 ${SERVER_BINARY_TAR_URL} >&/dev/null; then - echo "!!! Can't find release at ${SERVER_BINARY_TAR_URL}" >&2 - exit 1 - fi - - echo "== Release ${version} validated ==" -} - master_upgrade=true node_upgrade=true local_binaries=false @@ -261,7 +210,7 @@ if [[ "${node_upgrade}" == "true" ]]; then if [[ "${local_binaries}" == "true" ]]; then echo "Upgrading nodes to local binaries is not yet supported." >&2 else - upgrade-nodes ${binary_version} + upgrade-nodes fi fi diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index da69733eb1..b926404ff7 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -36,6 +36,13 @@ ALLOCATE_NODE_CIDRS=true KUBE_PROMPT_FOR_UPDATE=y KUBE_SKIP_UPDATE=${KUBE_SKIP_UPDATE-"n"} +# VERSION_REGEX matches things like "v0.13.1" +readonly KUBE_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$" + +# CI_VERSION_REGEX matches things like "v0.14.1-341-ge0c9d9e" +readonly KUBE_CI_VERSION_REGEX="^v(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)-(.*)$" + + function join_csv { local IFS=','; echo "$*"; } @@ -824,12 +831,13 @@ function kube-down { set -e } -# Update a kubernetes cluster with latest source -function kube-push { +# Prepare to push new binaries to kubernetes cluster +# $1 - whether prepare push to node +function prepare-push() { #TODO(dawnchen): figure out how to upgrade coreos node if [[ "${OS_DISTRIBUTION}" != "debian" ]]; then echo "Updating a kubernetes cluster with ${OS_DISTRIBUTION} is not supported yet." >&2 - return + exit 1 fi OUTPUT=${KUBE_ROOT}/_output/logs @@ -843,17 +851,79 @@ function kube-push { get-bearer-token # Make sure we have the tar files staged on Google Storage - find-release-tars - upload-server-tars + tars_from_version + # Prepare node env vars and update MIG template + if [[ "${1-}" == "true" ]]; then + write-node-env + + # TODO(mbforbes): Refactor setting scope flags. + local -a scope_flags=() + if (( "${#MINION_SCOPES[@]}" > 0 )); then + scope_flags=("--scopes" "${MINION_SCOPES[@]}") + else + scope_flags=("--no-scopes") + fi + + # Ugly hack: Since it is not possible to delete instance-template that is currently + # being used, create a temp one, then delete the old one and recreate it once again. + create-node-instance-template "tmp" + + gcloud preview managed-instance-groups --zone "${ZONE}" \ + set-template "${NODE_INSTANCE_PREFIX}-group" \ + --project "${PROJECT}" \ + --template "${NODE_INSTANCE_PREFIX}-template-tmp" || true; + + gcloud compute instance-templates delete \ + --project "${PROJECT}" \ + --quiet \ + "${NODE_INSTANCE_PREFIX}-template" || true + + create-node-instance-template + + gcloud preview managed-instance-groups --zone "${ZONE}" \ + set-template "${NODE_INSTANCE_PREFIX}-group" \ + --project "${PROJECT}" \ + --template "${NODE_INSTANCE_PREFIX}-template" || true; + + gcloud compute instance-templates delete \ + --project "${PROJECT}" \ + --quiet \ + "${NODE_INSTANCE_PREFIX}-template-tmp" || true + fi +} + +# Push binaries to kubernetes master +function push-master { echo "Updating master metadata ..." write-master-env add-instance-metadata-from-file "${KUBE_MASTER}" "kube-env=${KUBE_TEMP}/master-kube-env.yaml" "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" - echo "Pushing to master (log at ${OUTPUT}/kube-push-${KUBE_MASTER}.log) ..." - cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${KUBE_MASTER}".log + echo "Pushing to master (log at ${OUTPUT}/push-${KUBE_MASTER}.log) ..." + cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/push-"${KUBE_MASTER}".log +} - kube-update-nodes push +# Push binaries to kubernetes node +function push-node() { + node=${1} + + echo "Updating node ${node} metadata... " + add-instance-metadata-from-file "${node}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" + + echo "Start upgrading node ${node} (log at ${OUTPUT}/push-${node}.log) ..." + cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${node}" --command "sudo bash -s -- --push" &> ${OUTPUT}/push-"${node}".log +} + +# Push binaries to kubernetes cluster +function kube-push { + prepare-push true + + push-master + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + push-node "${MINION_NAMES[$i]}" & + done + wait-for-jobs # TODO(zmerlynn): Re-create instance-template with the new # node-kube-env. This isn't important until the node-ip-range issue @@ -872,43 +942,6 @@ function kube-push { echo } -# Push or upgrade nodes. -# -# TODO: This really needs to trampoline somehow to the configure-vm.sh -# from the .tar.gz that we're actually pushing onto the node, because -# that configuration shifts over versions. Right now, we're blasting -# the configure-vm from our version instead. -# -# Assumed vars: -# KUBE_ROOT -# MINION_NAMES -# KUBE_TEMP -# PROJECT -# ZONE -function kube-update-nodes() { - action=${1} - - OUTPUT=${KUBE_ROOT}/_output/logs - mkdir -p ${OUTPUT} - - echo "Updating node metadata... " - write-node-env - for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - add-instance-metadata-from-file "${MINION_NAMES[$i]}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" "startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" & - done - wait-for-jobs - echo "Done" - - for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - echo "Starting ${action} on node (log at ${OUTPUT}/kube-${action}-${MINION_NAMES[$i]}.log) ..." - cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${MINION_NAMES[$i]}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-${action}-"${MINION_NAMES[$i]}".log & - done - - echo -n "Waiting..." - wait-for-jobs - echo "Done" -} - # ----------------------------------------------------------------------------- # Cluster specific test helpers used from hack/e2e-test.sh diff --git a/cluster/kube-push.sh b/cluster/kube-push.sh index 3e7292cabe..522e9cb8bd 100755 --- a/cluster/kube-push.sh +++ b/cluster/kube-push.sh @@ -27,10 +27,61 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/cluster/kube-env.sh" source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" -echo "Updating cluster using provider: $KUBERNETES_PROVIDER" +function usage() { + echo "${0} [-m|-n ] " + echo " Updates Kurnetes binaries. Can be done for all components (by default), master(-m) or specified node(-n)." + echo " If the version is not specified will try to use local binaries." + echo " Warning: upgrading single node is experimental" +} + +push_to_master=false +push_to_node=false + +while getopts "mn:h" opt; do + case ${opt} in + m) + push_to_master=true;; + n) + push_to_node=true + node_id="$OPTARG";; + h) + usage + exit 0;; + \?) + echo "Invalid option: -$OPTARG" >&2 + usage + exit 1;; + esac +done +shift $((OPTIND-1)) + +if [[ "${push_to_master}" == "true" ]] && [[ "${push_to_node}" == "true" ]]; then + echo "Only one of options -m -n should be specified" + usage + exit 1 +fi verify-prereqs -kube-push +KUBE_VERSION=${1-} + +if [[ "${push_to_master}" == "false" ]] && [[ "${push_to_node}" == "false" ]]; then + echo "Updating cluster using provider: $KUBERNETES_PROVIDER" + kube-push +fi + +if [[ "${push_to_master}" == "true" ]]; then + echo "Udating master to version ${KUBE_VERSION:-"dev"}" + prepare-push false + push-master +fi + +if [[ "${push_to_node}" == "true" ]]; then + echo "Updating node $node_id to version ${KUBE_VERSION:-"dev"}" + prepare-push true + push-node $node_id +fi + +echo "Validating cluster post-push..." "${KUBE_ROOT}/cluster/validate-cluster.sh" diff --git a/cluster/kube-util.sh b/cluster/kube-util.sh index 3f9b8168de..85f4566101 100644 --- a/cluster/kube-util.sh +++ b/cluster/kube-util.sh @@ -47,11 +47,26 @@ function kube-down { echo "TODO" } -# Update a kubernetes cluster with latest source +# Update a kubernetes cluster function kube-push { echo "TODO" } +# Prepare update a kubernetes component +function prepare-push { + echo "TODO" +} + +# Update a kubernetes master +function push-master { + echo "TODO" +} + +# Update a kubernetes node +function push-node { + echo "TODO" +} + # Execute prior to running tests to build a release if required for env function test-build-release { echo "TODO"