From 75e23a3845fd2bfaaddfe3ce188fa307e2822268 Mon Sep 17 00:00:00 2001 From: Andy Zheng Date: Tue, 8 Dec 2015 14:32:23 -0800 Subject: [PATCH] Add release support for trusty kube-system manifests. This is a follow-up work for PR 18115. It adds release support for kube-system manifests. --- build/common.sh | 32 ++++++++++++++++++++++ cluster/common.sh | 22 ++++++++++++--- cluster/gce/trusty/configure.sh | 11 ++++---- cluster/gce/trusty/node.yaml | 8 +++--- cluster/gce/util.sh | 47 +++++++++------------------------ 5 files changed, 72 insertions(+), 48 deletions(-) diff --git a/build/common.sh b/build/common.sh index 9af436e87b..f9999dd86a 100755 --- a/build/common.sh +++ b/build/common.sh @@ -653,6 +653,7 @@ function kube::release::package_tarballs() { kube::release::package_client_tarballs & kube::release::package_server_tarballs & kube::release::package_salt_tarball & + kube::release::package_kube_manifests_tarball & kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; } kube::release::package_full_tarball & # _full depends on all the previous phases @@ -849,6 +850,36 @@ function kube::release::package_salt_tarball() { kube::release::create_tarball "${package_name}" "${release_stage}/.." } +# This will pack kube-system manifests files for distros without using salt +# such as Ubuntu Trusty. +# +# There are two sources of manifests files: (1) some manifests in the directory +# cluster/saltbase/salt can be directly used on instances without salt, so we copy +# them from there; (2) for the ones containing salt config, we cannot directly +# use them. Therefore, we will maintain separate copies in cluster/gce/kube-manifests. +function kube::release::package_kube_manifests_tarball() { + kube::log::status "Building tarball: manifests" + + local release_stage="${RELEASE_STAGE}/manifests/kubernetes" + rm -rf "${release_stage}" + mkdir -p "${release_stage}" + + # Source 1: manifests from cluster/saltbase/salt. + # TODO(andyzheng0831): Add more manifests when supporting master on trusty. + local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt" + cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/" + cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/" + cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/" + # Source 2: manifests from cluster/gce/kube-manifests. + # TODO(andyzheng0831): Enable the following line after finishing issue #16702. + # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/*" "${release_stage}/" + + kube::release::clean_cruft + + local package_name="${RELEASE_DIR}/kubernetes-manifests.tar.gz" + kube::release::create_tarball "${package_name}" "${release_stage}/.." +} + # This is the stuff you need to run tests from the binary distribution. function kube::release::package_test_tarball() { kube::log::status "Building tarball: test" @@ -912,6 +943,7 @@ function kube::release::package_full_tarball() { mkdir -p "${release_stage}/server" cp "${RELEASE_DIR}/kubernetes-salt.tar.gz" "${release_stage}/server/" cp "${RELEASE_DIR}"/kubernetes-server-*.tar.gz "${release_stage}/server/" + cp "${RELEASE_DIR}/kubernetes-manifests.tar.gz" "${release_stage}/server/" mkdir -p "${release_stage}/third_party" cp -R "${KUBE_ROOT}/third_party/htpasswd" "${release_stage}/third_party/htpasswd" diff --git a/cluster/common.sh b/cluster/common.sh index 444ae5696a..6c63cad04d 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -313,24 +313,38 @@ function tars_from_version() { # Vars set: # SERVER_BINARY_TAR # SALT_TAR +# KUBE_MANIFESTS_TAR function find-release-tars() { SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" - if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + if [[ ! -f "${SERVER_BINARY_TAR}" ]]; then SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" fi - if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + if [[ ! -f "${SERVER_BINARY_TAR}" ]]; then echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" >&2 exit 1 fi SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz" - if [[ ! -f "$SALT_TAR" ]]; then + if [[ ! -f "${SALT_TAR}" ]]; then SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz" fi - if [[ ! -f "$SALT_TAR" ]]; then + if [[ ! -f "${SALT_TAR}" ]]; then echo "!!! Cannot find kubernetes-salt.tar.gz" >&2 exit 1 fi + + # This tarball is only used by Ubuntu Trusty. + KUBE_MANIFESTS_TAR= + if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then + KUBE_MANIFESTS_TAR="${KUBE_ROOT}/server/kuernetes-manifests.tar.gz" + if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then + KUBE_MANIFESTS_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-manifests.tar.gz" + fi + if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then + echo "!!! Cannot find kubernetes-manifests.tar.gz" >&2 + exit 1 + fi + fi } # Discover the git version of the current build package diff --git a/cluster/gce/trusty/configure.sh b/cluster/gce/trusty/configure.sh index a21ffff4c7..c7bafebfb7 100644 --- a/cluster/gce/trusty/configure.sh +++ b/cluster/gce/trusty/configure.sh @@ -192,9 +192,10 @@ install_kube_binary_config() { fi # Put kube-system pods manifests in /etc/kube-manifests/. - cd /etc + mkdir -p /run/kube-manifests + cd /run/kube-manifests manifests_sha1="${KUBE_MANIFESTS_TAR_URL##*/}.sha1" - echo "Downloading kube-manifests tar sha1 file ${manifests_sha1}" + echo "Downloading kube-system manifests tar sha1 file ${manifests_sha1}" download_or_bust "${manifests_sha1}" "${KUBE_MANIFESTS_TAR_URL}.sha1" manifests_tar="${KUBE_MANIFESTS_TAR_URL##*/}" echo "Downloading kube-manifest tar file ${manifests_tar}" @@ -206,9 +207,9 @@ install_kube_binary_config() { else echo "Validated ${KUBE_MANIFESTS_TAR_URL} SHA1 = ${KUBE_MANIFESTS_TAR_HASH}" fi - tar xzf "/etc/${manifests_tar}" -C /etc/ --overwrite - rm "/etc/${manifests_sha1}" - rm "/etc/${manifests_tar}" + tar xzf "/run/kube-manifests/${manifests_tar}" -C /run/kube-manifests/ --overwrite + rm "/run/kube-manifests/${manifests_sha1}" + rm "/run/kube-manifests/${manifests_tar}" } restart_docker_daemon() { diff --git a/cluster/gce/trusty/node.yaml b/cluster/gce/trusty/node.yaml index cc1c90427e..6c94a03c0e 100644 --- a/cluster/gce/trusty/node.yaml +++ b/cluster/gce/trusty/node.yaml @@ -218,19 +218,19 @@ script set -o errexit set -o nounset - # Kube-system pod manifest files are located at /etc/kube-manifests. + # Kube-system pod manifest files are located at /run/kube-manifests/kubernetes. . /etc/kube-env # Fluentd if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then - cp /etc/kube-manifests/fluentd-gcp.yaml /etc/kubernetes/manifests/ + cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/ elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then - cp /etc/kube-manifests/fluentd-es.yaml /etc/kubernetes/manifests/ + cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/ fi fi # Kube-registry-proxy if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then - cp /etc/kube-manifests/kube-registry-proxy.yaml /etc/kubernetes/manifests/ + cp /run/kube-manifests/kubernetes/kube-registry-proxy.yaml /etc/kubernetes/manifests/ fi end script diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 79fb621aa6..f7fbf15484 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -156,36 +156,6 @@ function copy-if-not-staged() { fi } -# Prepare a tarball of kube-system manifests for trusty based cluster. -# -# Vars set: -# KUBE_MANIFESTS_TAR_URL -# KUBE_MANIFESTS_TAR_HASH -function prepare-manifests-tar() { - KUBE_MANIFESTS_TAR_URL= - KUBE_MANIFESTS_TAR_HASH= - if [[ "${OS_DISTRIBUTION}" != "trusty" ]]; then - return - fi - local tmp_dir="${KUBE_TEMP}/kube-manifests" - mkdir -p ${tmp_dir} - # The manifests used by nodes can be directly used on non-salt system. - # We simply copy them from cluster/saltbase/salt. - local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt" - cp -f "${salt_dir}/fluentd-es/fluentd-es.yaml" "${tmp_dir}" - cp -f "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${tmp_dir}" - cp -f "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${tmp_dir}" - - local kube_manifests_tar="${KUBE_TEMP}/kube-manifests.tar.gz" - tar czf "${kube_manifests_tar}" -C "${KUBE_TEMP}" kube-manifests - KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${kube_manifests_tar}") - local kube_manifests_gs_url="${staging_path}/${kube_manifests_tar##*/}" - copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${kube_manifests_tar}" "${KUBE_MANIFESTS_TAR_HASH}" - # Convert from gs:// URL to an https:// URL - KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}" -} - - # Take the local tar files and upload them to Google Storage. They will then be # downloaded by the master as part of the start up script for the master. # If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest @@ -195,16 +165,21 @@ function prepare-manifests-tar() { # PROJECT # SERVER_BINARY_TAR # SALT_TAR +# KUBE_MANIFESTS_TAR # Vars set: # SERVER_BINARY_TAR_URL # SERVER_BINARY_TAR_HASH # SALT_TAR_URL # SALT_TAR_HASH +# KUBE_MANIFESTS_TAR_URL +# KUBE_MANIFESTS_TAR_HASH function upload-server-tars() { SERVER_BINARY_TAR_URL= SERVER_BINARY_TAR_HASH= SALT_TAR_URL= SALT_TAR_HASH= + KUBE_MANIFESTS_TAR_URL= + KUBE_MANIFESTS_TAR_HASH= local project_hash if which md5 > /dev/null 2>&1; then @@ -240,11 +215,13 @@ function upload-server-tars() { SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" - # Create a tar for kube-system manifests files and stage it. - # TODO(andyzheng0831): After finishing k8s master on trusty (issue #16702), - # we will not need to stage the salt tar for trusty anymore. - # TODO(andyzheng0831): Add release support for this tar, in case GKE will it. - prepare-manifests-tar + if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then + local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}" + KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}") + copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}" + # Convert from gs:// URL to an https:// URL + KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}" + fi } # Detect minions created in the minion group