Add release support for trusty kube-system manifests.

This is a follow-up work for PR 18115. It adds release support for
kube-system manifests.

build/common.sh

@@ -653,6 +653,7 @@ function kube::release::package_tarballs() {
   kube::release::package_client_tarballs &
   kube::release::package_server_tarballs &
   kube::release::package_salt_tarball &
+  kube::release::package_kube_manifests_tarball &
   kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; }
 
   kube::release::package_full_tarball & # _full depends on all the previous phases
@@ -849,6 +850,36 @@ function kube::release::package_salt_tarball() {
   kube::release::create_tarball "${package_name}" "${release_stage}/.."
 }
 
+# This will pack kube-system manifests files for distros without using salt
+# such as Ubuntu Trusty.
+#
+# There are two sources of manifests files: (1) some manifests in the directory
+# cluster/saltbase/salt can be directly used on instances without salt, so we copy
+# them from there; (2) for the ones containing salt config, we cannot directly
+# use them. Therefore, we will maintain separate copies in cluster/gce/kube-manifests.
+function kube::release::package_kube_manifests_tarball() {
+  kube::log::status "Building tarball: manifests"
+
+  local release_stage="${RELEASE_STAGE}/manifests/kubernetes"
+  rm -rf "${release_stage}"
+  mkdir -p "${release_stage}"
+
+  # Source 1: manifests from cluster/saltbase/salt.
+  # TODO(andyzheng0831): Add more manifests when supporting master on trusty.
+  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
+  cp "${salt_dir}/fluentd-es/fluentd-es.yaml" "${release_stage}/"
+  cp "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${release_stage}/"
+  cp "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${release_stage}/"
+  # Source 2: manifests from cluster/gce/kube-manifests.
+  # TODO(andyzheng0831): Enable the following line after finishing issue #16702.
+  # cp "${KUBE_ROOT}/cluster/gce/kube-manifests/*" "${release_stage}/"
+
+  kube::release::clean_cruft
+
+  local package_name="${RELEASE_DIR}/kubernetes-manifests.tar.gz"
+  kube::release::create_tarball "${package_name}" "${release_stage}/.."
+}
+
 # This is the stuff you need to run tests from the binary distribution.
 function kube::release::package_test_tarball() {
   kube::log::status "Building tarball: test"
@@ -912,6 +943,7 @@ function kube::release::package_full_tarball() {
   mkdir -p "${release_stage}/server"
   cp "${RELEASE_DIR}/kubernetes-salt.tar.gz" "${release_stage}/server/"
   cp "${RELEASE_DIR}"/kubernetes-server-*.tar.gz "${release_stage}/server/"
+  cp "${RELEASE_DIR}/kubernetes-manifests.tar.gz" "${release_stage}/server/"
 
   mkdir -p "${release_stage}/third_party"
   cp -R "${KUBE_ROOT}/third_party/htpasswd" "${release_stage}/third_party/htpasswd"

cluster/common.sh

@@ -313,24 +313,38 @@ function tars_from_version() {
 # Vars set:
 #   SERVER_BINARY_TAR
 #   SALT_TAR
+#   KUBE_MANIFESTS_TAR
 function find-release-tars() {
   SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz"
-  if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
+  if [[ ! -f "${SERVER_BINARY_TAR}" ]]; then
     SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz"
   fi
-  if [[ ! -f "$SERVER_BINARY_TAR" ]]; then
+  if [[ ! -f "${SERVER_BINARY_TAR}" ]]; then
     echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" >&2
     exit 1
   fi
 
   SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz"
-  if [[ ! -f "$SALT_TAR" ]]; then
+  if [[ ! -f "${SALT_TAR}" ]]; then
     SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz"
   fi
-  if [[ ! -f "$SALT_TAR" ]]; then
+  if [[ ! -f "${SALT_TAR}" ]]; then
     echo "!!! Cannot find kubernetes-salt.tar.gz" >&2
     exit 1
   fi
+
+  # This tarball is only used by Ubuntu Trusty.
+  KUBE_MANIFESTS_TAR=
+  if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
+    KUBE_MANIFESTS_TAR="${KUBE_ROOT}/server/kuernetes-manifests.tar.gz"
+    if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then
+      KUBE_MANIFESTS_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-manifests.tar.gz"
+    fi
+    if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then
+      echo "!!! Cannot find kubernetes-manifests.tar.gz" >&2
+      exit 1
+    fi
+  fi
 }
 
 # Discover the git version of the current build package

cluster/gce/trusty/configure.sh

@@ -192,9 +192,10 @@ install_kube_binary_config() {
   fi
 
   # Put kube-system pods manifests in /etc/kube-manifests/.
-  cd /etc
+  mkdir -p /run/kube-manifests
+  cd /run/kube-manifests
   manifests_sha1="${KUBE_MANIFESTS_TAR_URL##*/}.sha1"
-  echo "Downloading kube-manifests tar sha1 file ${manifests_sha1}"
+  echo "Downloading kube-system manifests tar sha1 file ${manifests_sha1}"
   download_or_bust "${manifests_sha1}" "${KUBE_MANIFESTS_TAR_URL}.sha1"
   manifests_tar="${KUBE_MANIFESTS_TAR_URL##*/}"
   echo "Downloading kube-manifest tar file ${manifests_tar}"
@@ -206,9 +207,9 @@ install_kube_binary_config() {
   else
     echo "Validated ${KUBE_MANIFESTS_TAR_URL} SHA1 = ${KUBE_MANIFESTS_TAR_HASH}"
   fi
-  tar xzf "/etc/${manifests_tar}" -C /etc/ --overwrite
+  tar xzf "/run/kube-manifests/${manifests_tar}" -C /run/kube-manifests/ --overwrite
-  rm "/etc/${manifests_sha1}"
+  rm "/run/kube-manifests/${manifests_sha1}"
-  rm "/etc/${manifests_tar}"
+  rm "/run/kube-manifests/${manifests_tar}"
 }
 
 restart_docker_daemon() {

cluster/gce/trusty/node.yaml

@@ -218,19 +218,19 @@ script
 	set -o errexit
 	set -o nounset
 
-	# Kube-system pod manifest files are located at /etc/kube-manifests.
+	# Kube-system pod manifest files are located at /run/kube-manifests/kubernetes.
 	. /etc/kube-env
 	# Fluentd
 	if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then
 		if [ "${LOGGING_DESTINATION:-}" = "gcp" ]; then
-			cp /etc/kube-manifests/fluentd-gcp.yaml /etc/kubernetes/manifests/
+			cp /run/kube-manifests/kubernetes/fluentd-gcp.yaml /etc/kubernetes/manifests/
 		elif [ "${LOGGING_DESTINATION:-}" = "elasticsearch" ]; then
-			cp /etc/kube-manifests/fluentd-es.yaml /etc/kubernetes/manifests/
+			cp /run/kube-manifests/kubernetes/fluentd-es.yaml /etc/kubernetes/manifests/
 		fi
 	fi
 	# Kube-registry-proxy
 	if [ "${ENABLE_CLUSTER_REGISTRY:-}" = "true" ]; then
-		cp /etc/kube-manifests/kube-registry-proxy.yaml /etc/kubernetes/manifests/
+		cp /run/kube-manifests/kubernetes/kube-registry-proxy.yaml /etc/kubernetes/manifests/
 	fi
 end script
 

cluster/gce/util.sh

@@ -156,36 +156,6 @@ function copy-if-not-staged() {
   fi
 }
 
-# Prepare a tarball of kube-system manifests for trusty based cluster.
-#
-# Vars set:
-#   KUBE_MANIFESTS_TAR_URL
-#   KUBE_MANIFESTS_TAR_HASH
-function prepare-manifests-tar() {
-  KUBE_MANIFESTS_TAR_URL=
-  KUBE_MANIFESTS_TAR_HASH=
-  if [[ "${OS_DISTRIBUTION}" != "trusty" ]]; then
-    return
-  fi
-  local tmp_dir="${KUBE_TEMP}/kube-manifests"
-  mkdir -p ${tmp_dir}
-  # The manifests used by nodes can be directly used on non-salt system.
-  # We simply copy them from cluster/saltbase/salt.
-  local salt_dir="${KUBE_ROOT}/cluster/saltbase/salt"
-  cp -f "${salt_dir}/fluentd-es/fluentd-es.yaml" "${tmp_dir}"
-  cp -f "${salt_dir}/fluentd-gcp/fluentd-gcp.yaml" "${tmp_dir}"
-  cp -f "${salt_dir}/kube-registry-proxy/kube-registry-proxy.yaml" "${tmp_dir}"
-
-  local kube_manifests_tar="${KUBE_TEMP}/kube-manifests.tar.gz"
-  tar czf "${kube_manifests_tar}" -C "${KUBE_TEMP}" kube-manifests
-  KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${kube_manifests_tar}")
-  local kube_manifests_gs_url="${staging_path}/${kube_manifests_tar##*/}"
-  copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${kube_manifests_tar}" "${KUBE_MANIFESTS_TAR_HASH}"
-  # Convert from gs:// URL to an https:// URL
-  KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}"
-}
-
-
 # Take the local tar files and upload them to Google Storage.  They will then be
 # downloaded by the master as part of the start up script for the master.
 # If running on Ubuntu trusty, we also pack the dir cluster/gce/trusty/kube-manifest
@@ -195,16 +165,21 @@ function prepare-manifests-tar() {
 #   PROJECT
 #   SERVER_BINARY_TAR
 #   SALT_TAR
+#   KUBE_MANIFESTS_TAR
 # Vars set:
 #   SERVER_BINARY_TAR_URL
 #   SERVER_BINARY_TAR_HASH
 #   SALT_TAR_URL
 #   SALT_TAR_HASH
+#   KUBE_MANIFESTS_TAR_URL
+#   KUBE_MANIFESTS_TAR_HASH
 function upload-server-tars() {
   SERVER_BINARY_TAR_URL=
   SERVER_BINARY_TAR_HASH=
   SALT_TAR_URL=
   SALT_TAR_HASH=
+  KUBE_MANIFESTS_TAR_URL=
+  KUBE_MANIFESTS_TAR_HASH=
 
   local project_hash
   if which md5 > /dev/null 2>&1; then
@@ -240,11 +215,13 @@ function upload-server-tars() {
   SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}"
   SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}"
 
-  # Create a tar for kube-system manifests files and stage it.
+  if [[ "${OS_DISTRIBUTION}" == "trusty" ]]; then
-  # TODO(andyzheng0831): After finishing k8s master on trusty (issue #16702),
+    local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}"
-  # we will not need to stage the salt tar for trusty anymore.
+    KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}")
-  # TODO(andyzheng0831): Add release support for this tar, in case GKE will it.
+    copy-if-not-staged "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}"
-  prepare-manifests-tar
+    # Convert from gs:// URL to an https:// URL
+    KUBE_MANIFESTS_TAR_URL="${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}"
+  fi
 }
 
 # Detect minions created in the minion group