diff --git a/build/lib/release.sh b/build/lib/release.sh index 18e2ca6544..b4a059e590 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -32,6 +32,11 @@ KUBE_BUILD_HYPERKUBE=${KUBE_BUILD_HYPERKUBE:-y} KUBE_BUILD_CONFORMANCE=${KUBE_BUILD_CONFORMANCE:-y} KUBE_BUILD_PULL_LATEST_IMAGES=${KUBE_BUILD_PULL_LATEST_IMAGES:-y} +# The mondo test tarball is deprecated as of Kubernetes 1.14, and the default +# will be set to 'n' in a future release. +# See KEP sig-testing/20190118-breaking-apart-the-kubernetes-test-tarball +KUBE_BUILD_MONDO_TEST_TARBALL=${KUBE_BUILD_MONDO_TEST_TARBALL:-y} + # Validate a ci version # # Globals: @@ -89,7 +94,7 @@ function kube::release::package_tarballs() { kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; } kube::release::package_final_tarball & # _final depends on some of the previous phases - kube::release::package_test_tarball & # _test doesn't depend on anything + kube::release::package_test_tarballs & # _test doesn't depend on anything kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; } } @@ -136,7 +141,7 @@ function kube::release::package_client_tarballs() { # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the - # KUBE_CLIENT_BINARIES array. + # client_bins array. cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/client/bin/" @@ -169,7 +174,7 @@ function kube::release::package_node_tarballs() { fi # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the - # KUBE_NODE_BINARIES array. + # node_bins array. cp "${node_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/node/bin/" @@ -181,6 +186,9 @@ function kube::release::package_node_tarballs() { if [[ "${platform%/*}" == "windows" ]]; then client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") fi + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # client_bins array. cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/node/bin/" @@ -248,6 +256,9 @@ function kube::release::package_server_tarballs() { if [[ "${platform%/*}" == "windows" ]]; then client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}") fi + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # client_bins array. cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ "${release_stage}/server/bin/" @@ -466,39 +477,111 @@ function kube::release::package_kube_manifests_tarball() { kube::release::create_tarball "${package_name}" "${release_stage}/.." } +# Builds tarballs for each test platform containing the appropriate binaries. +function kube::release::package_test_platform_tarballs() { + local platform + rm -rf "${RELEASE_STAGE}/test" + # KUBE_TEST_SERVER_PLATFORMS is a subset of KUBE_TEST_PLATFORMS, + # so process it first. + for platform in "${KUBE_TEST_SERVER_PLATFORMS[@]}"; do + local platform_tag=${platform/\//-} # Replace a "/" for a "-" + local release_stage="${RELEASE_STAGE}/test/${platform_tag}/kubernetes" + mkdir -p "${release_stage}/test/bin" + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # KUBE_TEST_SERVER_BINARIES array. + cp "${KUBE_TEST_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/test/bin/" + done + for platform in "${KUBE_TEST_PLATFORMS[@]}"; do + ( + local platform_tag=${platform/\//-} # Replace a "/" for a "-" + kube::log::status "Starting tarball: test $platform_tag" + local release_stage="${RELEASE_STAGE}/test/${platform_tag}/kubernetes" + mkdir -p "${release_stage}/test/bin" + + local test_bins=("${KUBE_TEST_BINARIES[@]}") + if [[ "${platform%/*}" == "windows" ]]; then + test_bins=("${KUBE_TEST_BINARIES_WIN[@]}") + fi + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # test_bins array. + cp "${test_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/test/bin/" + + local package_name="${RELEASE_TARS}/kubernetes-test-${platform_tag}.tar.gz" + kube::release::create_tarball "${package_name}" "${release_stage}/.." + ) & + done + + kube::log::status "Waiting on test tarballs" + kube::util::wait-for-jobs || { kube::log::error "test tarball creation failed"; exit 1; } +} + + # This is the stuff you need to run tests from the binary distribution. -function kube::release::package_test_tarball() { - kube::log::status "Building tarball: test" +function kube::release::package_test_tarballs() { + kube::release::package_test_platform_tarballs + + kube::log::status "Building tarball: test portable" local release_stage="${RELEASE_STAGE}/test/kubernetes" rm -rf "${release_stage}" mkdir -p "${release_stage}" - local platform - for platform in "${KUBE_TEST_PLATFORMS[@]}"; do - local test_bins=("${KUBE_TEST_BINARIES[@]}") - if [[ "${platform%/*}" == "windows" ]]; then - test_bins=("${KUBE_TEST_BINARIES_WIN[@]}") - fi - mkdir -p "${release_stage}/platforms/${platform}" - cp "${test_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ - "${release_stage}/platforms/${platform}" - done - for platform in "${KUBE_TEST_SERVER_PLATFORMS[@]}"; do - mkdir -p "${release_stage}/platforms/${platform}" - cp "${KUBE_TEST_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ - "${release_stage}/platforms/${platform}" - done - - # Add the test image files + # First add test image files and other portable sources so we can create + # the portable test tarball. mkdir -p "${release_stage}/test/images" cp -fR "${KUBE_ROOT}/test/images" "${release_stage}/test/" tar c "${KUBE_TEST_PORTABLE[@]}" | tar x -C "${release_stage}" kube::release::clean_cruft - local package_name="${RELEASE_TARS}/kubernetes-test.tar.gz" - kube::release::create_tarball "${package_name}" "${release_stage}/.." + local portable_tarball_name="${RELEASE_TARS}/kubernetes-test-portable.tar.gz" + kube::release::create_tarball "${portable_tarball_name}" "${release_stage}/.." + + if [[ "${KUBE_BUILD_MONDO_TEST_TARBALL}" =~ [yY] ]]; then + kube::log::status "Building tarball: test mondo (deprecated by KEP sig-testing/20190118-breaking-apart-the-kubernetes-test-tarball)" + local platform + for platform in "${KUBE_TEST_PLATFORMS[@]}"; do + local test_bins=("${KUBE_TEST_BINARIES[@]}") + if [[ "${platform%/*}" == "windows" ]]; then + test_bins=("${KUBE_TEST_BINARIES_WIN[@]}") + fi + mkdir -p "${release_stage}/platforms/${platform}" + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # test_bins array. + cp "${test_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/platforms/${platform}" + done + for platform in "${KUBE_TEST_SERVER_PLATFORMS[@]}"; do + mkdir -p "${release_stage}/platforms/${platform}" + # This fancy expression will expand to prepend a path + # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the + # KUBE_TEST_SERVER_BINARIES array. + cp "${KUBE_TEST_SERVER_BINARIES[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \ + "${release_stage}/platforms/${platform}" + done + + cat < "${release_stage}/DEPRECATION_NOTICE" +The mondo test tarball containing binaries for all platforms is +DEPRECATED as of Kubernetes 1.14. + +Users of this tarball should migrate to using the platform-specific +tarballs in combination with the "portable" tarball which contains +scripts, test images, and other manifests. + +For more details, please see KEP +sig-testing/20190118-breaking-apart-the-kubernetes-test-tarball. +EOF + + kube::release::clean_cruft + + local package_name="${RELEASE_TARS}/kubernetes-test.tar.gz" + kube::release::create_tarball "${package_name}" "${release_stage}/.." + fi } # This is all the platform-independent stuff you need to run/install kubernetes. diff --git a/build/release-tars/BUILD b/build/release-tars/BUILD index cc79f4de78..6435c57ba2 100644 --- a/build/release-tars/BUILD +++ b/build/release-tars/BUILD @@ -152,8 +152,9 @@ pkg_tar( ], ) +# The mondo test tarball is deprecated. pkg_tar( - name = "_test-bin", + name = "_test-mondo-bin", srcs = ["//build:test-targets"], mode = "0755", package_dir = "platforms/" + PLATFORM_ARCH_STRING.replace("-", "/"), @@ -162,13 +163,70 @@ pkg_tar( visibility = ["//visibility:private"], ) +genrule( + name = "kubernetes-test-mondo-deprecation", + outs = ["DEPRECATION_NOTICE"], + cmd = """cat <$@ +The mondo test tarball containing binaries for all platforms is +DEPRECATED as of Kubernetes 1.14. + +Users of this tarball should migrate to using the platform-specific +tarballs in combination with the "portable" tarball which contains +scripts, test images, and other manifests. + +For more details, please see KEP +sig-testing/20190118-breaking-apart-the-kubernetes-test-tarball. +EOF +""", + visibility = ["//visibility:private"], +) + +# The mondo test tarball is deprecated. +# This one was never really correct, anyway, since we can't include +# binaries from multiple platforms in a single tarball. pkg_tar( name = "kubernetes-test", + srcs = [ + ":kubernetes-test-mondo-deprecation", + "//build:test-portable-targets", + ], + extension = "tar.gz", + package_dir = "kubernetes", + remap_paths = { + "build/release-tars/DEPRECATION_NOTICE": "DEPRECATION_NOTICE", + }, + strip_prefix = "//", + tags = ["no-cache"], + deps = [ + # TODO: how to make this multiplatform? + ":_test-mondo-bin", + ], +) + +pkg_tar( + name = "kubernetes-test-portable", srcs = ["//build:test-portable-targets"], extension = "tar.gz", package_dir = "kubernetes", strip_prefix = "//", tags = ["no-cache"], +) + +pkg_tar( + name = "_test-bin", + srcs = ["//build:test-targets"], + mode = "0755", + package_dir = "test/bin", + tags = ["no-cache"], + visibility = ["//visibility:private"], +) + +pkg_tar( + name = "kubernetes-test-%s" % PLATFORM_ARCH_STRING, + extension = "tar.gz", + package_dir = "kubernetes", + strip_prefix = "//", + tags = ["no-cache"], deps = [ # TODO: how to make this multiplatform? ":_test-bin", @@ -222,6 +280,8 @@ release_filegroup( ":kubernetes-server-%s.tar.gz" % PLATFORM_ARCH_STRING, ":kubernetes-manifests.tar.gz", ":kubernetes-src.tar.gz", + ":kubernetes-test-%s.tar.gz" % PLATFORM_ARCH_STRING, ":kubernetes-test.tar.gz", + ":kubernetes-test-portable.tar.gz", ], ) diff --git a/cluster/get-kube-binaries.sh b/cluster/get-kube-binaries.sh index 251b8f49d6..0f2e962759 100755 --- a/cluster/get-kube-binaries.sh +++ b/cluster/get-kube-binaries.sh @@ -150,11 +150,8 @@ function extract_arch_tarball() { platforms_dir="${KUBE_ROOT}/platforms/${platform}/${arch}" echo "Extracting ${tarfile} into ${platforms_dir}" mkdir -p "${platforms_dir}" - # Tarball looks like kubernetes/{client,server}/bin/BINARY" + # Tarball looks like kubernetes/{client,server,test}/bin/BINARY" tar -xzf "${tarfile}" --strip-components 3 -C "${platforms_dir}" - # Create convenience symlink - ln -sf "${platforms_dir}" "$(dirname "${tarfile}")/bin" - echo "Add '$(dirname "${tarfile}")/bin' to your PATH to use newly-installed binaries." } detect_kube_release @@ -177,21 +174,8 @@ echo "Server: ${SERVER_PLATFORM}/${SERVER_ARCH} (to override, set KUBERNETES_SE echo "Client: ${CLIENT_PLATFORM}/${CLIENT_ARCH} (autodetected)" echo -# TODO: remove this check and default to true when we stop shipping server -# tarballs in kubernetes.tar.gz -DOWNLOAD_SERVER_TAR=false -if [[ ! -e "${KUBE_ROOT}/server/${SERVER_TAR}" ]]; then - DOWNLOAD_SERVER_TAR=true - echo "Will download ${SERVER_TAR} from ${DOWNLOAD_URL_PREFIX}" -fi - -# TODO: remove this check and default to true when we stop shipping kubectl -# in kubernetes.tar.gz -DOWNLOAD_CLIENT_TAR=false -if [[ ! -x "${KUBE_ROOT}/platforms/${CLIENT_PLATFORM}/${CLIENT_ARCH}/kubectl" ]]; then - DOWNLOAD_CLIENT_TAR=true - echo "Will download and extract ${CLIENT_TAR} from ${DOWNLOAD_URL_PREFIX}" -fi +echo "Will download ${SERVER_TAR} from ${DOWNLOAD_URL_PREFIX}" +echo "Will download and extract ${CLIENT_TAR} from ${DOWNLOAD_URL_PREFIX}" DOWNLOAD_NODE_TAR=false if [[ -n "${NODE_TAR:-}" ]]; then @@ -199,18 +183,10 @@ if [[ -n "${NODE_TAR:-}" ]]; then echo "Will download and extract ${NODE_TAR} from ${DOWNLOAD_URL_PREFIX}" fi -TESTS_TAR="kubernetes-test.tar.gz" DOWNLOAD_TESTS_TAR=false if [[ -n "${KUBERNETES_DOWNLOAD_TESTS-}" ]]; then DOWNLOAD_TESTS_TAR=true - echo "Will download and extract ${TESTS_TAR} from ${DOWNLOAD_URL_PREFIX}" -fi - -if [[ "${DOWNLOAD_CLIENT_TAR}" == false && \ - "${DOWNLOAD_SERVER_TAR}" == false && \ - "${DOWNLOAD_TESTS_TAR}" == false ]]; then - echo "Nothing additional to download." - exit 0 + echo "Will download and extract kubernetes-test tarball(s) from ${DOWNLOAD_URL_PREFIX}" fi if [[ -z "${KUBERNETES_SKIP_CONFIRM-}" ]]; then @@ -222,22 +198,47 @@ if [[ -z "${KUBERNETES_SKIP_CONFIRM-}" ]]; then fi fi -if "${DOWNLOAD_SERVER_TAR}"; then - download_tarball "${KUBE_ROOT}/server" "${SERVER_TAR}" -fi +download_tarball "${KUBE_ROOT}/server" "${SERVER_TAR}" if "${DOWNLOAD_NODE_TAR}"; then download_tarball "${KUBE_ROOT}/node" "${NODE_TAR}" fi -if "${DOWNLOAD_CLIENT_TAR}"; then - download_tarball "${KUBE_ROOT}/client" "${CLIENT_TAR}" - extract_arch_tarball "${KUBE_ROOT}/client/${CLIENT_TAR}" "${CLIENT_PLATFORM}" "${CLIENT_ARCH}" -fi +download_tarball "${KUBE_ROOT}/client" "${CLIENT_TAR}" +extract_arch_tarball "${KUBE_ROOT}/client/${CLIENT_TAR}" "${CLIENT_PLATFORM}" "${CLIENT_ARCH}" +ln -s "${KUBE_ROOT}/platforms/${CLIENT_PLATFORM}/${CLIENT_ARCH}" "${KUBE_ROOT}/client/bin" +echo "Add '${KUBE_ROOT}/client/bin' to your PATH to use newly-installed binaries." if "${DOWNLOAD_TESTS_TAR}"; then - download_tarball "${KUBE_ROOT}/test" "${TESTS_TAR}" - echo "Extracting ${TESTS_TAR} into ${KUBE_ROOT}" - # Strip leading "kubernetes/" - tar -xzf "${KUBE_ROOT}/test/${TESTS_TAR}" --strip-components 1 -C "${KUBE_ROOT}" + TESTS_PORTABLE_TAR="kubernetes-test-portable.tar.gz" + download_tarball "${KUBE_ROOT}/test" "${TESTS_PORTABLE_TAR}" || true + if [[ -f "${KUBE_ROOT}/test/${TESTS_PORTABLE_TAR}" ]]; then + echo "Extracting ${TESTS_PORTABLE_TAR} into ${KUBE_ROOT}" + # Strip leading "kubernetes/" + tar -xzf "${KUBE_ROOT}/test/${TESTS_PORTABLE_TAR}" --strip-components 1 -C "${KUBE_ROOT}" + + # Next, download platform-specific test tarballs for all relevant platforms + TEST_PLATFORM_TUPLES=( + "${CLIENT_PLATFORM}/${CLIENT_ARCH}" + "${SERVER_PLATFORM}/${SERVER_ARCH}" + ) + if [[ -n "${NODE_PLATFORM:-}" && -n "${NODE_ARCH:-}" ]]; then + TEST_PLATFORM_TUPLES+=("${NODE_PLATFORM}/${NODE_ARCH}") + fi + # Loop over only the unique tuples + for TUPLE in $(printf "%s\n" "${TEST_PLATFORM_TUPLES[@]}" | sort -u); do + OS=$(echo "${TUPLE}" | cut -d/ -f1) + ARCH=$(echo "${TUPLE}" | cut -d/ -f2) + TEST_PLATFORM_TAR="kubernetes-test-${OS}-${ARCH}.tar.gz" + download_tarball "${KUBE_ROOT}/test" "${TEST_PLATFORM_TAR}" + extract_arch_tarball "${KUBE_ROOT}/test/${TEST_PLATFORM_TAR}" "${OS}" "${ARCH}" + done + else + echo "Failed to download portable test tarball, falling back to mondo test tarball." + TESTS_MONDO_TAR="kubernetes-test.tar.gz" + download_tarball "${KUBE_ROOT}/test" "${TESTS_MONDO_TAR}" + echo "Extracting ${TESTS_MONDO_TAR} into ${KUBE_ROOT}" + # Strip leading "kubernetes/" + tar -xzf "${KUBE_ROOT}/test/${TESTS_MONDO_TAR}" --strip-components 1 -C "${KUBE_ROOT}" + fi fi