From c98b3edb88d6c8bf4eb3812cc2ec63e18f54e15e Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Sat, 23 Feb 2019 18:33:49 -0800 Subject: [PATCH 1/7] bazel: make conformance and hyperkube images multiarch --- cluster/images/conformance/BUILD | 24 +++++++++++++++++++----- cluster/images/hyperkube/BUILD | 24 +++++++++++++++++++----- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/cluster/images/conformance/BUILD b/cluster/images/conformance/BUILD index 399127f2df..00e8e33435 100644 --- a/cluster/images/conformance/BUILD +++ b/cluster/images/conformance/BUILD @@ -1,4 +1,5 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image", "container_layer") +load("//build:platforms.bzl", "SERVER_PLATFORMS", "for_platforms") container_layer( name = "cluster-srcs", @@ -19,7 +20,10 @@ container_layer( container_image( name = "conformance-internal", - base = "@debian-hyperkube-base-amd64//image", + base = select(for_platforms( + for_server = "@debian-hyperkube-base-{ARCH}//image", + only_os = "linux", + )), cmd = [ "/bin/bash", "-c", @@ -44,10 +48,20 @@ container_image( workdir = "/usr/local/bin", ) -container_bundle( - name = "conformance", - images = {"k8s.gcr.io/conformance-amd64:{STABLE_DOCKER_TAG}": "conformance-internal"}, -) +[container_bundle( + name = "conformance-%s" % arch, + images = {"k8s.gcr.io/conformance-%s:{STABLE_DOCKER_TAG}" % arch: "conformance-internal"}, + tags = ["manual"], + visibility = ["//visibility:public"], +) for arch in SERVER_PLATFORMS["linux"]] + +[alias( + name = "conformance%s" % suffix, + actual = select(for_platforms(for_server = "conformance-{ARCH}%s" % suffix)), +) for suffix in [ + "", + ".tar", +]] filegroup( name = "package-srcs", diff --git a/cluster/images/hyperkube/BUILD b/cluster/images/hyperkube/BUILD index 19c0c75218..201f5abd35 100644 --- a/cluster/images/hyperkube/BUILD +++ b/cluster/images/hyperkube/BUILD @@ -1,18 +1,32 @@ load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image") +load("//build:platforms.bzl", "SERVER_PLATFORMS", "for_platforms") container_image( name = "hyperkube-internal", - base = "@debian-hyperkube-base-amd64//image", + base = select(for_platforms( + for_server = "@debian-hyperkube-base-{ARCH}//image", + only_os = "linux", + )), files = [ "//cmd/hyperkube", ], stamp = True, ) -container_bundle( - name = "hyperkube", - images = {"k8s.gcr.io/hyperkube-amd64:{STABLE_DOCKER_TAG}": "hyperkube-internal"}, -) +[container_bundle( + name = "hyperkube-%s" % arch, + images = {"k8s.gcr.io/hyperkube-%s:{STABLE_DOCKER_TAG}" % arch: "hyperkube-internal"}, + tags = ["manual"], + visibility = ["//visibility:public"], +) for arch in SERVER_PLATFORMS["linux"]] + +[alias( + name = "hyperkube%s" % suffix, + actual = select(for_platforms(for_server = "hyperkube-{ARCH}%s" % suffix)), +) for suffix in [ + "", + ".tar", +]] filegroup( name = "package-srcs", From 84a5a176a52bc5192f374a39d28cef14c9c7ddcf Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Sat, 23 Feb 2019 18:34:13 -0800 Subject: [PATCH 2/7] bazel: add target for multi-arch docker tags on server images These aren't used in the images saved in the release tars, but could be used for images that are pushed to gcr.io. --- build/BUILD | 52 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/build/BUILD b/build/BUILD index 7befe25224..7512eef259 100644 --- a/build/BUILD +++ b/build/BUILD @@ -3,7 +3,7 @@ package(default_visibility = ["//visibility:public"]) load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image") load("@io_k8s_repo_infra//defs:build.bzl", "release_filegroup") load(":code_generation_test.bzl", "code_generation_test_suite") -load(":platforms.bzl", "for_platforms") +load(":platforms.bzl", "SERVER_PLATFORMS", "for_platforms") code_generation_test_suite( name = "code_generation_tests", @@ -67,27 +67,63 @@ DOCKERIZED_BINARIES = { }, } -[container_image( - name = binary + "-internal", +[[container_image( + name = "%s-internal-%s" % (binary, arch), base = meta["base"], cmd = ["/usr/bin/" + binary], - debs = select(for_platforms( - for_node = ["//build/debs:%s-{ARCH}.deb" % binary], - only_os = "linux", - )), + debs = ["//build/debs:%s-%s.deb" % (binary, arch)], stamp = True, symlinks = { # Some cluster startup scripts expect to find the binaries in /usr/local/bin, # but the debs install the binaries into /usr/bin. "/usr/local/bin/" + binary: "/usr/bin/" + binary, }, -) for binary, meta in DOCKERIZED_BINARIES.items()] + visibility = ["//visibility:private"], +) for binary, meta in DOCKERIZED_BINARIES.items()] for arch in SERVER_PLATFORMS["linux"]] + +# Also create aliases for the arch-specific images defined above. +# The alias doesn't create a new file (like a genrule would); +# instead, we are using it with a select() conditional to +# be able to refer to the correct architecture in a consistent way. +# (Notably, container_bundle does not seem to support using a select() +# in the images attribute, so we defer that selection to this rule.) +[alias( + name = "%s-internal" % binary, + actual = select(for_platforms( + for_server = ":%s-internal-{ARCH}" % binary, + only_os = "linux", + )), + visibility = ["//visibility:private"], +) for binary in DOCKERIZED_BINARIES.keys()] + +# We create two container bundles with the desired tags; one with an architecture in the name +# and one without. +# In the bash-based build (build/lib/release.sh), the images built for amd64 do not use +# an arch in their name (but other arches do), and the GCE cluster scripts +# (which sideload the images via tarfiles) expect there not to be an arch. +# When pushing to gcr.io, we want to use an arch, since the archless name is now used for a +# manifest list. Bazel doesn't support manifest lists (yet), so we can't do that either. +# Instead, for now, we use the archless name for the image tars saved in the server tarball, +# to satisfy GCE and other similar providers. (If one were to pull the images via the manifest +# list, the arch wouldn't appear in the name either.) +# The bundle with the arch isn't used currently, but might be at some point for pushing to gcr.io +# and to then create a manifest list (possibly outside of bazel). +[[container_bundle( + name = "%s-%s" % (binary, arch), + images = { + "k8s.gcr.io/%s-%s:{STABLE_DOCKER_TAG}" % (binary, arch): "%s-internal-%s" % (binary, arch), + }, + tags = ["manual"], + visibility = ["//visibility:public"], +) for binary in DOCKERIZED_BINARIES.keys()] for arch in SERVER_PLATFORMS["linux"]] [container_bundle( name = binary, images = { "k8s.gcr.io/%s:{STABLE_DOCKER_TAG}" % binary: binary + "-internal", }, + tags = ["manual"], + visibility = ["//visibility:private"], ) for binary in DOCKERIZED_BINARIES.keys()] [genrule( From e7888f2cf0077b9acf5137e044c9aaf57bc754eb Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Sat, 23 Feb 2019 18:39:16 -0800 Subject: [PATCH 3/7] bazel: add a small TODO comment --- build/root/BUILD.root | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/root/BUILD.root b/build/root/BUILD.root index 9fefdf382e..057769f684 100644 --- a/build/root/BUILD.root +++ b/build/root/BUILD.root @@ -32,6 +32,8 @@ filegroup( visibility = ["//visibility:private"], ) +# TODO: also add container_push rules, +# and don't forget about the conformance and hyperkube images gcs_upload( name = "push-build", data = [ From d7c2c2a606aae0edbff29f61ebda88b5e58c06ed Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 6 Mar 2019 11:40:33 -0800 Subject: [PATCH 4/7] bazel: refactor multi-arch container builds into starlark --- build/container.bzl | 72 ++++++++++++++++++++++++++++++++ cluster/images/conformance/BUILD | 35 ++++++---------- cluster/images/hyperkube/BUILD | 32 +++++--------- 3 files changed, 94 insertions(+), 45 deletions(-) create mode 100644 build/container.bzl diff --git a/build/container.bzl b/build/container.bzl new file mode 100644 index 0000000000..a11e14ffb1 --- /dev/null +++ b/build/container.bzl @@ -0,0 +1,72 @@ +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image") +load("//build:platforms.bzl", "go_platform_constraint") + +# multi_arch_container produces a private internal container_image, multiple +# arch-specific tagged container_bundles (named NAME-ARCH) and aliases +# from NAME and NAME.tar to the appropriately NAME-ARCH container_bundle target +# for the currently-configured architecture. +# Args: +# name: name used for the alias; the internal container_image and +# container_bundles are based on this name +# architectures: list of architectures (in GOARCH naming parlance) to +# configure +# base: base image to use for the containers. The format string {ARCH} will +# be replaced with the configured GOARCH. +# docker_tags: list of docker tags to apply to the image. The format string +# {ARCH} will be replaced with the configured GOARCH; any stamping variables +# should be escaped, e.g. {{STABLE_MY_VAR}}. +# tags: will be applied to all rules +# visiblity: will be applied only to the container_bundles; the internal +# container_image is private +# All other args will be applied to the internal container_image. +def multi_arch_container( + name, + architectures, + base, + docker_tags, + tags = None, + visibility = None, + **kwargs): + container_image( + name = "%s-internal" % name, + base = select({ + go_platform_constraint(os = "linux", arch = arch): base.format(ARCH = arch) + for arch in architectures + }), + tags = tags, + visibility = ["//visibility:private"], + **kwargs + ) + + for arch in architectures: + container_bundle( + name = "%s-%s" % (name, arch), + images = { + docker_tag.format(ARCH = arch): ":%s-internal" % name + for docker_tag in docker_tags + }, + tags = tags, + visibility = visibility, + ) + for suffix in ["", ".tar"]: + native.alias( + name = "%s%s" % (name, suffix), + actual = select({ + go_platform_constraint(os = "linux", arch = arch): "%s-%s%s" % (name, arch, suffix) + for arch in architectures + }), + ) diff --git a/cluster/images/conformance/BUILD b/cluster/images/conformance/BUILD index 00e8e33435..3f4fa46342 100644 --- a/cluster/images/conformance/BUILD +++ b/cluster/images/conformance/BUILD @@ -1,5 +1,6 @@ -load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image", "container_layer") -load("//build:platforms.bzl", "SERVER_PLATFORMS", "for_platforms") +load("@io_bazel_rules_docker//container:container.bzl", "container_layer") +load("//build:platforms.bzl", "SERVER_PLATFORMS") +load("//build:container.bzl", "multi_arch_container") container_layer( name = "cluster-srcs", @@ -18,17 +19,18 @@ container_layer( ], ) -container_image( - name = "conformance-internal", - base = select(for_platforms( - for_server = "@debian-hyperkube-base-{ARCH}//image", - only_os = "linux", - )), +multi_arch_container( + name = "conformance", + architectures = SERVER_PLATFORMS["linux"], + base = "@debian-hyperkube-base-{ARCH}//image", cmd = [ "/bin/bash", "-c", "/run_e2e.sh", ], + # {ARCH} is replaced by the macro, but STABLE_DOCKER_TAG is replaced by the + # build stamping, so we need to escape it + docker_tags = ["k8s.gcr.io/conformance-{ARCH}:{{STABLE_DOCKER_TAG}}"], env = { "E2E_FOCUS": "\[Conformance\]", "E2E_SKIP": "", @@ -45,23 +47,10 @@ container_image( ":bins", ], stamp = True, - workdir = "/usr/local/bin", -) - -[container_bundle( - name = "conformance-%s" % arch, - images = {"k8s.gcr.io/conformance-%s:{STABLE_DOCKER_TAG}" % arch: "conformance-internal"}, tags = ["manual"], visibility = ["//visibility:public"], -) for arch in SERVER_PLATFORMS["linux"]] - -[alias( - name = "conformance%s" % suffix, - actual = select(for_platforms(for_server = "conformance-{ARCH}%s" % suffix)), -) for suffix in [ - "", - ".tar", -]] + workdir = "/usr/local/bin", +) filegroup( name = "package-srcs", diff --git a/cluster/images/hyperkube/BUILD b/cluster/images/hyperkube/BUILD index 201f5abd35..f0e376edb0 100644 --- a/cluster/images/hyperkube/BUILD +++ b/cluster/images/hyperkube/BUILD @@ -1,32 +1,20 @@ -load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image") -load("//build:platforms.bzl", "SERVER_PLATFORMS", "for_platforms") +load("//build:container.bzl", "multi_arch_container") +load("//build:platforms.bzl", "SERVER_PLATFORMS") -container_image( - name = "hyperkube-internal", - base = select(for_platforms( - for_server = "@debian-hyperkube-base-{ARCH}//image", - only_os = "linux", - )), +multi_arch_container( + name = "hyperkube", + architectures = SERVER_PLATFORMS["linux"], + base = "@debian-hyperkube-base-{ARCH}//image", + # {ARCH} is replaced by the macro, but STABLE_DOCKER_TAG is replaced by the + # build stamping, so we need to escape it + docker_tags = ["k8s.gcr.io/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"], files = [ "//cmd/hyperkube", ], stamp = True, -) - -[container_bundle( - name = "hyperkube-%s" % arch, - images = {"k8s.gcr.io/hyperkube-%s:{STABLE_DOCKER_TAG}" % arch: "hyperkube-internal"}, tags = ["manual"], visibility = ["//visibility:public"], -) for arch in SERVER_PLATFORMS["linux"]] - -[alias( - name = "hyperkube%s" % suffix, - actual = select(for_platforms(for_server = "hyperkube-{ARCH}%s" % suffix)), -) for suffix in [ - "", - ".tar", -]] +) filegroup( name = "package-srcs", From 5b7602a5259f18ee69e02aea07f60140a1de1949 Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 6 Mar 2019 11:54:49 -0800 Subject: [PATCH 5/7] bazel: refactor core docker images to use multi_arch_container --- build/BUILD | 92 ++++++++++++++--------------------------------------- 1 file changed, 23 insertions(+), 69 deletions(-) diff --git a/build/BUILD b/build/BUILD index 7512eef259..d8dec66d8d 100644 --- a/build/BUILD +++ b/build/BUILD @@ -1,8 +1,8 @@ package(default_visibility = ["//visibility:public"]) -load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image") load("@io_k8s_repo_infra//defs:build.bzl", "release_filegroup") load(":code_generation_test.bzl", "code_generation_test_suite") +load(":container.bzl", "multi_arch_container") load(":platforms.bzl", "SERVER_PLATFORMS", "for_platforms") code_generation_test_suite( @@ -31,100 +31,54 @@ filegroup( # in build/common.sh. DOCKERIZED_BINARIES = { "cloud-controller-manager": { - "base": select(for_platforms( - for_node = "@debian-base-{ARCH}//image", - only_os = "linux", - )), + "base": "@debian-base-{ARCH}//image", "target": "//cmd/cloud-controller-manager:cloud-controller-manager", }, "kube-apiserver": { - "base": select(for_platforms( - for_node = "@debian-base-{ARCH}//image", - only_os = "linux", - )), + "base": "@debian-base-{ARCH}//image", "target": "//cmd/kube-apiserver:kube-apiserver", }, "kube-controller-manager": { - "base": select(for_platforms( - for_node = "@debian-base-{ARCH}//image", - only_os = "linux", - )), + "base": "@debian-base-{ARCH}//image", "target": "//cmd/kube-controller-manager:kube-controller-manager", }, "kube-scheduler": { - "base": select(for_platforms( - for_node = "@debian-base-{ARCH}//image", - only_os = "linux", - )), + "base": "@debian-base-{ARCH}//image", "target": "//cmd/kube-scheduler:kube-scheduler", }, "kube-proxy": { - "base": select(for_platforms( - for_node = "@debian-iptables-{ARCH}//image", - only_os = "linux", - )), + "base": "@debian-iptables-{ARCH}//image", "target": "//cmd/kube-proxy:kube-proxy", }, } -[[container_image( - name = "%s-internal-%s" % (binary, arch), +# In the bash-based build (build/lib/release.sh), the images built for amd64 do not use +# an arch in their name (but other arches do), and the GCE cluster scripts +# (which sideload the images via tarfiles) expect there not to be an arch. +# When pushing to gcr.io, we want to use an arch, since the archless name is now used for a +# manifest list. Bazel doesn't support manifest lists (yet), so we can't do that either. +# For now, we use the archless name for the image tars saved in the server tarball, +# to satisfy GCE and other similar providers. (If one were to pull the images via the manifest +# list, the arch wouldn't appear in the name either.) +[multi_arch_container( + name = binary, + architectures = SERVER_PLATFORMS["linux"], base = meta["base"], cmd = ["/usr/bin/" + binary], - debs = ["//build/debs:%s-%s.deb" % (binary, arch)], + debs = select(for_platforms( + for_server = ["//build/debs:%s-{ARCH}.deb" % binary], + only_os = "linux", + )), + docker_tags = ["k8s.gcr.io/%s:{{STABLE_DOCKER_TAG}}" % binary], stamp = True, symlinks = { # Some cluster startup scripts expect to find the binaries in /usr/local/bin, # but the debs install the binaries into /usr/bin. "/usr/local/bin/" + binary: "/usr/bin/" + binary, }, - visibility = ["//visibility:private"], -) for binary, meta in DOCKERIZED_BINARIES.items()] for arch in SERVER_PLATFORMS["linux"]] - -# Also create aliases for the arch-specific images defined above. -# The alias doesn't create a new file (like a genrule would); -# instead, we are using it with a select() conditional to -# be able to refer to the correct architecture in a consistent way. -# (Notably, container_bundle does not seem to support using a select() -# in the images attribute, so we defer that selection to this rule.) -[alias( - name = "%s-internal" % binary, - actual = select(for_platforms( - for_server = ":%s-internal-{ARCH}" % binary, - only_os = "linux", - )), - visibility = ["//visibility:private"], -) for binary in DOCKERIZED_BINARIES.keys()] - -# We create two container bundles with the desired tags; one with an architecture in the name -# and one without. -# In the bash-based build (build/lib/release.sh), the images built for amd64 do not use -# an arch in their name (but other arches do), and the GCE cluster scripts -# (which sideload the images via tarfiles) expect there not to be an arch. -# When pushing to gcr.io, we want to use an arch, since the archless name is now used for a -# manifest list. Bazel doesn't support manifest lists (yet), so we can't do that either. -# Instead, for now, we use the archless name for the image tars saved in the server tarball, -# to satisfy GCE and other similar providers. (If one were to pull the images via the manifest -# list, the arch wouldn't appear in the name either.) -# The bundle with the arch isn't used currently, but might be at some point for pushing to gcr.io -# and to then create a manifest list (possibly outside of bazel). -[[container_bundle( - name = "%s-%s" % (binary, arch), - images = { - "k8s.gcr.io/%s-%s:{STABLE_DOCKER_TAG}" % (binary, arch): "%s-internal-%s" % (binary, arch), - }, - tags = ["manual"], - visibility = ["//visibility:public"], -) for binary in DOCKERIZED_BINARIES.keys()] for arch in SERVER_PLATFORMS["linux"]] - -[container_bundle( - name = binary, - images = { - "k8s.gcr.io/%s:{STABLE_DOCKER_TAG}" % binary: binary + "-internal", - }, tags = ["manual"], visibility = ["//visibility:private"], -) for binary in DOCKERIZED_BINARIES.keys()] +) for binary, meta in DOCKERIZED_BINARIES.items()] [genrule( name = binary + "_docker_tag", From 8495f7c9eda718e9083a56f307c5999144a5ab3f Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 6 Mar 2019 12:33:42 -0800 Subject: [PATCH 6/7] bazel: add docker_push rules --- build/BUILD | 20 ++++++++++-- build/container.bzl | 52 +++++++++++++++++++++++++++++++- build/root/BUILD.root | 6 ++-- cluster/images/conformance/BUILD | 7 +++-- cluster/images/hyperkube/BUILD | 7 +++-- hack/print-workspace-status.sh | 2 ++ 6 files changed, 83 insertions(+), 11 deletions(-) diff --git a/build/BUILD b/build/BUILD index d8dec66d8d..3ac15d0705 100644 --- a/build/BUILD +++ b/build/BUILD @@ -2,7 +2,7 @@ package(default_visibility = ["//visibility:public"]) load("@io_k8s_repo_infra//defs:build.bzl", "release_filegroup") load(":code_generation_test.bzl", "code_generation_test_suite") -load(":container.bzl", "multi_arch_container") +load(":container.bzl", "multi_arch_container", "multi_arch_container_push") load(":platforms.bzl", "SERVER_PLATFORMS", "for_platforms") code_generation_test_suite( @@ -69,7 +69,12 @@ DOCKERIZED_BINARIES = { for_server = ["//build/debs:%s-{ARCH}.deb" % binary], only_os = "linux", )), - docker_tags = ["k8s.gcr.io/%s:{{STABLE_DOCKER_TAG}}" % binary], + # Since the multi_arch_container macro replaces the {ARCH} format string, + # we need to escape the stamping vars. + # Also see comment above about why the push tags use ARCH while the + # non-push tags do not. + docker_push_tags = ["{{STABLE_DOCKER_PUSH_REGISTRY}}/%s-{ARCH}:{{STABLE_DOCKER_TAG}}" % binary], + docker_tags = ["{{STABLE_DOCKER_REGISTRY}}/%s:{{STABLE_DOCKER_TAG}}" % binary], stamp = True, symlinks = { # Some cluster startup scripts expect to find the binaries in /usr/local/bin, @@ -80,6 +85,17 @@ DOCKERIZED_BINARIES = { visibility = ["//visibility:private"], ) for binary, meta in DOCKERIZED_BINARIES.items()] +# Also roll up all images into a single bundle to push with one target. +multi_arch_container_push( + name = "server-images", + architectures = SERVER_PLATFORMS["linux"], + docker_tags_images = { + "{{STABLE_DOCKER_PUSH_REGISTRY}}/%s-{ARCH}:{{STABLE_DOCKER_TAG}}" % binary: "%s-internal" % binary + for binary in DOCKERIZED_BINARIES.keys() + }, + tags = ["manual"], +) + [genrule( name = binary + "_docker_tag", srcs = [meta["target"]], diff --git a/build/container.bzl b/build/container.bzl index a11e14ffb1..445f1ee9c6 100644 --- a/build/container.bzl +++ b/build/container.bzl @@ -13,12 +13,16 @@ # limitations under the License. load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image") +load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push") load("//build:platforms.bzl", "go_platform_constraint") # multi_arch_container produces a private internal container_image, multiple # arch-specific tagged container_bundles (named NAME-ARCH) and aliases # from NAME and NAME.tar to the appropriately NAME-ARCH container_bundle target # for the currently-configured architecture. +# Additionally, if docker_push_tags is provided, uses multi_arch_container_push +# to create container_bundles named push-NAME-ARCH with the provided push tags, +# along with a push-NAME docker_push target. # Args: # name: name used for the alias; the internal container_image and # container_bundles are based on this name @@ -29,7 +33,10 @@ load("//build:platforms.bzl", "go_platform_constraint") # docker_tags: list of docker tags to apply to the image. The format string # {ARCH} will be replaced with the configured GOARCH; any stamping variables # should be escaped, e.g. {{STABLE_MY_VAR}}. -# tags: will be applied to all rules +# docker_push_tags: list of docker tags to apply to the image for pushing. +# The format string {ARCH} will be replaced with the configured GOARCH; +# any stamping variables should be escaped, e.g. {{STABLE_MY_VAR}}. +# tags: will be applied to all targets # visiblity: will be applied only to the container_bundles; the internal # container_image is private # All other args will be applied to the internal container_image. @@ -38,6 +45,7 @@ def multi_arch_container( architectures, base, docker_tags, + docker_push_tags = None, tags = None, visibility = None, **kwargs): @@ -70,3 +78,45 @@ def multi_arch_container( for arch in architectures }), ) + + if docker_push_tags: + multi_arch_container_push( + name = name, + architectures = architectures, + docker_tags_images = {docker_push_tag: ":%s-internal" % name for docker_push_tag in docker_push_tags}, + tags = tags, + ) + +# multi_arch_container_push creates container_bundles named push-NAME-ARCH for +# the provided architectures, populating them with the images directory. +# It additionally creates a push-NAME docker_push rule which can be run to +# push the images to a Docker repository. +# Args: +# name: name used for targets created by this macro; the internal +# container_bundles are based on this name +# architectures: list of architectures (in GOARCH naming parlance) to +# configure +# docker_tags_images: dictionary mapping docker tag to the corresponding +# container_image target. The format string {ARCH} will be replaced +# in tags with the configured GOARCH; any stamping variables should be +# escaped, e.g. {{STABLE_MY_VAR}}. +# tags: applied to container_bundle targets +def multi_arch_container_push( + name, + architectures, + docker_tags_images, + tags = None): + for arch in architectures: + container_bundle( + name = "push-%s-%s" % (name, arch), + images = {tag.format(ARCH = arch): image for tag, image in docker_tags_images.items()}, + tags = tags, + visibility = ["//visibility:private"], + ) + docker_push( + name = "push-%s" % name, + bundle = select({ + go_platform_constraint(os = "linux", arch = arch): "push-%s-%s" % (name, arch) + for arch in architectures + }), + ) diff --git a/build/root/BUILD.root b/build/root/BUILD.root index 057769f684..3469dfab6d 100644 --- a/build/root/BUILD.root +++ b/build/root/BUILD.root @@ -32,8 +32,10 @@ filegroup( visibility = ["//visibility:private"], ) -# TODO: also add container_push rules, -# and don't forget about the conformance and hyperkube images +# TODO: collect all relevant docker_push targets into one target that can be run: +# //build:push-server-images +# //cluster/images/conformance:push-conformance +# //cluster/images/hyperkube:push-hyperkube gcs_upload( name = "push-build", data = [ diff --git a/cluster/images/conformance/BUILD b/cluster/images/conformance/BUILD index 3f4fa46342..14fd8f9859 100644 --- a/cluster/images/conformance/BUILD +++ b/cluster/images/conformance/BUILD @@ -28,9 +28,10 @@ multi_arch_container( "-c", "/run_e2e.sh", ], - # {ARCH} is replaced by the macro, but STABLE_DOCKER_TAG is replaced by the - # build stamping, so we need to escape it - docker_tags = ["k8s.gcr.io/conformance-{ARCH}:{{STABLE_DOCKER_TAG}}"], + # {ARCH} is replaced by the macro, but STABLE_ vars are replaced by the + # build stamping, so we need to escape them + docker_push_tags = ["{{STABLE_DOCKER_PUSH_REGISTRY}}/conformance-{ARCH}:{{STABLE_DOCKER_TAG}}"], + docker_tags = ["{{STABLE_DOCKER_REGISTRY}}/conformance-{ARCH}:{{STABLE_DOCKER_TAG}}"], env = { "E2E_FOCUS": "\[Conformance\]", "E2E_SKIP": "", diff --git a/cluster/images/hyperkube/BUILD b/cluster/images/hyperkube/BUILD index f0e376edb0..cc2653bf9d 100644 --- a/cluster/images/hyperkube/BUILD +++ b/cluster/images/hyperkube/BUILD @@ -5,9 +5,10 @@ multi_arch_container( name = "hyperkube", architectures = SERVER_PLATFORMS["linux"], base = "@debian-hyperkube-base-{ARCH}//image", - # {ARCH} is replaced by the macro, but STABLE_DOCKER_TAG is replaced by the - # build stamping, so we need to escape it - docker_tags = ["k8s.gcr.io/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"], + # {ARCH} is replaced by the macro, but STABLE_ vars are replaced by the + # build stamping, so we need to escape them + docker_push_tags = ["{{STABLE_DOCKER_PUSH_REGISTRY}}/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"], + docker_tags = ["{{STABLE_DOCKER_REGISTRY}}/hyperkube-{ARCH}:{{STABLE_DOCKER_TAG}}"], files = [ "//cmd/hyperkube", ], diff --git a/hack/print-workspace-status.sh b/hack/print-workspace-status.sh index 1d334f88a1..fb1d8a5bb9 100755 --- a/hack/print-workspace-status.sh +++ b/hack/print-workspace-status.sh @@ -38,6 +38,8 @@ STABLE_BUILD_SCM_REVISION ${KUBE_GIT_VERSION-} STABLE_BUILD_MAJOR_VERSION ${KUBE_GIT_MAJOR-} STABLE_BUILD_MINOR_VERSION ${KUBE_GIT_MINOR-} STABLE_DOCKER_TAG ${KUBE_GIT_VERSION/+/_} +STABLE_DOCKER_REGISTRY ${KUBE_DOCKER_REGISTRY:-k8s.gcr.io} +STABLE_DOCKER_PUSH_REGISTRY ${KUBE_DOCKER_PUSH_REGISTRY:-${KUBE_DOCKER_REGISTRY:-staging-k8s.gcr.io}} gitCommit ${KUBE_GIT_COMMIT-} gitTreeState ${KUBE_GIT_TREE_STATE-} gitVersion ${KUBE_GIT_VERSION-} From a259fd2f695b36d185e56274e940bac73a533f42 Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Wed, 6 Mar 2019 12:56:10 -0800 Subject: [PATCH 7/7] bazel maintain support for arch-less-named tarballs --- build/container.bzl | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/build/container.bzl b/build/container.bzl index 445f1ee9c6..54b45b7423 100644 --- a/build/container.bzl +++ b/build/container.bzl @@ -17,9 +17,10 @@ load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push") load("//build:platforms.bzl", "go_platform_constraint") # multi_arch_container produces a private internal container_image, multiple -# arch-specific tagged container_bundles (named NAME-ARCH) and aliases -# from NAME and NAME.tar to the appropriately NAME-ARCH container_bundle target -# for the currently-configured architecture. +# arch-specific tagged container_bundles (named NAME-ARCH), an alias +# from NAME to the appropriately NAME-ARCH container_bundle target, and a +# genrule for NAME.tar copying the appropriate NAME-ARCH container bundle +# tarball output for the currently-configured architecture. # Additionally, if docker_push_tags is provided, uses multi_arch_container_push # to create container_bundles named push-NAME-ARCH with the provided push tags, # along with a push-NAME docker_push target. @@ -70,14 +71,23 @@ def multi_arch_container( tags = tags, visibility = visibility, ) - for suffix in ["", ".tar"]: - native.alias( - name = "%s%s" % (name, suffix), - actual = select({ - go_platform_constraint(os = "linux", arch = arch): "%s-%s%s" % (name, arch, suffix) - for arch in architectures - }), - ) + native.alias( + name = name, + actual = select({ + go_platform_constraint(os = "linux", arch = arch): "%s-%s" % (name, arch) + for arch in architectures + }), + ) + native.genrule( + name = "gen_%s.tar" % name, + outs = ["%s.tar" % name], + srcs = select({ + go_platform_constraint(os = "linux", arch = arch): ["%s-%s.tar" % (name, arch)] + for arch in architectures + }), + cmd = "cp $< $@", + output_to_bindir = True, + ) if docker_push_tags: multi_arch_container_push(