From 560268e036f9bbf40e03f39c9386e0a91bfb3938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Mon, 4 Apr 2016 23:33:32 +0300 Subject: [PATCH] Add a flannel image for arm and amd64. Cross-build debian-iptables for arm, arm64 and ppc64le. Build and push hyperkube for arm on every release --- build/common.sh | 49 ++++++++++++----------- build/debian-iptables/Dockerfile | 12 ++++-- build/debian-iptables/Makefile | 46 +++++++++++++++++++--- build/debian-iptables/README.md | 29 ++++++++++++++ cluster/images/flannel/Dockerfile | 20 ++++++++++ cluster/images/flannel/Makefile | 60 +++++++++++++++++++++++++++++ cluster/images/flannel/README.md | 22 +++++++++++ cluster/images/hyperkube/Dockerfile | 4 ++ cluster/images/hyperkube/Makefile | 33 ++++++++++------ cluster/images/hyperkube/README.md | 27 +++++++++++++ cluster/images/hyperkube/etcd.json | 2 +- 11 files changed, 257 insertions(+), 47 deletions(-) create mode 100644 build/debian-iptables/README.md create mode 100644 cluster/images/flannel/Dockerfile create mode 100644 cluster/images/flannel/Makefile create mode 100644 cluster/images/flannel/README.md create mode 100644 cluster/images/hyperkube/README.md diff --git a/build/common.sh b/build/common.sh index c85345c378..fc2f9c5ee3 100755 --- a/build/common.sh +++ b/build/common.sh @@ -102,28 +102,28 @@ kube::build::get_docker_wrapped_binaries() { kube-apiserver,busybox kube-controller-manager,busybox kube-scheduler,busybox - kube-proxy,gcr.io/google_containers/debian-iptables:v1 + kube-proxy,gcr.io/google_containers/debian-iptables-amd64:v2 );; - "arm") # TODO: Use image with iptables installed for kube-proxy for arm, arm64 and ppc64le + "arm") local targets=( - kube-apiserver,hypriot/armhf-busybox - kube-controller-manager,hypriot/armhf-busybox - kube-scheduler,hypriot/armhf-busybox - kube-proxy,hypriot/armhf-busybox + kube-apiserver,armel/busybox + kube-controller-manager,armel/busybox + kube-scheduler,armel/busybox + kube-proxy,gcr.io/google_containers/debian-iptables-arm:v2 );; "arm64") local targets=( kube-apiserver,aarch64/busybox kube-controller-manager,aarch64/busybox kube-scheduler,aarch64/busybox - kube-proxy,aarch64/busybox + kube-proxy,gcr.io/google_containers/debian-iptables-arm64:v2 );; "ppc64le") local targets=( kube-apiserver,ppc64le/busybox kube-controller-manager,ppc64le/busybox kube-scheduler,ppc64le/busybox - kube-proxy,ppc64le/busybox + kube-proxy,gcr.io/google_containers/debian-iptables-ppc64le:v2 );; esac @@ -671,7 +671,12 @@ function kube::release::clean_cruft() { function kube::release::package_hyperkube() { # If we have these variables set then we want to build all docker images. if [[ -n "${KUBE_DOCKER_IMAGE_TAG-}" && -n "${KUBE_DOCKER_REGISTRY-}" ]]; then - REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" make -C cluster/images/hyperkube/ build + for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do + + local arch=${platform##*/} + kube::log::status "Building hyperkube image for arch: ${arch}" + REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" ARCH="${arch}" make -C cluster/images/hyperkube/ build + done fi } @@ -737,7 +742,7 @@ function kube::release::package_client_tarballs() { # Package up all of the server binaries function kube::release::package_server_tarballs() { local platform - for platform in "${KUBE_SERVER_PLATFORMS[@]}" ; do + for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do local platform_tag=${platform/\//-} # Replace a "/" for a "-" local arch=$(basename ${platform}) kube::log::status "Building tarball: server $platform_tag" @@ -1536,24 +1541,18 @@ function kube::release::docker::release() { for arch in "${archs[@]}"; do for binary in "${binaries[@]}"; do - # Temporary fix. hyperkube-arm isn't built in the release process, so we can't push it - # This if statement skips the push for hyperkube-arm - if [[ ${arch} != "arm" || ${binary} != "hyperkube" ]]; then + local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}" + kube::log::status "Pushing ${binary} to ${docker_target}" + "${docker_push_cmd[@]}" push "${docker_target}" + # If we have a amd64 docker image. Tag it without -amd64 also and push it for compatibility with earlier versions + if [[ ${arch} == "amd64" ]]; then + local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}" - local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}" - kube::log::status "Pushing ${binary} to ${docker_target}" - "${docker_push_cmd[@]}" push "${docker_target}" + "${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null - # If we have a amd64 docker image. Tag it without -amd64 also and push it for compatibility with earlier versions - if [[ ${arch} == "amd64" ]]; then - local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}" - - "${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null - - kube::log::status "Pushing ${binary} to ${legacy_docker_target}" - "${docker_push_cmd[@]}" push "${legacy_docker_target}" - fi + kube::log::status "Pushing ${binary} to ${legacy_docker_target}" + "${docker_push_cmd[@]}" push "${legacy_docker_target}" fi done done diff --git a/build/debian-iptables/Dockerfile b/build/debian-iptables/Dockerfile index 36cba66ec0..80fa3ecc44 100644 --- a/build/debian-iptables/Dockerfile +++ b/build/debian-iptables/Dockerfile @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM debian:jessie +FROM BASEIMAGE + +# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY +# If we're building normally, for amd64, CROSS_BUILD lines are removed +CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/ # All apt-get's must be in one run command or the # cleanup has no effect. -RUN apt-get update && \ - apt-get install -y iptables && \ - ls /var/lib/apt/lists/*debian* | xargs rm +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y iptables \ + && rm -rf /var/lib/apt/lists/* diff --git a/build/debian-iptables/Makefile b/build/debian-iptables/Makefile index 9f50b8776b..d1a4e0d873 100644 --- a/build/debian-iptables/Makefile +++ b/build/debian-iptables/Makefile @@ -14,13 +14,47 @@ .PHONY: build push -IMAGE = debian-iptables -TAG = v1 +REGISTRY?="gcr.io/google_containers" +IMAGE=debian-iptables +TAG=v2 +ARCH?=amd64 +TEMP_DIR:=$(shell mktemp -d) + +ifeq ($(ARCH),amd64) + BASEIMAGE?=debian:jessie +endif +ifeq ($(ARCH),arm) + BASEIMAGE?=armel/debian:jessie + QEMUARCH=arm +endif +ifeq ($(ARCH),arm64) + BASEIMAGE?=aarch64/debian:jessie + QEMUARCH=aarch64 +endif +ifeq ($(ARCH),ppc64le) + BASEIMAGE?=ppc64le/debian:jessie + QEMUARCH=ppc64le +endif build: - docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) . + cp ./* $(TEMP_DIR) + cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile + cd $(TEMP_DIR) && sed -i "s|ARCH|$(QEMUARCH)|g" Dockerfile -push: build - gcloud docker --server=gcr.io push gcr.io/google_containers/$(IMAGE):$(TAG) +ifeq ($(ARCH),amd64) + # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image + cd $(TEMP_DIR) && sed -i "/CROSS_BUILD_/d" Dockerfile +else + # When cross-building, only the placeholder "CROSS_BUILD_" should be removed + # Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel + docker run --rm --privileged multiarch/qemu-user-static:register --reset + curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-$(QEMUARCH)-static.tar.xz | tar -xJ -C $(TEMP_DIR) + cd $(TEMP_DIR) && sed -i "s/CROSS_BUILD_//g" Dockerfile +endif -all: push + docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR) + +push: build + gcloud docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) + +all: push diff --git a/build/debian-iptables/README.md b/build/debian-iptables/README.md new file mode 100644 index 0000000000..65dcfc0a8a --- /dev/null +++ b/build/debian-iptables/README.md @@ -0,0 +1,29 @@ +### debian-iptables + +Serves as the base image for `gcr.io/google_containers/kube-proxy-${ARCH}` and multiarch (not `amd64`) `gcr.io/google_containers/flannel-${ARCH}` images. + +This image is compiled for multiple architectures. + +#### How to release + +If you're editing the Dockerfile or some other thing, please bump the `TAG` in the Makefile. + +```console +# Build for linux/amd64 (default) +$ make push ARCH=amd64 +# ---> gcr.io/google_containers/debian-iptables-amd64:TAG + +$ make push ARCH=arm +# ---> gcr.io/google_containers/debian-iptables-arm:TAG + +$ make push ARCH=arm64 +# ---> gcr.io/google_containers/debian-iptables-arm64:TAG + +$ make push ARCH=ppc64le +# ---> gcr.io/google_containers/debian-iptables-ppc64le:TAG +``` + +If you don't want to push the images, run `make` or `make build` instead + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/build/debian-iptables/README.md?pixel)]() diff --git a/cluster/images/flannel/Dockerfile b/cluster/images/flannel/Dockerfile new file mode 100644 index 0000000000..9f75907983 --- /dev/null +++ b/cluster/images/flannel/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +COPY flanneld /opt/bin/ +COPY mk-docker-opts.sh /opt/bin/ + +CMD ["/opt/bin/flanneld"] diff --git a/cluster/images/flannel/Makefile b/cluster/images/flannel/Makefile new file mode 100644 index 0000000000..376a1f20b2 --- /dev/null +++ b/cluster/images/flannel/Makefile @@ -0,0 +1,60 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the flannel image +# +# Usage: +# [TAG=0.5.5] [REGISTRY=gcr.io/google_containers] [ARCH=amd64] make build + +TAG?=0.5.5 +ARCH?=amd64 +REGISTRY?=gcr.io/google_containers +KUBE_CROSS_TAG=v1.4.2-1 +GOARM=6 +TEMP_DIR:=$(shell mktemp -d) +BASEIMAGE?=gcr.io/google_containers/debian-iptables-${ARCH}:v2 + +ifeq ($(ARCH),arm) + CC=arm-linux-gnueabi-gcc +endif + +build: +ifeq ($(ARCH),amd64) + # If we should build an amd64 flannel, go with the official one + docker pull quay.io/coreos/flannel:$(TAG) + + docker tag -f quay.io/coreos/flannel:$(TAG) $(REGISTRY)/flannel-$(ARCH):$(TAG) +else + # Copy the content in this dir to the temp dir + cp ./* $(TEMP_DIR) + + docker run -it -v $(TEMP_DIR):/flannel/bin gcr.io/google_containers/kube-cross:$(KUBE_CROSS_TAG) /bin/bash -c \ + "curl -sSL https://github.com/coreos/flannel/archive/v${TAG}.tar.gz | tar -C /flannel -xz --strip-components=1 \ + && cd /flannel && GOARM=$(GOARM) GOARCH=$(ARCH) CC=$(CC) CGO_ENABLED=1 ./build" + + # Replace BASEIMAGE with the real base image + cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile + + # Download mk-docker-opts.sh + curl -sSL https://raw.githubusercontent.com/coreos/flannel/v$(TAG)/dist/mk-docker-opts.sh > $(TEMP_DIR)/mk-docker-opts.sh + + # And build the image + docker build -t $(REGISTRY)/flannel-$(ARCH):$(TAG) $(TEMP_DIR) +endif + +push: build + gcloud docker push $(REGISTRY)/flannel-$(ARCH):$(TAG) + +all: build +.PHONY: build push diff --git a/cluster/images/flannel/README.md b/cluster/images/flannel/README.md new file mode 100644 index 0000000000..50e8289660 --- /dev/null +++ b/cluster/images/flannel/README.md @@ -0,0 +1,22 @@ +### flannel + +This is used mostly for the `docker-multinode` config, but also in other places where flannel runs in a container. + +For `amd64`, this image equals to `quay.io/coreos/flannel` to maintain official support. +For other architectures, `flannel` is cross-compiled. The `debian-iptables` image serves as base image. + +#### How to release + +```console +# Build for linux/amd64 (default) +$ make push ARCH=amd64 +# ---> gcr.io/google_containers/flannel-amd64:TAG + +$ make push ARCH=arm +# ---> gcr.io/google_containers/flannel-arm:TAG +``` + +If you don't want to push the images, run `make` or `make build` instead + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/flannel/README.md?pixel)]() diff --git a/cluster/images/hyperkube/Dockerfile b/cluster/images/hyperkube/Dockerfile index 7ed1e14f1e..a70f6b8308 100644 --- a/cluster/images/hyperkube/Dockerfile +++ b/cluster/images/hyperkube/Dockerfile @@ -14,6 +14,10 @@ FROM BASEIMAGE +# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY +# If we're building normally, for amd64, CROSS_BUILD lines are removed +CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/ + RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \ && DEBIAN_FRONTEND=noninteractive apt-get -yy -q \ install \ diff --git a/cluster/images/hyperkube/Makefile b/cluster/images/hyperkube/Makefile index a3bd203840..e6e4fbe521 100644 --- a/cluster/images/hyperkube/Makefile +++ b/cluster/images/hyperkube/Makefile @@ -15,17 +15,19 @@ # Build the hyperkube image. # # Usage: -# VERSION=v1.1.2 [REGISTRY="gcr.io/google_containers"] make build +# VERSION=v1.2.0 [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make build REGISTRY?="gcr.io/google_containers" -ARCH=amd64 -BASEIMAGE=debian:jessie +ARCH?=amd64 TEMP_DIR:=$(shell mktemp -d) -## Comment in for arm builds, must be run on an arm machine -# ARCH=arm -# need to escape '/' for the regexp below -# BASEIMAGE=armbuild\\/debian:jessie + +ifeq ($(ARCH),amd64) + BASEIMAGE?=debian:jessie +endif +ifeq ($(ARCH),arm) + BASEIMAGE?=armel/debian:jessie +endif all: build @@ -38,18 +40,27 @@ endif cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR} cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR} cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json - cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json + cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json Dockerfile cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile rm ${TEMP_DIR}/*.back - docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR} - # Backward compatibility. TODO: deprecate this image tag + ifeq ($(ARCH),amd64) - docker tag -f ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${REGISTRY}/hyperkube:${VERSION} + # When building "normally" for amd64, remove the whole line, it has no part in the amd64 image + cd ${TEMP_DIR} && sed -i "/CROSS_BUILD_/d" Dockerfile +else + # When cross-building, only the placeholder "CROSS_BUILD_" should be removed + # Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel + docker run --rm --privileged multiarch/qemu-user-static:register --reset + curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-${ARCH}-static.tar.xz | tar -xJ -C ${TEMP_DIR} + cd ${TEMP_DIR} && sed -i "s/CROSS_BUILD_//g" Dockerfile endif + docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR} + push: build gcloud docker push ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ifeq ($(ARCH),amd64) + docker tag -f ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${REGISTRY}/hyperkube:${VERSION} gcloud docker push ${REGISTRY}/hyperkube:${VERSION} endif diff --git a/cluster/images/hyperkube/README.md b/cluster/images/hyperkube/README.md new file mode 100644 index 0000000000..28a6eca2f9 --- /dev/null +++ b/cluster/images/hyperkube/README.md @@ -0,0 +1,27 @@ +### hyperkube + +`hyperkube` is an all-in-one binary for the Kubernetes server components +Also, it's very easy to run this `hyperkube` setup dockerized. +See http://kubernetes.io/docs/getting-started-guides/docker/ for up-to-date commands. + +`hyperkube` is built for multiple architectures and pushed on every release. + +#### How to release by hand + +```console +# First, build the +$ build/run.sh hack/build-cross.sh + +# Build for linux/amd64 (default) +$ make push VERSION={target_version} ARCH=amd64 +# ---> gcr.io/google_containers/hyperkube-amd64:VERSION +# ---> gcr.io/google_containers/hyperkube:VERSION (image with backwards-compatible naming) + +$ make push VERSION={target_version} ARCH=arm +# ---> gcr.io/google_containers/hyperkube-arm:VERSION +``` + +If you don't want to push the images, run `make` or `make build` instead + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/images/hyperkube/README.md?pixel)]() diff --git a/cluster/images/hyperkube/etcd.json b/cluster/images/hyperkube/etcd.json index 1f4a39280e..368e0c768b 100644 --- a/cluster/images/hyperkube/etcd.json +++ b/cluster/images/hyperkube/etcd.json @@ -7,7 +7,7 @@ "containers": [ { "name": "etcd", - "image": "gcr.io/google_containers/etcd:2.2.1", + "image": "gcr.io/google_containers/etcd-ARCH:2.2.1", "command": [ "/usr/local/bin/etcd", "--listen-client-urls=http://127.0.0.1:4001",