mirror of https://github.com/k3s-io/k3s
Merge pull request #21617 from luxas/hyperkube_for_arm
Automatic merge from submit-queue Cross-build hyperkube and debian-iptables for ARM. Also add a flannel image We have to be able to build complex docker images too on `amd64` hosts. Right now we can't build Dockerfiles with `RUN` commands when building for other architectures e.g. ARM. Resin has a tutorial about this here: https://resin.io/blog/building-arm-containers-on-any-x86-machine-even-dockerhub/ But it's a bit clumsy syntax. The other alternative would be running this command in a Makefile: ``` # This registers in the kernel that ARM binaries should be run by /usr/bin/qemu-{ARCH}-static docker run --rm --privileged multiarch/qemu-user-static:register --reset ``` and ``` ADD https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-arm-static.tar.xz /usr/bin ``` Then the kernel will be able to differ ARM binaries from amd64. When it finds a ARM binary, it will invoke `/usr/bin/qemu-arm-static` first and lets `qemu` translate the ARM syscalls to amd64 ones. Some code here: https://github.com/multiarch WDYT is the best approach? If registering `binfmt_misc` in the kernels of the machines is OK, then I think we should go with that. Otherwise, we'll have to wait for resin's patch to be merged into mainline qemu before we may use the code I have here now. @fgrzadkowski @david-mcmahon @brendandburns @zmerlynn @ixdy @ihmccreery @thockinpull/6/head
k8s-merge-robot
commit
b8d000853e
|
|
@ -103,28 +103,28 @@ kube::build::get_docker_wrapped_binaries() {
|
|||
kube-apiserver,busybox
|
||||
kube-controller-manager,busybox
|
||||
kube-scheduler,busybox
|
||||
kube-proxy,gcr.io/google_containers/debian-iptables:v1
|
||||
kube-proxy,gcr.io/google_containers/debian-iptables-amd64:v2
|
||||
);;
|
||||
"arm") # TODO: Use image with iptables installed for kube-proxy for arm, arm64 and ppc64le
|
||||
"arm")
|
||||
local targets=(
|
||||
kube-apiserver,hypriot/armhf-busybox
|
||||
kube-controller-manager,hypriot/armhf-busybox
|
||||
kube-scheduler,hypriot/armhf-busybox
|
||||
kube-proxy,hypriot/armhf-busybox
|
||||
kube-apiserver,armel/busybox
|
||||
kube-controller-manager,armel/busybox
|
||||
kube-scheduler,armel/busybox
|
||||
kube-proxy,gcr.io/google_containers/debian-iptables-arm:v2
|
||||
);;
|
||||
"arm64")
|
||||
local targets=(
|
||||
kube-apiserver,aarch64/busybox
|
||||
kube-controller-manager,aarch64/busybox
|
||||
kube-scheduler,aarch64/busybox
|
||||
kube-proxy,aarch64/busybox
|
||||
kube-proxy,gcr.io/google_containers/debian-iptables-arm64:v2
|
||||
);;
|
||||
"ppc64le")
|
||||
local targets=(
|
||||
kube-apiserver,ppc64le/busybox
|
||||
kube-controller-manager,ppc64le/busybox
|
||||
kube-scheduler,ppc64le/busybox
|
||||
kube-proxy,ppc64le/busybox
|
||||
kube-proxy,gcr.io/google_containers/debian-iptables-ppc64le:v2
|
||||
);;
|
||||
esac
|
||||
|
||||
|
|
@ -672,7 +672,12 @@ function kube::release::clean_cruft() {
|
|||
function kube::release::package_hyperkube() {
|
||||
# If we have these variables set then we want to build all docker images.
|
||||
if [[ -n "${KUBE_DOCKER_IMAGE_TAG-}" && -n "${KUBE_DOCKER_REGISTRY-}" ]]; then
|
||||
REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" make -C cluster/images/hyperkube/ build
|
||||
for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
|
||||
|
||||
local arch=${platform##*/}
|
||||
kube::log::status "Building hyperkube image for arch: ${arch}"
|
||||
REGISTRY="${KUBE_DOCKER_REGISTRY}" VERSION="${KUBE_DOCKER_IMAGE_TAG}" ARCH="${arch}" make -C cluster/images/hyperkube/ build
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -738,7 +743,7 @@ function kube::release::package_client_tarballs() {
|
|||
# Package up all of the server binaries
|
||||
function kube::release::package_server_tarballs() {
|
||||
local platform
|
||||
for platform in "${KUBE_SERVER_PLATFORMS[@]}" ; do
|
||||
for platform in "${KUBE_SERVER_PLATFORMS[@]}"; do
|
||||
local platform_tag=${platform/\//-} # Replace a "/" for a "-"
|
||||
local arch=$(basename ${platform})
|
||||
kube::log::status "Building tarball: server $platform_tag"
|
||||
|
|
@ -1538,24 +1543,18 @@ function kube::release::docker::release() {
|
|||
for arch in "${archs[@]}"; do
|
||||
for binary in "${binaries[@]}"; do
|
||||
|
||||
# Temporary fix. hyperkube-arm isn't built in the release process, so we can't push it
|
||||
# This if statement skips the push for hyperkube-arm
|
||||
if [[ ${arch} != "arm" || ${binary} != "hyperkube" ]]; then
|
||||
local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}"
|
||||
kube::log::status "Pushing ${binary} to ${docker_target}"
|
||||
"${docker_push_cmd[@]}" push "${docker_target}"
|
||||
|
||||
# If we have a amd64 docker image. Tag it without -amd64 also and push it for compatibility with earlier versions
|
||||
if [[ ${arch} == "amd64" ]]; then
|
||||
local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}"
|
||||
|
||||
local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}"
|
||||
kube::log::status "Pushing ${binary} to ${docker_target}"
|
||||
"${docker_push_cmd[@]}" push "${docker_target}"
|
||||
"${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null
|
||||
|
||||
# If we have a amd64 docker image. Tag it without -amd64 also and push it for compatibility with earlier versions
|
||||
if [[ ${arch} == "amd64" ]]; then
|
||||
local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}"
|
||||
|
||||
"${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null
|
||||
|
||||
kube::log::status "Pushing ${binary} to ${legacy_docker_target}"
|
||||
"${docker_push_cmd[@]}" push "${legacy_docker_target}"
|
||||
fi
|
||||
kube::log::status "Pushing ${binary} to ${legacy_docker_target}"
|
||||
"${docker_push_cmd[@]}" push "${legacy_docker_target}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
|
|
|||
|
|
@ -12,10 +12,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM debian:jessie
|
||||
FROM BASEIMAGE
|
||||
|
||||
# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
|
||||
# If we're building normally, for amd64, CROSS_BUILD lines are removed
|
||||
CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
|
||||
|
||||
# All apt-get's must be in one run command or the
|
||||
# cleanup has no effect.
|
||||
RUN apt-get update && \
|
||||
apt-get install -y iptables && \
|
||||
ls /var/lib/apt/lists/*debian* | xargs rm
|
||||
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y iptables \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
|
|
|||
|
|
@ -14,13 +14,47 @@
|
|||
|
||||
.PHONY: build push
|
||||
|
||||
IMAGE = debian-iptables
|
||||
TAG = v1
|
||||
REGISTRY?="gcr.io/google_containers"
|
||||
IMAGE=debian-iptables
|
||||
TAG=v2
|
||||
ARCH?=amd64
|
||||
TEMP_DIR:=$(shell mktemp -d)
|
||||
|
||||
ifeq ($(ARCH),amd64)
|
||||
BASEIMAGE?=debian:jessie
|
||||
endif
|
||||
ifeq ($(ARCH),arm)
|
||||
BASEIMAGE?=armel/debian:jessie
|
||||
QEMUARCH=arm
|
||||
endif
|
||||
ifeq ($(ARCH),arm64)
|
||||
BASEIMAGE?=aarch64/debian:jessie
|
||||
QEMUARCH=aarch64
|
||||
endif
|
||||
ifeq ($(ARCH),ppc64le)
|
||||
BASEIMAGE?=ppc64le/debian:jessie
|
||||
QEMUARCH=ppc64le
|
||||
endif
|
||||
|
||||
build:
|
||||
docker build -t gcr.io/google_containers/$(IMAGE):$(TAG) .
|
||||
cp ./* $(TEMP_DIR)
|
||||
cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
|
||||
cd $(TEMP_DIR) && sed -i "s|ARCH|$(QEMUARCH)|g" Dockerfile
|
||||
|
||||
push: build
|
||||
gcloud docker --server=gcr.io push gcr.io/google_containers/$(IMAGE):$(TAG)
|
||||
ifeq ($(ARCH),amd64)
|
||||
# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
|
||||
cd $(TEMP_DIR) && sed -i "/CROSS_BUILD_/d" Dockerfile
|
||||
else
|
||||
# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
|
||||
# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
|
||||
docker run --rm --privileged multiarch/qemu-user-static:register --reset
|
||||
curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-$(QEMUARCH)-static.tar.xz | tar -xJ -C $(TEMP_DIR)
|
||||
cd $(TEMP_DIR) && sed -i "s/CROSS_BUILD_//g" Dockerfile
|
||||
endif
|
||||
|
||||
all: push
|
||||
docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
|
||||
|
||||
push: build
|
||||
gcloud docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG)
|
||||
|
||||
all: push
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
### debian-iptables
|
||||
|
||||
Serves as the base image for `gcr.io/google_containers/kube-proxy-${ARCH}` and multiarch (not `amd64`) `gcr.io/google_containers/flannel-${ARCH}` images.
|
||||
|
||||
This image is compiled for multiple architectures.
|
||||
|
||||
#### How to release
|
||||
|
||||
If you're editing the Dockerfile or some other thing, please bump the `TAG` in the Makefile.
|
||||
|
||||
```console
|
||||
# Build for linux/amd64 (default)
|
||||
$ make push ARCH=amd64
|
||||
# ---> gcr.io/google_containers/debian-iptables-amd64:TAG
|
||||
|
||||
$ make push ARCH=arm
|
||||
# ---> gcr.io/google_containers/debian-iptables-arm:TAG
|
||||
|
||||
$ make push ARCH=arm64
|
||||
# ---> gcr.io/google_containers/debian-iptables-arm64:TAG
|
||||
|
||||
$ make push ARCH=ppc64le
|
||||
# ---> gcr.io/google_containers/debian-iptables-ppc64le:TAG
|
||||
```
|
||||
|
||||
If you don't want to push the images, run `make` or `make build` instead
|
||||
|
||||
|
||||
[]()
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM BASEIMAGE
|
||||
|
||||
COPY flanneld /opt/bin/
|
||||
COPY mk-docker-opts.sh /opt/bin/
|
||||
|
||||
CMD ["/opt/bin/flanneld"]
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
# Copyright 2016 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Build the flannel image
|
||||
#
|
||||
# Usage:
|
||||
# [TAG=0.5.5] [REGISTRY=gcr.io/google_containers] [ARCH=amd64] make build
|
||||
|
||||
TAG?=0.5.5
|
||||
ARCH?=amd64
|
||||
REGISTRY?=gcr.io/google_containers
|
||||
KUBE_CROSS_TAG=v1.4.2-1
|
||||
GOARM=6
|
||||
TEMP_DIR:=$(shell mktemp -d)
|
||||
BASEIMAGE?=gcr.io/google_containers/debian-iptables-${ARCH}:v2
|
||||
|
||||
ifeq ($(ARCH),arm)
|
||||
CC=arm-linux-gnueabi-gcc
|
||||
endif
|
||||
|
||||
build:
|
||||
ifeq ($(ARCH),amd64)
|
||||
# If we should build an amd64 flannel, go with the official one
|
||||
docker pull quay.io/coreos/flannel:$(TAG)
|
||||
|
||||
docker tag -f quay.io/coreos/flannel:$(TAG) $(REGISTRY)/flannel-$(ARCH):$(TAG)
|
||||
else
|
||||
# Copy the content in this dir to the temp dir
|
||||
cp ./* $(TEMP_DIR)
|
||||
|
||||
docker run -it -v $(TEMP_DIR):/flannel/bin gcr.io/google_containers/kube-cross:$(KUBE_CROSS_TAG) /bin/bash -c \
|
||||
"curl -sSL https://github.com/coreos/flannel/archive/v${TAG}.tar.gz | tar -C /flannel -xz --strip-components=1 \
|
||||
&& cd /flannel && GOARM=$(GOARM) GOARCH=$(ARCH) CC=$(CC) CGO_ENABLED=1 ./build"
|
||||
|
||||
# Replace BASEIMAGE with the real base image
|
||||
cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
|
||||
|
||||
# Download mk-docker-opts.sh
|
||||
curl -sSL https://raw.githubusercontent.com/coreos/flannel/v$(TAG)/dist/mk-docker-opts.sh > $(TEMP_DIR)/mk-docker-opts.sh
|
||||
|
||||
# And build the image
|
||||
docker build -t $(REGISTRY)/flannel-$(ARCH):$(TAG) $(TEMP_DIR)
|
||||
endif
|
||||
|
||||
push: build
|
||||
gcloud docker push $(REGISTRY)/flannel-$(ARCH):$(TAG)
|
||||
|
||||
all: build
|
||||
.PHONY: build push
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
### flannel
|
||||
|
||||
This is used mostly for the `docker-multinode` config, but also in other places where flannel runs in a container.
|
||||
|
||||
For `amd64`, this image equals to `quay.io/coreos/flannel` to maintain official support.
|
||||
For other architectures, `flannel` is cross-compiled. The `debian-iptables` image serves as base image.
|
||||
|
||||
#### How to release
|
||||
|
||||
```console
|
||||
# Build for linux/amd64 (default)
|
||||
$ make push ARCH=amd64
|
||||
# ---> gcr.io/google_containers/flannel-amd64:TAG
|
||||
|
||||
$ make push ARCH=arm
|
||||
# ---> gcr.io/google_containers/flannel-arm:TAG
|
||||
```
|
||||
|
||||
If you don't want to push the images, run `make` or `make build` instead
|
||||
|
||||
|
||||
[]()
|
||||
|
|
@ -14,6 +14,10 @@
|
|||
|
||||
FROM BASEIMAGE
|
||||
|
||||
# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
|
||||
# If we're building normally, for amd64, CROSS_BUILD lines are removed
|
||||
CROSS_BUILD_COPY qemu-ARCH-static /usr/bin/
|
||||
|
||||
RUN DEBIAN_FRONTEND=noninteractive apt-get update -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get -yy -q \
|
||||
install \
|
||||
|
|
|
|||
|
|
@ -15,17 +15,19 @@
|
|||
# Build the hyperkube image.
|
||||
#
|
||||
# Usage:
|
||||
# VERSION=v1.1.2 [REGISTRY="gcr.io/google_containers"] make build
|
||||
# VERSION=v1.2.0 [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] make build
|
||||
|
||||
REGISTRY?="gcr.io/google_containers"
|
||||
ARCH=amd64
|
||||
BASEIMAGE=debian:jessie
|
||||
ARCH?=amd64
|
||||
TEMP_DIR:=$(shell mktemp -d)
|
||||
|
||||
## Comment in for arm builds, must be run on an arm machine
|
||||
# ARCH=arm
|
||||
# need to escape '/' for the regexp below
|
||||
# BASEIMAGE=armbuild\\/debian:jessie
|
||||
|
||||
ifeq ($(ARCH),amd64)
|
||||
BASEIMAGE?=debian:jessie
|
||||
endif
|
||||
ifeq ($(ARCH),arm)
|
||||
BASEIMAGE?=armel/debian:jessie
|
||||
endif
|
||||
|
||||
all: build
|
||||
|
||||
|
|
@ -38,18 +40,27 @@ endif
|
|||
cp ../../saltbase/salt/generate-cert/make-ca-cert.sh ${TEMP_DIR}
|
||||
cp ../../../_output/dockerized/bin/linux/${ARCH}/hyperkube ${TEMP_DIR}
|
||||
cd ${TEMP_DIR} && sed -i.back "s|VERSION|${VERSION}|g" master-multi.json master.json kube-proxy.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json
|
||||
cd ${TEMP_DIR} && sed -i.back "s|ARCH|${ARCH}|g" master-multi.json master.json kube-proxy.json etcd.json Dockerfile
|
||||
cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
|
||||
rm ${TEMP_DIR}/*.back
|
||||
docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR}
|
||||
# Backward compatibility. TODO: deprecate this image tag
|
||||
|
||||
ifeq ($(ARCH),amd64)
|
||||
docker tag -f ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${REGISTRY}/hyperkube:${VERSION}
|
||||
# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
|
||||
cd ${TEMP_DIR} && sed -i "/CROSS_BUILD_/d" Dockerfile
|
||||
else
|
||||
# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
|
||||
# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
|
||||
docker run --rm --privileged multiarch/qemu-user-static:register --reset
|
||||
curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/v2.5.0/x86_64_qemu-${ARCH}-static.tar.xz | tar -xJ -C ${TEMP_DIR}
|
||||
cd ${TEMP_DIR} && sed -i "s/CROSS_BUILD_//g" Dockerfile
|
||||
endif
|
||||
|
||||
docker build -t ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${TEMP_DIR}
|
||||
|
||||
push: build
|
||||
gcloud docker push ${REGISTRY}/hyperkube-${ARCH}:${VERSION}
|
||||
ifeq ($(ARCH),amd64)
|
||||
docker tag -f ${REGISTRY}/hyperkube-${ARCH}:${VERSION} ${REGISTRY}/hyperkube:${VERSION}
|
||||
gcloud docker push ${REGISTRY}/hyperkube:${VERSION}
|
||||
endif
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
### hyperkube
|
||||
|
||||
`hyperkube` is an all-in-one binary for the Kubernetes server components
|
||||
Also, it's very easy to run this `hyperkube` setup dockerized.
|
||||
See http://kubernetes.io/docs/getting-started-guides/docker/ for up-to-date commands.
|
||||
|
||||
`hyperkube` is built for multiple architectures and pushed on every release.
|
||||
|
||||
#### How to release by hand
|
||||
|
||||
```console
|
||||
# First, build the
|
||||
$ build/run.sh hack/build-cross.sh
|
||||
|
||||
# Build for linux/amd64 (default)
|
||||
$ make push VERSION={target_version} ARCH=amd64
|
||||
# ---> gcr.io/google_containers/hyperkube-amd64:VERSION
|
||||
# ---> gcr.io/google_containers/hyperkube:VERSION (image with backwards-compatible naming)
|
||||
|
||||
$ make push VERSION={target_version} ARCH=arm
|
||||
# ---> gcr.io/google_containers/hyperkube-arm:VERSION
|
||||
```
|
||||
|
||||
If you don't want to push the images, run `make` or `make build` instead
|
||||
|
||||
|
||||
[]()
|
||||
|
|
@ -7,7 +7,7 @@
|
|||
"containers": [
|
||||
{
|
||||
"name": "etcd",
|
||||
"image": "gcr.io/google_containers/etcd:2.2.1",
|
||||
"image": "gcr.io/google_containers/etcd-ARCH:2.2.1",
|
||||
"command": [
|
||||
"/usr/local/bin/etcd",
|
||||
"--listen-client-urls=http://127.0.0.1:4001",
|
||||
|
|
|
|||
Loading…
Reference in New Issue