Merge pull request #69820 from ixdy/fork-multiarch-register

Use a forked copy of multiarch/qemu-user-static scripts instead of a privileged container
pull/58/head
k8s-ci-robot 2018-11-07 19:10:15 -08:00 committed by GitHub
commit 3bf41a210b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 435 additions and 4 deletions

View File

@ -27,6 +27,8 @@ ALL_ARCH = amd64 arm arm64 ppc64le s390x
TEMP_DIR:=$(shell mktemp -d)
QEMUVERSION=v2.9.1
SUDO=$(if $(filter 0,$(shell id -u)),,sudo)
# This option is for running docker manifest command
export DOCKER_CLI_EXPERIMENTAL := enabled
@ -80,7 +82,7 @@ ifeq ($(ARCH),amd64)
else
# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
docker run --rm --privileged multiarch/qemu-user-static:register --reset
$(SUDO) ../../third_party/multiarch/qemu-user-static/register/register.sh --reset
curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMUVERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(TEMP_DIR)
# Ensure we don't get surprised by umask settings
chmod 0755 $(TEMP_DIR)/qemu-$(QEMUARCH)-static

View File

@ -33,6 +33,8 @@ CNI_TARBALL=cni-plugins-$(ARCH)-$(CNI_VERSION).tgz
# This option is for running docker manifest command
export DOCKER_CLI_EXPERIMENTAL := enabled
SUDO=$(if $(filter 0,$(shell id -u)),,sudo)
.PHONY: all build push clean all-build all-push-images all-push push-manifest
all: all-push
@ -74,7 +76,7 @@ endif
ifneq ($(ARCH),amd64)
# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
docker run --rm --privileged multiarch/qemu-user-static:register --reset
$(SUDO) ../../third_party/multiarch/qemu-user-static/register/register.sh --reset
endif
docker build --pull -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)
rm -rf $(TEMP_DIR)

View File

@ -26,13 +26,15 @@ BASEIMAGE?=k8s.gcr.io/debian-base-$(ARCH):0.4.0
# This option is for running docker manifest command
export DOCKER_CLI_EXPERIMENTAL := enabled
SUDO=$(if $(filter 0,$(shell id -u)),,sudo)
build:
cp ./* $(TEMP_DIR)
cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile
ifneq ($(ARCH),amd64)
# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
docker run --rm --privileged multiarch/qemu-user-static:register --reset
$(SUDO) ../../third_party/multiarch/qemu-user-static/register/register.sh --reset
endif
docker build --pull -t $(IMAGE)-$(ARCH):$(TAG) $(TEMP_DIR)

View File

@ -83,7 +83,12 @@ build() {
else
${SED} -i "s|QEMUARCH|${QEMUARCHS[$arch]}|g" Dockerfile
# Register qemu-*-static for all supported processors except the current one
docker run --rm --privileged multiarch/qemu-user-static:register --reset
echo "Registering qemu-*-static binaries in the kernel"
local sudo=""
if [[ $(id -u) != 0 ]]; then
sudo=sudo
fi
"${sudo}" "${KUBE_ROOT}/third_party/multiarch/qemu-user-static/register/register.sh" --reset
curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/${QEMUVERSION}/x86_64_qemu-${QEMUARCHS[$arch]}-static.tar.gz | tar -xz -C ${temp_dir}
# Ensure we don't get surprised by umask settings
chmod 0755 "${temp_dir}/qemu-${QEMUARCHS[$arch]}-static"

View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2015-2016 Manfred Touron
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -0,0 +1 @@
Files copied from https://github.com/multiarch/qemu-user-static at commit 22b0013668d2aed4a2cfd21650e85c664b1f21c6.

View File

@ -0,0 +1,367 @@
#!/bin/sh
# enable automatic i386/ARM/M68K/MIPS/SPARC/PPC/s390/HPPA/Xtensa/microblaze
# program execution by the kernel
#
# downloaded from https://raw.githubusercontent.com/qemu/qemu/master/scripts/qemu-binfmt-conf.sh
qemu_target_list="i386 i486 alpha arm armeb sparc32plus ppc ppc64 ppc64le m68k \
mips mipsel mipsn32 mipsn32el mips64 mips64el \
sh4 sh4eb s390x aarch64 aarch64_be hppa riscv32 riscv64 xtensa xtensaeb microblaze microblazeel"
i386_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00'
i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
i386_family=i386
i486_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00'
i486_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
i486_family=i386
alpha_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90'
alpha_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
alpha_family=alpha
arm_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00'
arm_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
arm_family=arm
armeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28'
armeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
armeb_family=armeb
sparc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02'
sparc_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sparc_family=sparc
sparc32plus_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x12'
sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sparc32plus_family=sparc
ppc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14'
ppc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
ppc_family=ppc
ppc64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15'
ppc64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
ppc64_family=ppc
ppc64le_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00'
ppc64le_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00'
ppc64le_family=ppcle
m68k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04'
m68k_mask='\xff\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
m68k_family=m68k
# FIXME: We could use the other endianness on a MIPS host.
mips_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
mips_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
mips_family=mips
mipsel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
mipsel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
mipsel_family=mips
mipsn32_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
mipsn32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
mipsn32_family=mips
mipsn32el_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
mipsn32el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
mipsn32el_family=mips
mips64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
mips64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
mips64_family=mips
mips64el_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
mips64el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
mips64el_family=mips
sh4_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00'
sh4_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
sh4_family=sh4
sh4eb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a'
sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
sh4eb_family=sh4
s390x_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16'
s390x_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
s390x_family=s390x
aarch64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00'
aarch64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
aarch64_family=arm
aarch64_be_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7'
aarch64_be_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
aarch64_be_family=armeb
hppa_magic='\x7f\x45\x4c\x46\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x0f'
hppa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
hppa_family=hppa
riscv32_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00'
riscv32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
riscv32_family=riscv
riscv64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xf3\x00'
riscv64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
riscv64_family=riscv
xtensa_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e\x00'
xtensa_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
xtensa_family=xtensa
xtensaeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x5e'
xtensaeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
xtensaeb_family=xtensaeb
microblaze_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xba\xab'
microblaze_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
microblaze_family=microblaze
microblazeel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xab\xba'
microblazeel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
microblazeel_family=microblazeel
qemu_get_family() {
cpu=${HOST_ARCH:-$(uname -m)}
case "$cpu" in
amd64|i386|i486|i586|i686|i86pc|BePC|x86_64)
echo "i386"
;;
mips*)
echo "mips"
;;
"Power Macintosh"|ppc64|powerpc|ppc)
echo "ppc"
;;
ppc64el|ppc64le)
echo "ppcle"
;;
arm|armel|armhf|arm64|armv[4-9]*l|aarch64)
echo "arm"
;;
armeb|armv[4-9]*b|aarch64_be)
echo "armeb"
;;
sparc*)
echo "sparc"
;;
riscv*)
echo "riscv"
;;
*)
echo "$cpu"
;;
esac
}
usage() {
cat <<EOF
Usage: qemu-binfmt-conf.sh [--qemu-path PATH][--debian][--systemd CPU]
[--help][--credential yes|no][--exportdir PATH]
Configure binfmt_misc to use qemu interpreter
--help: display this usage
--qemu-path: set path to qemu interpreter ($QEMU_PATH)
--debian: don't write into /proc,
instead generate update-binfmts templates
--systemd: don't write into /proc,
instead generate file for systemd-binfmt.service
for the given CPU. If CPU is "ALL", generate a
file for all known cpus
--exportdir: define where to write configuration files
(default: $SYSTEMDDIR or $DEBIANDIR)
--credential: if yes, credential and security tokens are
calculated according to the binary to interpret
To import templates with update-binfmts, use :
sudo update-binfmts --importdir ${EXPORTDIR:-$DEBIANDIR} --import qemu-CPU
To remove interpreter, use :
sudo update-binfmts --package qemu-CPU --remove qemu-CPU $QEMU_PATH
With systemd, binfmt files are loaded by systemd-binfmt.service
The environment variable HOST_ARCH allows to override 'uname' to generate
configuration files for a different architecture than the current one.
where CPU is one of:
$qemu_target_list
EOF
}
qemu_check_access() {
if [ ! -w "$1" ] ; then
echo "ERROR: cannot write to $1" 1>&2
exit 1
fi
}
qemu_check_bintfmt_misc() {
# load the binfmt_misc module
if [ ! -d /proc/sys/fs/binfmt_misc ]; then
if ! /sbin/modprobe binfmt_misc ; then
exit 1
fi
fi
if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then
exit 1
fi
fi
qemu_check_access /proc/sys/fs/binfmt_misc/register
}
installed_dpkg() {
dpkg --status "$1" > /dev/null 2>&1
}
qemu_check_debian() {
if [ ! -e /etc/debian_version ] ; then
echo "WARNING: your system is not a Debian based distro" 1>&2
elif ! installed_dpkg binfmt-support ; then
echo "WARNING: package binfmt-support is needed" 1>&2
fi
qemu_check_access "$EXPORTDIR"
}
qemu_check_systemd() {
if ! systemctl -q is-enabled systemd-binfmt.service ; then
echo "WARNING: systemd-binfmt.service is missing or disabled" 1>&2
fi
qemu_check_access "$EXPORTDIR"
}
qemu_generate_register() {
echo ":qemu-$cpu:M::$magic:$mask:$qemu:$FLAGS"
}
qemu_register_interpreter() {
echo "Setting $qemu as binfmt interpreter for $cpu"
qemu_generate_register > /proc/sys/fs/binfmt_misc/register
}
qemu_generate_systemd() {
echo "Setting $qemu as binfmt interpreter for $cpu for systemd-binfmt.service"
qemu_generate_register > "$EXPORTDIR/qemu-$cpu.conf"
}
qemu_generate_debian() {
cat > "$EXPORTDIR/qemu-$cpu" <<EOF
package qemu-$cpu
interpreter $qemu
magic $magic
mask $mask
EOF
if [ "$FLAGS" = "OC" ] ; then
echo "credentials yes" >> "$EXPORTDIR/qemu-$cpu"
fi
}
qemu_set_binfmts() {
# probe cpu type
host_family=$(qemu_get_family)
# register the interpreter for each cpu except for the native one
for cpu in ${qemu_target_list} ; do
magic=$(eval echo \$${cpu}_magic)
mask=$(eval echo \$${cpu}_mask)
family=$(eval echo \$${cpu}_family)
if [ "$magic" = "" ] || [ "$mask" = "" ] || [ "$family" = "" ] ; then
echo "INTERNAL ERROR: unknown cpu $cpu" 1>&2
continue
fi
qemu="$QEMU_PATH/qemu-$cpu-static"
if [ "$cpu" = "i486" ] ; then
qemu="$QEMU_PATH/qemu-i386-static"
fi
if [ "$host_family" != "$family" ] ; then
$BINFMT_SET
fi
done
}
CHECK=qemu_check_bintfmt_misc
BINFMT_SET=qemu_register_interpreter
SYSTEMDDIR="/etc/binfmt.d"
DEBIANDIR="/usr/share/binfmts"
QEMU_PATH=/usr/local/bin
FLAGS=""
options=$(getopt -o ds:Q:e:hc: -l debian,systemd:,qemu-path:,exportdir:,help,credential: -- "$@")
eval set -- "$options"
while true ; do
case "$1" in
-d|--debian)
CHECK=qemu_check_debian
BINFMT_SET=qemu_generate_debian
EXPORTDIR=${EXPORTDIR:-$DEBIANDIR}
;;
-s|--systemd)
CHECK=qemu_check_systemd
BINFMT_SET=qemu_generate_systemd
EXPORTDIR=${EXPORTDIR:-$SYSTEMDDIR}
shift
# check given cpu is in the supported CPU list
if [ "$1" != "ALL" ] ; then
for cpu in ${qemu_target_list} ; do
if [ "$cpu" = "$1" ] ; then
break
fi
done
if [ "$cpu" = "$1" ] ; then
qemu_target_list="$1"
else
echo "ERROR: unknown CPU \"$1\"" 1>&2
usage
exit 1
fi
fi
;;
-Q|--qemu-path)
shift
QEMU_PATH="$1"
;;
-e|--exportdir)
shift
EXPORTDIR="$1"
;;
-h|--help)
usage
exit 1
;;
-c|--credential)
shift
if [ "$1" = "yes" ] ; then
FLAGS="OC"
else
FLAGS=""
fi
;;
*)
break
;;
esac
shift
done
$CHECK
qemu_set_binfmts

View File

@ -0,0 +1,31 @@
#!/usr/bin/env bash
QEMU_BIN_DIR=${QEMU_BIN_DIR:-/usr/bin}
if [ ! -d /proc/sys/fs/binfmt_misc ]; then
echo "No binfmt support in the kernel."
echo " Try: '/sbin/modprobe binfmt_misc' from the host"
exit 1
fi
if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
fi
entries="aarch64 aarch64_be alpha arm armeb hppa m68k microblaze microblazeel mips mips64 mips64el mipsel mipsn32 mipsn32el ppc ppc64 ppc64le riscv32 riscv64 s390x sh4 sh4eb sparc sparc32plus sparc64 xtensa xtensaeb"
if [ "${1}" = "--reset" ]; then
shift
(
cd /proc/sys/fs/binfmt_misc
for file in $entries; do
if [ -f qemu-${file} ]; then
echo -1 > qemu-${file}
fi
done
)
fi
exec $(dirname "${BASH_SOURCE}")/qemu-binfmt-conf.sh --qemu-path="${QEMU_BIN_DIR}" $@