mirror of https://github.com/k3s-io/k3s
upstream heptio/kube-conformance
Pick up some code from https://github.com/heptio/kube-conformance Fix up build scripts for the new conformance image Fix Header template and Copyright to make verify job go green update README and add execute permissions for script Change-Id: Ib6509acd816cc2fb3a516bfb8e0ff9e32bff8f79pull/58/head
Davanum Srinivas
parent
6b9de2bdc7
commit
6830bad2a6
|
|
@ -29,6 +29,7 @@ readonly RELEASE_TARS="${LOCAL_OUTPUT_ROOT}/release-tars"
|
|||
readonly RELEASE_IMAGES="${LOCAL_OUTPUT_ROOT}/release-images"
|
||||
|
||||
KUBE_BUILD_HYPERKUBE=${KUBE_BUILD_HYPERKUBE:-y}
|
||||
KUBE_BUILD_CONFORMANCE=${KUBE_BUILD_CONFORMANCE:-y}
|
||||
|
||||
# Validate a ci version
|
||||
#
|
||||
|
|
@ -289,15 +290,28 @@ function kube::release::build_hyperkube_image() {
|
|||
if [[ -n "${save_dir}" ]]; then
|
||||
"${DOCKER[@]}" save "${hyperkube_tag}" > "${save_dir}/hyperkube-${arch}.tar"
|
||||
fi
|
||||
if [[ -z "${KUBE_DOCKER_IMAGE_TAG-}" || -z "${KUBE_DOCKER_REGISTRY-}" ]]; then
|
||||
# not a release
|
||||
kube::log::status "Deleting hyperkube image ${hyperkube_tag}"
|
||||
"${DOCKER[@]}" rmi "${hyperkube_tag}" &>/dev/null || true
|
||||
fi
|
||||
kube::log::status "Deleting hyperkube image ${hyperkube_tag}"
|
||||
"${DOCKER[@]}" rmi "${hyperkube_tag}" &>/dev/null || true
|
||||
}
|
||||
|
||||
# This will take binaries that run on master and creates Docker images
|
||||
# that wrap the binary in them. (One docker image per binary)
|
||||
function kube::release::build_conformance_image() {
|
||||
local -r arch="$1"
|
||||
local -r registry="$2"
|
||||
local -r version="$3"
|
||||
local -r save_dir="${4-}"
|
||||
kube::log::status "Building conformance image for arch: ${arch}"
|
||||
ARCH="${arch}" REGISTRY="${registry}" VERSION="${version}" \
|
||||
make -C cluster/images/conformance/ build >/dev/null
|
||||
|
||||
local conformance_tag="${registry}/conformance-${arch}:${version}"
|
||||
if [[ -n "${save_dir}" ]]; then
|
||||
"${DOCKER[@]}" save "${conformance_tag}" > "${save_dir}/conformance-${arch}.tar"
|
||||
fi
|
||||
kube::log::status "Deleting conformance image ${conformance_tag}"
|
||||
"${DOCKER[@]}" rmi "${conformance_tag}" &>/dev/null || true
|
||||
}
|
||||
|
||||
# This builds all the release docker images (One docker image per binary)
|
||||
# Args:
|
||||
# $1 - binary_dir, the directory to save the tared images to.
|
||||
# $2 - arch, architecture for which we are building docker images.
|
||||
|
|
@ -395,6 +409,10 @@ EOF
|
|||
kube::release::build_hyperkube_image "${arch}" "${docker_registry}" \
|
||||
"${docker_tag}" "${images_dir}" &
|
||||
fi
|
||||
if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
|
||||
kube::release::build_conformance_image "${arch}" "${docker_registry}" \
|
||||
"${docker_tag}" "${images_dir}" &
|
||||
fi
|
||||
|
||||
kube::util::wait-for-jobs || { kube::log::error "previous Docker build failed"; return 1; }
|
||||
kube::log::status "Docker builds done"
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ CMD_TARGETS="${KUBE_SERVER_IMAGE_TARGETS[*]}"
|
|||
if [[ "${KUBE_BUILD_HYPERKUBE}" =~ [yY] ]]; then
|
||||
CMD_TARGETS="${CMD_TARGETS} cmd/hyperkube"
|
||||
fi
|
||||
if [[ "${KUBE_BUILD_CONFORMANCE}" =~ [yY] ]]; then
|
||||
CMD_TARGETS="${CMD_TARGETS} ${KUBE_CONFORMANCE_IMAGE_TARGETS[*]}"
|
||||
fi
|
||||
|
||||
kube::build::verify_prereqs
|
||||
kube::build::build_image
|
||||
|
|
|
|||
|
|
@ -391,6 +391,7 @@ define RELEASE_IMAGES_HELP_INFO
|
|||
#
|
||||
# Args:
|
||||
# KUBE_BUILD_HYPERKUBE: Whether to build hyperkube image as well. Set to 'n' to skip.
|
||||
# KUBE_BUILD_CONFORMANCE: Whether to build conformance testing image as well. Set to 'n' to skip.
|
||||
#
|
||||
# Example:
|
||||
# make release-images
|
||||
|
|
@ -432,6 +433,7 @@ define QUICK_RELEASE_IMAGES_HELP_INFO
|
|||
# Args:
|
||||
# KUBE_FASTBUILD: Whether to cross-compile for other architectures. Set to 'false' to do so.
|
||||
# KUBE_BUILD_HYPERKUBE: Whether to build hyperkube image as well. Set to 'n' to skip.
|
||||
# KUBE_BUILD_CONFORMANCE: Whether to build conformance testing image as well. Set to 'n' to skip.
|
||||
#
|
||||
# Example:
|
||||
# make quick-release-images
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ filegroup(
|
|||
":package-srcs",
|
||||
"//cluster/addons:all-srcs",
|
||||
"//cluster/gce:all-srcs",
|
||||
"//cluster/images/conformance:all-srcs",
|
||||
"//cluster/images/etcd-version-monitor:all-srcs",
|
||||
"//cluster/images/etcd/migrate:all-srcs",
|
||||
"//cluster/images/hyperkube:all-srcs",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
load("@io_bazel_rules_docker//container:container.bzl", "container_bundle", "container_image", "container_layer")
|
||||
|
||||
container_layer(
|
||||
name = "cluster-srcs",
|
||||
data_path = "/",
|
||||
directory = "/kubernetes",
|
||||
files = ["//cluster:all-srcs"],
|
||||
)
|
||||
|
||||
container_layer(
|
||||
name = "bins",
|
||||
directory = "/usr/local/bin",
|
||||
files = [
|
||||
"//cmd/kubectl",
|
||||
"//test/e2e:e2e.test",
|
||||
"//vendor/github.com/onsi/ginkgo/ginkgo",
|
||||
],
|
||||
)
|
||||
|
||||
container_image(
|
||||
name = "conformance-internal",
|
||||
base = "@debian-hyperkube-base-amd64//image",
|
||||
cmd = [
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
"/run_e2e.sh",
|
||||
],
|
||||
env = {
|
||||
"E2E_FOCUS": "\[Conformance\]",
|
||||
"E2E_SKIP": "",
|
||||
"E2E_PARALLEL": "1",
|
||||
"E2E_PROVIDER": "local",
|
||||
"RESULTS_DIR": "/tmp/results",
|
||||
"KUBECONFIG": "",
|
||||
},
|
||||
files = [
|
||||
":run_e2e.sh",
|
||||
],
|
||||
layers = [
|
||||
":cluster-srcs",
|
||||
":bins",
|
||||
],
|
||||
stamp = True,
|
||||
workdir = "/usr/local/bin",
|
||||
)
|
||||
|
||||
container_bundle(
|
||||
name = "conformance",
|
||||
images = {"k8s.gcr.io/conformance-amd64:{STABLE_DOCKER_TAG}": "conformance-internal"},
|
||||
stamp = True,
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Copyright 2018 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM BASEIMAGE
|
||||
|
||||
COPY ginkgo /usr/local/bin/
|
||||
COPY e2e.test /usr/local/bin/
|
||||
COPY kubectl /usr/local/bin/
|
||||
COPY run_e2e.sh /run_e2e.sh
|
||||
COPY cluster /kubernetes/cluster
|
||||
WORKDIR /usr/local/bin
|
||||
|
||||
ENV E2E_FOCUS="\[Conformance\]"
|
||||
ENV E2E_SKIP=""
|
||||
ENV E2E_PROVIDER="local"
|
||||
ENV E2E_PARALLEL="1"
|
||||
ENV RESULTS_DIR="/tmp/results"
|
||||
ENV KUBECONFIG=""
|
||||
|
||||
CMD [ "/bin/bash", "-c", "/run_e2e.sh" ]
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Build the conformance image.
|
||||
#
|
||||
# Usage:
|
||||
# [ARCH=amd64] [REGISTRY="staging-k8s.gcr.io"] make (build|push) VERSION={some_released_version_of_kubernetes}
|
||||
|
||||
REGISTRY?=staging-k8s.gcr.io
|
||||
ARCH?=amd64
|
||||
OUT_DIR?=_output
|
||||
GINKGO_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/ginkgo
|
||||
KUBECTL_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/kubectl
|
||||
E2E_TEST_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/e2e.test
|
||||
CLUSTER_DIR?=$(shell pwd)/../../../cluster/
|
||||
|
||||
BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.10.2
|
||||
TEMP_DIR:=$(shell mktemp -d -t conformanceXXXXXX)
|
||||
|
||||
all: build
|
||||
|
||||
build:
|
||||
|
||||
ifndef VERSION
|
||||
$(error VERSION is undefined)
|
||||
endif
|
||||
cp -r ./* ${TEMP_DIR}
|
||||
|
||||
cp ${GINKGO_BIN} ${TEMP_DIR}
|
||||
cp ${KUBECTL_BIN} ${TEMP_DIR}
|
||||
cp ${E2E_TEST_BIN} ${TEMP_DIR}
|
||||
cp -r ${CLUSTER_DIR} ${TEMP_DIR}/cluster
|
||||
|
||||
chmod a+rx ${TEMP_DIR}/ginkgo
|
||||
chmod a+rx ${TEMP_DIR}/kubectl
|
||||
chmod a+rx ${TEMP_DIR}/e2e.test
|
||||
|
||||
cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
|
||||
|
||||
ifneq ($(ARCH),amd64)
|
||||
# Register /usr/bin/qemu-ARCH-static as the handler for non-x86 binaries in the kernel
|
||||
docker run --rm --privileged multiarch/qemu-user-static:register --reset
|
||||
endif
|
||||
docker build --pull -t ${REGISTRY}/conformance-${ARCH}:${VERSION} ${TEMP_DIR}
|
||||
rm -rf "${TEMP_DIR}"
|
||||
|
||||
push: build
|
||||
docker push ${REGISTRY}/conformance-${ARCH}:${VERSION}
|
||||
ifeq ($(ARCH),amd64)
|
||||
docker rmi ${REGISTRY}/conformance:${VERSION} 2>/dev/null || true
|
||||
docker tag ${REGISTRY}/conformance-${ARCH}:${VERSION} ${REGISTRY}/conformance:${VERSION}
|
||||
docker push ${REGISTRY}/conformance:${VERSION}
|
||||
endif
|
||||
|
||||
.PHONY: build push all
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
reviewers:
|
||||
- timothysc
|
||||
- dims
|
||||
- ixdy
|
||||
- spiffxp
|
||||
approvers:
|
||||
- timothysc
|
||||
- dims
|
||||
- ixdy
|
||||
- spiffxp
|
||||
labels:
|
||||
- sig/release
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
### conformance
|
||||
|
||||
`conformance` is a standalone container to launch Kubernetes end-to-end tests, for the purposes of conformance testing.
|
||||
`conformance` is built for multiple architectures and _the image is pushed automatically on every release._
|
||||
|
||||
#### How to release by hand
|
||||
|
||||
```console
|
||||
# First, build the binaries
|
||||
$ build/run.sh make cross
|
||||
|
||||
# Build for linux/amd64 (default)
|
||||
# export REGISTRY=$HOST/$ORG to switch from staging-k8s.gcr.io
|
||||
|
||||
$ make push VERSION={target_version} ARCH=amd64
|
||||
# ---> staging-k8s.gcr.io/conformance-amd64:VERSION
|
||||
# ---> staging-k8s.gcr.io/conformance:VERSION (image with backwards-compatible naming)
|
||||
|
||||
$ make push VERSION={target_version} ARCH=arm
|
||||
# ---> staging-k8s.gcr.io/conformance-arm:VERSION
|
||||
|
||||
$ make push VERSION={target_version} ARCH=arm64
|
||||
# ---> staging-k8s.gcr.io/conformance-arm64:VERSION
|
||||
|
||||
$ make push VERSION={target_version} ARCH=ppc64le
|
||||
# ---> staging-k8s.gcr.io/conformance-ppc64le:VERSION
|
||||
|
||||
$ make push VERSION={target_version} ARCH=s390x
|
||||
# ---> staging-k8s.gcr.io/conformance-s390x:VERSION
|
||||
```
|
||||
|
||||
If you don't want to push the images, run `make` or `make build` instead
|
||||
|
||||
|
||||
#### How to setup RBAC needed
|
||||
|
||||
```
|
||||
kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=default:default
|
||||
```
|
||||
|
||||
#### How to run a single test
|
||||
|
||||
```
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: e2e-producer-consumer-test
|
||||
spec:
|
||||
containers:
|
||||
- name: conformance-container
|
||||
image: gcr.io/heptio-images/kube-conformance:latest
|
||||
image: staging-k8s.gcr.io/conformance-amd64:v1.12.1
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: E2E_FOCUS
|
||||
value: "Pods should be submitted and removed"
|
||||
volumeMounts:
|
||||
- name: output-volume
|
||||
mountPath: /tmp/results
|
||||
volumes:
|
||||
- name: output-volume
|
||||
hostPath:
|
||||
path: /tmp/results
|
||||
restartPolicy: Never
|
||||
```
|
||||
|
||||
|
||||
[]()
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2018 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# Shutdown the tests gracefully then save the results
|
||||
shutdown () {
|
||||
E2E_SUITE_PID=$(pgrep e2e.test)
|
||||
echo "sending TERM to ${E2E_SUITE_PID}"
|
||||
kill -s TERM "${E2E_SUITE_PID}"
|
||||
|
||||
# Kind of a hack to wait for this pid to finish.
|
||||
# Since it's not a child of this shell we cannot use wait.
|
||||
tail --pid ${E2E_SUITE_PID} -f /dev/null
|
||||
saveResults
|
||||
}
|
||||
|
||||
saveResults() {
|
||||
cd "${RESULTS_DIR}" || exit
|
||||
tar -czf e2e.tar.gz ./*
|
||||
# mark the done file as a termination notice.
|
||||
echo -n "${RESULTS_DIR}/e2e.tar.gz" > "${RESULTS_DIR}/done"
|
||||
}
|
||||
|
||||
# We get the TERM from kubernetes and handle it gracefully
|
||||
trap shutdown TERM
|
||||
|
||||
ginkgo_args=(
|
||||
"--focus=${E2E_FOCUS}"
|
||||
"--skip=${E2E_SKIP}"
|
||||
"--noColor=true"
|
||||
)
|
||||
|
||||
case ${E2E_PARALLEL} in
|
||||
'y'|'Y') ginkgo_args+=("--nodes=25") ;;
|
||||
[1-9]|[1-9][0-9]*) ginkgo_args+=("--nodes=${E2E_PARALLEL}") ;;
|
||||
esac
|
||||
|
||||
echo "/usr/local/bin/ginkgo ${ginkgo_args[@]} /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider=\"${E2E_PROVIDER}\" --report-dir=\"${RESULTS_DIR}\" --kubeconfig=\"${KUBECONFIG}\""
|
||||
/usr/local/bin/ginkgo "${ginkgo_args[@]}" /usr/local/bin/e2e.test -- --disable-log-dump --repo-root=/kubernetes --provider="${E2E_PROVIDER}" --report-dir="${RESULTS_DIR}" --kubeconfig="${KUBECONFIG}" | tee ${RESULTS_DIR}/e2e.log &
|
||||
# $! is the pid of tee, not ginkgo
|
||||
wait $(pgrep ginkgo)
|
||||
saveResults
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2014 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script builds some binaries and then the conformance image.
|
||||
# REGISTRY and VERSION must be set.
|
||||
# Example usage:
|
||||
# $ export REGISTRY=gcr.io/someone
|
||||
# $ export VERSION=v1.4.0-testfix
|
||||
# ./hack/dev-push-conformance.sh
|
||||
# That will build and push gcr.io/someone/conformance-amd64:v1.4.0-testfix
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
KUBE_ROOT="$(dirname "${BASH_SOURCE}")/.."
|
||||
source "${KUBE_ROOT}/build/common.sh"
|
||||
|
||||
if [[ -z "${REGISTRY:-}" ]]; then
|
||||
echo "REGISTRY must be set"
|
||||
exit -1
|
||||
fi
|
||||
if [[ -z "${VERSION:-}" ]]; then
|
||||
echo "VERSION must be set"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
IMAGE="${REGISTRY}/conformance-amd64:${VERSION}"
|
||||
|
||||
kube::build::verify_prereqs
|
||||
kube::build::build_image
|
||||
kube::build::run_build_command make WHAT="vendor/github.com/onsi/ginkgo/ginkgo test/e2e/e2e.test cmd/kubectl"
|
||||
kube::build::copy_output
|
||||
|
||||
make -C "${KUBE_ROOT}/cluster/images/conformance" build
|
||||
docker push "${IMAGE}"
|
||||
|
|
@ -57,6 +57,20 @@ IFS=" " read -ra KUBE_SERVER_IMAGE_TARGETS <<< "$(kube::golang::server_image_tar
|
|||
readonly KUBE_SERVER_IMAGE_TARGETS
|
||||
readonly KUBE_SERVER_IMAGE_BINARIES=("${KUBE_SERVER_IMAGE_TARGETS[@]##*/}")
|
||||
|
||||
# The set of conformance targets we build docker image for
|
||||
kube::golang::conformance_image_targets() {
|
||||
# NOTE: this contains cmd targets for kube::release::build_conformance_image
|
||||
local targets=(
|
||||
vendor/github.com/onsi/ginkgo/ginkgo
|
||||
test/e2e/e2e.test
|
||||
cmd/kubectl
|
||||
)
|
||||
echo "${targets[@]}"
|
||||
}
|
||||
|
||||
IFS=" " read -ra KUBE_CONFORMANCE_IMAGE_TARGETS <<< "$(kube::golang::conformance_image_targets)"
|
||||
readonly KUBE_CONFORMANCE_IMAGE_TARGETS
|
||||
|
||||
# The set of server targets that we are only building for Kubernetes nodes
|
||||
# If you update this list, please also update build/BUILD.
|
||||
kube::golang::node_targets() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue