From f2959ab2b00f5e20d2ec08db835741f2f4d3fd65 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Thu, 12 Apr 2018 21:37:15 -0400 Subject: [PATCH] Support containerized kubelet in CI * Adjust hyperkube Makefile to allow HYPERKUBE_BIN to be specified from outside * new DOCKER_ROOT variable to specify --root-dir and associated volume for kubelet to know where docker directory lives * Save the docker logs when we kill the kubelet container * Allow customized image for dockerized kubelet and build a Default image if one is not specified We should allow a custom build using script we have: `VERSION="latest" REGISTRY="k8s.gcr.io" hack/dev-push-hyperkube.sh` to be used quickly using DOCKERIZE_KUBELET. To do this we remove the hard coded `k8s.gcr.io/kubelet` image and introduce a new environment variable `KUBELET_IMAGE`. Note that we are switching to hyperkube from kubelet as we have a quick and easy script to build hyperkube image and load it into local docker daemon. This reduces bad hacks like `docker tag k8s.gcr.io/hyperkube-amd64: k8s.gcr.io/kubelet:latest` being used by folks today (see 62057) * Better cope with PID 0 when running "docker inspect" to look for the process id for kubelet container --- cluster/images/hyperkube/Makefile | 4 ++-- hack/local-up-cluster.sh | 36 ++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cluster/images/hyperkube/Makefile b/cluster/images/hyperkube/Makefile index 5fc09b0000..4c8a645d92 100644 --- a/cluster/images/hyperkube/Makefile +++ b/cluster/images/hyperkube/Makefile @@ -20,7 +20,7 @@ REGISTRY?=staging-k8s.gcr.io ARCH?=amd64 OUT_DIR?=_output -HYPERKUBE_BIN?=$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/hyperkube +HYPERKUBE_BIN?=$(shell pwd)/../../../$(OUT_DIR)/dockerized/bin/linux/$(ARCH)/hyperkube BASEIMAGE=k8s.gcr.io/debian-hyperkube-base-$(ARCH):0.10 TEMP_DIR:=$(shell mktemp -d -t hyperkubeXXXXXX) @@ -33,7 +33,7 @@ ifndef VERSION $(error VERSION is undefined) endif cp -r ./* ${TEMP_DIR} - cp ../../../${HYPERKUBE_BIN} ${TEMP_DIR} + cp ${HYPERKUBE_BIN} ${TEMP_DIR} chmod a+rx ${TEMP_DIR}/hyperkube diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index a3d5312d2f..fdab7e76b6 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -22,6 +22,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. DOCKER_OPTS=${DOCKER_OPTS:-""} DOCKER=(docker ${DOCKER_OPTS}) DOCKERIZE_KUBELET=${DOCKERIZE_KUBELET:-""} +DOCKER_ROOT=${DOCKER_ROOT:-""} ALLOW_PRIVILEGED=${ALLOW_PRIVILEGED:-""} DENY_SECURITY_CONTEXT_ADMISSION=${DENY_SECURITY_CONTEXT_ADMISSION:-""} PSP_ADMISSION=${PSP_ADMISSION:-""} @@ -31,6 +32,7 @@ KUBELET_AUTHORIZATION_WEBHOOK=${KUBELET_AUTHORIZATION_WEBHOOK:-""} KUBELET_AUTHENTICATION_WEBHOOK=${KUBELET_AUTHENTICATION_WEBHOOK:-""} POD_MANIFEST_PATH=${POD_MANIFEST_PATH:-"/var/run/kubernetes/static-pods"} KUBELET_FLAGS=${KUBELET_FLAGS:-""} +KUBELET_IMAGE=${KUBELET_IMAGE:-""} # many dev environments run with swap on, so we don't fail in this env FAIL_SWAP_ON=${FAIL_SWAP_ON:-"false"} # Name of the network plugin, eg: "kubenet" @@ -340,6 +342,13 @@ cleanup_dockerized_kubelet() if [[ -e $KUBELET_CIDFILE ]]; then docker kill $(<$KUBELET_CIDFILE) > /dev/null rm -f $KUBELET_CIDFILE + + # Save the docker logs + if [[ -f /var/log/docker.log ]]; then + sudo cp /var/log/docker.log ${LOG_DIR}/docker.log + elif command -v journalctl &>/dev/null; then + journalctl -u docker --no-pager > ${LOG_DIR}/docker.log + fi fi } @@ -778,6 +787,13 @@ function start_kubelet { sudo -E "${GO_OUT}/hyperkube" kubelet "${all_kubelet_flags[@]}" >"${KUBELET_LOG}" 2>&1 & KUBELET_PID=$! else + + # Build the hyperkube container image if necessary + if [[ -z "$KUBELET_IMAGE" && -n "$DOCKERIZE_KUBELET" ]]; then + HYPERKUBE_BIN="${GO_OUT}/hyperkube" REGISTRY="k8s.gcr.io" VERSION="latest" make -C "${KUBE_ROOT}/cluster/images/hyperkube" build + KUBELET_IMAGE="k8s.gcr.io/hyperkube-amd64:latest" + fi + # Docker won't run a container with a cidfile (container id file) # unless that file does not already exist; clean up an existing # dockerized kubelet that might be running. @@ -799,7 +815,7 @@ function start_kubelet { fi all_kubelet_flags+=(--containerized) - docker run --rm --name kubelet \ + all_kubelet_volumes=( --volume=/:/rootfs:ro,rslave \ --volume=/var/run:/var/run:rw \ --volume=/sys:/sys:ro \ @@ -807,19 +823,28 @@ function start_kubelet { --volume=/var/lib/kubelet/:/var/lib/kubelet:rslave \ --volume=/dev:/dev \ --volume=/run/xtables.lock:/run/xtables.lock:rw \ + ) + + if [[ -n "${DOCKER_ROOT}" ]]; then + all_kubelet_flags+=(--root-dir="${DOCKER_ROOT}") + all_kubelet_volumes+=(--volume="${DOCKER_ROOT}:${DOCKER_ROOT}:rslave") + fi + + docker run --rm --name kubelet \ + "${all_kubelet_volumes[@]}" \ ${cred_bind} \ --net=host \ --pid=host \ --privileged=true \ -i \ --cidfile=$KUBELET_CIDFILE \ - k8s.gcr.io/kubelet \ + "${KUBELET_IMAGE}" \ /kubelet "${all_kubelet_flags[@]}" >"${KUBELET_LOG}" 2>&1 & # Get PID of kubelet container. for i in {1..3}; do echo -n "Trying to get PID of kubelet container..." KUBELET_PID=$(docker inspect kubelet -f '{{.State.Pid}}' 2>/dev/null || true) - if [ -n "$KUBELET_PID" ]; then + if [[ -n ${KUBELET_PID} && ${KUBELET_PID} -gt 0 ]]; then echo " ok, $KUBELET_PID." break else @@ -989,9 +1014,14 @@ if [[ "${KUBETEST_IN_DOCKER:-}" == "true" ]]; then ${KUBE_ROOT}/hack/install-etcd.sh export PATH="${KUBE_ROOT}/third_party/etcd:${PATH}" KUBE_FASTBUILD=true make ginkgo cross + apt install -y sudo + # configure shared mounts to prevent failure in DIND scenarios mount --make-rshared / + + # kubekins has a special directory for docker root + DOCKER_ROOT="/docker-graph" fi # validate that etcd is: not running, in path, and has minimum required version.