From 4ace8280a0bbd50b93e7514ea662de7719e5dc8f Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Tue, 17 Nov 2015 18:13:24 -0800 Subject: [PATCH] add an option to push/pull component images from a registry using kube-up --- cluster/common.sh | 45 ++++++++++++++++++- cluster/gce/configure-vm.sh | 9 ++-- cluster/gce/util.sh | 2 + cluster/kube-up.sh | 5 +++ cluster/saltbase/install.sh | 7 +++ .../kube-apiserver/kube-apiserver.manifest | 2 +- .../kube-controller-manager.manifest | 2 +- .../salt/kube-proxy/kube-proxy.manifest | 2 +- .../kube-scheduler/kube-scheduler.manifest | 2 +- 9 files changed, 66 insertions(+), 10 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 7ed0b87622..444ae5696a 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -344,8 +344,8 @@ function find-release-version() { if [[ -f "${KUBE_ROOT}/version" ]]; then KUBE_GIT_VERSION="$(cat ${KUBE_ROOT}/version)" fi - if [[ -f "${KUBE_ROOT}/_output/full/kubernetes/version" ]]; then - KUBE_GIT_VERSION="$(cat ${KUBE_ROOT}/_output/full/kubernetes/version)" + if [[ -f "${KUBE_ROOT}/_output/release-stage/full/kubernetes/version" ]]; then + KUBE_GIT_VERSION="$(cat ${KUBE_ROOT}/_output/release-stage/full/kubernetes/version)" fi if [[ -z "${KUBE_GIT_VERSION}" ]]; then @@ -353,3 +353,44 @@ function find-release-version() { exit 1 fi } + +function stage-images() { + find-release-version + find-release-tars + + KUBE_IMAGE_TAG="$(echo """${KUBE_GIT_VERSION}""" | sed 's/+/-/g')" + + local docker_wrapped_binaries=( + "kube-apiserver" + "kube-controller-manager" + "kube-scheduler" + "kube-proxy" + ) + + local docker_cmd=("docker") + + if [[ "${KUBE_DOCKER_REGISTRY}" == "gcr.io/"* ]]; then + local docker_push_cmd=("gcloud" "docker") + fi + + local temp_dir="$(mktemp -d -t 'kube-server-XXXX')" + + tar xzfv "${SERVER_BINARY_TAR}" -C "${temp_dir}" &> /dev/null + + for binary in "${docker_wrapped_binaries[@]}"; do + local docker_tag="$(cat ${temp_dir}/kubernetes/server/bin/${binary}.docker_tag)" + ( + "${docker_cmd[@]}" load -i "${temp_dir}/kubernetes/server/bin/${binary}.tar" + "${docker_cmd[@]}" tag -f "gcr.io/google_containers/${binary}:${docker_tag}" "${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_IMAGE_TAG}" + "${docker_push_cmd[@]}" push "${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_IMAGE_TAG}" + ) &> "${temp_dir}/${binary}-push.log" & + done + + kube::util::wait-for-jobs || { + kube::log::error "unable to push images. see ${temp_dir}/*.log for more info." + return 1 + } + + rm -rf "${temp_dir}" + return 0 +} diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 5ee5a5d673..019629598a 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -77,12 +77,13 @@ function set-kube-env() { done # kube-env has all the environment variables we care about, in a flat yaml format - eval $(python -c ''' + eval "$(python -c ' import pipes,sys,yaml for k,v in yaml.load(sys.stdin).iteritems(): - print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v))) -''' < "${kube_env_yaml}") + print """readonly {var}={value}""".format(var = k, value = pipes.quote(str(v))) + print """export {var}""".format(var = k) + ' < """${kube_env_yaml}""")" } function remove-docker-artifacts() { @@ -524,7 +525,7 @@ function download-release() { done echo "Running release install script" - sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}" + kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}" } function fix-apt-sources() { diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 42b356ce98..306c1b8778 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1302,6 +1302,8 @@ OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-}) OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-}) OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-}) E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-}) +KUBE_IMAGE_TAG: $(yaml-quote ${KUBE_IMAGE_TAG:-}) +KUBE_DOCKER_REGISTRY: $(yaml-quote ${KUBE_DOCKER_REGISTRY:-}) EOF if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then cat >>$file <&2 echo "... calling verify-prereqs" >&2 verify-prereqs +if [[ "${KUBE_STAGE_IMAGES:-}" == "true" ]]; then + echo "... staging images" >&2 + stage-images +fi + echo "... calling kube-up" >&2 kube-up diff --git a/cluster/saltbase/install.sh b/cluster/saltbase/install.sh index 21a7d127ee..be755b63e9 100755 --- a/cluster/saltbase/install.sh +++ b/cluster/saltbase/install.sh @@ -73,9 +73,16 @@ kube_bin_dir="/srv/salt-new/salt/kube-bins"; docker_images_sls_file="/srv/salt-new/pillar/docker-images.sls"; for docker_file in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do docker_tag=$(cat ${kube_bin_dir}/${docker_file}.docker_tag); + if [[ ! -z "${KUBE_IMAGE_TAG}" ]]; then + docker_tag="${KUBE_IMAGE_TAG}" + fi sed -i "s/#${docker_file}_docker_tag_value#/${docker_tag}/" "${docker_images_sls_file}"; done +cat <>"${docker_images_sls_file}" +kube_docker_registry: '$(echo ${KUBE_DOCKER_REGISTRY:-gcr.io/google_containers})' +EOF + echo "+++ Swapping in new configs" for dir in "${SALTDIRS[@]}"; do diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 1191e0766d..6bb4588f24 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -111,7 +111,7 @@ "containers":[ { "name": "kube-apiserver", - "image": "gcr.io/google_containers/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}", + "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}", "resources": { "limits": { "cpu": "250m" diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index bd6e454edd..ce5649cd7b 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -65,7 +65,7 @@ "containers":[ { "name": "kube-controller-manager", - "image": "gcr.io/google_containers/kube-controller-manager:{{pillar['kube-controller-manager_docker_tag']}}", + "image": "{{pillar['kube_docker_registry']}}/kube-controller-manager:{{pillar['kube-controller-manager_docker_tag']}}", "resources": { "limits": { "cpu": "200m" diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest index 751d077a51..e039d32273 100644 --- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest +++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest @@ -25,7 +25,7 @@ spec: hostNetwork: true containers: - name: kube-proxy - image: gcr.io/google_containers/kube-proxy:{{pillar['kube-proxy_docker_tag']}} + image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}} command: - /bin/sh - -c diff --git a/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest b/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest index 259b45540c..799fbe4717 100644 --- a/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest +++ b/cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest @@ -21,7 +21,7 @@ "containers":[ { "name": "kube-scheduler", - "image": "gcr.io/google_containers/kube-scheduler:{{pillar['kube-scheduler_docker_tag']}}", + "image": "{{pillar['kube_docker_registry']}}/kube-scheduler:{{pillar['kube-scheduler_docker_tag']}}", "resources": { "limits": { "cpu": "100m"