Merge pull request #17580 from mikedanese/stage-image

add an option to push/pull component images from a registry using kube-up

cluster/common.sh

@@ -344,8 +344,8 @@ function find-release-version() {
   if [[ -f "${KUBE_ROOT}/version" ]]; then
     KUBE_GIT_VERSION="$(cat ${KUBE_ROOT}/version)"
   fi
-  if [[ -f "${KUBE_ROOT}/_output/full/kubernetes/version" ]]; then
-    KUBE_GIT_VERSION="$(cat ${KUBE_ROOT}/_output/full/kubernetes/version)"
+  if [[ -f "${KUBE_ROOT}/_output/release-stage/full/kubernetes/version" ]]; then
+    KUBE_GIT_VERSION="$(cat ${KUBE_ROOT}/_output/release-stage/full/kubernetes/version)"
   fi
 
   if [[ -z "${KUBE_GIT_VERSION}" ]]; then
@@ -353,3 +353,44 @@ function find-release-version() {
     exit 1
   fi
 }
+
+function stage-images() {
+  find-release-version
+  find-release-tars
+
+  KUBE_IMAGE_TAG="$(echo """${KUBE_GIT_VERSION}""" | sed 's/+/-/g')"
+
+  local docker_wrapped_binaries=(
+    "kube-apiserver"
+    "kube-controller-manager"
+    "kube-scheduler"
+    "kube-proxy"
+  )
+
+  local docker_cmd=("docker")
+
+  if [[ "${KUBE_DOCKER_REGISTRY}" == "gcr.io/"* ]]; then
+    local docker_push_cmd=("gcloud" "docker")
+  fi
+
+  local temp_dir="$(mktemp -d -t 'kube-server-XXXX')"
+
+  tar xzfv "${SERVER_BINARY_TAR}" -C "${temp_dir}" &> /dev/null
+
+  for binary in "${docker_wrapped_binaries[@]}"; do
+    local docker_tag="$(cat ${temp_dir}/kubernetes/server/bin/${binary}.docker_tag)"
+    (
+      "${docker_cmd[@]}" load -i "${temp_dir}/kubernetes/server/bin/${binary}.tar"
+      "${docker_cmd[@]}" tag -f "gcr.io/google_containers/${binary}:${docker_tag}" "${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_IMAGE_TAG}"
+      "${docker_push_cmd[@]}" push "${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_IMAGE_TAG}"
+    ) &> "${temp_dir}/${binary}-push.log" &
+  done
+
+  kube::util::wait-for-jobs || {
+    kube::log::error "unable to push images. see ${temp_dir}/*.log for more info."
+    return 1
+  }
+
+  rm -rf "${temp_dir}"
+  return 0
+}

cluster/gce/configure-vm.sh

@@ -77,12 +77,13 @@ function set-kube-env() {
   done
 
   # kube-env has all the environment variables we care about, in a flat yaml format
-  eval $(python -c '''
+  eval "$(python -c '
 import pipes,sys,yaml
 
 for k,v in yaml.load(sys.stdin).iteritems():
-  print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))
-''' < "${kube_env_yaml}")
+  print """readonly {var}={value}""".format(var = k, value = pipes.quote(str(v)))
+  print """export {var}""".format(var = k)
+  ' < """${kube_env_yaml}""")"
 }
 
 function remove-docker-artifacts() {
@@ -528,7 +529,7 @@ function download-release() {
   done
 
   echo "Running release install script"
-  sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"
+  kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"
 }
 
 function fix-apt-sources() {

cluster/gce/util.sh

@@ -1340,6 +1340,8 @@ OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})
 OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-})
 OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})
 E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-})
+KUBE_IMAGE_TAG: $(yaml-quote ${KUBE_IMAGE_TAG:-})
+KUBE_DOCKER_REGISTRY: $(yaml-quote ${KUBE_DOCKER_REGISTRY:-})
 EOF
   if [ -n "${KUBELET_PORT:-}" ]; then
     cat >>$file <<EOF

cluster/kube-up.sh

@@ -38,6 +38,11 @@ echo "... Starting cluster using provider: $KUBERNETES_PROVIDER" >&2
 echo "... calling verify-prereqs" >&2
 verify-prereqs
 
+if [[ "${KUBE_STAGE_IMAGES:-}" == "true" ]]; then
+  echo "... staging images" >&2
+  stage-images
+fi
+
 echo "... calling kube-up" >&2
 kube-up
 

cluster/saltbase/install.sh

@@ -73,9 +73,16 @@ kube_bin_dir="/srv/salt-new/salt/kube-bins";
 docker_images_sls_file="/srv/salt-new/pillar/docker-images.sls";
 for docker_file in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do
   docker_tag=$(cat ${kube_bin_dir}/${docker_file}.docker_tag);
+  if [[ ! -z "${KUBE_IMAGE_TAG}" ]]; then
+    docker_tag="${KUBE_IMAGE_TAG}"
+  fi
   sed -i "s/#${docker_file}_docker_tag_value#/${docker_tag}/" "${docker_images_sls_file}";
 done
 
+cat <<EOF >>"${docker_images_sls_file}"
+kube_docker_registry: '$(echo ${KUBE_DOCKER_REGISTRY:-gcr.io/google_containers})'
+EOF
+
 
 echo "+++ Swapping in new configs"
 for dir in "${SALTDIRS[@]}"; do

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -111,7 +111,7 @@
 "containers":[
     {
     "name": "kube-apiserver",
-    "image": "gcr.io/google_containers/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
+    "image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
     "resources": {
       "limits": {
         "cpu": "250m"

cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest

@@ -65,7 +65,7 @@
 "containers":[
     {
     "name": "kube-controller-manager",
-    "image": "gcr.io/google_containers/kube-controller-manager:{{pillar['kube-controller-manager_docker_tag']}}",
+    "image": "{{pillar['kube_docker_registry']}}/kube-controller-manager:{{pillar['kube-controller-manager_docker_tag']}}",
     "resources": {
       "limits": {
         "cpu": "200m"

cluster/saltbase/salt/kube-proxy/kube-proxy.manifest

@@ -25,7 +25,7 @@ spec:
   hostNetwork: true
   containers:
   - name: kube-proxy
-    image: gcr.io/google_containers/kube-proxy:{{pillar['kube-proxy_docker_tag']}}
+    image: {{pillar['kube_docker_registry']}}/kube-proxy:{{pillar['kube-proxy_docker_tag']}}
     command:
     - /bin/sh
     - -c

cluster/saltbase/salt/kube-scheduler/kube-scheduler.manifest

@@ -21,7 +21,7 @@
 "containers":[
     {
     "name": "kube-scheduler",
-    "image": "gcr.io/google_containers/kube-scheduler:{{pillar['kube-scheduler_docker_tag']}}",
+    "image": "{{pillar['kube_docker_registry']}}/kube-scheduler:{{pillar['kube-scheduler_docker_tag']}}",
     "resources": {
       "limits": {
         "cpu": "100m"