Merge pull request #12833 from uluyol/insecure-reg

Launch a cluster-local registry.

build/common.sh

@@ -100,6 +100,12 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=(
   kube-scheduler
 )
 
+# The set of addons images that should be prepopulated
+readonly KUBE_ADDON_PATHS=(
+  gcr.io/google_containers/pause:0.8.0
+  gcr.io/google_containers/kube-registry-proxy:0.3
+)
+
 # ---------------------------------------------------------------------------
 # Basic setup functions
 
@@ -602,6 +608,7 @@ function kube::release::package_server_tarballs() {
     local release_stage="${RELEASE_STAGE}/server/${platform_tag}/kubernetes"
     rm -rf "${release_stage}"
     mkdir -p "${release_stage}/server/bin"
+    mkdir -p "${release_stage}/addons"
 
     # This fancy expression will expand to prepend a path
     # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
@@ -610,6 +617,7 @@ function kube::release::package_server_tarballs() {
       "${release_stage}/server/bin/"
     
     kube::release::create_docker_images_for_server "${release_stage}/server/bin";
+    kube::release::write_addon_docker_images_for_server "${release_stage}/addons"
 
     # Include the client binaries here too as they are useful debugging tools.
     local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
@@ -681,6 +689,27 @@ function kube::release::create_docker_images_for_server() {
   )
 }
 
+# This will pull and save docker images for addons which need to placed
+# on the nodes directly.
+function kube::release::write_addon_docker_images_for_server() {
+  # Create a sub-shell so that we don't pollute the outer environment
+  (
+    local addon_path
+    for addon_path in "${KUBE_ADDON_PATHS[@]}"; do
+      (
+        kube::log::status "Pulling and writing Docker image for addon: ${addon_path}"
+
+        local dest_name="${addon_path//\//\~}"
+        docker pull "${addon_path}"
+        docker save "${addon_path}" > "${1}/${dest_name}.tar"
+      ) &
+    done
+
+    kube::util::wait-for-jobs || { kube::log::error "unable to pull or write addon image"; return 1; }
+    kube::log::status "Addon images done"
+  )
+}
+
 # Package up the salt configuration tree.  This is an optional helper to getting
 # a cluster up and running.
 function kube::release::package_salt_tarball() {

cluster/addons/registry/images/Dockerfile

@@ -0,0 +1,12 @@
+FROM haproxy:1.5
+MAINTAINER Muhammed Uluyol <uluyol@google.com>
+
+RUN apt-get update && apt-get install -y dnsutils
+
+ADD proxy.conf.insecure.in /proxy.conf.in
+ADD run_proxy.sh /usr/bin/run_proxy
+
+RUN chown root:users /usr/bin/run_proxy
+RUN chmod 755 /usr/bin/run_proxy
+
+CMD ["/usr/bin/run_proxy"]

cluster/addons/registry/images/Makefile

@@ -0,0 +1,10 @@
+.PHONY: build push vet test clean
+
+TAG = 0.3
+REPO = gcr.io/google_containers/kube-registry-proxy
+
+build:
+	docker build -t $(REPO):$(TAG) .
+
+push:
+	gcloud docker push $(REPO):$(TAG)

cluster/addons/registry/images/proxy.conf.in

@@ -0,0 +1,17 @@
+global
+  maxconn 1024
+
+defaults
+  mode http
+  retries 3
+  option redispatch
+  timeout client 1s
+  timeout server 5s
+  timeout connect 5s
+
+frontend forwarder
+  bind *:%FWDPORT%
+  default_backend registry
+
+backend registry
+  server kube-registry %HOST%:%PORT% ssl verify required ca-file %CA_FILE%

cluster/addons/registry/images/proxy.conf.insecure.in

@@ -0,0 +1,17 @@
+global
+  maxconn 1024
+
+defaults
+  mode http
+  retries 3
+  option redispatch
+  timeout client 1s
+  timeout server 5s
+  timeout connect 5s
+
+frontend forwarder
+  bind *:%FWDPORT%
+  default_backend registry
+
+backend registry
+  server kube-registry %HOST%:%PORT%

cluster/addons/registry/images/run_proxy.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REGISTRY_HOST=${REGISTRY_HOST:?no host}
+REGISTRY_PORT=${REGISTRY_PORT:-5000}
+REGISTRY_CA=${REGISTRY_CA:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt}
+FORWARD_PORT=${FORWARD_PORT:-5000}
+sed -e "s/%HOST%/$REGISTRY_HOST/g" \
+	-e "s/%PORT%/$REGISTRY_PORT/g" \
+	-e "s/%FWDPORT%/$FORWARD_PORT/g" \
+	-e "s|%CA_FILE%|$REGISTRY_CA|g" \
+	</proxy.conf.in >/proxy.conf
+
+# wait for registry to come online
+while ! host "$REGISTRY_HOST" &>/dev/null; do
+	printf "waiting for %s to come online\n" "$REGISTRY_HOST"
+	sleep 1
+done
+
+printf "starting proxy\n"
+exec haproxy -f /proxy.conf "$@"

cluster/addons/registry/registry-pv.yaml.in

@@ -0,0 +1,16 @@
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  name: kube-system-kube-registry-pv
+  labels:
+    kubernetes.io/cluster-service: "true"
+spec:
+{% if pillar.get('cluster_registry_disk_type', '') == 'gce' %}
+  capacity:
+    storage: {{ pillar['cluster_registry_disk_size'] }}
+  accessModes:
+    - ReadWriteOnce
+  gcePersistentDisk:
+    pdName: "{{ pillar['cluster_registry_disk_name'] }}"
+    fsType: "ext4"
+{% endif %}

cluster/addons/registry/registry-pvc.yaml.in

@@ -0,0 +1,13 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: kube-registry-pvc
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ pillar['cluster_registry_disk_size'] }}

cluster/addons/registry/registry-rc.yaml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kube-registry-v0
+  namespace: kube-system
+  labels:
+    k8s-app: kube-registry
+    version: v0
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: kube-registry
+    version: v0
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-registry
+        version: v0
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: registry
+        image: registry:2
+        resources:
+          limits:
+            cpu: 100m
+            memory: 100Mi
+        env:
+        - name: REGISTRY_HTTP_ADDR
+          value: :5000
+        - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
+          value: /var/lib/registry
+        volumeMounts:
+        - name: image-store
+          mountPath: /var/lib/registry
+        ports:
+        - containerPort: 5000
+          name: registry
+          protocol: TCP
+      # TODO: use a persistent volume claim
+      volumes:
+      - name: image-store
+        persistentVolumeClaim:
+          claimName: kube-registry-pvc

cluster/addons/registry/registry-svc.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-registry
+  namespace: kube-system
+  labels:
+    k8s-app: kube-registry
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "KubeRegistry"
+spec:
+  selector:
+    k8s-app: kube-registry
+  ports:
+  - name: registry
+    port: 5000
+    protocol: TCP

cluster/gce/config-default.sh

@@ -77,6 +77,12 @@ DNS_SERVER_IP="10.0.0.10"
 DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 
+# Optional: Install cluster docker registry.
+ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}"
+CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_PD:-${INSTANCE_PREFIX}-kube-system-kube-registry}"
+CLUSTER_REGISTRY_DISK_SIZE="${CLUSTER_REGISTRY_DISK_SIZE:-200GB}"
+CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}"
+
 # Optional: Install Kubernetes UI
 ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
 

cluster/gce/config-test.sh

@@ -82,6 +82,12 @@ DNS_SERVER_IP="10.0.0.10"
 DNS_DOMAIN="cluster.local"
 DNS_REPLICAS=1
 
+# Optional: Install cluster docker registry.
+ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}"
+CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_DISK:-${INSTANCE_PREFIX}-kube-system-kube-registry}"
+CLUSTER_REGISTRY_DISK_SIZE="${CLUSTER_REGISTRY_DISK_SIZE:-200GB}"
+CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}"
+
 # Optional: Install Kubernetes UI
 ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
 

cluster/gce/configure-vm.sh

@@ -272,6 +272,7 @@ enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")'
 logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")'
 elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")'
 enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")'
+enable_cluster_registry: '$(echo "$ENABLE_CLUSTER_REGISTRY" | sed -e "s/'/''/g")'
 dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")'
 dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
 dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
@@ -303,6 +304,28 @@ EOF
 kubeproxy_test_args: '$(echo "$KUBEPROXY_TEST_ARGS" | sed -e "s/'/''/g")'
 EOF
     fi
+    # TODO: Replace this  with a persistent volume (and create it).
+    if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then
+      cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
+cluster_registry_disk_type: gce
+cluster_registry_disk_size: $(convert-bytes-gce-kube ${CLUSTER_REGISTRY_DISK_SIZE})
+cluster_registry_disk_name: ${CLUSTER_REGISTRY_DISK}
+EOF
+    fi
+}
+
+# The job of this function is simple, but the basic regular expression syntax makes
+# this difficult to read. What we want to do is convert from [0-9]+B, KB, KiB, MB, etc
+# into [0-9]+, Ki, Mi, Gi, etc.
+# This is done in two steps:
+#   1. Convert from [0-9]+X?i?B into [0-9]X? (X denotes the prefix, ? means the field
+#      is optional.
+#   2. Attach an 'i' to the end of the string if we find a letter.
+# The two step process is needed to handle the edge case in which we want to convert
+# a raw byte count, as the result should be a simple number (e.g. 5B -> 5).
+function convert-bytes-gce-kube() {
+  local -r storage_space=$1
+  echo "${storage_space}" | sed -e 's/^\([0-9]\+\)\([A-Z]\)\?i\?B$/\1\2/g' -e 's/\([A-Z]\)$/\1i/'
 }
 
 # This should only happen on cluster initialization.

cluster/gce/coreos/helper.sh

@@ -43,6 +43,7 @@ ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
 LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
 ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
 ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
+ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false})
 DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
 DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
 DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})
@@ -80,6 +81,7 @@ ENABLE_NODE_LOGGING=${ENABLE_NODE_LOGGING:-false}
 LOGGING_DESTINATION=${LOGGING_DESTINATION:-}
 ELASTICSEARCH_LOGGING_REPLICAS=${ELASTICSEARCH_LOGGING_REPLICAS:-}
 ENABLE_CLUSTER_DNS=${ENABLE_CLUSTER_DNS:-false}
+ENABLE_CLUSTER_REGISTRY=${ENABLE_CLUSTER_REGISTRY:-false})
 DNS_REPLICAS=${DNS_REPLICAS:-}
 DNS_SERVER_IP=${DNS_SERVER_IP:-}
 DNS_DOMAIN=${DNS_DOMAIN:-}

cluster/gce/debian/helper.sh

@@ -41,6 +41,9 @@ ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false})
 LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-})
 ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-})
 ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false})
+ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false})
+CLUSTER_REGISTRY_DISK: $(yaml-quote ${CLUSTER_REGISTRY_DISK})
+CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE})
 DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-})
 DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-})
 DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-})

cluster/gce/util.sh

@@ -643,6 +643,15 @@ function kube-up {
     --type "${MASTER_DISK_TYPE}" \
     --size "${MASTER_DISK_SIZE}"
 
+  # Create disk for cluster registry if enabled
+  if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then
+    gcloud compute disks create "${CLUSTER_REGISTRY_DISK}" \
+      --project "${PROJECT}" \
+      --zone "${ZONE}" \
+      --type "${CLUSTER_REGISTRY_DISK_TYPE_GCE}" \
+      --size "${CLUSTER_REGISTRY_DISK_SIZE}" &
+  fi
+
   # Generate a bearer token for this cluster. We push this separately
   # from the other cluster variables so that the client (this
   # computer) can forget it later. This should disappear with
@@ -837,6 +846,17 @@ function kube-down {
       "${MASTER_NAME}"-pd
   fi
 
+  # Delete disk for cluster registry if enabled
+  if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then
+    if gcloud compute disks describe "${CLUSTER_REGISTRY_DISK}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then
+      gcloud compute disks delete \
+        --project "${PROJECT}" \
+        --quiet \
+        --zone "${ZONE}" \
+        "${CLUSTER_REGISTRY_DISK}"
+    fi
+  fi
+
   # Find out what minions are running.
   local -a minions
   minions=( $(gcloud compute instances list \
@@ -955,6 +975,11 @@ function check-resources {
     return 1
   fi
 
+  if gcloud compute disks describe --project "${PROJECT}" "${CLUSTER_REGISTRY_DISK}" --zone "${ZONE}" &>/dev/null; then
+    KUBE_RESOURCE_FOUND="Persistent disk ${CLUSTER_REGISTRY_DISK}"
+    return 1
+  fi
+
   # Find out what minions are running.
   local -a minions
   minions=( $(gcloud compute instances list \

cluster/saltbase/install.sh

@@ -64,7 +64,9 @@ done
 echo "+++ Install binaries from tar: $1"
 tar -xz -C "${KUBE_TEMP}" -f "$1"
 mkdir -p /srv/salt-new/salt/kube-bins
+mkdir -p /srv/salt-new/salt/kube-addons-images
 cp -v "${KUBE_TEMP}/kubernetes/server/bin/"* /srv/salt-new/salt/kube-bins/
+cp -v "${KUBE_TEMP}/kubernetes/addons/"* /srv/salt-new/salt/kube-addons-images/
 
 kube_bin_dir="/srv/salt-new/salt/kube-bins";
 docker_images_sls_file="/srv/salt-new/pillar/docker-images.sls";

cluster/saltbase/salt/kube-addons/init.sls

@@ -81,6 +81,42 @@ addon-dir-create:
     - makedirs: True
 {% endif %}
 
+{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %}
+/etc/kubernetes/addons/registry/registry-svc.yaml:
+  file.managed:
+    - source: salt://kube-addons/registry/registry-svc.yaml
+    - user: root
+    - group: root
+    - file_mode: 644
+    - makedirs: True
+
+/etc/kubernetes/addons/registry/registry-rc.yaml:
+  file.managed:
+    - source: salt://kube-addons/registry/registry-rc.yaml
+    - user: root
+    - group: root
+    - file_mode: 644
+    - makedirs: True
+
+/etc/kubernetes/addons/registry/registry-pv.yaml:
+  file.managed:
+    - source: salt://kube-addons/registry/registry-pv.yaml.in
+    - template: jinja
+    - user: root
+    - group: root
+    - file_mode: 644
+    - makedirs: True
+
+/etc/kubernetes/addons/registry/registry-pvc.yaml:
+  file.managed:
+    - source: salt://kube-addons/registry/registry-pvc.yaml.in
+    - template: jinja
+    - user: root
+    - group: root
+    - file_mode: 644
+    - makedirs: True
+{% endif %}
+
 {% if pillar.get('enable_node_logging', '').lower() == 'true'
    and pillar.get('logging_destination').lower() == 'elasticsearch'
    and pillar.get('enable_cluster_logging', '').lower() == 'true' %}

cluster/saltbase/salt/kube-addons/kube-addon-update.sh

@@ -470,12 +470,14 @@ function update-addons() {
     # be careful, reconcile-objects uses global variables
     reconcile-objects ${addon_path} ReplicationController "-" &
 
-    # We don't expect service names to be versioned, so
-    # we match entire name, ignoring version suffix.
+    # We don't expect names to be versioned for the following kinds, so
+    # we match the entire name, ignoring version suffix.
     # That's why we pass an empty string as the version separator.
-    # If the service description differs on disk, the service should be recreated.
+    # If the description differs on disk, the object should be recreated.
     # This is not implemented in this version.
     reconcile-objects ${addon_path} Service "" &
+    reconcile-objects ${addon_path} PersistentVolume "" &
+    reconcile-objects ${addon_path} PersistentVolumeClaim "" &
 
     wait-for-jobs
     if [[ $? -eq 0 ]]; then

cluster/saltbase/salt/kube-addons/kube-addons.sh

@@ -125,6 +125,28 @@ function create-resource-from-string() {
   return 1;
 }
 
+# $1 is the directory containing all of the docker images
+function load-docker-images() {
+  local success
+  local restart_docker
+  while true; do
+    success=true
+    restart_docker=false
+    for image in "$1/"*; do
+      timeout 30 docker load -i "${image}" &>/dev/null
+      rc=$?
+      if [[ "$rc" == 124 ]]; then
+        restart_docker=true
+      elif [[ "$rc" != 0 ]]; then
+        success=false
+      fi
+    done
+    if [[ "$success" == "true" ]]; then break; fi
+    if [[ "$restart_docker" == "true" ]]; then service docker restart; fi
+    sleep 15
+  done
+}
+
 # The business logic for whether a given object should be created
 # was already enforced by salt, and /etc/kubernetes/addons is the
 # managed result is of that. Start everything below that directory.
@@ -142,6 +164,9 @@ for k,v in yaml.load(sys.stdin).iteritems():
 ''' < "${kube_env_yaml}")
 fi
 
+# Load any images that we may need
+load-docker-images /srv/salt/kube-addons-images
+
 # Create the namespace that will be used to host the cluster-level add-ons.
 start_addon /etc/kubernetes/addons/namespace.yaml 100 10 "" &
 
@@ -175,13 +200,13 @@ while read line; do
     # do not have DNS available will have to override the server.
     create-kubeconfig-secret "${token}" "${username}" "https://kubernetes.default"
   fi
-done < ${token_dir}/known_tokens.csv
+done < "${token_dir}/known_tokens.csv"
 
 # Create admission_control objects if defined before any other addon services. If the limits
 # are defined in a namespace other than default, we should still create the limits for the
 # default namespace.
 for obj in $(find /etc/kubernetes/admission-controls \( -name \*.yaml -o -name \*.json \)); do
-  start_addon ${obj} 100 10 default &
+  start_addon "${obj}" 100 10 default &
   echo "++ obj ${obj} is created ++"
 done
 

cluster/saltbase/salt/kube-registry-proxy/init.sls

@@ -0,0 +1,8 @@
+/etc/kubernetes/manifests/kube-registry-proxy.yaml:
+  file.managed:
+    - source: salt://kube-registry-proxy/kube-registry-proxy.yaml
+    - user: root
+    - group: root
+    - mode: 644
+    - makedirs: True
+    - dir_mode: 755

cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-registry-proxy
+  namespace: kube-system
+spec:
+  containers:
+  - name: kube-registry-proxy
+    image: gcr.io/google_containers/kube-registry-proxy:0.3
+    resources:
+      limits:
+        cpu: 100m
+        memory: 50Mi
+    env:
+    - name: REGISTRY_HOST
+      value: kube-registry.kube-system.svc.cluster.local
+    - name: REGISTRY_PORT
+      value: "5000"
+    - name: FORWARD_PORT
+      value: "5000"
+    ports:
+    - name: registry
+      containerPort: 5000
+      hostPort: 5000

cluster/saltbase/salt/top.sls

@@ -24,6 +24,9 @@ base:
   {% elif pillar['logging_destination'] == 'gcp' %}
     - fluentd-gcp
   {% endif %}
+{% endif %}
+{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %}
+    - kube-registry-proxy
 {% endif %}
     - logrotate
 {% if grains['cloud'] is defined and grains.cloud == 'gce' %}