From 7129d477d3d6e5f514332cc8ac6e4c09a24a5d11 Mon Sep 17 00:00:00 2001 From: Muhammed Uluyol Date: Mon, 27 Jul 2015 11:50:31 -0700 Subject: [PATCH 1/5] Launch a cluster-local registry. This registry can be accessed through proxies that run on each node listening on port 5000. We send the proxy images to the nodes directly to avoid requests that hit the network during cluster launch. For now, we continue to pull the registry itself over the network, especially given its large size (we should be able to dramatically shrink the image). On GCE we create a PD and use that for storage, otherwise we use an emptyDir. The registry is not enabled outside of GCE. All communication is currently plain HTTP. In order to use SSL, we will need to be able to request a certificate/key from the apiserver signed by the apiserver's CA cert. --- build/common.sh | 29 ++++++++++ cluster/addons/registry/images/Dockerfile | 12 +++++ cluster/addons/registry/images/Makefile | 10 ++++ cluster/addons/registry/images/proxy.conf.in | 17 ++++++ .../registry/images/proxy.conf.insecure.in | 17 ++++++ cluster/addons/registry/images/run_proxy.sh | 33 ++++++++++++ cluster/addons/registry/registry-rc.yaml.in | 53 +++++++++++++++++++ cluster/addons/registry/registry-svc.yaml | 16 ++++++ cluster/gce/config-default.sh | 6 +++ cluster/gce/config-test.sh | 6 +++ cluster/gce/configure-vm.sh | 10 ++++ cluster/gce/coreos/helper.sh | 2 + cluster/gce/debian/helper.sh | 2 + cluster/gce/util.sh | 25 +++++++++ cluster/saltbase/install.sh | 2 + cluster/saltbase/salt/kube-addons/init.sls | 19 +++++++ .../saltbase/salt/kube-addons/kube-addons.sh | 25 +++++++++ .../salt/kube-registry-proxy/init.sls | 8 +++ .../kube-registry-proxy.yaml | 24 +++++++++ cluster/saltbase/salt/top.sls | 3 ++ 20 files changed, 319 insertions(+) create mode 100644 cluster/addons/registry/images/Dockerfile create mode 100644 cluster/addons/registry/images/Makefile create mode 100644 cluster/addons/registry/images/proxy.conf.in create mode 100644 cluster/addons/registry/images/proxy.conf.insecure.in create mode 100644 cluster/addons/registry/images/run_proxy.sh create mode 100644 cluster/addons/registry/registry-rc.yaml.in create mode 100644 cluster/addons/registry/registry-svc.yaml create mode 100644 cluster/saltbase/salt/kube-registry-proxy/init.sls create mode 100644 cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml diff --git a/build/common.sh b/build/common.sh index 0853a7c6b1..b73b7f5d82 100755 --- a/build/common.sh +++ b/build/common.sh @@ -100,6 +100,12 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=( kube-scheduler ) +# The set of addons images that should be prepopulated +readonly KUBE_ADDON_PATHS=( + gcr.io/google_containers/pause:0.8.0 + uluyol/kube-registry-proxy:0.2.3 +) + # --------------------------------------------------------------------------- # Basic setup functions @@ -602,6 +608,7 @@ function kube::release::package_server_tarballs() { local release_stage="${RELEASE_STAGE}/server/${platform_tag}/kubernetes" rm -rf "${release_stage}" mkdir -p "${release_stage}/server/bin" + mkdir -p "${release_stage}/addons" # This fancy expression will expand to prepend a path # (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the @@ -610,6 +617,7 @@ function kube::release::package_server_tarballs() { "${release_stage}/server/bin/" kube::release::create_docker_images_for_server "${release_stage}/server/bin"; + kube::release::write_addon_docker_images_for_server "${release_stage}/addons" # Include the client binaries here too as they are useful debugging tools. local client_bins=("${KUBE_CLIENT_BINARIES[@]}") @@ -681,6 +689,27 @@ function kube::release::create_docker_images_for_server() { ) } +# This will pull and save docker images for addons which need to placed +# on the nodes directly. +function kube::release::write_addon_docker_images_for_server() { + # Create a sub-shell so that we don't pollute the outer environment + ( + local addon_path + for addon_path in "${KUBE_ADDON_PATHS[@]}"; do + ( + kube::log::status "Pulling and writing Docker image for addon: ${addon_path}" + + local dest_name="${addon_path//\//\~}" + docker pull "${addon_path}" + docker save "${addon_path}" > "${1}/${dest_name}.tar" + ) & + done + + kube::util::wait-for-jobs || { kube::log::error "unable to pull or write addon image"; return 1; } + kube::log::status "Addon images done" + ) +} + # Package up the salt configuration tree. This is an optional helper to getting # a cluster up and running. function kube::release::package_salt_tarball() { diff --git a/cluster/addons/registry/images/Dockerfile b/cluster/addons/registry/images/Dockerfile new file mode 100644 index 0000000000..f296449bad --- /dev/null +++ b/cluster/addons/registry/images/Dockerfile @@ -0,0 +1,12 @@ +FROM haproxy:1.5 +MAINTAINER Muhammed Uluyol + +RUN apt-get update && apt-get install -y dnsutils + +ADD proxy.conf.insecure.in /proxy.conf.in +ADD run_proxy.sh /usr/bin/run_proxy + +RUN chown root:users /usr/bin/run_proxy +RUN chmod 755 /usr/bin/run_proxy + +CMD ["/usr/bin/run_proxy"] diff --git a/cluster/addons/registry/images/Makefile b/cluster/addons/registry/images/Makefile new file mode 100644 index 0000000000..17429b5032 --- /dev/null +++ b/cluster/addons/registry/images/Makefile @@ -0,0 +1,10 @@ +.PHONY: build push vet test clean + +TAG = 0.2.3 +REPO = uluyol/kube-registry-proxy + +build: + docker build -t $(REPO):$(TAG) . + +push: + docker push $(REPO):$(TAG) diff --git a/cluster/addons/registry/images/proxy.conf.in b/cluster/addons/registry/images/proxy.conf.in new file mode 100644 index 0000000000..b7a18f93de --- /dev/null +++ b/cluster/addons/registry/images/proxy.conf.in @@ -0,0 +1,17 @@ +global + maxconn 1024 + +defaults + mode http + retries 3 + option redispatch + timeout client 1s + timeout server 5s + timeout connect 5s + +frontend forwarder + bind *:%FWDPORT% + default_backend registry + +backend registry + server kube-registry %HOST%:%PORT% ssl verify required ca-file %CA_FILE% diff --git a/cluster/addons/registry/images/proxy.conf.insecure.in b/cluster/addons/registry/images/proxy.conf.insecure.in new file mode 100644 index 0000000000..d70ff567b2 --- /dev/null +++ b/cluster/addons/registry/images/proxy.conf.insecure.in @@ -0,0 +1,17 @@ +global + maxconn 1024 + +defaults + mode http + retries 3 + option redispatch + timeout client 1s + timeout server 5s + timeout connect 5s + +frontend forwarder + bind *:%FWDPORT% + default_backend registry + +backend registry + server kube-registry %HOST%:%PORT% diff --git a/cluster/addons/registry/images/run_proxy.sh b/cluster/addons/registry/images/run_proxy.sh new file mode 100644 index 0000000000..ad52ae8702 --- /dev/null +++ b/cluster/addons/registry/images/run_proxy.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REGISTRY_HOST=${REGISTRY_HOST:?no host} +REGISTRY_PORT=${REGISTRY_PORT:-5000} +REGISTRY_CA=${REGISTRY_CA:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt} +FORWARD_PORT=${FORWARD_PORT:-5000} +sed -e "s/%HOST%/$REGISTRY_HOST/g" \ + -e "s/%PORT%/$REGISTRY_PORT/g" \ + -e "s/%FWDPORT%/$FORWARD_PORT/g" \ + -e "s|%CA_FILE%|$REGISTRY_CA|g" \ + /proxy.conf + +# wait for registry to come online +while ! host "$REGISTRY_HOST" &>/dev/null; do + printf "waiting for %s to come online\n" "$REGISTRY_HOST" + sleep 1 +done + +printf "starting proxy\n" +exec haproxy -f /proxy.conf "$@" diff --git a/cluster/addons/registry/registry-rc.yaml.in b/cluster/addons/registry/registry-rc.yaml.in new file mode 100644 index 0000000000..0af47a36eb --- /dev/null +++ b/cluster/addons/registry/registry-rc.yaml.in @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kube-registry-v0 + namespace: kube-system + labels: + k8s-app: kube-registry + version: v0 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kube-registry + version: v0 + template: + metadata: + labels: + k8s-app: kube-registry + version: v0 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: registry + image: registry:2 + resources: + limits: + cpu: 100m + memory: 100Mi + env: + - name: REGISTRY_HTTP_ADDR + value: :5000 + - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: /var/lib/registry + volumeMounts: + - name: image-store + mountPath: /var/lib/registry + ports: + - containerPort: 5000 + name: registry + protocol: TCP + # TODO: use a persistent volume claim + volumes: + - name: image-store + {%- if pillar.get('cluster_registry_disk', '') != '' -%} + {%- for disk in pillar['cluster_registry_disk'].keys() %} + {{ disk }}: + {%- for k, v in pillar['cluster_registry_disk'][disk].items() %} + {{k}}: {{v}} + {%- endfor -%} + {%- endfor -%} + {%- else %} + emptyDir: {} + {%- endif -%} diff --git a/cluster/addons/registry/registry-svc.yaml b/cluster/addons/registry/registry-svc.yaml new file mode 100644 index 0000000000..b9f1cc40b9 --- /dev/null +++ b/cluster/addons/registry/registry-svc.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-registry + namespace: kube-system + labels: + k8s-app: kube-registry + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeRegistry" +spec: + selector: + k8s-app: kube-registry + ports: + - name: registry + port: 5000 + protocol: TCP diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c12f136ffc..bc37499bd1 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -77,6 +77,12 @@ DNS_SERVER_IP="10.0.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Install cluster docker registry. +ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}" +CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_PD:-${INSTANCE_PREFIX}-kube-system-kube-registry}" +CLUSTER_REGISTRY_DISK_SIZE="${CLUSTER_REGISTRY_DISK_SIZE:-200GB}" +CLUSTER_REGISTRY_DISK_TYPE="${CLUSTER_REGISTRY_DISK_TYPE:-pd-standard}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 316d59f234..1c14e67b61 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -82,6 +82,12 @@ DNS_SERVER_IP="10.0.0.10" DNS_DOMAIN="cluster.local" DNS_REPLICAS=1 +# Optional: Install cluster docker registry. +ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}" +CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_DISK:-${INSTANCE_PREFIX}-kube-system-kube-registry}" +CLUSTER_REGISTRY_DISK_SIZE="${CLUSTER_REGISTRY_DISK_SIZE:-200GB}" +CLUSTER_REGISTRY_DISK_TYPE="${CLUSTER_REGISTRY_DISK_TYPE:-pd-standard}" + # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 07db65ad4e..3043e2cda2 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -272,6 +272,7 @@ enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' +enable_cluster_registry: '$(echo "$ENABLE_CLUSTER_REGISTRY" | sed -e "s/'/''/g")' dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' @@ -301,6 +302,15 @@ EOF if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls kubeproxy_test_args: '$(echo "$KUBEPROXY_TEST_ARGS" | sed -e "s/'/''/g")' +EOF + fi + # TODO: Replace this with a persistent volume (and create it). + if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +cluster_registry_disk: + gcePersistentDisk: + pdName: ${CLUSTER_REGISTRY_DISK} + fsType: ext4 EOF fi } diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 705d284227..dbf9a46a06 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -43,6 +43,7 @@ ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false}) LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-}) ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-}) ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false}) +ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false}) DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-}) DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) @@ -80,6 +81,7 @@ ENABLE_NODE_LOGGING=${ENABLE_NODE_LOGGING:-false} LOGGING_DESTINATION=${LOGGING_DESTINATION:-} ELASTICSEARCH_LOGGING_REPLICAS=${ELASTICSEARCH_LOGGING_REPLICAS:-} ENABLE_CLUSTER_DNS=${ENABLE_CLUSTER_DNS:-false} +ENABLE_CLUSTER_REGISTRY=${ENABLE_CLUSTER_REGISTRY:-false}) DNS_REPLICAS=${DNS_REPLICAS:-} DNS_SERVER_IP=${DNS_SERVER_IP:-} DNS_DOMAIN=${DNS_DOMAIN:-} diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 199adb3807..e224d57caf 100755 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -41,6 +41,8 @@ ENABLE_NODE_LOGGING: $(yaml-quote ${ENABLE_NODE_LOGGING:-false}) LOGGING_DESTINATION: $(yaml-quote ${LOGGING_DESTINATION:-}) ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-}) ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false}) +ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false}) +CLUSTER_REGISTRY_DISK: $(yaml-quote ${CLUSTER_REGISTRY_DISK}) DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-}) DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 7617cd773a..a6cedd1043 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -643,6 +643,15 @@ function kube-up { --type "${MASTER_DISK_TYPE}" \ --size "${MASTER_DISK_SIZE}" + # Create disk for cluster registry if enabled + if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then + gcloud compute disks create "${CLUSTER_REGISTRY_DISK}" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --type "${CLUSTER_REGISTRY_DISK_TYPE}" \ + --size "${CLUSTER_REGISTRY_DISK_SIZE}" & + fi + # Generate a bearer token for this cluster. We push this separately # from the other cluster variables so that the client (this # computer) can forget it later. This should disappear with @@ -837,6 +846,17 @@ function kube-down { "${MASTER_NAME}"-pd fi + # Delete disk for cluster registry if enabled + if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then + if gcloud compute disks describe "${CLUSTER_REGISTRY_DISK}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then + gcloud compute disks delete \ + --project "${PROJECT}" \ + --quiet \ + --zone "${ZONE}" \ + "${CLUSTER_REGISTRY_DISK}" + fi + fi + # Find out what minions are running. local -a minions minions=( $(gcloud compute instances list \ @@ -955,6 +975,11 @@ function check-resources { return 1 fi + if gcloud compute disks describe --project "${PROJECT}" "${CLUSTER_REGISTRY_DISK}" --zone "${ZONE}" &>/dev/null; then + KUBE_RESOURCE_FOUND="Persistent disk ${CLUSTER_REGISTRY_DISK}" + return 1 + fi + # Find out what minions are running. local -a minions minions=( $(gcloud compute instances list \ diff --git a/cluster/saltbase/install.sh b/cluster/saltbase/install.sh index b172f74f15..c7fe553f22 100755 --- a/cluster/saltbase/install.sh +++ b/cluster/saltbase/install.sh @@ -64,7 +64,9 @@ done echo "+++ Install binaries from tar: $1" tar -xz -C "${KUBE_TEMP}" -f "$1" mkdir -p /srv/salt-new/salt/kube-bins +mkdir -p /srv/salt-new/salt/kube-addons-images cp -v "${KUBE_TEMP}/kubernetes/server/bin/"* /srv/salt-new/salt/kube-bins/ +cp -v "${KUBE_TEMP}/kubernetes/addons/"* /srv/salt-new/salt/kube-addons-images/ kube_bin_dir="/srv/salt-new/salt/kube-bins"; docker_images_sls_file="/srv/salt-new/pillar/docker-images.sls"; diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls index 5dab9c8fa5..df9c3c739d 100644 --- a/cluster/saltbase/salt/kube-addons/init.sls +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -81,6 +81,25 @@ addon-dir-create: - makedirs: True {% endif %} +{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %} +/etc/kubernetes/addons/registry/registry-svc.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-svc.yaml + - user: root + - group: root + - file_mode: 644 + - makedirs: True + +/etc/kubernetes/addons/registry/registry-rc.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-rc.yaml.in + - template: jinja + - user: root + - group: root + - file_mode: 644 + - makedirs: True +{% endif %} + {% if pillar.get('enable_node_logging', '').lower() == 'true' and pillar.get('logging_destination').lower() == 'elasticsearch' and pillar.get('enable_cluster_logging', '').lower() == 'true' %} diff --git a/cluster/saltbase/salt/kube-addons/kube-addons.sh b/cluster/saltbase/salt/kube-addons/kube-addons.sh index 0bd24401fa..0655237362 100644 --- a/cluster/saltbase/salt/kube-addons/kube-addons.sh +++ b/cluster/saltbase/salt/kube-addons/kube-addons.sh @@ -125,6 +125,28 @@ function create-resource-from-string() { return 1; } +# $1 is the directory containing all of the docker images +function load-docker-images() { + local success + local restart_docker + while true; do + success=true + restart_docker=false + for image in "$1/"*; do + timeout 30 docker load -i "${image}" &>/dev/null + rc=$? + if [[ $rc == 124 ]]; then + restart_docker=true + elif [[ $rc != 0 ]]; then + success=false + fi + done + if [[ $success == true ]]; then break; fi + if [[ $restart_docker == true ]]; then service docker restart; fi + sleep 15 + done +} + # The business logic for whether a given object should be created # was already enforced by salt, and /etc/kubernetes/addons is the # managed result is of that. Start everything below that directory. @@ -142,6 +164,9 @@ for k,v in yaml.load(sys.stdin).iteritems(): ''' < "${kube_env_yaml}") fi +# Load any images that we may need +load-docker-images /srv/salt/kube-addons-images + # Create the namespace that will be used to host the cluster-level add-ons. start_addon /etc/kubernetes/addons/namespace.yaml 100 10 "" & diff --git a/cluster/saltbase/salt/kube-registry-proxy/init.sls b/cluster/saltbase/salt/kube-registry-proxy/init.sls new file mode 100644 index 0000000000..cc5d416a11 --- /dev/null +++ b/cluster/saltbase/salt/kube-registry-proxy/init.sls @@ -0,0 +1,8 @@ +/etc/kubernetes/manifests/kube-registry-proxy.yaml: + file.managed: + - source: salt://kube-registry-proxy/kube-registry-proxy.yaml + - user: root + - group: root + - mode: 644 + - makedirs: True + - dir_mode: 755 diff --git a/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml b/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml new file mode 100644 index 0000000000..d19878e3b1 --- /dev/null +++ b/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-registry-proxy + namespace: kube-system +spec: + containers: + - name: kube-registry-proxy + image: uluyol/kube-registry-proxy:0.2.3 + resources: + limits: + cpu: 100m + memory: 50Mi + env: + - name: REGISTRY_HOST + value: kube-registry.kube-system.svc.cluster.local + - name: REGISTRY_PORT + value: "5000" + - name: FORWARD_PORT + value: "5000" + ports: + - name: registry + containerPort: 5000 + hostPort: 5000 diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 4ffec5e8eb..8d55b139b0 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -24,6 +24,9 @@ base: {% elif pillar['logging_destination'] == 'gcp' %} - fluentd-gcp {% endif %} +{% endif %} +{% if pillar.get('enable_cluster_registry', '').lower() == 'true' %} + - kube-registry-proxy {% endif %} - logrotate {% if grains['cloud'] is defined and grains.cloud == 'gce' %} From 14b554cef63f6fc8ac911b17da1730599adf8a66 Mon Sep 17 00:00:00 2001 From: Muhammed Uluyol Date: Wed, 19 Aug 2015 15:47:55 -0700 Subject: [PATCH 2/5] Use a persistent volume for the docker registry. --- cluster/addons/registry/registry-pv.yaml.in | 16 ++++++++++++++++ cluster/addons/registry/registry-pvc.yaml.in | 13 +++++++++++++ .../{registry-rc.yaml.in => registry-rc.yaml} | 12 ++---------- cluster/gce/config-default.sh | 2 +- cluster/gce/config-test.sh | 2 +- cluster/gce/configure-vm.sh | 7 +++---- cluster/gce/debian/helper.sh | 1 + cluster/gce/util.sh | 2 +- cluster/saltbase/salt/kube-addons/init.sls | 19 ++++++++++++++++++- .../salt/kube-addons/kube-addon-update.sh | 8 +++++--- 10 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 cluster/addons/registry/registry-pv.yaml.in create mode 100644 cluster/addons/registry/registry-pvc.yaml.in rename cluster/addons/registry/{registry-rc.yaml.in => registry-rc.yaml} (74%) diff --git a/cluster/addons/registry/registry-pv.yaml.in b/cluster/addons/registry/registry-pv.yaml.in new file mode 100644 index 0000000000..b6fd2b3529 --- /dev/null +++ b/cluster/addons/registry/registry-pv.yaml.in @@ -0,0 +1,16 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: kube-system-kube-registry-pv + labels: + kubernetes.io/cluster-service: "true" +spec: +{% if pillar.get('cluster_registry_disk_type', '') == 'gce' %} + capacity: + storage: {{ pillar['cluster_registry_disk_size'] }} + accessModes: + - ReadWriteOnce + gcePersistentDisk: + pdName: "{{ pillar['cluster_registry_disk_name'] }}" + fsType: "ext4" +{% endif %} diff --git a/cluster/addons/registry/registry-pvc.yaml.in b/cluster/addons/registry/registry-pvc.yaml.in new file mode 100644 index 0000000000..92bfc69864 --- /dev/null +++ b/cluster/addons/registry/registry-pvc.yaml.in @@ -0,0 +1,13 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kube-registry-pvc + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ pillar['cluster_registry_disk_size'] }} diff --git a/cluster/addons/registry/registry-rc.yaml.in b/cluster/addons/registry/registry-rc.yaml similarity index 74% rename from cluster/addons/registry/registry-rc.yaml.in rename to cluster/addons/registry/registry-rc.yaml index 0af47a36eb..9ac12c5675 100644 --- a/cluster/addons/registry/registry-rc.yaml.in +++ b/cluster/addons/registry/registry-rc.yaml @@ -41,13 +41,5 @@ spec: # TODO: use a persistent volume claim volumes: - name: image-store - {%- if pillar.get('cluster_registry_disk', '') != '' -%} - {%- for disk in pillar['cluster_registry_disk'].keys() %} - {{ disk }}: - {%- for k, v in pillar['cluster_registry_disk'][disk].items() %} - {{k}}: {{v}} - {%- endfor -%} - {%- endfor -%} - {%- else %} - emptyDir: {} - {%- endif -%} + persistentVolumeClaim: + claimName: kube-registry-pvc diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index bc37499bd1..7e62500c6c 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -81,7 +81,7 @@ DNS_REPLICAS=1 ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}" CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_PD:-${INSTANCE_PREFIX}-kube-system-kube-registry}" CLUSTER_REGISTRY_DISK_SIZE="${CLUSTER_REGISTRY_DISK_SIZE:-200GB}" -CLUSTER_REGISTRY_DISK_TYPE="${CLUSTER_REGISTRY_DISK_TYPE:-pd-standard}" +CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 1c14e67b61..9c09a42c89 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -86,7 +86,7 @@ DNS_REPLICAS=1 ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}" CLUSTER_REGISTRY_DISK="${CLUSTER_REGISTRY_DISK:-${INSTANCE_PREFIX}-kube-system-kube-registry}" CLUSTER_REGISTRY_DISK_SIZE="${CLUSTER_REGISTRY_DISK_SIZE:-200GB}" -CLUSTER_REGISTRY_DISK_TYPE="${CLUSTER_REGISTRY_DISK_TYPE:-pd-standard}" +CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" # Optional: Install Kubernetes UI ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 3043e2cda2..bfed226d4c 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -307,10 +307,9 @@ EOF # TODO: Replace this with a persistent volume (and create it). if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls -cluster_registry_disk: - gcePersistentDisk: - pdName: ${CLUSTER_REGISTRY_DISK} - fsType: ext4 +cluster_registry_disk_type: gce +cluster_registry_disk_size: ${CLUSTER_REGISTRY_DISK_SIZE//GB/Gi} +cluster_registry_disk_name: ${CLUSTER_REGISTRY_DISK} EOF fi } diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index e224d57caf..ad717bc106 100755 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -43,6 +43,7 @@ ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote ${ELASTICSEARCH_LOGGING_REPLICAS:-} ENABLE_CLUSTER_DNS: $(yaml-quote ${ENABLE_CLUSTER_DNS:-false}) ENABLE_CLUSTER_REGISTRY: $(yaml-quote ${ENABLE_CLUSTER_REGISTRY:-false}) CLUSTER_REGISTRY_DISK: $(yaml-quote ${CLUSTER_REGISTRY_DISK}) +CLUSTER_REGISTRY_DISK_SIZE: $(yaml-quote ${CLUSTER_REGISTRY_DISK_SIZE}) DNS_REPLICAS: $(yaml-quote ${DNS_REPLICAS:-}) DNS_SERVER_IP: $(yaml-quote ${DNS_SERVER_IP:-}) DNS_DOMAIN: $(yaml-quote ${DNS_DOMAIN:-}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index a6cedd1043..d8ff6494ac 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -648,7 +648,7 @@ function kube-up { gcloud compute disks create "${CLUSTER_REGISTRY_DISK}" \ --project "${PROJECT}" \ --zone "${ZONE}" \ - --type "${CLUSTER_REGISTRY_DISK_TYPE}" \ + --type "${CLUSTER_REGISTRY_DISK_TYPE_GCE}" \ --size "${CLUSTER_REGISTRY_DISK_SIZE}" & fi diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls index df9c3c739d..b9d4e93f53 100644 --- a/cluster/saltbase/salt/kube-addons/init.sls +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -92,7 +92,24 @@ addon-dir-create: /etc/kubernetes/addons/registry/registry-rc.yaml: file.managed: - - source: salt://kube-addons/registry/registry-rc.yaml.in + - source: salt://kube-addons/registry/registry-rc.yaml + - user: root + - group: root + - file_mode: 644 + - makedirs: True + +/etc/kubernetes/addons/registry/registry-pv.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-pv.yaml.in + - template: jinja + - user: root + - group: root + - file_mode: 644 + - makedirs: True + +/etc/kubernetes/addons/registry/registry-pvc.yaml: + file.managed: + - source: salt://kube-addons/registry/registry-pvc.yaml.in - template: jinja - user: root - group: root diff --git a/cluster/saltbase/salt/kube-addons/kube-addon-update.sh b/cluster/saltbase/salt/kube-addons/kube-addon-update.sh index c04815ff21..0caa4c0a8b 100755 --- a/cluster/saltbase/salt/kube-addons/kube-addon-update.sh +++ b/cluster/saltbase/salt/kube-addons/kube-addon-update.sh @@ -470,12 +470,14 @@ function update-addons() { # be careful, reconcile-objects uses global variables reconcile-objects ${addon_path} ReplicationController "-" & - # We don't expect service names to be versioned, so - # we match entire name, ignoring version suffix. + # We don't expect names to be versioned for the following kinds, so + # we match the entire name, ignoring version suffix. # That's why we pass an empty string as the version separator. - # If the service description differs on disk, the service should be recreated. + # If the description differs on disk, the object should be recreated. # This is not implemented in this version. reconcile-objects ${addon_path} Service "" & + reconcile-objects ${addon_path} PersistentVolume "" & + reconcile-objects ${addon_path} PersistentVolumeClaim "" & wait-for-jobs if [[ $? -eq 0 ]]; then From 82638f8b29232ac654f328aaead97d78824c85ca Mon Sep 17 00:00:00 2001 From: Muhammed Uluyol Date: Wed, 19 Aug 2015 17:19:37 -0700 Subject: [PATCH 3/5] Add conversion function from GCE storage units to kubernetes units. --- cluster/gce/configure-vm.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index bfed226d4c..6e41453649 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -308,12 +308,26 @@ EOF if [[ "${ENABLE_CLUSTER_REGISTRY}" == true && -n "${CLUSTER_REGISTRY_DISK}" ]]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls cluster_registry_disk_type: gce -cluster_registry_disk_size: ${CLUSTER_REGISTRY_DISK_SIZE//GB/Gi} +cluster_registry_disk_size: $(convert-bytes-gce-kube ${CLUSTER_REGISTRY_DISK_SIZE}) cluster_registry_disk_name: ${CLUSTER_REGISTRY_DISK} EOF fi } +# The job of this function is simple, but the basic regular expression syntax makes +# this difficult to read. What we want to do is convert from [0-9]+B, KB, KiB, MB, etc +# into [0-9]+, Ki, Mi, Gi, etc. +# This is done in two steps: +# 1. Convert from [0-9]+X?i?B into [0-9]X? (X denotes the prefix, ? means the field +# is optional. +# 2. Attach an 'i' to the end of the string if we find a letter. +# The two step process is needed to handle the edge case in which we want to convert +# a raw byte count, as the result should be a simple number (e.g. 5B -> 5). +function convert-bytes-gce-kube() { + local -r storage_space=$1 + echo "${storage_space}" | sed -e 's/^\([0-9]\+\)\([A-Z]\)\?i\?B$/\1\2/g' -e 's/\([A-Z]\)$/\1i/' +} + # This should only happen on cluster initialization. # # - Uses KUBE_PASSWORD and KUBE_USER to generate basic_auth.csv. From 2fb4e7b7a4f4620ac682100dde29d85405585e27 Mon Sep 17 00:00:00 2001 From: Muhammed Uluyol Date: Thu, 20 Aug 2015 18:41:36 -0700 Subject: [PATCH 4/5] Use quotes more consistently --- cluster/saltbase/salt/kube-addons/kube-addons.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster/saltbase/salt/kube-addons/kube-addons.sh b/cluster/saltbase/salt/kube-addons/kube-addons.sh index 0655237362..45b1ee42b2 100644 --- a/cluster/saltbase/salt/kube-addons/kube-addons.sh +++ b/cluster/saltbase/salt/kube-addons/kube-addons.sh @@ -135,14 +135,14 @@ function load-docker-images() { for image in "$1/"*; do timeout 30 docker load -i "${image}" &>/dev/null rc=$? - if [[ $rc == 124 ]]; then + if [[ "$rc" == 124 ]]; then restart_docker=true - elif [[ $rc != 0 ]]; then + elif [[ "$rc" != 0 ]]; then success=false fi done - if [[ $success == true ]]; then break; fi - if [[ $restart_docker == true ]]; then service docker restart; fi + if [[ "$success" == "true" ]]; then break; fi + if [[ "$restart_docker" == "true" ]]; then service docker restart; fi sleep 15 done } @@ -200,13 +200,13 @@ while read line; do # do not have DNS available will have to override the server. create-kubeconfig-secret "${token}" "${username}" "https://kubernetes.default" fi -done < ${token_dir}/known_tokens.csv +done < "${token_dir}/known_tokens.csv" # Create admission_control objects if defined before any other addon services. If the limits # are defined in a namespace other than default, we should still create the limits for the # default namespace. for obj in $(find /etc/kubernetes/admission-controls \( -name \*.yaml -o -name \*.json \)); do - start_addon ${obj} 100 10 default & + start_addon "${obj}" 100 10 default & echo "++ obj ${obj} is created ++" done From 3dc10a209ba457aa8fd28a5e0593e1608c6babc4 Mon Sep 17 00:00:00 2001 From: Muhammed Uluyol Date: Fri, 21 Aug 2015 11:36:22 -0700 Subject: [PATCH 5/5] Use gcr.io for proxy image --- build/common.sh | 2 +- cluster/addons/registry/images/Makefile | 6 +++--- .../salt/kube-registry-proxy/kube-registry-proxy.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/common.sh b/build/common.sh index b73b7f5d82..268277f12f 100755 --- a/build/common.sh +++ b/build/common.sh @@ -103,7 +103,7 @@ readonly KUBE_DOCKER_WRAPPED_BINARIES=( # The set of addons images that should be prepopulated readonly KUBE_ADDON_PATHS=( gcr.io/google_containers/pause:0.8.0 - uluyol/kube-registry-proxy:0.2.3 + gcr.io/google_containers/kube-registry-proxy:0.3 ) # --------------------------------------------------------------------------- diff --git a/cluster/addons/registry/images/Makefile b/cluster/addons/registry/images/Makefile index 17429b5032..bb776b656a 100644 --- a/cluster/addons/registry/images/Makefile +++ b/cluster/addons/registry/images/Makefile @@ -1,10 +1,10 @@ .PHONY: build push vet test clean -TAG = 0.2.3 -REPO = uluyol/kube-registry-proxy +TAG = 0.3 +REPO = gcr.io/google_containers/kube-registry-proxy build: docker build -t $(REPO):$(TAG) . push: - docker push $(REPO):$(TAG) + gcloud docker push $(REPO):$(TAG) diff --git a/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml b/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml index d19878e3b1..14f9225601 100644 --- a/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml +++ b/cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml @@ -6,7 +6,7 @@ metadata: spec: containers: - name: kube-registry-proxy - image: uluyol/kube-registry-proxy:0.2.3 + image: gcr.io/google_containers/kube-registry-proxy:0.3 resources: limits: cpu: 100m