From fa9d79df75d0a5326fb6ca72308ebbaf5bde603e Mon Sep 17 00:00:00 2001
From: Alain Roy <alainr@vmware.com>
Date: Tue, 8 Mar 2016 11:25:41 -0800
Subject: [PATCH] Initial kube-up support for VMware's Photon Controller

This is for: https://github.com/kubernetes/kubernetes/issues/24121

Photon Controller is an open-source cloud management platform. More
information is available at:
http://vmware.github.io/photon-controller/

This commit provides initial support for Photon Controller. The
following features are tested and working:
- kube-up and kube-down
- Basic pod and service management
- Networking within the Kubernetes cluster
- UI and DNS addons

It has been tested with a Kubernetes cluster of up to 10
nodes. Further work on scaling is planned for the near future.

Internally we have implemented continuous integration testing and will
run it multiple times per day against the Kubernetes master branch
once this is integrated so we can quickly react to problems.

A few things have not yet been implemented, but are planned:
- Support for kube-push
- Support for test-build-release, test-setup, test-teardown

Assuming this is accepted for inclusion, we will write documentation
for the kubernetes.io site.

We have included a script to help users configure Photon Controller
for use with Kubernetes. While not required, it will help some
users get started more quickly. It will be documented.

We are aware of the kube-deploy efforts and will track them and
support them as appropriate.
---
 cluster/addons/dns/skydns-rc.yaml.in          |    3 +
 cluster/get-kube.sh                           |    2 +
 cluster/photon-controller/config-common.sh    |   72 ++
 cluster/photon-controller/config-default.sh   |   92 ++
 cluster/photon-controller/config-test.sh      |   20 +
 cluster/photon-controller/setup-prereq.sh     |  239 ++++
 cluster/photon-controller/templates/README    |    4 +
 .../templates/create-dynamic-salt-files.sh    |  130 ++
 .../photon-controller/templates/hostname.sh   |   22 +
 .../templates/install-release.sh              |   26 +
 .../templates/salt-master.sh                  |   56 +
 .../templates/salt-minion.sh                  |   51 +
 cluster/photon-controller/util.sh             | 1092 +++++++++++++++++
 cluster/saltbase/salt/docker/init.sls         |    3 +-
 cluster/saltbase/salt/generate-cert/init.sls  |    2 +-
 cluster/saltbase/salt/kube-apiserver/init.sls |    4 +-
 .../kube-apiserver/kube-apiserver.manifest    |    8 +-
 .../kube-controller-manager.manifest          |    4 +-
 .../salt/kube-proxy/kube-proxy.manifest       |    2 +-
 cluster/saltbase/salt/kubelet/default         |    6 +-
 cluster/saltbase/salt/top.sls                 |    2 +-
 hack/verify-flags/exceptions.txt              |    4 +
 22 files changed, 1829 insertions(+), 15 deletions(-)
 create mode 100644 cluster/photon-controller/config-common.sh
 create mode 100755 cluster/photon-controller/config-default.sh
 create mode 100755 cluster/photon-controller/config-test.sh
 create mode 100755 cluster/photon-controller/setup-prereq.sh
 create mode 100644 cluster/photon-controller/templates/README
 create mode 100755 cluster/photon-controller/templates/create-dynamic-salt-files.sh
 create mode 100755 cluster/photon-controller/templates/hostname.sh
 create mode 100755 cluster/photon-controller/templates/install-release.sh
 create mode 100755 cluster/photon-controller/templates/salt-master.sh
 create mode 100755 cluster/photon-controller/templates/salt-minion.sh
 create mode 100755 cluster/photon-controller/util.sh

diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in
index 5185176aa1..db581572a9 100644
--- a/cluster/addons/dns/skydns-rc.yaml.in
+++ b/cluster/addons/dns/skydns-rc.yaml.in
@@ -19,6 +19,9 @@ spec:
         version: v11
         kubernetes.io/cluster-service: "true"
     spec:
+{% if grains['cloud'] is defined and grains['cloud'] in [ 'vsphere', 'photon-controller' ] %}
+      hostNetwork: true
+{% endif %}
       containers:
       - name: etcd
         image: gcr.io/google_containers/etcd-amd64:2.2.1
diff --git a/cluster/get-kube.sh b/cluster/get-kube.sh
index 60f85b843b..f74b752c1d 100755
--- a/cluster/get-kube.sh
+++ b/cluster/get-kube.sh
@@ -34,6 +34,8 @@
 #   * export KUBERNETES_PROVIDER=vagrant; wget -q -O - https://get.k8s.io | bash
 #  VMWare VSphere
 #   * export KUBERNETES_PROVIDER=vsphere; wget -q -O - https://get.k8s.io | bash
+#  VMWare Photon Controller
+#   * export KUBERNETES_PROVIDER=photon-controller; wget -q -O - https://get.k8s.io | bash
 #  Rackspace
 #   * export KUBERNETES_PROVIDER=rackspace; wget -q -O - https://get.k8s.io | bash
 #
diff --git a/cluster/photon-controller/config-common.sh b/cluster/photon-controller/config-common.sh
new file mode 100644
index 0000000000..6f4d65e4c7
--- /dev/null
+++ b/cluster/photon-controller/config-common.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##########################################################
+#
+# These parameters describe objects we are using from
+# Photon Controller. They are all assumed to be pre-existing.
+#
+# Note: if you want help in creating them, you can use 
+# the setup-prereq.sh script, which will create any of these
+# that do not already exist.
+#
+##########################################################
+
+# Pre-created tenant for Kubernetes to use
+PHOTON_TENANT=kube-tenant
+
+# Pre-created project in PHOTON_TENANT for Kubernetes to use
+PHOTON_PROJECT=kube-project
+
+# Pre-created VM flavor for Kubernetes master to use
+# Can be same as master
+# We recommend at least 1GB of memory
+PHOTON_MASTER_FLAVOR=kube-vm
+
+# Pre-created VM flavor for Kubernetes node to use
+# Can be same as master
+# We recommend at least 2GB of memory
+PHOTON_NODE_FLAVOR=kube-vm
+
+# Pre-created disk flavor for Kubernetes to use
+PHOTON_DISK_FLAVOR=kube-disk
+
+# Pre-created Debian 8 image with kube user uploaded to Photon Controller
+# Note: While Photon Controller allows multiple images to have the same
+# name, we assume that there is exactly one image with this name.
+PHOTON_IMAGE=kube
+
+##########################################################
+# 
+# Parameters just for the setup-prereq.sh script: not used
+# elsewhere. If you create the above objects by hand, you 
+# do not need to edit these. 
+# 
+# Note that setup-prereq.sh also creates the objects 
+# above.
+# 
+##########################################################
+
+# The specifications for the master and node flavors
+SETUP_MASTER_FLAVOR_SPEC="vm 1 COUNT, vm.cpu 1 COUNT, vm.memory 2 GB"
+SETUP_NODE_FLAVOR_SPEC=${SETUP_MASTER_FLAVOR_SPEC}
+
+# The specification for the ephemeral disk flavor. 
+SETUP_DISK_FLAVOR_SPEC="ephemeral-disk 1 COUNT"
+
+# The specification for the tenant resource ticket and the project resources
+SETUP_TICKET_SPEC="vm.memory 1000 GB, vm 1000 COUNT"
+SETUP_PROJECT_SPEC="${SETUP_TICKET_SPEC}"
diff --git a/cluster/photon-controller/config-default.sh b/cluster/photon-controller/config-default.sh
new file mode 100755
index 0000000000..4117eac33e
--- /dev/null
+++ b/cluster/photon-controller/config-default.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##########################################################
+#
+# Common parameters for Kubernetes
+#
+##########################################################
+
+# Default number of nodes to make. You can change this as needed
+NUM_NODES=3
+
+# Range of IPs assigned to pods
+NODE_IP_RANGES="10.244.0.0/16"
+
+# IPs used by Kubernetes master
+MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
+
+# Range of IPs assigned by Kubernetes to services
+SERVICE_CLUSTER_IP_RANGE="10.244.240.0/20"
+
+##########################################################
+#
+# Advanced parameters for Kubernetes
+#
+##########################################################
+
+# The instance prefix is the beginning of the name given to each VM we create
+# If this is changed, you can have multiple kubernetes clusters per project
+# Note that even if you don't change it, each tenant/project can have its own
+# Kubernetes cluster
+INSTANCE_PREFIX=kubernetes
+
+# Name of the user used to configure the VM
+# We use cloud-init to create the user
+VM_USER=kube
+
+# SSH options for how we connect to the Kubernetes VMs
+# We set the user known hosts file to /dev/null because we are connecting to new VMs.
+# When working in an environment where there is a lot of VM churn, VM IP addresses
+# will be reused, and the ssh keys will be different. This prevents us from seeing error
+# due to this, and it will not save the SSH key to the known_hosts file, so users will
+# still have standard ssh security checks.
+SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR"
+
+# Optional: Enable node logging.
+# Note: currently untested
+ENABLE_NODE_LOGGING=false
+LOGGING_DESTINATION=elasticsearch
+
+# Optional: When set to true, Elasticsearch and Kibana will be setup
+# Note: currently untested
+ENABLE_CLUSTER_LOGGING=false
+ELASTICSEARCH_LOGGING_REPLICAS=1
+
+# Optional: Cluster monitoring to setup as part of the cluster bring up:
+#   none     - No cluster monitoring setup
+#   influxdb - Heapster, InfluxDB, and Grafana
+#   google   - Heapster, Google Cloud Monitoring, and Google Cloud Logging
+# Note: currently untested
+ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
+
+# Optional: Install cluster DNS.
+ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
+DNS_SERVER_IP="10.244.240.240"
+DNS_DOMAIN="cluster.local"
+DNS_REPLICAS=1
+
+# Optional: Install Kubernetes UI
+ENABLE_CLUSTER_UI=true
+
+# We need to configure subject alternate names (SANs) for the master's certificate
+# we generate.  While users will connect via the external IP, pods (like the UI)
+# will connect via the cluster IP, from the SERVICE_CLUSTER_IP_RANGE.
+# In addition to the extra SANS here, we'll also add one for for the service IP.
+MASTER_EXTRA_SANS="DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
diff --git a/cluster/photon-controller/config-test.sh b/cluster/photon-controller/config-test.sh
new file mode 100755
index 0000000000..04b79d3aa3
--- /dev/null
+++ b/cluster/photon-controller/config-test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+NUM_NODES=2
+NODE_IP_RANGES="10.244.0.0/16"
+MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
+SERVICE_CLUSTER_IP_RANGE="10.244.240.0/20"
diff --git a/cluster/photon-controller/setup-prereq.sh b/cluster/photon-controller/setup-prereq.sh
new file mode 100755
index 0000000000..2c8ac9cfba
--- /dev/null
+++ b/cluster/photon-controller/setup-prereq.sh
@@ -0,0 +1,239 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This sets up a Photon Controller with the tenant, project, flavors
+# and image that are needed to deploy Kubernetes with kube-up.
+#
+# This is not meant to be used in production: it creates resource tickets
+# (quotas) that are arbitrary and not likely to work in your environment.
+# However, it may be a quick way to get your environment set up to try out
+# a Kubernetes installation.
+#
+# It uses the names for the tenant, project, and flavors as specified in the
+# config-common.sh file
+#
+# If you want to do this by hand, this script is equivalent to the following
+# Photon Controller commands (assuming you haven't edited config-common.sh
+# to change the names)
+#
+# photon target set https://192.0.2.2
+# photon tenant create kube-tenant
+# photon tenant set kube-tenant
+# photon resource-ticket create --tenant kube-tenant --name kube-resources --limits "vm.memory 1000 GB, vm 1000 COUNT"
+# photon project create --tenant kube-tenant --resource-ticket kube-resources --name kube-project --limits "vm.memory 1000 GB, vm 1000 COUNT"
+# photon project set kube-project
+# photon -n flavor create --name "kube-vm" --kind "vm" --cost "vm 1 COUNT, vm.cpu 1 COUNT, vm.memory 2 GB"
+# photon -n flavor create --name "kube-disk" --kind "ephemeral-disk" --cost "ephemeral-disk 1 COUNT"
+# photon image create kube.vmdk -n kube-image -i EAGER
+#
+# Note that the kube.vmdk can be downloaded as specified in the documentation.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
+# shellcheck source=./util.sh
+source "${KUBE_ROOT}/cluster/photon-controller/util.sh"
+
+function main {
+  verify-cmd-in-path photon
+  set-target
+  create-tenant
+  create-project
+  create-vm-flavor "${PHOTON_MASTER_FLAVOR}" "${SETUP_MASTER_FLAVOR_SPEC}"
+  if [ "${PHOTON_MASTER_FLAVOR}" != "${PHOTON_NODE_FLAVOR}" ]; then
+    create-vm-flavor "${PHOTON_NODE_FLAVOR}" "${SETUP_NODE_FLAVOR_SPEC}"
+  fi
+  create-disk-flavor
+  create-image
+}
+
+function parse-cmd-line {
+  PHOTON_TARGET=${1:-""}
+  PHOTON_VMDK=${2:-""}
+
+  if [[ "${PHOTON_TARGET}" = "" || "${PHOTON_VMDK}" = "" ]]; then
+    echo "Usage: setup-prereq <photon target> <path-to-kube-vmdk>"
+    echo "Target should be a URL like https://192.0.2.1"
+    echo ""
+    echo "This will create the following, based on the configuration in config-common.sh"
+    echo "    * A tenant named ${PHOTON_TENANT}"
+    echo "    * A project named ${PHOTON_PROJECT}"
+    echo "    * A VM flavor named ${PHOTON_MASTER_FLAVOR}"
+    echo "    * A disk flavor named ${PHOTON_DISK_FLAVOR}"
+    echo "It will also upload the Kube VMDK"
+    echo ""
+    echo "It creates the tenant with a resource ticket (quota) that may"
+    echo "be inappropriate for your environment. For a production"
+    echo "environment, you should configure these to match your"
+    echo "environment."
+    exit 1
+  fi
+
+  echo "Photon Target: ${PHOTON_TARGET}"
+  echo "Photon VMDK: ${PHOTON_VMDK}"
+}
+
+function set-target {
+  ${PHOTON} target set "${PHOTON_TARGET}" > /dev/null 2>&1
+}
+
+function create-tenant {
+  local rc=0
+  local output
+
+  ${PHOTON} tenant list | grep -q "\t${PHOTON_TENANT}$" > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    echo "Tenant ${PHOTON_TENANT} already made, skipping"
+  else
+    echo "Making tenant ${PHOTON_TENANT}"
+    rc=0
+    output=$(${PHOTON} tenant create "${PHOTON_TENANT}" 2>&1) || {
+      echo "ERROR: Could not create tenant \"${PHOTON_TENANT}\", exiting"
+      echo "Output from tenant creation:"
+      echo "${output}"
+      exit 1
+    }
+  fi
+  ${PHOTON} tenant set "${PHOTON_TENANT}" > /dev/null 2>&1
+}
+
+function create-project {
+  local rc=0
+  local output
+
+  ${PHOTON} project list | grep -q "\t${PHOTON_PROJECT}\t" > /dev/null 2>&1  || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    echo "Project ${PHOTON_PROJECT} already made, skipping"
+  else
+    echo "Making project ${PHOTON_PROJECT}"
+    rc=0
+    output=$(${PHOTON} resource-ticket create --tenant "${PHOTON_TENANT}" --name "${PHOTON_TENANT}-resources" --limits "${SETUP_TICKET_SPEC}" 2>&1) || {
+      echo "ERROR: Could not create resource ticket, exiting"
+      echo "Output from resource ticket creation:"
+      echo "${output}"
+      exit 1
+    }
+
+    rc=0
+    output=$(${PHOTON} project create --tenant "${PHOTON_TENANT}" --resource-ticket "${PHOTON_TENANT}-resources" --name "${PHOTON_PROJECT}" --limits "${SETUP_PROJECT_SPEC}" 2>&1) || {
+      echo "ERROR: Could not create project \"${PHOTON_PROJECT}\", exiting"
+      echo "Output from project creation:"
+      echo "${output}"
+      exit 1
+    }
+  fi
+  ${PHOTON} project set "${PHOTON_PROJECT}"
+}
+
+function create-vm-flavor {
+  local flavor_name=${1}
+  local flavor_spec=${2}
+  local rc=0
+  local output
+
+  ${PHOTON} flavor list | grep -q "\t${flavor_name}\t" > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    check-flavor-ready "${flavor_name}"
+    echo "Flavor ${flavor_name} already made, skipping"
+  else
+    echo "Making VM flavor ${flavor_name}"
+    rc=0
+    output=$(${PHOTON} -n flavor create --name "${flavor_name}" --kind "vm" --cost "${flavor_spec}" 2>&1) || {
+      echo "ERROR: Could not create vm flavor \"${flavor_name}\", exiting"
+      echo "Output from flavor creation:"
+      echo "${output}"
+      exit 1
+    }
+  fi
+}
+
+function create-disk-flavor {
+  local rc=0
+  local output
+
+  ${PHOTON} flavor list | grep -q "\t${PHOTON_DISK_FLAVOR}\t" > /dev/null 2>&1  || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    check-flavor-ready "${PHOTON_DISK_FLAVOR}"
+    echo "Flavor ${PHOTON_DISK_FLAVOR} already made, skipping"
+  else
+    echo "Making disk flavor ${PHOTON_DISK_FLAVOR}"
+    rc=0
+    output=$(${PHOTON} -n flavor create --name "${PHOTON_DISK_FLAVOR}" --kind "ephemeral-disk" --cost "${SETUP_DISK_FLAVOR_SPEC}" 2>&1) || {
+      echo "ERROR: Could not create disk flavor \"${PHOTON_DISK_FLAVOR}\", exiting"
+      echo "Output from flavor creation:"
+      echo "${output}"
+      exit 1
+    }
+  fi
+}
+
+function check-flavor-ready {
+  local flavor_name=${1}
+  local rc=0
+
+  local flavor_id
+  flavor_id=$(${PHOTON} flavor list | grep  "\t${flavor_name}\t" | awk '{print $1}') || {
+    echo "ERROR: Found ${flavor_name} but cannot find it's id"
+    exit 1
+  }
+
+  ${PHOTON} flavor show "${flavor_id}" | grep "\tREADY\$" > /dev/null 2>&1  || {
+    echo "ERROR: Flavor \"${flavor_name}\" already exists but is not READY. Please delete or fix it."
+    exit 1
+  }
+}
+
+function create-image {
+  local rc=0
+  local num_images
+  local output
+
+  ${PHOTON} image list | grep "\t${PHOTON_IMAGE}\t" | grep -q ERROR > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    echo "Warning: You have at least one ${PHOTON_IMAGE} image in the ERROR state. You may want to investigate."
+    echo "Images in the ERROR state will be ignored."
+  fi
+
+  rc=0
+  # We don't use grep -c because it exists non-zero when there are no matches, tell shellcheck
+  # shellcheck disable=SC2126 
+  num_images=$(${PHOTON} image list | grep "\t${PHOTON_IMAGE}\t" | grep READY | wc -l)
+  if [[ "${num_images}" -gt 1 ]]; then
+    echo "Warning: You have more than one good ${PHOTON_IMAGE} image. You may want to remove duplicates."
+  fi
+
+  ${PHOTON} image list | grep "\t${PHOTON_IMAGE}\t" | grep -q READY > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    echo "Image ${PHOTON_VMDK} already uploaded, skipping"
+  else
+    echo "Uploading image ${PHOTON_VMDK}"
+    rc=0
+    output=$(${PHOTON} image create "${PHOTON_VMDK}" -n "${PHOTON_IMAGE}" -i EAGER 2>&1) || {
+      echo "ERROR: Could not upload image, exiting"
+      echo "Output from image create:"
+      echo "${output}"
+      exit 1
+    }
+  fi
+}
+
+# We don't want silent pipeline failure: we check for failure
+set +o pipefail
+
+parse-cmd-line "$@"
+main
diff --git a/cluster/photon-controller/templates/README b/cluster/photon-controller/templates/README
new file mode 100644
index 0000000000..b91d629fa0
--- /dev/null
+++ b/cluster/photon-controller/templates/README
@@ -0,0 +1,4 @@
+The scripts in this directory are not meant to be invoked
+directly. Instead they are partial scripts that are combined into full
+scripts by util.sh and are run on the Kubernetes nodes are part of the
+setup. 
diff --git a/cluster/photon-controller/templates/create-dynamic-salt-files.sh b/cluster/photon-controller/templates/create-dynamic-salt-files.sh
new file mode 100755
index 0000000000..af69f71388
--- /dev/null
+++ b/cluster/photon-controller/templates/create-dynamic-salt-files.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#generate token files
+
+KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
+if [[ ! -f "${known_tokens_file}" ]]; then
+
+  mkdir -p /srv/salt-overlay/salt/kube-apiserver
+  known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
+  (umask u=rw,go= ;
+   echo "$KUBELET_TOKEN,kubelet,kubelet" > $known_tokens_file;
+   echo "$KUBE_PROXY_TOKEN,kube_proxy,kube_proxy" >> $known_tokens_file)
+
+  mkdir -p /srv/salt-overlay/salt/kubelet
+  kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
+  (umask u=rw,go= ; echo "{\"BearerToken\": \"$KUBELET_TOKEN\", \"Insecure\": true }" > $kubelet_auth_file)
+  kubelet_kubeconfig_file="/srv/salt-overlay/salt/kubelet/kubeconfig"
+
+  mkdir -p /srv/salt-overlay/salt/kubelet
+  (umask 077;
+  cat > "${kubelet_kubeconfig_file}" << EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    insecure-skip-tls-verify: true
+  name: local
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: service-account-context
+current-context: service-account-context
+users:
+- name: kubelet
+  user:
+    token: ${KUBELET_TOKEN}
+EOF
+)
+
+
+  mkdir -p /srv/salt-overlay/salt/kube-proxy
+  kube_proxy_kubeconfig_file="/srv/salt-overlay/salt/kube-proxy/kubeconfig"
+  # Make a kubeconfig file with the token.
+  # TODO(etune): put apiserver certs into secret too, and reference from authfile,
+  # so that "Insecure" is not needed.
+  (umask 077;
+  cat > "${kube_proxy_kubeconfig_file}" << EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    insecure-skip-tls-verify: true
+  name: local
+contexts:
+- context:
+    cluster: local
+    user: kube-proxy
+  name: service-account-context
+current-context: service-account-context
+users:
+- name: kube-proxy
+  user:
+    token: ${KUBE_PROXY_TOKEN}
+EOF
+)
+
+  # Generate tokens for other "service accounts".  Append to known_tokens.
+  #
+  # NB: If this list ever changes, this script actually has to
+  # change to detect the existence of this file, kill any deleted
+  # old tokens and add any new tokens (to handle the upgrade case).
+  service_accounts=("system:scheduler" "system:controller_manager" "system:logging" "system:monitoring" "system:dns")
+  for account in "${service_accounts[@]}"; do
+    token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
+    echo "${token},${account},${account}" >> "${known_tokens_file}"
+  done
+fi
+
+readonly BASIC_AUTH_FILE="/srv/salt-overlay/salt/kube-apiserver/basic_auth.csv"
+if [[ ! -e "${BASIC_AUTH_FILE}" ]]; then
+  mkdir -p /srv/salt-overlay/salt/kube-apiserver
+  (umask 077;
+    echo "${KUBE_PASSWORD},${KUBE_USER},admin" > "${BASIC_AUTH_FILE}")
+fi
+
+
+# Create the overlay files for the salt tree.  We create these in a separate
+# place so that we can blow away the rest of the salt configs on a kube-push and
+# re-apply these.
+
+mkdir -p /srv/salt-overlay/pillar
+cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
+instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
+node_instance_prefix: $NODE_INSTANCE_PREFIX
+service_cluster_ip_range: $SERVICE_CLUSTER_IP_RANGE
+enable_cluster_monitoring: "${ENABLE_CLUSTER_MONITORING:-none}"
+enable_cluster_logging: "${ENABLE_CLUSTER_LOGGING:false}"
+enable_cluster_ui: "${ENABLE_CLUSTER_UI:true}"
+enable_node_logging: "${ENABLE_NODE_LOGGING:false}"
+logging_destination: $LOGGING_DESTINATION
+elasticsearch_replicas: $ELASTICSEARCH_LOGGING_REPLICAS
+enable_cluster_dns: "${ENABLE_CLUSTER_DNS:-false}"
+dns_replicas: ${DNS_REPLICAS:-1}
+dns_server: $DNS_SERVER_IP
+dns_domain: $DNS_DOMAIN
+e2e_storage_test_environment: "${E2E_STORAGE_TEST_ENVIRONMENT:-false}"
+cluster_cidr: "$NODE_IP_RANGES"
+allocate_node_cidrs: "${ALLOCATE_NODE_CIDRS:-true}"
+admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+EOF
+
+mkdir -p /srv/salt-overlay/salt/nginx
+echo ${MASTER_HTPASSWD} > /srv/salt-overlay/salt/nginx/htpasswd
diff --git a/cluster/photon-controller/templates/hostname.sh b/cluster/photon-controller/templates/hostname.sh
new file mode 100755
index 0000000000..32ec5606c8
--- /dev/null
+++ b/cluster/photon-controller/templates/hostname.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remove kube.vm from /etc/hosts
+sed -i -e 's/\b\w\+.vm\b//' /etc/hosts
+
+# Update hostname in /etc/hosts and /etc/hostname
+sed -i -e "s/\\bkube\\b/${MY_NAME}/g" /etc/host{s,name}
+hostname ${MY_NAME}
diff --git a/cluster/photon-controller/templates/install-release.sh b/cluster/photon-controller/templates/install-release.sh
new file mode 100755
index 0000000000..8d4229f015
--- /dev/null
+++ b/cluster/photon-controller/templates/install-release.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script assumes that the environment variable SERVER_BINARY_TAR contains
+# the release tar to download and unpack. It is meant to be pushed to the
+# master and run.
+
+echo "Unpacking Salt tree"
+rm -rf kubernetes
+tar xzf "${SALT_TAR}"
+
+echo "Running release install script"
+sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR}"
diff --git a/cluster/photon-controller/templates/salt-master.sh b/cluster/photon-controller/templates/salt-master.sh
new file mode 100755
index 0000000000..d83992a4ab
--- /dev/null
+++ b/cluster/photon-controller/templates/salt-master.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use other Debian mirror
+sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
+
+# Prepopulate the name of the Master
+mkdir -p /etc/salt/minion.d
+echo "master: ${MASTER_NAME}" > /etc/salt/minion.d/master.conf
+
+cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  roles:
+    - kubernetes-master
+  cbr-cidr: $MASTER_IP_RANGE
+  cloud: photon-controller
+  master_extra_sans: $MASTER_EXTRA_SANS
+EOF
+
+# Auto accept all keys from minions that try to join
+mkdir -p /etc/salt/master.d
+cat <<EOF >/etc/salt/master.d/auto-accept.conf
+auto_accept: True
+EOF
+
+cat <<EOF >/etc/salt/master.d/reactor.conf
+# React to new minions starting by running highstate on them.
+reactor:
+  - 'salt/minion/*/start':
+    - /srv/reactor/highstate-new.sls
+    - /srv/reactor/highstate-masters.sls
+    - /srv/reactor/highstate-minions.sls
+EOF
+
+# Install Salt
+#
+# We specify -X to avoid a race condition that can cause minion failure to
+# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
+#
+# -M installs the master
+set +x
+curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -M -X
+set -x
diff --git a/cluster/photon-controller/templates/salt-minion.sh b/cluster/photon-controller/templates/salt-minion.sh
new file mode 100755
index 0000000000..599f246fff
--- /dev/null
+++ b/cluster/photon-controller/templates/salt-minion.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use other Debian mirror
+sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list
+
+# Resolve hostname of master
+if ! grep -q $KUBE_MASTER /etc/hosts; then
+  echo "Adding host entry for $KUBE_MASTER"
+  echo "${KUBE_MASTER_IP} ${KUBE_MASTER}" >> /etc/hosts
+fi
+
+# Prepopulate the name of the Master
+mkdir -p /etc/salt/minion.d
+echo "master: ${KUBE_MASTER}" > /etc/salt/minion.d/master.conf
+
+# Turn on debugging for salt-minion
+# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion
+
+# Our minions will have a pool role to distinguish them from the master.
+#
+# Setting the "minion_ip" here causes the kubelet to use its IP for
+# identification instead of its hostname.
+#
+cat <<EOF >/etc/salt/minion.d/grains.conf
+grains:
+  hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}')
+  roles:
+    - kubernetes-pool
+    - kubernetes-pool-photon-controller
+  cloud: photon-controller
+EOF
+
+# Install Salt
+#
+# We specify -X to avoid a race condition that can cause minion failure to
+# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
+curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X
diff --git a/cluster/photon-controller/util.sh b/cluster/photon-controller/util.sh
new file mode 100755
index 0000000000..55bf9e9733
--- /dev/null
+++ b/cluster/photon-controller/util.sh
@@ -0,0 +1,1092 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
+# shellcheck source=./config-common.sh
+source "${KUBE_ROOT}/cluster/photon-controller/config-common.sh"
+# shellcheck source=./config-default.sh
+source "${KUBE_ROOT}/cluster/photon-controller/${KUBE_CONFIG_FILE-"config-default.sh"}"
+# shellcheck source=../common.sh
+source "${KUBE_ROOT}/cluster/common.sh"
+
+readonly PHOTON="photon -n"
+
+# Naming scheme for VMs (masters & nodes)
+readonly MASTER_NAME="${INSTANCE_PREFIX}-master"
+
+# shell check claims this doesn't work because you can't use a variable in a brace
+# range. It does work because we're calling eval. 
+# shellcheck disable=SC2051 
+readonly NODE_NAMES=($(eval echo "${INSTANCE_PREFIX}"-node-{1.."${NUM_NODES}"}))
+
+#####################################################################
+#
+# Public API
+#
+#####################################################################
+
+#
+# detect-master will query Photon Controller for the Kubernetes master.
+# It assumes that the VM name for the master is unique.
+# It will set KUBE_MASTER_ID to be the VM ID of the master
+# It will set KUBE_MASTER_IP to be the IP address of the master
+# If the silent parameter is passed, it will not print when the master
+# is found: this is used internally just to find the MASTER
+#
+function detect-master {
+  local silent=${1:-""}
+  local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
+
+  KUBE_MASTER=${MASTER_NAME}
+  KUBE_MASTER_ID=${KUBE_MASTER_ID:-""}
+  KUBE_MASTER_IP=${KUBE_MASTER_IP:-""}
+
+  # We don't want silent failure: we check for failure
+  set +o pipefail
+  if [[ -z ${KUBE_MASTER_ID} ]]; then
+    KUBE_MASTER_ID=$(${PHOTON} vm list ${tenant_args} | grep $'\t'"kubernetes-master"$'\t' | awk '{print $1}')
+  fi
+  if [[ -z ${KUBE_MASTER_ID} ]]; then
+    kube::log::error "Could not find Kubernetes master node ID. Make sure you've launched a cluster with kube-up.sh"
+    exit 1
+  fi
+
+  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
+      # Make sure to ignore lines where it's not attached to a portgroup
+      # Make sure to ignore lines that have a network interface but no address
+    KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
+  fi
+  if [[ -z "${KUBE_MASTER_IP-}" ]]; then
+    kube::log::error "Could not find Kubernetes master node IP. Make sure you've launched a cluster with 'kube-up.sh'" >&2
+    exit 1
+  fi
+  if [[ -z ${silent} ]]; then
+    kube::log::status "Master: $KUBE_MASTER ($KUBE_MASTER_IP)"
+  fi
+  # Reset default set in common.sh
+  set -o pipefail
+}
+
+#
+# detect-nodes will query Photon Controller for the Kubernetes nodes
+# It assumes that the VM name for the nodes are unique.
+# It assumes that NODE_NAMES has been set
+# It will set KUBE_NODE_IP_ADDRESSES to be the VM IPs of the nodes
+# It will set the KUBE_NODE_IDS to be the VM IDs of the nodes
+# If the silent parameter is passed, it will not print when the nodes
+# are found: this is used internally just to find the MASTER
+#
+function detect-nodes {
+  local silent=${1:-""}
+  local failure=0
+  local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
+
+  KUBE_NODE_IP_ADDRESSES=()
+  KUBE_NODE_IDS=()
+  # We don't want silent failure: we check for failure
+  set +o pipefail
+  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
+
+    local node_id
+    node_id=$(${PHOTON} vm list ${tenant_args} | grep $'\t'"${NODE_NAMES[${i}]}"$'\t' | awk '{print $1}')
+    if [[ -z ${node_id} ]]; then
+      kube::log::error "Could not find ${NODE_NAMES[${i}]}"
+      failure=1
+    fi
+    KUBE_NODE_IDS+=("${node_id}")
+
+    # Make sure to ignore lines where it's not attached to a portgroup
+    # Make sure to ignore lines that have a network interface but no address
+    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
+    KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
+
+    if [[ -z ${silent} ]]; then
+      kube::log::status "Node: ${NODE_NAMES[${i}]} (${KUBE_NODE_IP_ADDRESSES[${i}]})"
+    fi
+  done
+
+  if [[ ${failure} -ne 0 ]]; then
+    exit 1
+  fi
+  # Reset default set in common.sh
+  set -o pipefail
+}
+
+# Get node names if they are not static.
+function detect-node-names {
+  echo "TODO: detect-node-names" 1>&2
+}
+
+#
+# Verifies that this computer has sufficient software installed
+# so that it can run the rest of the script.
+#
+function verify-prereqs {
+  verify-cmd-in-path photon
+  verify-cmd-in-path ssh
+  verify-cmd-in-path scp
+  verify-cmd-in-path ssh-add
+  verify-cmd-in-path openssl
+  verify-cmd-in-path mkisofs
+}
+
+#
+# The entry point for bringing up a Kubernetes cluster
+#
+function kube-up {
+  verify-prereqs
+  verify-ssh-prereqs
+  verify-photon-config
+  ensure-temp-dir
+
+  find-release-tars
+  find-image-id
+
+  load-or-gen-kube-basicauth
+  gen-cloud-init-iso
+  gen-master-start
+  create-master-vm
+  install-salt-on-master
+
+  gen-node-start
+  install-salt-on-nodes
+
+  detect-nodes -s
+
+  install-kubernetes-on-master
+  install-kubernetes-on-nodes
+
+  wait-master-api
+  wait-node-apis
+
+  setup-pod-routes
+
+  copy-kube-certs
+  kube::log::status "Creating kubeconfig..."
+  create-kubeconfig
+}
+
+# Delete a kubernetes cluster
+function kube-down {
+  detect-master
+  detect-nodes
+
+  pc-delete-vm "${KUBE_MASTER}" "${KUBE_MASTER_ID}"
+  for (( node=0; node<${#KUBE_NODE_IDS[@]}; node++)); do
+    pc-delete-vm "${NODE_NAMES[${node}]}" "${KUBE_NODE_IDS[${node}]}"
+  done
+}
+
+# Update a kubernetes cluster
+function kube-push {
+  echo "TODO: kube-push" 1>&2
+}
+
+# Prepare update a kubernetes component
+function prepare-push {
+  echo "TODO: prepare-push" 1>&2
+}
+
+# Update a kubernetes master
+function push-master {
+  echo "TODO: push-master" 1>&2
+}
+
+# Update a kubernetes node
+function push-node {
+  echo "TODO: push-node" 1>&2
+}
+
+# Execute prior to running tests to build a release if required for env
+function test-build-release {
+  echo "TODO: test-build-release" 1>&2
+}
+
+# Execute prior to running tests to initialize required structure
+function test-setup {
+  echo "TODO: test-setup" 1>&2
+}
+
+# Execute after running tests to perform any required clean-up
+function test-teardown {
+  echo "TODO: test-teardown" 1>&2
+}
+
+#####################################################################
+#
+# Internal functions
+#
+#####################################################################
+
+#
+# Uses Photon Controller to make a VM
+# Takes two parameters:
+#   - The name of the VM (Assumed to be unique)
+#   - The name of the flavor to create the VM (Assumed to be unique)
+#
+# It assumes that the variables in config-common.sh (PHOTON_TENANT, etc)
+# are set correctly.
+#
+# It also assumes the cloud-init ISO has been generated
+#
+# When it completes, it sets two environment variables for use by the
+# caller: _VM_ID (the ID of the created VM) and _VM_IP (the IP address
+# of the created VM)
+#
+function pc-create-vm {
+  local vm_name="${1}"
+  local vm_flavor="${2}"
+  local rc=0
+  local i=0
+
+  # Create the VM
+  local tenant_args="--tenant ${PHOTON_TENANT} --project ${PHOTON_PROJECT}"
+  local vm_args="--name ${vm_name} --image ${PHOTON_IMAGE_ID} --flavor ${vm_flavor}"
+  local disk_args="disk-1 ${PHOTON_DISK_FLAVOR} boot=true"
+
+  rc=0
+  _VM_ID=$(${PHOTON} vm create ${tenant_args} ${vm_args} --disks "${disk_args}" 2>&1) || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    kube::log::error "Failed to create VM. Error output:"
+    echo "${_VM_ID}"
+    exit 1
+  fi
+  kube::log::status "Created VM ${vm_name}: ${_VM_ID}"
+
+  # Start the VM
+  # Note that the VM has cloud-init in it, and we attach an ISO that
+  # contains a user-data.txt file for cloud-init. When the VM starts,
+  # cloud-init will temporarily mount the ISO and configure the VM
+  # Our user-data will configure the 'kube' user and set up the ssh
+  # authorized keys to allow us to ssh to the VM and do further work.
+  run-cmd "${PHOTON} vm attach-iso -p ${KUBE_TEMP}/cloud-init.iso ${_VM_ID}"
+  run-cmd "${PHOTON} vm start ${_VM_ID}"
+  kube::log::status "Started VM ${vm_name}, waiting for network address..."
+
+   # Wait for the VM to be started and connected to the network
+  have_network=0
+  for i in $(seq 120); do
+    # photon -n vm networks print several fields:
+    # NETWORK MAC IP GATEWAY CONNECTED?
+    # We wait until CONNECTED is True
+    rc=0
+    networks=$(${PHOTON} vm networks "${_VM_ID}") || rc=$?
+    if [[ ${rc} -ne 0 ]]; then
+      kube::log::error "'${PHOTON} vm networks ${_VM_ID}' failed. Error output: "
+      echo "${networks}"
+    fi
+    networks=$(echo "${networks}" | grep True) || rc=$?
+    if [[ ${rc} -eq 0 ]]; then
+      have_network=1
+      break;
+    fi
+    sleep 1
+  done
+
+  # Fail if the VM didn't come up
+  if [[ ${have_network} -eq 0 ]]; then
+    kube::log::error "VM ${vm_name} failed to start up: no IP was found"
+    exit 1
+  fi
+
+  # Find the IP address of the VM
+  _VM_IP=$(${PHOTON} vm networks "${_VM_ID}" | head -1 | awk -F'\t' '{print $3}')
+  kube::log::status "VM ${vm_name} has IP: ${_VM_IP}"
+}
+
+#
+# Delete one of our VMs
+# If it is STARTED, it will be stopped first.
+#
+function pc-delete-vm {
+  local vm_name="${1}"
+  local vm_id="${2}"
+  local rc=0
+
+  kube::log::status "Deleting VM ${vm_name}"
+  ${PHOTON} vm show "${vm_id}" | head -1 | grep STARTED > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    ${PHOTON} vm stop "${vm_id}" > /dev/null 2>&1 || rc=$?
+    if [[ ${rc} -ne 0 ]]; then
+      kube::log::error "Error: could not stop ${vm_name} ($vm_id)"
+      kube::log::error "Please investigate and stop manually"
+      return
+    fi
+  fi
+
+  rc=0
+  ${PHOTON} vm delete "${vm_id}" > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    kube::log::error "Error: could not delete ${vm_name} ($vm_id)"
+    kube::log::error "Please investigate and delete manually"
+  fi
+}
+
+#
+# Looks for the image named PHOTON_IMAGE
+# Sets PHOTON_IMAGE_ID to be the id of that image.
+# We currently assume there is exactly one image with name
+#
+function find-image-id {
+  local rc=0
+  PHOTON_IMAGE_ID=$(${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | head -1 | grep READY | awk -F'\t' '{print $1}')
+  if [[ ${rc} -ne 0 ]]; then
+    kube::log::error "Cannot find image \"${PHOTON_IMAGE}\""
+    fail=1
+  fi
+}
+
+#
+# Generate an ISO with a single file called user-data.txt
+# This ISO will be used to configure cloud-init (which is already
+# on the VM). We will tell cloud-init to create the kube user/group
+# and give ourselves the ability to ssh to the VM with ssh. We also
+# allow people to ssh with the same password that was randomly
+# generated for access to Kubernetes as a backup method.
+#
+# Assumes environment variables:
+#   - VM_USER
+#   - KUBE_PASSWORD (randomly generated password)
+#
+function gen-cloud-init-iso {
+  local password_hash
+  password_hash=$(openssl passwd -1 "${KUBE_PASSWORD}")
+
+  local ssh_key
+  ssh_key=$(ssh-add -L | head -1)
+
+  # Make the user-data file that will be used by cloud-init
+  (
+    echo "#cloud-config"
+    echo ""
+    echo "groups:"
+    echo "  - ${VM_USER}"
+    echo ""
+    echo "users:"
+    echo "  - name: ${VM_USER}"
+    echo "    gecos: Kubernetes"
+    echo "    primary-group: ${VM_USER}"
+    echo "    lock-passwd: false"
+    echo "    passwd: ${password_hash}"
+    echo "    ssh-authorized-keys: "
+    echo "      - ${ssh_key}"
+    echo "    sudo: ALL=(ALL) NOPASSWD:ALL"
+    echo "    shell: /bin/bash"
+    echo ""
+    echo "hostname:"
+    echo "  - hostname: kube"
+  ) > "${KUBE_TEMP}/user-data.txt"
+
+  # Make the ISO that will contain the user-data
+  # The -rock option means that we'll generate real filenames (long and with case)
+  run-cmd "mkisofs -rock -o ${KUBE_TEMP}/cloud-init.iso ${KUBE_TEMP}/user-data.txt"
+}
+
+#
+# Generate a script used to install salt on the master
+# It is placed into $KUBE_TEMP/master-start.sh
+#
+function gen-master-start {
+  python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
+    -b -c "${KUBE_TEMP}/htpasswd" "${KUBE_USER}" "${KUBE_PASSWORD}"
+  local htpasswd
+  htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
+
+  # This calculation of the service IP should work, but if you choose an
+  # alternate subnet, there's a small chance you'd need to modify the
+  # service_ip, below.  We'll choose an IP like 10.244.240.1 by taking
+  # the first three octets of the SERVICE_CLUSTER_IP_RANGE and tacking
+  # on a .1
+  local octets
+  local service_ip
+  octets=($(echo "${SERVICE_CLUSTER_IP_RANGE}" | sed -e 's|/.*||' -e 's/\./ /g'))
+  ((octets[3]+=1))
+  service_ip=$(echo "${octets[*]}" | sed 's/ /./g')
+  MASTER_EXTRA_SANS="IP:${service_ip},DNS:${MASTER_NAME},${MASTER_EXTRA_SANS}"
+
+  (
+    echo "#! /bin/bash"
+    echo "readonly MY_NAME=${MASTER_NAME}"
+    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh"
+    echo "cd /home/kube/cache/kubernetes-install"
+    echo "readonly MASTER_NAME='${MASTER_NAME}'"
+    echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE}'"
+    echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
+    echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-node'"
+    echo "readonly NODE_IP_RANGES='${NODE_IP_RANGES}'"
+    echo "readonly SERVICE_CLUSTER_IP_RANGE='${SERVICE_CLUSTER_IP_RANGE}'"
+    echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
+    echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
+    echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
+    echo "readonly ENABLE_CLUSTER_UI='${ENABLE_CLUSTER_UI:-false}'"
+    echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
+    echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
+    echo "readonly KUBE_USER='${KUBE_USER:-}'"
+    echo "readonly KUBE_PASSWORD='${KUBE_PASSWORD:-}'"
+    echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'"
+    echo "readonly SALT_TAR='${SALT_TAR##*/}'"
+    echo "readonly MASTER_HTPASSWD='${htpasswd}'"
+    echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
+    echo "readonly MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
+    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/create-dynamic-salt-files.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/install-release.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/salt-master.sh"
+  ) > "${KUBE_TEMP}/master-start.sh"
+}
+
+#
+# Generate the scripts for each node to install salt
+#
+function gen-node-start {
+  local i
+  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
+    (
+      echo "#! /bin/bash"
+      echo "readonly MY_NAME=${NODE_NAMES[${i}]}"
+      grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/hostname.sh"
+      echo "KUBE_MASTER=${KUBE_MASTER}"
+      echo "KUBE_MASTER_IP=${KUBE_MASTER_IP}"
+      echo "NODE_IP_RANGE=$NODE_IP_RANGES"
+      grep -v "^#" "${KUBE_ROOT}/cluster/photon-controller/templates/salt-minion.sh"
+    ) > "${KUBE_TEMP}/node-start-${i}.sh"
+  done
+}
+
+#
+# Create a script that will run on the Kubernetes master and will run salt
+# to configure the master. We make it a script instead of just running a
+# single ssh command so that we can get logging.
+#
+function gen-master-salt {
+  gen-salt "kubernetes-master"
+}
+
+#
+# Create scripts that will be run on the Kubernetes master. Each of these
+# will invoke salt to configure one of the nodes
+#
+function gen-node-salt {
+  local i
+  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
+    gen-salt "${NODE_NAMES[${i}]}"
+  done
+}
+
+# 
+# Shared implementation for gen-master-salt and gen-node-salt
+# Writes a script that installs Kubernetes with salt
+# The core of the script is simple (run 'salt ... state.highstate')
+# We also do a bit of logging so we can debug problems
+#
+# There is also a funky workaround for an issue with docker 1.9 
+# (elsewhere we peg ourselves to docker 1.9). It's fixed in 1.10, 
+# so we should be able to remove it in the future
+# https://github.com/docker/docker/issues/18113
+# The problem is that sometimes the install (with apt-get) of
+# docker fails. Deleting a file and retrying fixes it.
+#
+# Tell shellcheck to ignore our variables within single quotes: 
+# We're writing a script, not executing it, so this is normal
+# shellcheck disable=SC2016
+function gen-salt {
+  node_name=${1}
+    (
+      echo '#!/bin/bash'
+      echo ''
+      echo "node=${node_name}"
+      echo 'out=/tmp/${node}-salt.out'
+      echo 'log=/tmp/${node}-salt.log'
+      echo ''
+      echo 'echo $(date) >> $log'
+      echo 'salt ${node} state.highstate -t 30 --no-color > ${out}'
+      echo 'grep -E "Failed:[[:space:]]+0" ${out}'
+      echo 'success=$?'
+      echo 'cat ${out} >> ${log}'
+      echo ''
+      echo 'if [[ ${success} -ne 0 ]]; then'
+      echo '  # Did we try to install docker-engine?'
+      echo '  attempted=$(grep docker-engine ${out} | wc -l)'
+      echo '  # Is docker-engine installed?'
+      echo '  installed=$(salt --output=txt ${node} pkg.version docker-engine | wc -l)'
+      echo '  if [[ ${attempted} -ne 0 && ${installed} -eq 0 ]]; then'
+      echo '    echo "Unwedging docker-engine install" >> ${log}'
+      echo '    salt ${node} cmd.run "rm -f /var/lib/docker/network/files/local-kv.db"'
+      echo '  fi'
+      echo 'fi'
+      echo 'exit ${success}'
+    ) > "${KUBE_TEMP}/${node_name}-salt.sh"
+}
+
+#
+# Create the Kubernetes master VM
+# Sets global variables: 
+# - KUBE_MASTER    (Name)
+# - KUBE_MASTER_ID (Photon VM ID)
+# - KUBE_MASTER_IP (IP address)
+#
+function create-master-vm {
+  kube::log::status "Starting master VM..."
+  pc-create-vm "${MASTER_NAME}" "${PHOTON_MASTER_FLAVOR}"
+  KUBE_MASTER=${MASTER_NAME}
+  KUBE_MASTER_ID=${_VM_ID}
+  KUBE_MASTER_IP=${_VM_IP}
+}
+
+# 
+# Install salt on the Kubernetes master
+# Relies on the master-start.sh script created in gen-master-start
+#
+function install-salt-on-master {
+  kube::log::status "Installing salt on master..."
+  upload-server-tars "${MASTER_NAME}" "${KUBE_MASTER_IP}"
+  run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/master-start.sh"
+}
+
+#
+# Installs salt on Kubernetes nodes in parallel
+# Relies on the node-start script created in gen-node-start
+#
+function install-salt-on-nodes {
+  kube::log::status "Creating nodes and installing salt on them..."
+
+  # Start each of the VMs in parallel
+  # In the future, we'll batch this because it doesn't scale well
+  # past 10 or 20 nodes
+  local node
+  for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
+  (
+    pc-create-vm "${NODE_NAMES[${node}]}" "${PHOTON_NODE_FLAVOR}"
+    run-script-remotely "${_VM_IP}" "${KUBE_TEMP}/node-start-${node}.sh"
+  ) &
+  done
+
+  # Wait for the node VM startups to complete
+  local fail=0
+  local job
+  for job in $(jobs -p); do
+    wait "${job}" || fail=$((fail + 1))
+  done
+  if (( fail != 0 )); then
+    kube::log::error "Failed to start ${fail}/${NUM_NODES} nodes"
+    exit 1
+  fi
+}
+
+#
+# Install Kubernetes on the master.
+# This uses the kubernetes-master-salt.sh script created by gen-master-salt
+# That script uses salt to install Kubernetes
+#
+function install-kubernetes-on-master {
+  # Wait until salt-master is running: it may take a bit
+  try-until-success-ssh "${KUBE_MASTER_IP}" \
+    "Waiting for salt-master to start on ${KUBE_MASTER}" \
+    "pgrep salt-master"
+  gen-master-salt
+  copy-file-to-vm "${_VM_IP}" "${KUBE_TEMP}/kubernetes-master-salt.sh" "/tmp/kubernetes-master-salt.sh"
+  try-until-success-ssh "${KUBE_MASTER_IP}" \
+    "Installing Kubernetes on ${KUBE_MASTER} via salt" \
+    "sudo /bin/bash /tmp/kubernetes-master-salt.sh"
+}
+
+#
+# Install Kubernetes on the the nodes in parallel
+# This uses the kubernetes-master-salt.sh script created by gen-node-salt
+# That script uses salt to install Kubernetes
+#
+function install-kubernetes-on-nodes {
+  gen-node-salt
+
+  # Run in parallel to bring up the cluster faster
+  # TODO: Batch this so that we run up to N in parallel, so
+  # we don't overload this machine or the salt master
+  local node
+  for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
+    (
+      copy-file-to-vm "${_VM_IP}" "${KUBE_TEMP}/${NODE_NAMES[${node}]}-salt.sh" "/tmp/${NODE_NAMES[${node}]}-salt.sh"
+      try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
+        "Waiting for salt-master to start on ${NODE_NAMES[${node}]}" \
+        "pgrep salt-minion"
+      try-until-success-ssh "${KUBE_MASTER_IP}" \
+        "Installing Kubernetes on ${NODE_NAMES[${node}]} via salt" \
+        "sudo /bin/bash /tmp/${NODE_NAMES[${node}]}-salt.sh"
+    ) &
+  done
+
+  # Wait for the Kubernetes installations to complete
+  local fail=0
+  local job
+  for job in $(jobs -p); do
+    wait "${job}" || fail=$((fail + 1))
+  done
+  if (( fail != 0 )); then
+    kube::log::error "Failed to start install Kubernetes on ${fail} out of ${NUM_NODES} nodess"
+    exit 1
+  fi
+}
+
+#
+# Upload the Kubernetes tarballs to the master
+#
+function upload-server-tars {
+  vm_name=${1}
+  vm_ip=${2}
+
+  run-ssh-cmd "${vm_ip}" "mkdir -p /home/kube/cache/kubernetes-install"
+
+  local tar
+  for tar in "${SERVER_BINARY_TAR}" "${SALT_TAR}"; do
+    local base_tar
+    base_tar=$(basename "${tar}")
+    kube::log::status "Uploading ${base_tar} to ${vm_name}..."
+    copy-file-to-vm "${vm_ip}" "${tar}" "/home/kube/cache/kubernetes-install/${tar##*/}"
+  done
+}
+
+#
+# Wait for the Kubernets healthz API to be responsive on the master
+#
+function wait-master-api {
+  local curl_creds="--insecure --user ${KUBE_USER}:${KUBE_PASSWORD}"
+  local curl_output="--fail --output /dev/null --silent"
+  local curl_net="--max-time 1"
+
+  try-until-success "Waiting for Kubernetes API on ${KUBE_MASTER}" \
+    "curl ${curl_creds} ${curl_output} ${curl_net} https://${KUBE_MASTER_IP}/healthz"
+}
+
+#
+# Wait for the Kubernetes healthz API to be responsive on each node
+#
+function wait-node-apis {
+  local curl_output="--fail --output /dev/null --silent"
+  local curl_net="--max-time 1"
+
+  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
+    try-until-success "Waiting for Kubernetes API on ${NODE_NAMES[${i}]}..." \
+      "curl ${curl_output} ${curl_net} http://${KUBE_NODE_IP_ADDRESSES[${i}]}:10250/healthz"
+  done
+}
+
+#
+# Configure the nodes so the pods can communicate
+# Each node will have a bridge named cbr0 for the NODE_IP_RANGES
+# defined in config-default.sh. This finds the IP subnet (assigned
+# by Kubernetes) to nodes and configures routes so they can communicate
+#
+# Also configure the master to be able to talk to the nodes. This is
+# useful so that you can get to the UI from the master.
+#
+function setup-pod-routes {
+  local node
+
+  KUBE_NODE_BRIDGE_NETWORK=()
+  for (( node=0; node<${#NODE_NAMES[@]}; node++)); do
+
+    # This happens in two steps (wait for an address, wait for a non 172.x.x.x address)
+    # because it's both simpler and more clear what's happening.
+    try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
+      "Waiting for cbr0 bridge on ${NODE_NAMES[${node}]} to have an address"  \
+      'sudo ifconfig cbr0  | grep -oP "inet addr:\K\S+"'
+
+    try-until-success-ssh "${KUBE_NODE_IP_ADDRESSES[${node}]}" \
+      "Waiting for cbr0 bridge on ${NODE_NAMES[${node}]} to have correct address"  \
+      'sudo ifconfig cbr0  | grep -oP "inet addr:\K\S+" | grep -v  "^172."'
+
+    run-ssh-cmd "${KUBE_NODE_IP_ADDRESSES[${node}]}" 'sudo ip route show | grep -E "dev cbr0" | cut -d " " -f1'
+    KUBE_NODE_BRIDGE_NETWORK+=(${_OUTPUT})
+    kube::log::status "cbr0 on ${NODE_NAMES[${node}]} is ${_OUTPUT}"
+  done
+
+  local i
+  local j
+  for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
+    kube::log::status "Configuring pod routes on ${NODE_NAMES[${i}]}..."
+    run-ssh-cmd "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
+    for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
+      if [[ "${i}" != "${j}" ]]; then
+        run-ssh-cmd "${KUBE_NODE_IP_ADDRESSES[${i}]}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${j}]} gw ${KUBE_NODE_IP_ADDRESSES[${j}]}"
+      fi
+    done
+  done
+}
+
+#
+# Copy the certificate/key from the Kubernetes master
+# These are used to create the kubeconfig file, which allows
+# users to use kubectl easily
+#
+# We also set KUBE_CERT, KUBE_KEY, CA_CERT, and CONTEXT because they
+# are needed by create-kubeconfig from common.sh to generate
+# the kube config file.
+#
+function copy-kube-certs {
+  local cert="kubecfg.crt"
+  local key="kubecfg.key"
+  local ca="ca.crt"
+  local cert_dir="/srv/kubernetes"
+
+  kube::log::status "Copying credentials from ${KUBE_MASTER}"
+
+  # Set global environment variables: needed by create-kubeconfig
+  # in common.sh
+  export KUBE_CERT="${KUBE_TEMP}/${cert}"
+  export KUBE_KEY="${KUBE_TEMP}/${key}"
+  export CA_CERT="${KUBE_TEMP}/${ca}"
+  export CONTEXT="photon-${INSTANCE_PREFIX}"
+
+  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${cert}"
+  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${key}"
+  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 644 ${cert_dir}/${ca}"
+
+  copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${cert}" "${KUBE_CERT}"
+  copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${key}"  "${KUBE_KEY}"
+  copy-file-from-vm "${KUBE_MASTER_IP}" "${cert_dir}/${ca}"   "${CA_CERT}"
+
+  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${cert}"
+  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${key}"
+  run-ssh-cmd "${KUBE_MASTER_IP}" "sudo chmod 600 ${cert_dir}/${ca}"
+}
+
+#
+# Copies a script to a VM and runs it
+# Parameters:
+#   - IP of VM
+#   - Path to local file
+#
+function run-script-remotely {
+  local vm_ip=${1}
+  local local_file="${2}"
+  local base_file
+  local remote_file
+
+  base_file=$(basename "${local_file}")
+  remote_file="/tmp/${base_file}"
+
+  copy-file-to-vm "${vm_ip}" "${local_file}" "${remote_file}"
+  run-ssh-cmd "${vm_ip}" "chmod 700 ${remote_file}"
+  run-ssh-cmd "${vm_ip}" "nohup sudo ${remote_file} < /dev/null 1> ${remote_file}.out 2>&1 &"
+}
+
+#
+# Runs an command on a VM using ssh
+# Parameters:
+#   - (optional) -i to ignore failure
+#   - IP address of the VM
+#   - Command to run
+# Assumes environment variables:
+#   - VM_USER
+#   - SSH_OPTS
+#
+function run-ssh-cmd {
+  local ignore_failure=""
+  if [[ "${1}" = "-i" ]]; then
+    ignore_failure="-i"
+    shift
+  fi
+
+  local vm_ip=${1}
+  shift
+  local cmd=${1}
+
+
+  run-cmd ${ignore_failure} "ssh ${SSH_OPTS} $VM_USER@${vm_ip} $1"
+}
+
+#
+# Uses scp to copy file to VM
+# Parameters:
+#   - IP address of the VM
+#   - Path to local file
+#   - Path to remote file
+# Assumes environment variables:
+#   - VM_USER
+#   - SSH_OPTS
+#
+function copy-file-to-vm {
+  local vm_ip=${1}
+  local local_file=${2}
+  local remote_file=${3}
+
+  run-cmd "scp ${SSH_OPTS} ${local_file} ${VM_USER}@${vm_ip}:${remote_file}"
+}
+
+function copy-file-from-vm {
+  local vm_ip=${1}
+  local remote_file=${2}
+  local local_file=${3}
+
+  run-cmd "scp ${SSH_OPTS} ${VM_USER}@${vm_ip}:${remote_file} ${local_file}"
+}
+
+#
+# Run a command, print nice error output
+# Used by copy-file-to-vm and run-ssh-cmd
+#
+function run-cmd {
+  local rc=0
+  local ignore_failure=""
+  if [[ "${1}" = "-i" ]]; then
+    ignore_failure=${1}
+    shift
+  fi
+
+  local cmd=$1
+  local output
+  output=$(${cmd} 2>&1) || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    if [[ -z "${ignore_failure}" ]]; then
+      kube::log::error "Failed to run command: ${cmd} Output:"
+      echo "${output}"
+      exit 1
+    fi
+  fi
+  _OUTPUT=${output}
+  return ${rc}
+}
+
+#
+# After the initial VM setup, we use SSH with keys to access the VMs
+# This requires an SSH agent, so we verify that it's running
+#
+function verify-ssh-prereqs {
+  kube::log::status "Validating SSH configuration..."
+  local rc
+
+  rc=0
+  ssh-add -L 1> /dev/null 2> /dev/null || rc=$?
+  # "Could not open a connection to your authentication agent."
+  if [[ "${rc}" -eq 2 ]]; then
+    # ssh agent wasn't running, so start it and ensure we stop it
+    eval "$(ssh-agent)" > /dev/null
+    trap-add "kill ${SSH_AGENT_PID}" EXIT
+  fi
+
+  rc=0
+  ssh-add -L 1> /dev/null 2> /dev/null || rc=$?
+  # "The agent has no identities."
+  if [[ "${rc}" -eq 1 ]]; then
+  # Try adding one of the default identities, with or without passphrase.
+    ssh-add || true
+  fi
+
+  # Expect at least one identity to be available.
+  if ! ssh-add -L 1> /dev/null 2> /dev/null; then
+    kube::log::error "Could not find or add an SSH identity."
+    kube::log::error "Please start ssh-agent, add your identity, and retry."
+    exit 1
+  fi
+}
+
+#
+# Verify that Photon Controller has been configured in the way we expect. Specifically
+# - Have the flavors been created?
+# - Has the image been uploaded?
+# TODO: Check the tenant and project as well.
+function verify-photon-config {
+  kube::log::status "Validating Photon configuration..."
+
+  # We don't want silent failure: we check for failure
+  set +o pipefail
+
+  verify-photon-flavors
+  verify-photon-image
+  verify-photon-tenant
+
+  # Reset default set in common.sh
+  set -o pipefail
+}
+
+#
+# Verify that the VM and disk flavors have been created
+#
+function verify-photon-flavors {
+  local rc=0
+
+  ${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_MASTER_FLAVOR}$" > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    kube::log::error "ERROR: Cannot find VM flavor named ${PHOTON_MASTER_FLAVOR}"
+    exit 1
+  fi
+
+  if [[ "${PHOTON_MASTER_FLAVOR}" != "${PHOTON_NODE_FLAVOR}" ]]; then
+    rc=0
+    ${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_NODE_FLAVOR}$" > /dev/null 2>&1 || rc=$?
+    if [[ ${rc} -ne 0 ]]; then
+      kube::log::error "ERROR: Cannot find VM flavor named ${PHOTON_NODE_FLAVOR}"
+      exit 1
+    fi
+  fi
+
+  ${PHOTON} flavor list | awk -F'\t' '{print $2}' | grep -q "^${PHOTON_DISK_FLAVOR}$" > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    kube::log::error "ERROR: Cannot find disk flavor named ${PHOTON_DISK_FLAVOR}"
+    exit 1
+  fi
+}
+
+#
+# Verify that we have the image we need, and it's not in error state or
+# multiple copies
+#
+function verify-photon-image {
+  local rc
+
+  rc=0
+  ${PHOTON} image list | grep -q $'\t'"${PHOTON_IMAGE}"$'\t'  > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    kube::log::error "ERROR: Cannot find image \"${PHOTON_IMAGE}\""
+    exit 1
+  fi
+
+  rc=0
+  ${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | grep ERROR > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -eq 0 ]]; then
+    echo "Warning: You have at least one ${PHOTON_IMAGE} image in the ERROR state. You may want to investigate."
+    echo "Images in the ERROR state will be ignored."
+  fi
+
+  rc=0
+  num_images=$(${PHOTON} image list | grep $'\t'"${PHOTON_IMAGE}"$'\t' | grep -c READY)
+  if [[ "${num_images}" -gt 1 ]]; then
+    echo "ERROR: You have more than one READY ${PHOTON_IMAGE} image. Ensure there is only one"
+    exit 1
+  fi
+}
+
+function verify-photon-tenant {
+  local rc
+
+  rc=0
+  ${PHOTON} tenant list | grep -q $'\t'"${PHOTON_TENANT}"  > /dev/null 2>&1 || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    echo "ERROR: Cannot find tenant \"${PHOTON_TENANT}\""
+    exit 1
+  fi
+
+  ${PHOTON} project list --tenant "${PHOTON_TENANT}" | grep -q $'\t'"${PHOTON_PROJECT}"$'\t' > /dev/null 2>&1  || rc=$?
+  if [[ ${rc} -ne 0 ]]; then
+    echo "ERROR: Cannot find project \"${PHOTON_PROJECT}\""
+    exit 1
+  fi
+}
+
+#
+# Verifies that a given command is in the PATH
+#
+function verify-cmd-in-path {
+  cmd=${1}
+  which "${cmd}" >/dev/null || {
+    kube::log::error "Can't find ${cmd} in PATH, please install and retry."
+    exit 1
+  }
+}
+
+#
+# Checks that KUBE_TEMP is set, or sets it
+# If it sets it, it also creates the temporary directory
+# and sets up a trap so that we delete it when we exit
+#
+function ensure-temp-dir {
+  if [[ -z ${KUBE_TEMP-} ]]; then
+    KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
+    trap-add "rm -rf '${KUBE_TEMP}'" EXIT
+  fi
+}
+
+#
+# Repeatedly try a command over ssh until it succeeds or until five minutes have passed
+# The timeout isn't exact, since we assume the command runs instantaneously, and
+# it doesn't.
+#
+function try-until-success-ssh {
+  local vm_ip=${1}
+  local cmd_description=${2}
+  local cmd=${3}
+  local timeout=600
+  local sleep_time=5
+  local max_attempts
+
+  ((max_attempts=timeout/sleep_time))
+
+  kube::log::status "${cmd_description} for up to 10 minutes..."
+  local attempt=0
+  while true; do
+    local rc=0
+    run-ssh-cmd -i "${vm_ip}" "${cmd}" || rc=1
+    if [[ ${rc} != 0 ]]; then
+      if (( attempt == max_attempts )); then
+        kube::log::error "Failed, cannot proceed: you may need to retry to log into the VM to debug"
+        exit 1
+      fi
+    else
+      break
+    fi
+    attempt=$((attempt+1))
+    sleep ${sleep_time}
+  done
+}
+
+function try-until-success {
+  local cmd_description=${1}
+  local cmd=${2}
+  local timeout=600
+  local sleep_time=5
+  local max_attempts
+
+  ((max_attempts=timeout/sleep_time))
+
+  kube::log::status "${cmd_description} for up to 10 minutes..."
+  local attempt=0
+  while true; do
+    local rc=0
+    run-cmd -i "${cmd}" || rc=1
+    if [[ ${rc} != 0 ]]; then
+      if (( attempt == max_attempts )); then
+        kube::log::error "Failed, cannot proceed"
+        exit 1
+      fi
+    else
+      break
+    fi
+    attempt=$((attempt+1))
+    sleep ${sleep_time}
+  done
+}
+
+#
+# Sets up a trap handler
+#
+function trap-add {
+  local handler="${1}"
+  local signal="${2-EXIT}"
+  local cur
+
+  cur="$(eval "sh -c 'echo \$3' -- $(trap -p ${signal})")"
+  if [[ -n "${cur}" ]]; then
+    handler="${cur}; ${handler}"
+  fi
+
+  # We want ${handler} to expand now, so tell shellcheck
+  # shellcheck disable=SC2064
+  trap "${handler}" ${signal}
+}
diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls
index 493ea6a77e..bf80af111e 100644
--- a/cluster/saltbase/salt/docker/init.sls
+++ b/cluster/saltbase/salt/docker/init.sls
@@ -47,7 +47,7 @@ docker:
       - pkg: docker-io
 
 {% endif %}
-{% elif grains.cloud is defined and grains.cloud == 'vsphere' and grains.os == 'Debian' and grains.osrelease_info[0] >=8 %}
+{% elif grains.cloud is defined and grains.cloud in ['vsphere', 'photon-controller'] and grains.os == 'Debian' and grains.osrelease_info[0] >=8 %}
 
 {% if pillar.get('is_systemd') %}
 
@@ -69,6 +69,7 @@ docker:
         environment_file: {{ environment_file }}
     - require:
       - file: /opt/kubernetes/helpers/docker-prestart
+      - pkg: docker-engine
 
 # The docker service.running block below doesn't work reliably
 # Instead we run our script which e.g. does a systemd daemon-reload
diff --git a/cluster/saltbase/salt/generate-cert/init.sls b/cluster/saltbase/salt/generate-cert/init.sls
index 5d66fa29d2..b70a1c2440 100644
--- a/cluster/saltbase/salt/generate-cert/init.sls
+++ b/cluster/saltbase/salt/generate-cert/init.sls
@@ -6,7 +6,7 @@
   {% if grains.cloud == 'aws' %}
     {% set cert_ip='_use_aws_external_ip_' %}
   {% endif %}
-  {% if grains.cloud == 'vsphere' %}
+  {% if grains.cloud == 'vsphere' or grains.cloud == 'photon-controller' %}
     {% set cert_ip=grains.ip_interfaces.eth0[0] %}
   {% endif %}
 {% endif %}
diff --git a/cluster/saltbase/salt/kube-apiserver/init.sls b/cluster/saltbase/salt/kube-apiserver/init.sls
index 60c95a165b..92a5208416 100644
--- a/cluster/saltbase/salt/kube-apiserver/init.sls
+++ b/cluster/saltbase/salt/kube-apiserver/init.sls
@@ -1,5 +1,5 @@
 {% if grains.cloud is defined %}
-{% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] %}
+{% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller'] %}
 # TODO: generate and distribute tokens on other cloud providers.
 /srv/kubernetes/known_tokens.csv:
   file.managed:
@@ -12,7 +12,7 @@
 {% endif %}
 {% endif %}
 
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ,'vsphere']  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant' ,'vsphere', 'photon-controller']  %}
 /srv/kubernetes/basic_auth.csv:
   file.managed:
     - source: salt://kube-apiserver/basic_auth.csv
diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
index 52b0089a00..518d8959c1 100644
--- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
+++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest
@@ -14,7 +14,7 @@
 {% set srv_sshproxy_path = "/srv/sshproxy" -%}
 
 {% if grains.cloud is defined -%}
-  {% if grains.cloud not in ['vagrant', 'vsphere'] -%}
+  {% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller'] -%}
     {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
   {% endif -%}
 
@@ -58,7 +58,7 @@
 {% set client_ca_file = "" -%}
 
 {% set secure_port = "6443" -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]    %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]    %}
   {% set secure_port = "443" -%}
   {% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}
@@ -72,12 +72,12 @@
 {% endif -%}
 
 {% if grains.cloud is defined -%}
-{% if grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ] -%}
+{% if grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ] -%}
     {% set token_auth_file = "--token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
 {% endif -%}
 {% endif -%}
 
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere']  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]  %}
   {% set basic_auth_file = "--basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
 {% endif -%}
 
diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest
index 3e857ac887..966dd4f395 100644
--- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest
+++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest
@@ -32,7 +32,7 @@
 {% set srv_kube_path = "/srv/kubernetes" -%}
 
 {% if grains.cloud is defined -%}
-  {% if grains.cloud not in ['vagrant', 'vsphere'] -%}
+  {% if grains.cloud not in ['vagrant', 'vsphere', 'photon-controller'] -%}
     {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
   {% endif -%}
   {% set service_account_key = "--service-account-private-key-file=/srv/kubernetes/server.key" -%}
@@ -46,7 +46,7 @@
 
 {% set root_ca_file = "" -%}
 
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]    %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]    %}
    {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%}
 {% endif -%}
 
diff --git a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
index ff462473c5..94639a1c18 100644
--- a/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
+++ b/cluster/saltbase/salt/kube-proxy/kube-proxy.manifest
@@ -5,7 +5,7 @@
   {% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
   {% set api_servers = "--master=https://" + ips[0][0] -%}
 {% endif -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]  %}
   {% set api_servers_with_port = api_servers -%}
 {% else -%}
   {% set api_servers_with_port = api_servers + ":6443" -%}
diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default
index a13278b60c..5462c761a0 100644
--- a/cluster/saltbase/salt/kubelet/default
+++ b/cluster/saltbase/salt/kubelet/default
@@ -16,7 +16,7 @@
 {% endif -%}
 
 # TODO: remove nginx for other cloud providers.
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere' ]  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller' ]  %}
   {% set api_servers_with_port = api_servers -%}
 {% else -%}
   {% set api_servers_with_port = api_servers + ":6443" -%}
@@ -28,7 +28,7 @@
 
 {% set reconcile_cidr_args = "" -%}
 {% if grains['roles'][0] == 'kubernetes-master' -%}
-  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere'] -%}
+  {% if grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller'] -%}
 
     # Unless given a specific directive, disable registration for the kubelet
     # running on the master.
@@ -48,7 +48,7 @@
 {% endif -%}
 
 {% set cloud_provider = "" -%}
-{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere'] -%}
+{% if grains.cloud is defined and grains.cloud not in ['vagrant', 'vsphere', 'photon-controller'] -%}
   {% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
 {% endif -%}
 
diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls
index e6a2bce62f..4b84cefccf 100644
--- a/cluster/saltbase/salt/top.sls
+++ b/cluster/saltbase/salt/top.sls
@@ -72,7 +72,7 @@ base:
     - logrotate
 {% endif %}
     - kube-addons
-{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere' ] %}
+{% if grains['cloud'] is defined and grains['cloud'] in [ 'vagrant', 'gce', 'aws', 'vsphere', 'photon-controller' ] %}
     - docker
     - kubelet
 {% endif %}
diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt
index 82caf87825..f4eb3db969 100644
--- a/hack/verify-flags/exceptions.txt
+++ b/hack/verify-flags/exceptions.txt
@@ -26,6 +26,10 @@ cluster/log-dump.sh:    for node_name in "${NODE_NAMES[@]}"; do
 cluster/log-dump.sh:    local -r node_name="${1}"
 cluster/log-dump.sh:readonly report_dir="${1:-_artifacts}"
 cluster/mesos/docker/km/build.sh:  km_path=$(find-binary km darwin/amd64)
+cluster/photon-controller/templates/salt-minion.sh:  hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}')
+cluster/photon-controller/util.sh:    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
+cluster/photon-controller/util.sh:  local cert_dir="/srv/kubernetes"
+cluster/photon-controller/util.sh:  node_name=${1}
 cluster/rackspace/util.sh:    local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \
 cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control objects if defined before any other addon services. If the limits
 cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %}