Fixes and improvements to Photon Controller backend for kube-up

- Improve reliability of network address detection by using MAC
  address. VMware has a MAC OUI that reliably distinguishes the VM's
  NICs from the other NICs (like the CBR). This doesn't rely on the
  unreliable reporting of the portgroup.
- Persist route changes. We configure routes on the master and nodes,
  but previously we didn't persist them so they didn't last across
  reboots. This persists them in /etc/network/interfaces
- Fix regression that didn't configure auth for kube-apiserver with
  Photon Controller.
- Reliably run apt-get update: Not doing this can cause apt to fail.
- Remove unused nginx config in salt

cluster/photon-controller/templates/create-dynamic-salt-files.sh

@@ -125,6 +125,3 @@ cluster_cidr: "$NODE_IP_RANGES"
 allocate_node_cidrs: "${ALLOCATE_NODE_CIDRS:-true}"
 admission_control: NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
 EOF
-
-mkdir -p /srv/salt-overlay/salt/nginx
-echo ${MASTER_HTPASSWD} > /srv/salt-overlay/salt/nginx/htpasswd

cluster/photon-controller/templates/salt-master.sh

@@ -28,6 +28,7 @@ grains:
   cbr-cidr: $MASTER_IP_RANGE
   cloud: photon-controller
   master_extra_sans: $MASTER_EXTRA_SANS
+  api_servers: $MASTER_NAME
 EOF
 
 # Auto accept all keys from minions that try to join

cluster/photon-controller/util.sh

@@ -71,9 +71,9 @@ function detect-master {
   fi
 
   if [[ -z "${KUBE_MASTER_IP-}" ]]; then
-      # Make sure to ignore lines where it's not attached to a portgroup
+      # Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29)
       # Make sure to ignore lines that have a network interface but no address
-    KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
+    KUBE_MASTER_IP=$(${PHOTON} vm networks "${KUBE_MASTER_ID}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
   fi
   if [[ -z "${KUBE_MASTER_IP-}" ]]; then
     kube::log::error "Could not find Kubernetes master node IP. Make sure you've launched a cluster with 'kube-up.sh'" >&2
@@ -114,9 +114,9 @@ function detect-nodes {
     fi
     KUBE_NODE_IDS+=("${node_id}")
 
-    # Make sure to ignore lines where it's not attached to a portgroup
+    # Pick out the NICs that have a MAC address owned VMware (with OUI 00:0C:29)
     # Make sure to ignore lines that have a network interface but no address
-    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
+    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
     KUBE_NODE_IP_ADDRESSES+=("${node_ip}")
 
     if [[ -z ${silent} ]]; then
@@ -323,7 +323,11 @@ function pc-delete-vm {
   local rc=0
 
   kube::log::status "Deleting VM ${vm_name}"
+  # In some cases, head exits before photon, so the pipline exits with
+  # SIGPIPE. We disable the pipefile option to hide that failure.
+  set +o pipefail
   ${PHOTON} vm show "${vm_id}" | head -1 | grep STARTED > /dev/null 2>&1 || rc=$?
+  set +o pipefail
   if [[ ${rc} -eq 0 ]]; then
     ${PHOTON} vm stop "${vm_id}" > /dev/null 2>&1 || rc=$?
     if [[ ${rc} -ne 0 ]]; then
@@ -536,6 +540,28 @@ function gen-salt {
     ) > "${KUBE_TEMP}/${node_name}-salt.sh"
 }
 
+#
+# Generate a script to add a route to a host (master or node)
+# The script will do two things:
+# 1. Add the route immediately with the route command
+# 2. Persist the route by saving it in /etc/network/interfaces
+# This was done with a script because it was easier to get the quoting right
+# and make it clear.
+#
+function gen-add-route {
+  route=${1}
+  gateway=${2}
+  (
+      echo '#!/bin/bash'
+      echo ''
+      echo '# Immediately add route'
+      echo "sudo route add -net ${route} gw ${gateway}"
+      echo ''
+      echo '# Persist route so it lasts over restarts'
+      echo 'sed -in "s|^iface eth0.*|&\n    post-up route add -net' "${route} gw ${gateway}|"'" /etc/network/interfaces'
+  ) > "${KUBE_TEMP}/add-route.sh"
+}
+
 #
 # Create the Kubernetes master VM
 # Sets global variables: 
@@ -721,10 +747,13 @@ function setup-pod-routes {
   local j
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
     kube::log::status "Configuring pod routes on ${NODE_NAMES[${i}]}..."
-    run-ssh-cmd "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
+    gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${i}]}" "${KUBE_NODE_IP_ADDRESSES[${i}]}"
+    run-script-remotely "${KUBE_MASTER_IP}" "${KUBE_TEMP}/add-route.sh"
+
     for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
       if [[ "${i}" != "${j}" ]]; then
-        run-ssh-cmd "${KUBE_NODE_IP_ADDRESSES[${i}]}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${j}]} gw ${KUBE_NODE_IP_ADDRESSES[${j}]}"
+        gen-add-route "${KUBE_NODE_BRIDGE_NETWORK[${j}]}" "${KUBE_NODE_IP_ADDRESSES[${j}]}"
+        run-script-remotely "${KUBE_NODE_IP_ADDRESSES[${i}]}" "${KUBE_TEMP}/add-route.sh"
       fi
     done
   done

cluster/saltbase/salt/docker/init.sls

@@ -94,13 +94,13 @@ fix-service-docker:
     - require:
       - pkg: docker-engine
 
-'apt-key':
+apt-key:
    cmd.run:
      - name: 'apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D'
      - unless: 'apt-key finger | grep "5811 8E89"'
 
-'apt-update':
-  cmd.wait:
+apt-update:
+  cmd.run:
     - name: '/usr/bin/apt-get update -y'
     - require:
        - cmd : 'apt-key'

cluster/saltbase/salt/kube-apiserver/init.sls

@@ -1,4 +1,4 @@
-{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'openstack'] %}
+{% if grains['cloud'] is defined and grains.cloud in ['aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack'] %}
 # TODO: generate and distribute tokens on other cloud providers.
 /srv/kubernetes/known_tokens.csv:
   file.managed:

cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest

@@ -76,7 +76,7 @@
 {% set basic_auth_file = "" -%}
 {% set authz_mode = "" -%}
 {% set abac_policy_file = "" -%}
-{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'openstack']  %}
+{% if grains['cloud'] is defined and grains.cloud in [ 'aws', 'gce', 'vagrant', 'vsphere', 'photon-controller', 'openstack']  %}
   {% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
   {% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
   {% set authz_mode = " --authorization-mode=ABAC" -%}

hack/verify-flags/exceptions.txt

@@ -29,8 +29,9 @@ cluster/log-dump.sh:    for node_name in "${NODE_NAMES[@]}"; do
 cluster/log-dump.sh:    local -r node_name="${1}"
 cluster/log-dump.sh:readonly report_dir="${1:-_artifacts}"
 cluster/mesos/docker/km/build.sh:  km_path=$(find-binary km darwin/amd64)
+cluster/photon-controller/templates/salt-master.sh:  api_servers: $MASTER_NAME
 cluster/photon-controller/templates/salt-minion.sh:  hostname_override: $(ip route get 1.1.1.1 | awk '{print $7}')
-cluster/photon-controller/util.sh:    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -v "^-" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
+cluster/photon-controller/util.sh:    node_ip=$(${PHOTON} vm networks "${node_id}" | grep -i $'\t'"00:0C:29" | grep -E '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 | awk -F'\t' '{print $3}')
 cluster/photon-controller/util.sh:  local cert_dir="/srv/kubernetes"
 cluster/photon-controller/util.sh:  node_name=${1}
 cluster/rackspace/util.sh:    local node_ip=$(nova show --minimal ${NODE_NAMES[$i]} \