GCE will properly regenerate basic_auth.csv on kube-apiserver start.

If the file does not exist we will generate it as normal.
If the file exists we will remove the old admin password before adding
the "new" one. (Turns in to a no-op if the password exists).
This did not work properly before because we were replacing by key,
where the key was the password. New password would not match and so
not replace the old password.
Added a METADATA_CLOBBERS_CONFIG flag
METADATA_CLOBBERS_CONFIG controls if we consider the values on disk or in
metadata to be the canonical source of truth. Currently defaulting to
false for GCE and forcing to true for GKE.
Added handling for older forms of the basic_auth.csv file.
Fixed comment to reflect new METADATA_CLOBBERS_CONFIG var.

cluster/gce/config-default.sh

@@ -201,3 +201,7 @@ ENABLE_DEFAULT_STORAGE_CLASS="${ENABLE_DEFAULT_STORAGE_CLASS:-true}"
 # TODO(dawn1107): Remove this once the flag is built into CVM image.
 # Kernel panic upon soft lockup issue
 SOFTLOCKUP_PANIC="${SOFTLOCKUP_PANIC:-false}" # true, false
+
+# Indicates if the values (eg. kube password) in metadata should be treated as
+# canonical, and therefore disk copies ought to be recreated/clobbered.
+METADATA_CLOBBERS_CONFIG=${METADATA_CLOBBERS_CONFIG:-false}

cluster/gce/gci/configure-helper.sh

@@ -273,12 +273,19 @@ function create-master-pki {
 
 # After the first boot and on upgrade, these files exist on the master-pd
 # and should never be touched again (except perhaps an additional service
-# account, see NB below.)
+# account, see NB below.) One exception is if METADATA_CLOBBERS_CONFIG is
+# enabled. In that case the basic_auth.csv file will be rewritten to make
+# sure it matches the metadata source of truth. 
 function create-master-auth {
   echo "Creating master auth files"
   local -r auth_dir="/etc/srv/kubernetes"
   local -r basic_auth_csv="${auth_dir}/basic_auth.csv"
   if [[ -n "${KUBE_PASSWORD:-}" && -n "${KUBE_USER:-}" ]]; then
+    if [[ -e "${basic_auth_csv}" && "${METADATA_CLOBBERS_CONFIG:-false}" == "true" ]]; then
+      sed -i "/,${KUBE_USER},admin,system:masters$/d" "${basic_auth_csv}"
+      # The following is for the legacy form of the password line.
+      sed -i "/,${KUBE_USER},admin$/d" "${basic_auth_csv}"
+    fi
     replace_prefixed_line "${basic_auth_csv}" "${KUBE_PASSWORD},${KUBE_USER}," "admin,system:masters"
   fi
   local -r known_tokens_csv="${auth_dir}/known_tokens.csv"

cluster/gke/config-default.sh

@@ -42,3 +42,7 @@ ENABLE_L7_LOADBALANCING="${KUBE_ENABLE_L7_LOADBALANCING:-glbc}"
 ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-standalone}"
 
 KUBE_DELETE_NETWORK=${KUBE_DELETE_NETWORK:-false}
+
+# Indicates if the values (eg. kube password) in metadata should be treated as
+# canonical, and therefore disk copies ought to be recreated/clobbered.
+METADATA_CLOBBERS_CONFIG=true