Modified etcd manifest to support clustering.

Modified etcd manifest to support clustering.

cluster/common.sh

@@ -644,6 +644,7 @@ ENABLE_MANIFEST_URL: $(yaml-quote ${ENABLE_MANIFEST_URL:-false})
 MANIFEST_URL: $(yaml-quote ${MANIFEST_URL:-})
 MANIFEST_URL_HEADER: $(yaml-quote ${MANIFEST_URL_HEADER:-})
 NUM_NODES: $(yaml-quote ${NUM_NODES})
+INITIAL_ETCD_CLUSTER: $(yaml-quote ${MASTER_NAME})
 EOF
     if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
       cat >>$file <<EOF

cluster/gce/configure-vm.sh

@@ -452,10 +452,11 @@ network_policy_provider: '$(echo "$NETWORK_POLICY_PROVIDER" | sed -e "s/'/''/g")
 enable_manifest_url: '$(echo "${ENABLE_MANIFEST_URL:-}" | sed -e "s/'/''/g")'
 manifest_url: '$(echo "${MANIFEST_URL:-}" | sed -e "s/'/''/g")'
 manifest_url_header: '$(echo "${MANIFEST_URL_HEADER:-}" | sed -e "s/'/''/g")'
-master_name: '$(echo "${MASTER_NAME:-}" | sed -e "s/'/''/g")'
+master_name: '$(echo "${KUBERNETES_MASTER_NAME:-}" | sed -e "s/'/''/g")'
 num_nodes: $(echo "${NUM_NODES:-}" | sed -e "s/'/''/g")
 e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 kube_uid: '$(echo "${KUBE_UID}" | sed -e "s/'/''/g")'
+initial_etcd_cluster: '$(echo "${KUBERNETES_MASTER_NAME:-}" | sed -e "s/'/''/g")'
 EOF
     if [ -n "${KUBELET_PORT:-}" ]; then
       cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls

cluster/gce/gci/configure-helper.sh

@@ -507,12 +507,27 @@ function start-kube-proxy {
 # $4: value for variable 'cpulimit'
 # $5: pod name, which should be either etcd or etcd-events
 function prepare-etcd-manifest {
+  local host_name=$(hostname)
+  local etcd_cluster=""
+  local cluster_state="new"
+  for host in $(echo "${INITIAL_ETCD_CLUSTER}" | tr "," "\n"); do
+    etcd_host="etcd-${host}=http://${host}:$3"
+    if [[ -n "${etcd_cluster}" ]]; then
+      etcd_cluster+=","
+      cluster_state="existing"
+    fi
+    etcd_cluster+="${etcd_host}"
+  done
   local -r temp_file="/tmp/$5"
   cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd.manifest" "${temp_file}"
+  remove-salt-config-comments "${temp_file}"
   sed -i -e "s@{{ *suffix *}}@$1@g" "${temp_file}"
   sed -i -e "s@{{ *port *}}@$2@g" "${temp_file}"
   sed -i -e "s@{{ *server_port *}}@$3@g" "${temp_file}"
   sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${temp_file}"
+  sed -i -e "s@{{ *host_name *}}@$host_name@g" "${temp_file}"
+  sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
+  sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${temp_file}"
   # Replace the volume host path.
   sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${temp_file}"
   mv "${temp_file}" /etc/kubernetes/manifests

cluster/gce/trusty/configure-helper.sh

@@ -409,12 +409,27 @@ create_master_kubelet_auth() {
 # $4: value for variable 'cpulimit'
 # $5: pod name, which should be either etcd or etcd-events
 prepare_etcd_manifest() {
+  local host_name=$(hostname)
+  local etcd_cluster=""
+  local cluster_state="new"
+  for host in $(echo "${INITIAL_ETCD_CLUSTER}" | tr "," "\n"); do
+    etcd_host="etcd-${host}=http://${host}:$3"
+    if [[ -n "${etcd_cluster}" ]]; then
+      etcd_cluster+=","
+      cluster_state="existing"
+    fi
+    etcd_cluster+="${etcd_host}"
+  done
   etcd_temp_file="/tmp/$5"
   cp /home/kubernetes/kube-manifests/kubernetes/gci-trusty/etcd.manifest "${etcd_temp_file}"
+  remove_salt_config_comments "${etcd_temp_file}"
   sed -i -e "s@{{ *suffix *}}@$1@g" "${etcd_temp_file}"
   sed -i -e "s@{{ *port *}}@$2@g" "${etcd_temp_file}"
   sed -i -e "s@{{ *server_port *}}@$3@g" "${etcd_temp_file}"
   sed -i -e "s@{{ *cpulimit *}}@\"$4\"@g" "${etcd_temp_file}"
+  sed -i -e "s@{{ *host_name *}}@$host_name@g" "${etcd_temp_file}"
+  sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${etcd_temp_file}"
+  sed -i -e "s@{{ *cluster_state *}}@$cluster_state@g" "${etcd_temp_file}"
   # Replace the volume host path
   sed -i -e "s@/mnt/master-pd/var/etcd@/mnt/disks/master-pd/var/etcd@g" "${etcd_temp_file}"
   mv "${etcd_temp_file}" /etc/kubernetes/manifests

cluster/saltbase/salt/etcd/etcd.manifest

@@ -1,3 +1,20 @@
+{% set cluster_state = 'new' -%}
+{% set host_name = pillar.get('master_name', '') -%}
+{% set etcd_cluster_array = pillar.get('initial_etcd_cluster', '').split(',') -%}
+{% set etcd_cluster = '' -%}
+{# We use vars dictionary to pass variables set inside the for loop, because jinja defines new variables inside the for loop that hide variables from the outside. #}
+{% set vars = {'etcd_cluster': '', 'cluster_state': cluster_state} -%}
+{% for host in etcd_cluster_array -%}
+  {% if etcd_cluster != '' -%}
+    {% set cluster_state = 'existing' -%}
+    {% set etcd_cluster = etcd_cluster ~ ',' -%}
+  {% endif -%}
+  {% set etcd_cluster = etcd_cluster ~ 'etcd-' ~ host ~ '=http://' ~ host ~ ':' ~ server_port -%}
+  {% do vars.update({'etcd_cluster': etcd_cluster, 'cluster_state': cluster_state}) -%}
+{% endfor -%}
+{% set etcd_cluster = vars.etcd_cluster -%}
+{% set cluster_state = vars.cluster_state -%}
+
 {
 "apiVersion": "v1",
 "kind": "Pod",
@@ -19,7 +36,7 @@
     "command": [
               "/bin/sh",
               "-c",
-              "/usr/local/bin/etcd --listen-peer-urls http://127.0.0.1:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} --data-dir /var/etcd/data{{ suffix }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
+              "/usr/local/bin/etcd --name etcd-{{ host_name }} --listen-peer-urls http://{{ host_name }}:{{ server_port }} --initial-advertise-peer-urls http://{{ host_name }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster {{ etcd_cluster }} --initial-cluster-state {{ cluster_state }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
             ],
     "livenessProbe": {
       "httpGet": {