Merge pull request #42994 from Shawyeok/features/full-tls-etcd-cluster

Automatic merge from submit-queue

Centos provider: generate SSL certificates for etcd cluster.

**What this PR does / why we need it**:
Support secure etcd cluster for centos provider, generate SSL certificates for etcd in default. Running it w/o SSL is exposing cluster data to everyone and is not recommended. [#39462](https://github.com/kubernetes/kubernetes/pull/39462#issuecomment-271601547)

/cc @jszczepkowski @zmerlynn 

**Release note**:
```release-note
Support secure etcd cluster for centos provider.
```

cluster/centos/.gitignore

@@ -1,5 +1,6 @@
 binaries
 ca-cert
+etcd-cert
 
 master/bin/etcd
 master/bin/etcdctl

cluster/centos/config-default.sh

@@ -71,7 +71,7 @@ function concat-etcd-servers() {
     if [ -n "$etcd_servers" ]; then
       prefix="${etcd_servers},"
     fi
-    etcd_servers="${prefix}http://${master_ip}:2379"
+    etcd_servers="${prefix}https://${master_ip}:2379"
   done
 
   echo "$etcd_servers"
@@ -89,7 +89,7 @@ function concat-etcd-initial-cluster() {
     if [ -n "$etcd_initial_cluster" ]; then
       etcd_initial_cluster+=","
     fi
-    etcd_initial_cluster+="infra${num_infra}=http://${master_ip}:2380"
+    etcd_initial_cluster+="infra${num_infra}=https://${master_ip}:2380"
     let ++num_infra
   done
 
@@ -97,7 +97,10 @@ function concat-etcd-initial-cluster() {
 }
 export ETCD_INITIAL_CLUSTER="$(concat-etcd-initial-cluster)"
 
-export CERT_DIR="${CERT_DIR:-$(cd "${root}/ca-cert" && pwd)}"
+CERT_DIR="${CERT_DIR:-${root}/ca-cert}"
+mkdir -p "${CERT_DIR}"
+# CERT_DIR path must be absolute.
+export CERT_DIR="$(cd "${CERT_DIR}"; pwd)"
 
 # define the IP range used for service cluster IPs.
 # according to rfc 1918 ref: https://tools.ietf.org/html/rfc1918 choose a private ip range here.
@@ -117,7 +120,7 @@ export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"}
 
 # Admission Controllers to invoke prior to persisting objects in cluster
 # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
-export ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,ResourceQuota
+export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,ResourceQuota"}
 
 # Extra options to set on the Docker command line.
 # This is useful for setting --insecure-registry for local registries.

cluster/centos/master/scripts/apiserver.sh

@@ -31,6 +31,15 @@ KUBE_LOG_LEVEL="--v=4"
 # comma separated. Mutually exclusive with -etcd-config
 KUBE_ETCD_SERVERS="--etcd-servers=${ETCD_SERVERS}"
 
+# --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication.
+KUBE_ETCD_CAFILE="--etcd-cafile=/srv/kubernetes/etcd/ca.pem"
+
+# --etcd-certfile="": SSL certification file used to secure etcd communication.
+KUBE_ETCD_CERTFILE="--etcd-certfile=/srv/kubernetes/etcd/client.pem"
+
+# --etcd-keyfile="": key file used to secure etcd communication.
+KUBE_ETCD_KEYFILE="--etcd-keyfile=/srv/kubernetes/etcd/client-key.pem"
+
 # --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port.
 KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
 
@@ -77,6 +86,9 @@ EOF
 KUBE_APISERVER_OPTS="   \${KUBE_LOGTOSTDERR}         \\
                         \${KUBE_LOG_LEVEL}           \\
                         \${KUBE_ETCD_SERVERS}        \\
+                        \${KUBE_ETCD_CAFILE}         \\
+                        \${KUBE_ETCD_CERTFILE}       \\
+                        \${KUBE_ETCD_KEYFILE}        \\
                         \${KUBE_API_ADDRESS}         \\
                         \${KUBE_API_PORT}            \\
                         \${NODE_PORT}                \\

cluster/centos/master/scripts/etcd.sh

@@ -31,20 +31,20 @@ ETCD_DATA_DIR="${etcd_data_dir}/default.etcd"
 #ETCD_SNAPSHOT_COUNTER="10000"
 #ETCD_HEARTBEAT_INTERVAL="100"
 #ETCD_ELECTION_TIMEOUT="1000"
-ETCD_LISTEN_PEER_URLS="http://${ETCD_LISTEN_IP}:2380"
+ETCD_LISTEN_PEER_URLS="https://${ETCD_LISTEN_IP}:2380"
-ETCD_LISTEN_CLIENT_URLS="http://${ETCD_LISTEN_IP}:2379,http://127.0.0.1:2379"
+ETCD_LISTEN_CLIENT_URLS="https://${ETCD_LISTEN_IP}:2379,https://127.0.0.1:2379"
 #ETCD_MAX_SNAPSHOTS="5"
 #ETCD_MAX_WALS="5"
 #ETCD_CORS=""
 #
 #[cluster]
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${ETCD_LISTEN_IP}:2380"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_LISTEN_IP}:2380"
 # if you use different ETCD_NAME (e.g. test),
 # set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
 ETCD_INITIAL_CLUSTER="${ETCD_INITIAL_CLUSTER}"
 ETCD_INITIAL_CLUSTER_STATE="new"
 ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
-ETCD_ADVERTISE_CLIENT_URLS="http://${ETCD_LISTEN_IP}:2379"
+ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_LISTEN_IP}:2379"
 #ETCD_DISCOVERY=""
 #ETCD_DISCOVERY_SRV=""
 #ETCD_DISCOVERY_FALLBACK="proxy"
@@ -54,12 +54,14 @@ ETCD_ADVERTISE_CLIENT_URLS="http://${ETCD_LISTEN_IP}:2379"
 #ETCD_PROXY="off"
 #
 #[security]
-#ETCD_CA_FILE=""
+CLIENT_CERT_AUTH="true"
-#ETCD_CERT_FILE=""
+ETCD_CA_FILE="/srv/kubernetes/etcd/ca.pem"
-#ETCD_KEY_FILE=""
+ETCD_CERT_FILE="/srv/kubernetes/etcd/server-${ETCD_NAME}.pem"
-#ETCD_PEER_CA_FILE=""
+ETCD_KEY_FILE="/srv/kubernetes/etcd/server-${ETCD_NAME}-key.pem"
-#ETCD_PEER_CERT_FILE=""
+PEER_CLIENT_CERT_AUTH="true"
-#ETCD_PEER_KEY_FILE=""
+ETCD_PEER_CA_FILE="/srv/kubernetes/etcd/ca.pem"
+ETCD_PEER_CERT_FILE="/srv/kubernetes/etcd/peer-${ETCD_NAME}.pem"
+ETCD_PEER_KEY_FILE="/srv/kubernetes/etcd/peer-${ETCD_NAME}-key.pem"
 EOF
 
 cat <<EOF >//usr/lib/systemd/system/etcd.service

cluster/centos/master/scripts/flannel.sh

@@ -18,10 +18,16 @@
 ETCD_SERVERS=${1:-"http://8.8.8.18:4001"}
 FLANNEL_NET=${2:-"172.16.0.0/16"}
 
+CA_FILE="/srv/kubernetes/etcd/ca.pem"
+CERT_FILE="/srv/kubernetes/etcd/client.pem"
+KEY_FILE="/srv/kubernetes/etcd/client-key.pem"
 
 cat <<EOF >/opt/kubernetes/cfg/flannel
 FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}"
 FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network"
+FLANNEL_ETCD_CAFILE="--etcd-cafile=${CA_FILE}"
+FLANNEL_ETCD_CERTFILE="--etcd-certfile=${CERT_FILE}"
+FLANNEL_ETCD_KEYFILE="--etcd-keyfile=${KEY_FILE}"
 EOF
 
 cat <<EOF >/usr/lib/systemd/system/flannel.service
@@ -31,7 +37,7 @@ After=network.target
 
 [Service]
 EnvironmentFile=-/opt/kubernetes/cfg/flannel
-ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY}
+ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} \${FLANNEL_ETCD_CAFILE} \${FLANNEL_ETCD_CERTFILE} \${FLANNEL_ETCD_KEYFILE}
 
 Type=notify
 
@@ -42,7 +48,8 @@ EOF
 # Store FLANNEL_NET to etcd.
 attempt=0
 while true; do
-  /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+  /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+    --no-sync -C ${ETCD_SERVERS} \
     get /coreos.com/network/config >/dev/null 2>&1
   if [[ "$?" == 0 ]]; then
     break
@@ -52,7 +59,8 @@ while true; do
       exit 2
     fi
 
-    /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+    /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+      --no-sync -C ${ETCD_SERVERS} \
       mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1
     attempt=$((attempt+1))
     sleep 3

cluster/centos/node/scripts/flannel.sh

@@ -18,10 +18,16 @@
 ETCD_SERVERS=${1:-"http://8.8.8.18:2379"}
 FLANNEL_NET=${2:-"172.16.0.0/16"}
 
+CA_FILE="/srv/kubernetes/etcd/ca.pem"
+CERT_FILE="/srv/kubernetes/etcd/client.pem"
+KEY_FILE="/srv/kubernetes/etcd/client-key.pem"
 
 cat <<EOF >/opt/kubernetes/cfg/flannel
 FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}"
 FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network"
+FLANNEL_ETCD_CAFILE="--etcd-cafile=${CA_FILE}"
+FLANNEL_ETCD_CERTFILE="--etcd-certfile=${CERT_FILE}"
+FLANNEL_ETCD_KEYFILE="--etcd-keyfile=${KEY_FILE}"
 EOF
 
 cat <<EOF >/usr/lib/systemd/system/flannel.service
@@ -33,7 +39,7 @@ Before=docker.service
 [Service]
 EnvironmentFile=-/opt/kubernetes/cfg/flannel
 ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh
-ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY}
+ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} \${FLANNEL_ETCD_CAFILE} \${FLANNEL_ETCD_CERTFILE} \${FLANNEL_ETCD_KEYFILE}
 ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker
 
 Type=notify
@@ -46,7 +52,8 @@ EOF
 # Store FLANNEL_NET to etcd.
 attempt=0
 while true; do
-  /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+  /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+    --no-sync -C ${ETCD_SERVERS} \
     get /coreos.com/network/config >/dev/null 2>&1
   if [[ "$?" == 0 ]]; then
     break
@@ -56,7 +63,8 @@ while true; do
       exit 2
     fi
 
-    /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+    /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+      --no-sync -C ${ETCD_SERVERS} \
       mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1
     attempt=$((attempt+1))
     sleep 3

cluster/centos/util.sh

@@ -208,6 +208,7 @@ echo "[INFO] tear-down-master on $1"
         fi"
   done
   kube-ssh "${1}" "sudo rm -rf /opt/kubernetes"
+  kube-ssh "${1}" "sudo rm -rf /srv/kubernetes"
   kube-ssh "${1}" "sudo rm -rf ${KUBE_TEMP}"
   kube-ssh "${1}" "sudo rm -rf /var/lib/etcd"
 }
@@ -226,6 +227,7 @@ echo "[INFO] tear-down-node on $1"
   done
   kube-ssh "$1" "sudo rm -rf /run/flannel"
   kube-ssh "$1" "sudo rm -rf /opt/kubernetes"
+  kube-ssh "$1" "sudo rm -rf /srv/kubernetes"
   kube-ssh "$1" "sudo rm -rf ${KUBE_TEMP}"
 }
 
@@ -239,6 +241,7 @@ function make-ca-cert() {
 #
 # Assumed vars:
 #   $1 (master)
+#   $2 (etcd_name)
 #   KUBE_TEMP
 #   ETCD_SERVERS
 #   ETCD_INITIAL_CLUSTER
@@ -250,12 +253,21 @@ function provision-master() {
   local master_ip="${master#*@}"
   local etcd_name="$2"
   ensure-setup-dir "${master}"
+  ensure-etcd-cert "${etcd_name}" "${master_ip}"
 
   kube-scp "${master}" "${ROOT}/ca-cert ${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}"
+  kube-scp "${master}" "${ROOT}/etcd-cert/ca.pem \
+    ${ROOT}/etcd-cert/client.pem \
+    ${ROOT}/etcd-cert/client-key.pem \
+    ${ROOT}/etcd-cert/server-${etcd_name}.pem \
+    ${ROOT}/etcd-cert/server-${etcd_name}-key.pem \
+    ${ROOT}/etcd-cert/peer-${etcd_name}.pem \
+    ${ROOT}/etcd-cert/peer-${etcd_name}-key.pem" "${KUBE_TEMP}/etcd-cert"
   kube-ssh "${master}" " \
     sudo rm -rf /opt/kubernetes/bin; \
     sudo cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes; \
-    sudo mkdir -p /srv/kubernetes; sudo cp -f ${KUBE_TEMP}/ca-cert/* /srv/kubernetes; \
+    sudo mkdir -p /srv/kubernetes/; sudo cp -f ${KUBE_TEMP}/ca-cert/* /srv/kubernetes/; \
+    sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \
     sudo chmod -R +x /opt/kubernetes/bin; \
     sudo ln -sf /opt/kubernetes/bin/* /usr/local/bin/; \
     sudo bash ${KUBE_TEMP}/master/scripts/etcd.sh ${etcd_name} ${master_ip} ${ETCD_INITIAL_CLUSTER}; \
@@ -298,12 +310,17 @@ function provision-node() {
   local dns_domain=${DNS_DOMAIN#*@}
   ensure-setup-dir ${node}
 
-  kube-scp ${node} "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP}
+  kube-scp "${node}" "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}"
+  kube-scp "${node}" "${ROOT}/etcd-cert/ca.pem \
+    ${ROOT}/etcd-cert/client.pem \
+    ${ROOT}/etcd-cert/client-key.pem" "${KUBE_TEMP}/etcd-cert"
   kube-ssh "${node}" " \
     rm -rf /opt/kubernetes/bin; \
     sudo cp -r ${KUBE_TEMP}/node/bin /opt/kubernetes; \
     sudo chmod -R +x /opt/kubernetes/bin; \
+    sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \
     sudo ln -s /opt/kubernetes/bin/* /usr/local/bin/; \
+    sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \
     sudo bash ${KUBE_TEMP}/node/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \
     sudo bash ${KUBE_TEMP}/node/scripts/docker.sh \"${DOCKER_OPTS}\"; \
     sudo bash ${KUBE_TEMP}/node/scripts/kubelet.sh ${MASTER_ADVERTISE_ADDRESS} ${node_ip} ${dns_ip} ${dns_domain}; \
@@ -316,10 +333,29 @@ function provision-node() {
 #   KUBE_TEMP
 function ensure-setup-dir() {
   kube-ssh "${1}" "mkdir -p ${KUBE_TEMP}; \
+                   mkdir -p ${KUBE_TEMP}/etcd-cert; \
                    sudo mkdir -p /opt/kubernetes/bin; \
                    sudo mkdir -p /opt/kubernetes/cfg"
 }
 
+# Generate certificates for etcd cluster
+#
+# Assumed vars:
+#   $1 (etcd member name)
+#   $2 (master ip)
+function ensure-etcd-cert() {
+  local etcd_name="$1"
+  local master_ip="$2"
+  local cert_dir="${ROOT}/etcd-cert"
+
+  if [[ ! -r "${cert_dir}/client.pem" || ! -r "${cert_dir}/client-key.pem" ]]; then
+    generate-etcd-cert "${cert_dir}" "${master_ip}" "client" "client"
+  fi
+
+  generate-etcd-cert "${cert_dir}" "${master_ip}" "server" "server-${etcd_name}"
+  generate-etcd-cert "${cert_dir}" "${master_ip}" "peer" "peer-${etcd_name}"
+}
+
 # Run command over ssh
 function kube-ssh() {
   local host="$1"

cluster/common.sh

@@ -901,14 +901,14 @@ function sha1sum-file() {
   fi
 }
 
-# Downloads cfssl into ${KUBE_TEMP}/cfssl directory
+# Downloads cfssl into $1 directory
 #
 # Assumed vars:
-#   KUBE_TEMP: temporary directory
+#   $1 (cfssl directory)
 #
 function download-cfssl {
-  mkdir -p "${KUBE_TEMP}/cfssl"
+  mkdir -p "$1"
-  pushd "${KUBE_TEMP}/cfssl"
+  pushd "$1"
 
   kernel=$(uname -s)
   case "${kernel}" in
@@ -1023,7 +1023,7 @@ function generate-certs {
     ./easyrsa --subject-alt-name="${SANS}" build-server-full "${MASTER_NAME}" nopass
     ./easyrsa build-client-full kube-apiserver nopass
 
-    download-cfssl
+    download-cfssl "${KUBE_TEMP}/cfssl"
 
     # make the config for the signer
     echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json"
@@ -1046,6 +1046,132 @@ function generate-certs {
   }
 }
 
+# Run the cfssl command to generates certificate files for etcd service, the
+# certificate files will save in $1 directory.
+#
+# Optional vars:
+#   GEN_ETCD_CA_CERT (CA cert encode with base64 and ZIP compression)
+#   GEN_ETCD_CA_KEY (CA key encode with base64)
+#
+# If GEN_ETCD_CA_CERT or GEN_ETCD_CA_KEY is not specified, it will generates certs for CA.
+#
+# Args:
+#   $1 (the directory that certificate files to save)
+#   $2 (the ip of etcd member)
+#   $3 (the type of etcd certificates, must be one of client, server, peer)
+#   $4 (the prefix of the certificate filename, default is $3)
+function generate-etcd-cert() {
+  local cert_dir=${1}
+  local member_ip=${2}
+  local type_cert=${3}
+  local prefix=${4:-"${type_cert}"}
+
+  local GEN_ETCD_CA_CERT=${GEN_ETCD_CA_CERT:-}
+  local GEN_ETCD_CA_KEY=${GEN_ETCD_CA_KEY:-}
+
+  mkdir -p "${cert_dir}"
+  pushd "${cert_dir}"
+
+  if [ ! -x cfssl ] || [ ! -x cfssljson ]; then
+    echo "Download cfssl & cfssljson ..."
+    download-cfssl .
+  fi
+
+  if [ ! -r "ca-config.json" ]; then
+    cat >ca-config.json <<EOF
+{
+    "signing": {
+        "default": {
+            "expiry": "43800h"
+        },
+        "profiles": {
+            "server": {
+                "expiry": "43800h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "server auth"
+                ]
+            },
+            "client": {
+                "expiry": "43800h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "client auth"
+                ]
+            },
+            "peer": {
+                "expiry": "43800h",
+                "usages": [
+                    "signing",
+                    "key encipherment",
+                    "server auth",
+                    "client auth"
+                ]
+            }
+        }
+    }
+}
+EOF
+  fi
+
+  if [ ! -r "ca-csr.json" ]; then
+    cat >ca-csr.json <<EOF
+{
+    "CN": "Kubernetes",
+    "key": {
+        "algo": "ecdsa",
+        "size": 256
+    },
+    "names": [
+        {
+            "C": "US",
+            "L": "CA",
+            "O": "kubernetes.io"
+        }
+    ]
+}
+EOF
+  fi
+
+  if [[ -n "${GEN_ETCD_CA_CERT}" && -n "${GEN_ETCD_CA_KEY}" ]]; then
+    echo "${ca_cert}" | base64 --decode | gunzip > ca.pem
+    echo "${ca_key}" | base64 --decode > ca-key.pem
+  fi
+
+  if [[ ! -r "ca.pem" || ! -r "ca-key.pem" ]]; then
+    ./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca -
+  fi
+
+  case "${type_cert}" in
+    client)
+      echo "Generate client certificates..."
+      echo '{"CN":"client","hosts":["*"],"key":{"algo":"ecdsa","size":256}}' \
+       | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client - \
+       | ./cfssljson -bare "${prefix}"
+      ;;
+    server)
+      echo "Generate server certificates..."
+      echo '{"CN":"'${member_ip}'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
+       | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname="${member_ip},127.0.0.1" - \
+       | ./cfssljson -bare "${prefix}"
+      ;;
+    peer)
+      echo "Generate peer certificates..."
+      echo '{"CN":"'${member_ip}'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
+       | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer -hostname="${member_ip},127.0.0.1" - \
+       | ./cfssljson -bare "${prefix}"
+      ;;
+    *)
+      echo "Unknow, unsupported etcd certs type: ${type_cert}" >&2
+      echo "Supported type: client, server, peer" >&2
+      exit 2
+  esac
+
+  popd
+}
+
 #
 # Using provided master env, extracts value from provided key.
 #
@@ -1149,3 +1275,13 @@ function verify-kube-binaries() {
   fi
   "${get_binaries_script}"
 }
+
+# Run pushd without stack output
+function pushd() {
+  command pushd $@ > /dev/null
+}
+
+# Run popd without stack output
+function popd() {
+  command popd $@ > /dev/null
+}

cluster/gce/upgrade.sh

@@ -130,7 +130,7 @@ function backfile-kubeletauth-certs() {
   echo "${CA_KEY_BASE64}" | base64 -d > "${KUBE_TEMP}/pki/ca.key"
   echo "${CA_CERT_BASE64}" | base64 -d > "${KUBE_TEMP}/pki/ca.crt"
   (cd "${KUBE_TEMP}/pki"
-    download-cfssl
+    download-cfssl "${KUBE_TEMP}/cfssl"
     cat <<EOF > ca-config.json
 {
   "signing": {

cluster/gce/util.sh

@@ -742,44 +742,14 @@ function create-etcd-certs {
   local ca_cert=${2:-}
   local ca_key=${3:-}
 
-  download-cfssl
+  GEN_ETCD_CA_CERT="${ca_cert}" GEN_ETCD_CA_KEY="${ca_key}" \
+    generate-etcd-cert "${KUBE_TEMP}/cfssl" "${host}" "peer" "peer"
 
   pushd "${KUBE_TEMP}/cfssl"
-
-  cat >ca-config.json <<EOF
-{
-    "signing": {
-        "default": {
-            "expiry": "168h"
-        },
-        "profiles": {
-            "client-server": {
-                "expiry": "43800h",
-                "usages": [
-                    "signing",
-                    "key encipherment"
-                ]
-            }
-        }
-    }
-}
-EOF
-  if [[ ! -z "${ca_key}" && ! -z "${ca_cert}" ]]; then
-    echo "${ca_key}" | base64 --decode > ca-key.pem
-    echo "${ca_cert}" | base64 --decode | gunzip > ca.pem
-  else
-    ./cfssl print-defaults csr > ca-csr.json
-    ./cfssl gencert -initca ca-csr.json | ./cfssljson -bare ca -
-  fi
-
-  echo '{"CN":"'"${host}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \
-      | ./cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server -hostname="${host}" - \
-      | ./cfssljson -bare etcd
-
   ETCD_CA_KEY_BASE64=$(cat "ca-key.pem" | base64 | tr -d '\r\n')
   ETCD_CA_CERT_BASE64=$(cat "ca.pem" | gzip | base64 | tr -d '\r\n')
-  ETCD_PEER_KEY_BASE64=$(cat "etcd-key.pem" | base64 | tr -d '\r\n')
+  ETCD_PEER_KEY_BASE64=$(cat "peer-key.pem" | base64 | tr -d '\r\n')
-  ETCD_PEER_CERT_BASE64=$(cat "etcd.pem" | gzip | base64 | tr -d '\r\n')
+  ETCD_PEER_CERT_BASE64=$(cat "peer.pem" | gzip | base64 | tr -d '\r\n')
   popd
 }