diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 153f7a5644..629a2fd332 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1126,6 +1126,30 @@ function create-master-etcd-auth { fi } +function create-master-etcd-apiserver-auth { + if [[ -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then + local -r auth_dir="/etc/srv/kubernetes/pki" + + ETCD_APISERVER_CA_KEY_PATH="${auth_dir}/etcd-apiserver-ca.key" + echo "${ETCD_APISERVER_CA_KEY}" | base64 --decode > "${ETCD_APISERVER_CA_KEY_PATH}" + + ETCD_APISERVER_CA_CERT_PATH="${auth_dir}/etcd-apiserver-ca.crt" + echo "${ETCD_APISERVER_CA_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-apiserver-ca.crt" + + ETCD_APISERVER_SERVER_KEY_PATH="${auth_dir}/etcd-apiserver-server.key" + echo "${ETCD_APISERVER_SERVER_KEY}" | base64 --decode > "${ETCD_APISERVER_SERVER_KEY_PATH}" + + ETCD_APISERVER_SERVER_CERT_PATH="${auth_dir}/etcd-apiserver-server.crt" + echo "${ETCD_APISERVER_SERVER_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-apiserver-server.crt" + + ETCD_APISERVER_CLIENT_KEY_PATH="${auth_dir}/etcd-apiserver-client.key" + echo "${ETCD_APISERVER_CLIENT_KEY}" | base64 --decode > "${auth_dir}/etcd-apiserver-client.key" + + ETCD_APISERVER_CLIENT_CERT_PATH="${auth_dir}/etcd-apiserver-client.crt" + echo "${ETCD_APISERVER_CLIENT_CERT}" | base64 --decode | gunzip > "${auth_dir}/etcd-apiserver-client.crt" + fi +} + function assemble-docker-flags { echo "Assemble docker command line flags" local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" @@ -1357,6 +1381,7 @@ function prepare-etcd-manifest { local cluster_state="new" local etcd_protocol="http" local etcd_creds="" + local etcd_apiserver_creds="${ETCD_APISERVER_CREDS:-}" local etcd_extra_args="${ETCD_EXTRA_ARGS:-}" if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then @@ -1367,6 +1392,10 @@ function prepare-etcd-manifest { etcd_protocol="https" fi + if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" ]]; then + etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} " + fi + for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3" if [[ -n "${etcd_cluster}" ]]; then @@ -1412,6 +1441,7 @@ function prepare-etcd-manifest { fi sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}" sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}" + sed -i -e "s@{{ *etcd_apiserver_creds *}}@$etcd_apiserver_creds@g" "${temp_file}" sed -i -e "s@{{ *etcd_extra_args *}}@$etcd_extra_args@g" "${temp_file}" if [[ -n "${ETCD_VERSION:-}" ]]; then sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}" @@ -1521,6 +1551,11 @@ function start-kube-apiserver { elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" fi + if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then + params+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}" + params+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}" + params+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}" + fi params+=" --secure-port=443" params+=" --tls-cert-file=${APISERVER_SERVER_CERT_PATH}" params+=" --tls-private-key-file=${APISERVER_SERVER_KEY_PATH}" @@ -2831,6 +2866,7 @@ function main() { create-master-auth create-master-kubelet-auth create-master-etcd-auth + create-master-etcd-apiserver-auth override-pv-recycler gke-master-start else diff --git a/cluster/gce/gci/master-helper.sh b/cluster/gce/gci/master-helper.sh index 76261f6621..04225afc35 100755 --- a/cluster/gce/gci/master-helper.sh +++ b/cluster/gce/gci/master-helper.sh @@ -62,6 +62,19 @@ function replicate-master-instance() { kube_env="$(echo "${kube_env}" | grep -v "ETCD_PEER_CERT")" kube_env="$(echo -e "${kube_env}\nETCD_PEER_CERT: '${ETCD_PEER_CERT_BASE64}'")" + ETCD_APISERVER_CA_KEY="$(echo "${kube_env}" | grep "ETCD_APISERVER_CA_KEY" | sed "s/^.*: '//" | sed "s/'$//")" + ETCD_APISERVER_CA_CERT="$(echo "${kube_env}" | grep "ETCD_APISERVER_CA_CERT" | sed "s/^.*: '//" | sed "s/'$//")" + create-etcd-apiserver-certs "etcd-${REPLICA_NAME}" "${REPLICA_NAME}" "${ETCD_APISERVER_CA_CERT}" "${ETCD_APISERVER_CA_KEY}" + + kube_env="$(echo "${kube_env}" | grep -v "ETCD_APISERVER_SERVER_KEY")" + kube_env="$(echo -e "${kube_env}\nETCD_APISERVER_SERVER_KEY: '${ETCD_APISERVER_SERVER_KEY_BASE64}'")" + kube_env="$(echo "${kube_env}" | grep -v "ETCD_APISERVER_SERVER_CERT")" + kube_env="$(echo -e "${kube_env}\nETCD_APISERVER_SERVER_CERT: '${ETCD_APISERVER_SERVER_CERT_BASE64}'")" + kube_env="$(echo "${kube_env}" | grep -v "ETCD_APISERVER_CLIENT_KEY")" + kube_env="$(echo -e "${kube_env}\nETCD_APISERVER_CLIENT_KEY: '${ETCD_APISERVER_CLIENT_KEY_BASE64}'")" + kube_env="$(echo "${kube_env}" | grep -v "ETCD_APISERVER_CLIENT_CERT")" + kube_env="$(echo -e "${kube_env}\nETCD_APISERVER_CLIENT_CERT: '${ETCD_APISERVER_CLIENT_CERT_BASE64}'")" + echo "${kube_env}" > ${KUBE_TEMP}/master-kube-env.yaml get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-name > "${KUBE_TEMP}/cluster-name.txt" get-metadata "${existing_master_zone}" "${existing_master_name}" gci-update-strategy > "${KUBE_TEMP}/gci-update.txt" diff --git a/cluster/gce/manifests/etcd.manifest b/cluster/gce/manifests/etcd.manifest index 361eeef150..0211aa1dbb 100644 --- a/cluster/gce/manifests/etcd.manifest +++ b/cluster/gce/manifests/etcd.manifest @@ -23,7 +23,7 @@ "command": [ "/bin/sh", "-c", - "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + "if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls http://127.0.0.1:{{ port }} --listen-client-urls http://127.0.0.1:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" ], "env": [ { "name": "TARGET_STORAGE", @@ -47,6 +47,9 @@ { "name": "ETCD_CREDS", "value": "{{ etcd_creds }}" }, + { "name": "ETCD_APISERVER_CREDS", + "value": "{{ etcd_apiserver_creds }}" + }, { "name": "ETCD_SNAPSHOT_COUNT", "value": "10000" } diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index c907d1dee7..de359618c7 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -818,6 +818,12 @@ AGGREGATOR_CA_KEY: $(yaml-quote ${AGGREGATOR_CA_KEY_BASE64:-}) REQUESTHEADER_CA_CERT: $(yaml-quote ${REQUESTHEADER_CA_CERT_BASE64:-}) PROXY_CLIENT_CERT: $(yaml-quote ${PROXY_CLIENT_CERT_BASE64:-}) PROXY_CLIENT_KEY: $(yaml-quote ${PROXY_CLIENT_KEY_BASE64:-}) +ETCD_APISERVER_CA_KEY: $(yaml-quote ${ETCD_APISERVER_CA_KEY_BASE64:-}) +ETCD_APISERVER_CA_CERT: $(yaml-quote ${ETCD_APISERVER_CA_CERT_BASE64:-}) +ETCD_APISERVER_SERVER_KEY: $(yaml-quote ${ETCD_APISERVER_SERVER_KEY_BASE64:-}) +ETCD_APISERVER_SERVER_CERT: $(yaml-quote ${ETCD_APISERVER_SERVER_CERT_BASE64:-}) +ETCD_APISERVER_CLIENT_KEY: $(yaml-quote ${ETCD_APISERVER_CLIENT_KEY_BASE64:-}) +ETCD_APISERVER_CLIENT_CERT: $(yaml-quote ${ETCD_APISERVER_CLIENT_CERT_BASE64:-}) EOF } @@ -1470,6 +1476,12 @@ function parse-master-env() { PROXY_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_CERT") PROXY_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_KEY") ENABLE_LEGACY_ABAC=$(get-env-val "${master_env}" "ENABLE_LEGACY_ABAC") + ETCD_APISERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_KEY") + ETCD_APISERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_CERT") + ETCD_APISERVER_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_KEY") + ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") + ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") + ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") } # Update or verify required gcloud components are installed @@ -2047,11 +2059,10 @@ function delete-subnetworks() { fi } -# Generates SSL certificates for etcd cluster. Uses cfssl program. +# Generates SSL certificates for etcd cluster peer to peer communication. Uses cfssl program. # # Assumed vars: # KUBE_TEMP: temporary directory -# NUM_NODES: #nodes in the cluster # # Args: # $1: host name @@ -2082,6 +2093,48 @@ function create-etcd-certs { popd } +# Generates SSL certificates for etcd-client and kube-apiserver communication. Uses cfssl program. +# +# Assumed vars: +# KUBE_TEMP: temporary directory +# +# Args: +# $1: host server name +# $2: host client name +# $3: CA certificate +# $4: CA key +# +# If CA cert/key is empty, the function will also generate certs for CA. +# +# Vars set: +# ETCD_APISERVER_CA_KEY_BASE64 +# ETCD_APISERVER_CA_CERT_BASE64 +# ETCD_APISERVER_SERVER_KEY_BASE64 +# ETCD_APISERVER_SERVER_CERT_BASE64 +# ETCD_APISERVER_CLIENT_KEY_BASE64 +# ETCD_APISERVER_CLIENT_CERT_BASE64 +# +function create-etcd-apiserver-certs { + local hostServer=${1} + local hostClient=${2} + local etcd_apiserver_ca_cert=${3:-} + local etcd_apiserver_ca_key=${4:-} + + GEN_ETCD_CA_CERT="${etcd_apiserver_ca_cert}" GEN_ETCD_CA_KEY="${etcd_apiserver_ca_key}" \ + generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostServer}" "server" "etcd-apiserver-server" + generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostClient}" "client" "etcd-apiserver-client" + + pushd "${KUBE_TEMP}/cfssl" + ETCD_APISERVER_CA_KEY_BASE64=$(cat "ca-key.pem" | base64 | tr -d '\r\n') + ETCD_APISERVER_CA_CERT_BASE64=$(cat "ca.pem" | gzip | base64 | tr -d '\r\n') + ETCD_APISERVER_SERVER_KEY_BASE64=$(cat "etcd-apiserver-server-key.pem" | base64 | tr -d '\r\n') + ETCD_APISERVER_SERVER_CERT_BASE64=$(cat "etcd-apiserver-server.pem" | gzip | base64 | tr -d '\r\n') + ETCD_APISERVER_CLIENT_KEY_BASE64=$(cat "etcd-apiserver-client-key.pem" | base64 | tr -d '\r\n') + ETCD_APISERVER_CLIENT_CERT_BASE64=$(cat "etcd-apiserver-client.pem" | gzip | base64 | tr -d '\r\n') + popd +} + + function create-master() { echo "Starting master and configuring firewalls" gcloud compute firewall-rules create "${MASTER_NAME}-https" \ @@ -2132,6 +2185,7 @@ function create-master() { create-certs "${MASTER_RESERVED_IP}" create-etcd-certs ${MASTER_NAME} + create-etcd-apiserver-certs "etcd-${MASTER_NAME}" ${MASTER_NAME} if [[ "${NUM_NODES}" -ge "50" ]]; then # We block on master creation for large clusters to avoid doing too much