Use EBS for Master Data

Signed-off-by: Adam Sunderland <iterion@gmail.com>

cluster/aws/config-default.sh

@@ -37,6 +37,9 @@ IAM_PROFILE_MINION="kubernetes-minion"
 
 LOG="/dev/null"
 
+MASTER_DISK_TYPE="${MASTER_DISK_TYPE:-gp2}"
+MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20}
+
 MASTER_NAME="${INSTANCE_PREFIX}-master"
 MASTER_TAG="${INSTANCE_PREFIX}-master"
 MINION_TAG="${INSTANCE_PREFIX}-minion"

cluster/aws/config-test.sh

@@ -33,6 +33,9 @@ IAM_PROFILE_MINION="kubernetes-minion"
 
 LOG="/dev/null"
 
+MASTER_DISK_TYPE="${MASTER_DISK_TYPE:-gp2}"
+MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20}
+
 MASTER_NAME="${INSTANCE_PREFIX}-master"
 MASTER_TAG="${INSTANCE_PREFIX}-master"
 MINION_TAG="${INSTANCE_PREFIX}-minion"

cluster/aws/templates/setup-master-pd.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Format and mount the disk, create directories on it for all of the master's
+# persistent data, and link them to where they're used.
+
+# Mount Master Persistent Disk
+echo "Mounting master-pd"
+mkdir -p /mnt/master-pd
+mkfs -t ext4 /dev/xvdh
+echo "/dev/xvdh  /mnt/master-pd  ext4  noatime  0 0" >> /etc/fstab
+
+# Contains all the data stored in etcd
+mkdir -m 700 -p /mnt/master-pd/var/etcd
+# Contains the dynamically generated apiserver auth certs and keys
+mkdir -p /mnt/master-pd/srv/kubernetes
+# Contains the cluster's initial config parameters and auth tokens
+mkdir -p /mnt/master-pd/srv/salt-overlay
+# Directory for kube-apiserver to store SSH key (if necessary)
+mkdir -p /mnt/master-pd/srv/sshproxy
+
+ln -s -f /mnt/master-pd/var/etcd /var/etcd
+ln -s -f /mnt/master-pd/srv/kubernetes /srv/kubernetes
+ln -s -f /mnt/master-pd/srv/sshproxy /srv/sshproxy
+ln -s -f /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
+
+# This is a bit of a hack to get around the fact that salt has to run after the
+# PD and mounted directory are already set up. We can't give ownership of the
+# directory to etcd until the etcd user and group exist, but they don't exist
+# until salt runs if we don't create them here. We could alternatively make the
+# permissions on the directory more permissive, but this seems less bad.
+if ! id etcd &>/dev/null; then
+  useradd -s /sbin/nologin -d /var/etcd etcd
+fi
+chown -R etcd /mnt/master-pd/var/etcd
+chgrp -R etcd /mnt/master-pd/var/etcd

cluster/aws/util.sh

@@ -55,7 +55,7 @@ MINION_SG_NAME="kubernetes-minion-${CLUSTER_ID}"
 # Be sure to map all the ephemeral drives.  We can specify more than we actually have.
 # TODO: Actually mount the correct number (especially if we have more), though this is non-trivial, and
 #  only affects the big storage instance types, which aren't a typical use case right now.
-BLOCK_DEVICE_MAPPINGS="[{\"DeviceName\": \"/dev/sdb\",\"VirtualName\":\"ephemeral0\"},{\"DeviceName\": \"/dev/sdc\",\"VirtualName\":\"ephemeral1\"},{\"DeviceName\": \"/dev/sdd\",\"VirtualName\":\"ephemeral2\"},{\"DeviceName\": \"/dev/sde\",\"VirtualName\":\"ephemeral3\"}]"
+BLOCK_DEVICE_MAPPINGS="{\"DeviceName\": \"/dev/sdb\",\"VirtualName\":\"ephemeral0\"},{\"DeviceName\": \"/dev/sdc\",\"VirtualName\":\"ephemeral1\"},{\"DeviceName\": \"/dev/sdd\",\"VirtualName\":\"ephemeral2\"},{\"DeviceName\": \"/dev/sde\",\"VirtualName\":\"ephemeral3\"}"
 
 function json_val {
     python -c 'import json,sys;obj=json.load(sys.stdin);print obj'$1''
@@ -724,6 +724,9 @@ function kube-up {
   # HTTPS to the master is allowed (for API access)
   authorize-security-group-ingress "${MASTER_SG_ID}" "--protocol tcp --port 443 --cidr 0.0.0.0/0"
 
+  master_volume_mapping="{\"DeviceName\": \"/dev/sdh\",\"Ebs\":{\"DeleteOnTermination\":false,\"VolumeSize\":${MASTER_DISK_SIZE},\"VolumeType\":\"${MASTER_DISK_TYPE}\"}}"
+  master_block_device_mappings="[${master_volume_mapping},${BLOCK_DEVICE_MAPPINGS}]"
+
   (
     # We pipe this to the ami as a startup script in the user-data field.  Requires a compatible ami
     echo "#! /bin/bash"
@@ -756,6 +759,7 @@ function kube-up {
     echo "readonly KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN}'"
     echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
+    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/setup-master-pd.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/download-release.sh"
@@ -772,7 +776,7 @@ function kube-up {
     --key-name ${AWS_SSH_KEY_NAME} \
     --security-group-ids ${MASTER_SG_ID} \
     --associate-public-ip-address \
-    --block-device-mappings "${BLOCK_DEVICE_MAPPINGS}" \
+    --block-device-mappings "${master_block_device_mappings}" \
     --user-data file://${KUBE_TEMP}/master-start.sh | json_val '["Instances"][0]["InstanceId"]')
   add-tag $master_id Name $MASTER_NAME
   add-tag $master_id Role $MASTER_TAG
@@ -810,6 +814,16 @@ function kube-up {
     sleep 10
   done
 
+  # Master is ready, find the id of the root ebs device
+  # This assumes that the root device will be the first device listed in the response"
+  master_volume_id=$($AWS_CMD describe-instance-attribute \
+    --instance-id $master_id \
+    --attribute blockDeviceMapping | \
+    json_val '["BlockDeviceMappings"][0]["Ebs"]["VolumeId"]')
+
+  add-tag $master_volume_id Name ${MASTER_NAME}-pd
+  add-tag $master_volume_id KubernetesCluster ${CLUSTER_ID}
+
   # Check for SSH connectivity
   attempt=0
   while true; do
@@ -878,7 +892,7 @@ function kube-up {
       --key-name ${AWS_SSH_KEY_NAME} \
       --security-groups ${MINION_SG_ID} \
       ${public_ip_option} \
-      --block-device-mappings "${BLOCK_DEVICE_MAPPINGS}" \
+      --block-device-mappings "[${BLOCK_DEVICE_MAPPINGS}]" \
       --user-data "file://${KUBE_TEMP}/minion-user-data"
 
   echo "Creating autoscaling group"