Merge pull request #9297 from justinsb/aws_filesystem_type

AWS: Support ext4 or btrfs by setting FILESYSTEM env var

cluster/aws/config-default.sh

@@ -26,6 +26,9 @@ NUM_MINIONS=${NUM_MINIONS:-4}
 # Because regions are globally named, we want to create in a single region; default to us-east-1
 AWS_S3_REGION=${AWS_S3_REGION:-us-east-1}
 
+# Which docker storage mechanism to use.
+DOCKER_STORAGE=${DOCKER_STORAGE:-aufs}
+
 INSTANCE_PREFIX="${KUBE_AWS_INSTANCE_PREFIX:-kubernetes}"
 CLUSTER_ID=${INSTANCE_PREFIX}
 AWS_SSH_KEY=${AWS_SSH_KEY:-$HOME/.ssh/kube_aws_rsa}

cluster/aws/config-test.sh

@@ -22,6 +22,9 @@ NUM_MINIONS=${NUM_MINIONS:-2}
 # Because regions are globally named, we want to create in a single region; default to us-east-1
 AWS_S3_REGION=${AWS_S3_REGION:-us-east-1}
 
+# Which docker storage mechanism to use.
+DOCKER_STORAGE=${DOCKER_STORAGE:-aufs}
+
 INSTANCE_PREFIX="${KUBE_AWS_INSTANCE_PREFIX:-e2e-test-${USER}}"
 CLUSTER_ID=${INSTANCE_PREFIX}
 AWS_SSH_KEY=${AWS_SSH_KEY:-$HOME/.ssh/kube_aws_rsa}

cluster/aws/options.md

@@ -50,5 +50,24 @@ Please note: Do not set this to "false" unless you...
 - ... already configured a route for "YOUR_IP/32" to an AWS internet gateway (for the master instance to reach your
   client directly during setup)
 
+## DOCKER_STORAGE
+
+Choose the docker storage driver to use.  This is an advanced option; most people should leave it as the default aufs
+for parity with GCE.
+
+Supported values: btrfs, aufs, devicemapper, aufs-nolvm
+
+This will also configure your ephemeral storage in a compatible way, and your Docker containers
+will run on this storage if available, as typically the root disk is comparatively small.
+
+* `btrfs` will combine your ephemeral disks into a btrfs volume.  This is a good option if you have a recent kernel
+  with a reliable btrfs.
+* `aufs` uses the aufs driver, but also installs LVM to combine your disks. `aufs-nolvm` will not use LVM,
+ meaning that only your first ephemeral disk will be used.
+* `devicemapper` sets up LVM across all your ephemeral disks and sets Docker to drive it directly.  This is a
+  similar option to btrfs, but without relying on the btrfs filesystem.  Sadly, it does not work with most
+  configurations - see [this docker bug](https://github.com/docker/docker/issues/4036)
+
+If your machines don't have any ephemeral disks, this will default to the aufs driver on your root disk (with no LVM).
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/aws/options.md?pixel)]()

cluster/aws/templates/format-disks.sh

@@ -42,39 +42,139 @@ for ephemeral_device in $ephemeral_devices; do
   fi
 done
 
+# These are set if we should move where docker/kubelet store data
+# Note this gets set to the parent directory
+move_docker=""
+move_kubelet=""
+
+apt-get update
+
+docker_storage=${DOCKER_STORAGE:-aufs}
+
 # Format the ephemeral disks
 if [[ ${#block_devices[@]} == 0 ]]; then
-  echo "No ephemeral block devices found"
+  echo "No ephemeral block devices found; will use aufs on root"
+  docker_storage="aufs"
 else
-  echo "Block devices: ${block_devices}"
+  echo "Block devices: ${block_devices[@]}"
 
-  apt-get install --yes btrfs-tools
+  if [[ ${docker_storage} == "btrfs" ]]; then
+    apt-get install --yes btrfs-tools
 
-  if [[ ${#block_devices[@]} == 1 ]]; then
+    if [[ ${#block_devices[@]} == 1 ]]; then
-    echo "One ephemeral block device found; formatting with btrfs"
+      echo "One ephemeral block device found; formatting with btrfs"
-    mkfs.btrfs -f ${block_devices[0]}
+      mkfs.btrfs -f ${block_devices[0]}
+    else
+      echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
+      mkfs.btrfs -f --data raid0 ${block_devices[@]}
+    fi
+    mount -t btrfs ${block_devices[0]} /mnt
+
+    mkdir -p /mnt/kubernetes
+
+    move_docker="/mnt"
+    move_kubelet="/mnt/kubernetes"
+  elif [[ ${docker_storage} == "aufs-nolvm" ]]; then
+    if [[ ${#block_devices[@]} != 1 ]]; then
+      echo "aufs-nolvm selected, but multiple ephemeral devices were found; only the first will be available"
+    fi
+
+    /bin/umount ${block_devices[0]}
+    mkfs -t ext4 ${block_devices[0]}
+    mount -t ext4 ${block_devices[0]} /mnt
+
+    mkdir -p /mnt/kubernetes
+
+    move_docker="/mnt"
+    move_kubelet="/mnt/kubernetes"
+  elif [[ ${docker_storage} == "devicemapper" || ${docker_storage} == "aufs" ]]; then
+    # We always use LVM, even with one device
+    # In devicemapper mode, Docker can use LVM directly
+    # Also, fewer code paths are good
+    echo "Using LVM2 and ext4"
+    apt-get install --yes lvm2
+
+    # Don't output spurious "File descriptor X leaked on vgcreate invocation."
+    # Known bug: e.g. Ubuntu #591823
+    export LVM_SUPPRESS_FD_WARNINGS=1
+
+    for block_device in ${block_devices}; do
+      /bin/umount ${block_device}
+      pvcreate ${block_device}
+    done
+    vgcreate vg-ephemeral ${block_devices[@]}
+
+    if [[ ${docker_storage} == "devicemapper" ]]; then
+      # devicemapper thin provisioning, managed by docker
+      # This is the best option, but it is sadly broken on most distros
+      # Bug: https://github.com/docker/docker/issues/4036
+
+      # 95% goes to the docker thin-pool
+      lvcreate -l 95%VG --thinpool docker-thinpool vg-ephemeral
+
+      DOCKER_OPTS="${DOCKER_OPTS} --storage-opt dm.thinpooldev=/dev/mapper/vg--ephemeral-docker--thinpool"
+      # Note that we don't move docker; docker goes direct to the thinpool
+    else
+      # aufs
+
+      # Create a docker lv, use docker on it
+      # 95% goes to the docker thin-pool
+      lvcreate -l 95%VG --thinpool docker-thinpool vg-ephemeral
+
+      THINPOOL_SIZE=$(lvs vg-ephemeral/docker-thinpool -o LV_SIZE --noheadings --units M --nosuffix)
+      lvcreate -V${THINPOOL_SIZE}M -T vg-ephemeral/docker-thinpool -n docker
+
+      mkfs -t ext4 /dev/vg-ephemeral/docker
+      mkdir -p /mnt/docker
+      mount -t ext4 /dev/vg-ephemeral/docker /mnt/docker
+      move_docker="/mnt"
+    fi
+
+    # Remaining 5% is for kubernetes data
+    # TODO: Should this be a thin pool?  e.g. would we ever want to snapshot this data?
+    lvcreate -l 100%FREE -n kubernetes vg-ephemeral
+    mkfs -t ext4 /dev/vg-ephemeral/kubernetes
+    mkdir -p /mnt/kubernetes
+    mount -t ext4 /dev/vg-ephemeral/kubernetes /mnt/kubernetes
+    move_kubelet="/mnt/kubernetes"
   else
-    echo "Found multiple ephemeral block devices, formatting with btrfs as RAID-0"
+    echo "Ignoring unknown DOCKER_STORAGE: ${docker_storage}"
-    mkfs.btrfs -f --data raid0 ${block_devices[@]}
   fi
-  mount -t btrfs ${block_devices[0]} /mnt
+fi
-
+
-  # Move docker to /mnt if we have it
+
-  if [[ -d /var/lib/docker ]]; then
+if [[ ${docker_storage} == "btrfs" ]]; then
-    mv /var/lib/docker /mnt/
+  DOCKER_OPTS="${DOCKER_OPTS} -s btrfs"
-  fi
+elif [[ ${docker_storage} == "aufs-nolvm" || ${docker_storage} == "aufs" ]]; then
-  mkdir -p /mnt/docker
+  # Install aufs kernel module
-  ln -s /mnt/docker /var/lib/docker
+  apt-get install --yes linux-image-extra-$(uname -r)
-  DOCKER_ROOT="/mnt/docker"
+
-  DOCKER_OPTS="${DOCKER_OPTS} -g /mnt/docker"
+  DOCKER_OPTS="${DOCKER_OPTS} -s aufs"
-
+elif [[ ${docker_storage} == "devicemapper" ]]; then
-  # Move /var/lib/kubelet to /mnt if we have it
+  DOCKER_OPTS="${DOCKER_OPTS} -s devicemapper"
-  # (the backing for empty-dir volumes can use a lot of space!)
+else
-  if [[ -d /var/lib/kubelet ]]; then
+  echo "Ignoring unknown DOCKER_STORAGE: ${docker_storage}"
-    mv /var/lib/kubelet /mnt/
+fi
-  fi
+
-  mkdir -p /mnt/kubelet
+if [[ -n "${move_docker}" ]]; then
-  ln -s /mnt/kubelet /var/lib/kubelet
+  # Move docker to e.g. /mnt
-  KUBELET_ROOT="/mnt/kubelet"
+  if [[ -d /var/lib/docker ]]; then
+    mv /var/lib/docker ${move_docker}/
+  fi
+  mkdir -p ${move_docker}/docker
+  ln -s ${move_docker}/docker /var/lib/docker
+  DOCKER_ROOT="${move_docker}/docker"
+  DOCKER_OPTS="${DOCKER_OPTS} -g ${DOCKER_ROOT}"
+fi
+
+if [[ -n "${move_kubelet}" ]]; then
+  # Move /var/lib/kubelet to e.g. /mnt
+  # (the backing for empty-dir volumes can use a lot of space!)
+  if [[ -d /var/lib/kubelet ]]; then
+    mv /var/lib/kubelet ${move_kubelet}/
+  fi
+  mkdir -p ${move_kubelet}/kubelet
+  ln -s ${move_kubelet}/kubelet /var/lib/kubelet
+  KUBELET_ROOT="${move_kubelet}/kubelet"
 fi
 

cluster/aws/ubuntu/util.sh

@@ -31,6 +31,7 @@ function generate-minion-user-data {
   echo "SALT_MASTER='${MASTER_INTERNAL_IP}'"
   echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'"
   echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'"
+  echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'"
   grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
   grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
   grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-minion.sh"

cluster/aws/util.sh

@@ -674,6 +674,7 @@ function kube-up {
     echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE:-}'"
     echo "readonly KUBELET_TOKEN='${KUBELET_TOKEN}'"
     echo "readonly KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN}'"
+    echo "readonly DOCKER_STORAGE='${DOCKER_STORAGE:-}'"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
     grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh"