Configure cluster for e2e tests.

When KUBE_E2E_STORAGE_TEST_ENVIRONMENT is set to 'true', kube-up.sh script will: - Install the right packages for all storage volumes. - Use devicemapper as docker storage backend. 'aufs', the default one on Debian, does not support extended attibutes required by Ceph RBD and Gluster server containers. Tested on GCE and Vagrant, e2e tests for storage volumes passes without any additional configuration.
2015-10-29 11:03:34 +01:00 · 2015-10-29 11:03:34 +01:00 · fe0741bffe
parent f2208a9855
commit fe0741bffe
20 changed files with 81 additions and 3 deletions
--- a/cluster/aws/config-default.sh
+++ b/cluster/aws/config-default.sh
@ -145,3 +145,6 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/aws/config-test.sh
+++ b/cluster/aws/config-test.sh
@ -137,3 +137,6 @@ KUBE_MINION_IMAGE="${KUBE_MINION_IMAGE:-}"
 COREOS_CHANNEL="${COREOS_CHANNEL:-alpha}"
 CONTAINER_RUNTIME="${KUBE_CONTAINER_RUNTIME:-docker}"
 RKT_VERSION="${KUBE_RKT_VERSION:-0.5.5}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/aws/templates/create-dynamic-salt-files.sh
+++ b/cluster/aws/templates/create-dynamic-salt-files.sh
@ -41,6 +41,7 @@ opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG")'
 opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG")'
 opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET")'
 num_nodes: $(echo "${NUM_MINIONS}")
+e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 EOF

 if [ -n "${ENABLE_EXPERIMENTAL_API:-}" ]; then
--- a/cluster/aws/util.sh
+++ b/cluster/aws/util.sh
@ -839,6 +839,7 @@ function kube-up {
    echo "readonly OPENCONTRAIL_TAG='${OPENCONTRAIL_TAG:-}'"
    echo "readonly OPENCONTRAIL_KUBERNETES_TAG='${OPENCONTRAIL_KUBERNETES_TAG:-}'"
    echo "readonly OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'"
+    echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh"
    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh"
    grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/setup-master-pd.sh"
--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@ -119,3 +119,6 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/gce/config-test.sh
+++ b/cluster/gce/config-test.sh
@ -134,3 +134,6 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/gce/configure-vm.sh
+++ b/cluster/gce/configure-vm.sh
@ -285,6 +285,7 @@ enable_manifest_url: '$(echo "$ENABLE_MANIFEST_URL" | sed -e "s/'/''/g")'
 manifest_url: '$(echo "$MANIFEST_URL" | sed -e "s/'/''/g")'
 manifest_url_header: '$(echo "$MANIFEST_URL_HEADER" | sed -e "s/'/''/g")'
 num_nodes: $(echo "${NUM_MINIONS}")
+e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 EOF

    if [ -n "${APISERVER_TEST_ARGS:-}" ]; then
--- a/cluster/gce/debian/helper.sh
+++ b/cluster/gce/debian/helper.sh
@ -60,6 +60,7 @@ NETWORK_PROVIDER: $(yaml-quote ${NETWORK_PROVIDER:-})
 OPENCONTRAIL_TAG: $(yaml-quote ${OPENCONTRAIL_TAG:-})
 OPENCONTRAIL_KUBERNETES_TAG: $(yaml-quote ${OPENCONTRAIL_KUBERNETES_TAG:-})
 OPENCONTRAIL_PUBLIC_SUBNET: $(yaml-quote ${OPENCONTRAIL_PUBLIC_SUBNET:-})
+E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote ${E2E_STORAGE_TEST_ENVIRONMENT:-})
 EOF
  if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then
    cat >>$file <<EOF
--- a/cluster/saltbase/salt/base.sls
+++ b/cluster/saltbase/salt/base.sls
@ -5,11 +5,9 @@ pkg-core:
 {% if grains['os_family'] == 'RedHat' %}
      - python
      - git
-      - glusterfs-fuse
 {% else %}
      - apt-transport-https
      - python-apt
-      - glusterfs-client
      - nfs-common
      - socat
 {% endif %}
--- a/cluster/saltbase/salt/docker/docker-defaults
+++ b/cluster/saltbase/salt/docker/docker-defaults
@ -1,3 +1,7 @@
 {% set grains_opts = grains.get('docker_opts', '') -%}
-DOCKER_OPTS="{{grains_opts}} --bridge=cbr0 --iptables=false --ip-masq=false --log-level=warn"
+{% set e2e_opts = '' -%}
+{% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' -%}
+{% set e2e_opts = '-s devicemapper' -%}
+{% endif -%}
+DOCKER_OPTS="{{grains_opts}} {{e2e_opts}} --bridge=cbr0 --iptables=false --ip-masq=false --log-level=warn"
 DOCKER_NOFILE=1000000
--- a/cluster/saltbase/salt/e2e/init.sls
+++ b/cluster/saltbase/salt/e2e/init.sls
@ -0,0 +1,40 @@
+e2e:
+
+{% if grains['os_family'] == 'Debian' and  grains['oscodename'] == 'wheezy' %}
+  # Add GlusterFS 3.5.2 Debian repo - Wheezy has Gluster 3.2 and that's too old for us.
+  pkgrepo.managed:
+    - name: deb http://download.gluster.org/pub/gluster/glusterfs/3.5/3.5.2/Debian/wheezy/apt wheezy main
+    - dist: wheezy
+    - file: /etc/apt/sources.list.d/gluster.list
+    - key_url: http://download.gluster.org/pub/gluster/glusterfs/3.5/3.5.2/Debian/wheezy/pubkey.gpg
+{% endif %}
+
+  # Install various packages required by e2e tests to all hosts.
+  # 'pkg.latest' is used to install updated glusterfs-client from the repo above,
+  # GCE image already has glusterfs-client 3.2.5, which is too old.
+  pkg.latest:
+    - refresh: true
+    - pkgs:
+      - targetcli
+      - ceph
+{% if grains['os_family'] == 'RedHat' %}
+      - glusterfs-fuse
+      - rbd-fuse
+      - iscsi-initiator-utils
+      - nfs-utils
+{% else %}
+      - glusterfs-client
+      - open-iscsi
+      - iscsitarget-dkms
+      - nfs-common
+{% endif %}
+
+
+
+{% if grains['os_family'] == 'Debian' %}
+# On Debian, re-start open-iscsi to generate unique
+# /etc/iscsi/initiatorname.iscsi
+open-iscsi:
+  cmd.run:
+    - name: 'service open-iscsi restart'
+{% endif %}
--- a/cluster/saltbase/salt/top.sls
+++ b/cluster/saltbase/salt/top.sls
@ -6,6 +6,9 @@ base:
 {% if grains.get('cloud') == 'aws' %}
    - ntp
 {% endif %}
+{% if pillar.get('e2e_storage_test_environment', '').lower() == 'true' %}
+    - e2e
+{% endif %}

  'roles:kubernetes-pool':
    - match: grain
--- a/cluster/vagrant/config-default.sh
+++ b/cluster/vagrant/config-default.sh
@ -105,3 +105,5 @@ NETWORK_PROVIDER="${NETWORK_PROVIDER:-none}" # opencontrail
 OPENCONTRAIL_TAG="${OPENCONTRAIL_TAG:-R2.20}"
 OPENCONTRAIL_KUBERNETES_TAG="${OPENCONTRAIL_KUBERNETES_TAG:-master}"
 OPENCONTRAIL_PUBLIC_SUBNET="${OPENCONTRAIL_PUBLIC_SUBNET:-10.1.0.0/16}"
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/vagrant/config-test.sh
+++ b/cluster/vagrant/config-test.sh
@ -23,3 +23,6 @@ source "${KUBE_ROOT}/cluster/vagrant/config-default.sh"

 # Do not register the master kubelet during testing
 REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false}
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/vagrant/provision-master.sh
+++ b/cluster/vagrant/provision-master.sh
@ -132,6 +132,7 @@ cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
  opencontrail_tag: '$(echo "$OPENCONTRAIL_TAG" | sed -e "s/'/''/g")'
  opencontrail_kubernetes_tag: '$(echo "$OPENCONTRAIL_KUBERNETES_TAG" | sed -e "s/'/''/g")'
  opencontrail_public_subnet: '$(echo "$OPENCONTRAIL_PUBLIC_SUBNET" | sed -e "s/'/''/g")'
+  e2e_storage_test_environment: '$(echo "$E2E_STORAGE_TEST_ENVIRONMENT" | sed -e "s/'/''/g")'
 EOF

 # Configure the salt-master
--- a/cluster/vagrant/util.sh
+++ b/cluster/vagrant/util.sh
@ -170,6 +170,7 @@ function create-provision-scripts {
    echo "OPENCONTRAIL_TAG='${OPENCONTRAIL_TAG:-}'"
    echo "OPENCONTRAIL_KUBERNETES_TAG='${OPENCONTRAIL_KUBERNETES_TAG:-}'"
    echo "OPENCONTRAIL_PUBLIC_SUBNET='${OPENCONTRAIL_PUBLIC_SUBNET:-}'"
+    echo "E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
    awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-network-master.sh"
    awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-master.sh"
  ) > "${KUBE_TEMP}/master-start.sh"
@ -195,6 +196,7 @@ function create-provision-scripts {
      echo "KUBELET_TOKEN='${KUBELET_TOKEN:-}'"
      echo "KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN:-}'"
      echo "MASTER_EXTRA_SANS='${MASTER_EXTRA_SANS:-}'"
+      echo "E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
      awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-network-minion.sh"
      awk '!/^#/' "${KUBE_ROOT}/cluster/vagrant/provision-minion.sh"
    ) > "${KUBE_TEMP}/minion-start-${i}.sh"
--- a/cluster/vsphere/config-default.sh
+++ b/cluster/vsphere/config-default.sh
@ -55,3 +55,6 @@ DNS_REPLICAS=1

 # Optional: Install Kubernetes UI
 ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}"
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/vsphere/config-test.sh
+++ b/cluster/vsphere/config-test.sh
@ -32,3 +32,6 @@ MINION_MEMORY_MB=1024
 MINION_CPU=1

 SERVICE_CLUSTER_IP_RANGE="10.244.240.0/20"  # formerly PORTAL_NET
+
+# Optional: if set to true, kube-up will configure the cluster to run e2e tests.
+E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
--- a/cluster/vsphere/templates/create-dynamic-salt-files.sh
+++ b/cluster/vsphere/templates/create-dynamic-salt-files.sh
@ -33,6 +33,8 @@ enable_cluster_dns: $ENABLE_CLUSTER_DNS
 dns_replicas: $DNS_REPLICAS
 dns_server: $DNS_SERVER_IP
 dns_domain: $DNS_DOMAIN
+e2e_storage_test_environment: $E2E_STORAGE_TEST_ENVIRONMENT
+
 EOF

 mkdir -p /srv/salt-overlay/salt/nginx
--- a/cluster/vsphere/util.sh
+++ b/cluster/vsphere/util.sh
@ -250,6 +250,7 @@ function kube-up {
    echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'"
    echo "readonly SALT_TAR='${SALT_TAR##*/}'"
    echo "readonly MASTER_HTPASSWD='${htpasswd}'"
+    echo "readonly E2E_STORAGE_TEST_ENVIRONMENT='${E2E_STORAGE_TEST_ENVIRONMENT:-}'"
    grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/create-dynamic-salt-files.sh"
    grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/install-release.sh"
    grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-master.sh"