From a2b7117ee440cd0e8ffca3853a88a72de1d1b0d8 Mon Sep 17 00:00:00 2001
From: Tim Hockin <thockin@google.com>
Date: Wed, 3 Jun 2015 12:09:54 -0700
Subject: [PATCH] Make services e2e more debuggable

---
 hack/e2e-suite/services.sh | 84 ++++++++++++++++++++------------------
 1 file changed, 44 insertions(+), 40 deletions(-)

diff --git a/hack/e2e-suite/services.sh b/hack/e2e-suite/services.sh
index ebe644a2ce..863c08dbf9 100755
--- a/hack/e2e-suite/services.sh
+++ b/hack/e2e-suite/services.sh
@@ -28,6 +28,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
 
 export KUBECTL KUBE_CONFIG_FILE
 
+TEST_NAMESPACE="services-test-${RANDOM}"
+KUBECTL="${KUBECTL} --namespace=${TEST_NAMESPACE}"
+
 source "${KUBE_ROOT}/cluster/kube-env.sh"
 source "${KUBE_VERSION_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
 
@@ -39,7 +42,9 @@ function error() {
 }
 
 function sort_args() {
-  printf "%s\n" "$@" | sort -n | tr '\n\r' ' ' | sed 's/  */ /g'
+  [ $# == 0 ] && return
+  a=($(printf "%s\n" "$@" | sort -n))
+  echo "${a[*]}"
 }
 
 # Join args $2... with $1 between them.
@@ -58,10 +63,20 @@ function join() {
 
 svcs_to_clean=()
 function do_teardown() {
-  local svc
-  for svc in "${svcs_to_clean[@]:+${svcs_to_clean[@]}}"; do
-    stop_service "${svc}"
-  done
+  ${KUBECTL} delete namespace "${TEST_NAMESPACE}"
+}
+
+function make_namespace() {
+  echo "Making namespace '${TEST_NAMESPACE}'"
+  ${KUBECTL} create -f - << __EOF__
+{
+    "kind": "Namespace",
+    "apiVersion": "v1beta3",
+    "metadata": {
+        "name": "${TEST_NAMESPACE}"
+    }
+}
+__EOF__
 }
 
 # Args:
@@ -70,7 +85,7 @@ function do_teardown() {
 #   $3: service replica count
 #   $4: public IPs (optional, string e.g. "1.2.3.4 5.6.7.8")
 function start_service() {
-  echo "Starting service '$1' on port $2 with $3 replicas"
+  echo "Starting service '${TEST_NAMESPACE}/$1' on port $2 with $3 replicas"
   svcs_to_clean+=("$1")
   ${KUBECTL} create -f - << __EOF__
 {
@@ -78,7 +93,6 @@ function start_service() {
   "apiVersion": "v1beta3",
   "metadata": {
     "name": "$1",
-    "namespace": "default",
     "labels": {
       "name": "$1"
     }
@@ -124,7 +138,6 @@ __EOF__
   "apiVersion": "v1beta3",
   "metadata": {
     "name": "$1",
-    "namespace": "default",
     "labels": {
       "name": "$1"
     }
@@ -218,30 +231,28 @@ function wait_for_pods() {
 #   $2: service IP
 #   $3: service port
 #   $4: pod count
-#   $5: pod IDs
+#   $5: pod IDs (sorted)
 function wait_for_service_up() {
   local i
   local found_pods
-  for i in $(seq 1 20); do
+  echo "waiting for $1 at $2:$3"
+  for i in $(seq 1 5); do
     results=($(ssh-to-node "${test_node}" "
         set -e;
-        for i in $(seq -s' ' 1 $4); do
+        for i in $(seq -s' ' 1 $(($4*3))); do
           curl -s --connect-timeout 1 http://$2:$3;
           echo;
-        done | sort | uniq
+        done | sort -n | uniq
         "))
 
     found_pods=$(sort_args "${results[@]:+${results[@]}}")
-    echo "Checking if ${found_pods} == ${5}"
     if [[ "${found_pods}" == "$5" ]]; then
-      break
+      return
     fi
-    echo "Waiting for endpoints to propagate"
-    sleep 3
+    echo "expected '$5', got '${found_pods}': will try again"
+    sleep 3  # wait for endpoints to propagate
   done
-  if [[ "${found_pods}" != "$5" ]]; then
-    error "Endpoints did not propagate in time"
-  fi
+  error "$1: failed to verify portal from host"
 }
 
 # Args:
@@ -264,35 +275,26 @@ function wait_for_service_down() {
 #   $2: service IP
 #   $3: service port
 #   $4: pod count
-#   $5: pod IDs
+#   $5: pod IDs (sorted)
 function verify_from_container() {
+  echo "waiting for $1 at $2:$3"
   results=($(ssh-to-node "${test_node}" "
       set -e;
       sudo docker pull busybox >/dev/null;
       sudo docker run busybox sh -c '
-          for i in $(seq -s' ' 1 $4); do
-            ok=false
-            for j in $(seq -s' ' 1 10); do
-              if wget -q -T 5 -O - http://$2:$3; then
-                echo
-                ok=true
-                break
-              fi
-              sleep 1
-            done
-            if [[ \${ok} == false ]]; then
+          for i in $(seq -s' ' 1 $(($4*3))); do
+            if wget -q -T 3 -O - http://$2:$3; then
+              echo
+            else
               exit 1
             fi
           done
-      '")) \
+      '" | sort -r -n | uniq)) \
       || error "testing $1 VIP from container failed"
   found_pods=$(sort_args "${results[@]}")
   if [[ "${found_pods}" != "$5" ]]; then
-    error -e "$1 VIP failed from container, expected:\n
-        $(printf '\t%s\n' $5)\n
-        got:\n
-        $(printf '\t%s\n' ${found_pods})
-        "
+    echo "expected '$5', got '${found_pods}'"
+    error "$1: failed to verify VIP from container"
   fi
 }
 
@@ -303,14 +305,17 @@ detect-minions
 test_node="${MINION_NAMES[0]}"
 master="${MASTER_NAME}"
 
+# Make our namespace
+make_namespace
+
 # Launch some pods and services.
-svc1_name="service-${RANDOM}"
+svc1_name="service1"
 svc1_port=80
 svc1_count=3
 svc1_publics="192.168.1.1 192.168.1.2"
 start_service "${svc1_name}" "${svc1_port}" "${svc1_count}" "${svc1_publics}"
 
-svc2_name="service-${RANDOM}"
+svc2_name="service2"
 svc2_port=80
 svc2_count=3
 start_service "${svc2_name}" "${svc2_port}" "${svc2_count}"
@@ -414,7 +419,6 @@ echo "Manually removing iptables rules"
 # Remove both the new and old style chains, in case we're testing on an old kubelet
 ssh-to-node "${test_node}" "sudo iptables -t nat -F KUBE-PORTALS-HOST || true"
 ssh-to-node "${test_node}" "sudo iptables -t nat -F KUBE-PORTALS-CONTAINER || true"
-ssh-to-node "${test_node}" "sudo iptables -t nat -F KUBE-PROXY || true"
 echo "Verifying the VIPs from the host"
 wait_for_service_up "${svc3_name}" "${svc3_ip}" "${svc3_port}" \
     "${svc3_count}" "${svc3_pods}"