Improve test script

add usage
verify flag value for -i is numeric
allow multiple targets on the command line
actually capture coverage output
fix lingering GOFLAGS undef issue
fix issue with -i not working at all: ((x++)) returns 1 when x is 0, which is
  incompatible with "set -e"

hack/build-go.sh

@@ -49,10 +49,13 @@ for arg; do
   binaries+=("${KUBE_GO_PACKAGE}/${arg}")
 done
 
+# Use eval to preserve embedded quoted strings.
+eval "goflags=(${GOFLAGS:-})"
+
 # Note that the flags to 'go build' are duplicated in the salt build setup
 # (release/build-release.sh) for our cluster deploy.  If we add more command
 # line options to our standard build we'll want to duplicate them there.  As we
 # move to distributing pre- built binaries we can eliminate this duplication.
-go install ${GOFLAGS:-} \
+go install "${goflags[@]:+${goflags[@]}}" \
     -ldflags "${version_ldflags}" \
     "${binaries[@]}"

hack/test-go.sh

@@ -37,7 +37,11 @@ find_test_dirs() {
         -o -wholename '*/third_party/*' \
         -o -wholename '*/Godeps/*' \
       \) -prune \
-    \) -name '*_test.go' -print0 | xargs -0n1 dirname | sort -u | xargs -n1 printf "${KUBE_GO_PACKAGE}/%s\n"
+    \) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u
+}
+
+find_test_pkgs() {
+  find_test_dirs | xargs -n1 printf "${KUBE_GO_PACKAGE}/%s\n"
 }
 
 # -covermode=atomic becomes default with -race in Go >=1.3
@@ -46,45 +50,76 @@ KUBE_TIMEOUT=${KUBE_TIMEOUT:--timeout 30s}
 
 cd "${KUBE_TARGET}"
 
-while getopts "i:" opt ; do
+usage() {
+  cat << EOF
+usage: $0 [OPTIONS] [TARGETS]
+
+OPTIONS:
+  -i <number>   : number of times to run each test, must be >= 1
+EOF
+}
+
+isnum() {
+  [[ "$1" =~ ^[0-9]+$ ]]
+}
+
+iterations=1
+while getopts "hi:" opt ; do
   case $opt in
+    h)
+      usage
+      exit 0
+      ;;
     i)
-      iterations=$OPTARG
+      iterations="$OPTARG"
+      if ! isnum "${iterations}" || [[ "${iterations}" -le 0 ]]; then
+        echo "$0": argument to -i must be numeric and greater than 0 >&2
+        usage >&2
+        exit 1
+      fi
       ;;
     ?)
-      echo "Invalid argument -$OPTARG"
+      usage >&2
       exit 1
       ;;
     :)
-      echo "Option -$OPTARG <value>"
+      echo "Option -$OPTARG <value>" >&2
+      usage >&2
       exit 1
       ;;
   esac
 done
 shift $((OPTIND - 1))
 
-if [[ -n "${iterations}" ]]; then
-  echo "Running ${iterations} times"
-  if [[ -n "$1" ]]; then
-    pkg=$KUBE_GO_PACKAGE/$1
+# Use eval to preserve embedded quoted strings.
+eval "goflags=(${GOFLAGS:-})"
+
+if [[ "${iterations}" -gt 1 ]]; then
+  if [[ $# -eq 0 ]]; then
+    set -- $(find_test_dirs)
   fi
-  rm -f *.test
-  # build a test binary
-  echo "${pkg}"
-  go test -c -race ${KUBE_TIMEOUT} "${pkg}"
-  # keep going, even if there are failures
-  pass=0
-  count=0
-  for i in $(seq 1 ${iterations}); do
-    for test_binary in *.test; do
-      if "./${test_binary}"; then
-        ((pass++))
+  echo "Running ${iterations} times"
+  fails=0
+  for arg; do
+    trap 'exit 1' SIGINT
+    echo
+    pkg=${KUBE_GO_PACKAGE}/${arg}
+    echo "${pkg}"
+    # keep going, even if there are failures
+    pass=0
+    count=0
+    for i in $(seq 1 ${iterations}); do
+      if go test "${goflags[@]:+${goflags[@]}}" \
+          -race ${KUBE_TIMEOUT} "${pkg}"; then
+        pass=$((pass + 1))
+      else
+        fails=$((fails + 1))
       fi
-      ((count++))
-    done
-  done 2>&1
-  echo "${pass}" / "${count}" passing
-  if [[ ${pass} != ${count} ]]; then
+      count=$((count + 1))
+    done 2>&1
+    echo "${pass}" / "${count}" passed
+  done
+  if [[ ${fails} -gt 0 ]]; then
     exit 1
   else
     exit 0
@@ -92,16 +127,22 @@ if [[ -n "${iterations}" ]]; then
 fi
 
 if [[ -n "$1" ]]; then
-  go test ${GOFLAGS} \
-      -race \
-      ${KUBE_TIMEOUT} \
-      ${KUBE_COVER} -coverprofile=tmp.out \
-      "${KUBE_GO_PACKAGE}/$1" "${@:2}"
+  covdir="/tmp/k8s_coverage/$(date "+%s")"
+  echo saving coverage output in "${covdir}"
+  for arg; do
+    trap 'exit 1' SIGINT
+    mkdir -p "${covdir}/${arg}"
+    pkg=${KUBE_GO_PACKAGE}/${arg}
+    go test "${goflags[@]:+${goflags[@]}}" \
+        -race \
+        ${KUBE_TIMEOUT} \
+        ${KUBE_COVER} -coverprofile="${covdir}/${arg}/coverage.out" \
+        "${pkg}"
+  done
   exit 0
 fi
 
-find_test_dirs | xargs go test ${GOFLAGS:-} \
+find_test_pkgs | xargs go test "${goflags[@]:+${goflags[@]}}" \
     -race \
-    -timeout 30s \
-    ${KUBE_COVER} \
-    "${@:2}"
+    ${KUBE_TIMEOUT} \
+    ${KUBE_COVER}

hack/test-integration.sh

@@ -36,7 +36,8 @@ trap cleanup EXIT SIGINT
 echo
 echo Integration test cases ...
 echo
-$(dirname $0)/../hack/test-go.sh test/integration -tags 'integration no-docker'
+GOFLAGS="-tags 'integration no-docker'" \
+    $(dirname $0)/../hack/test-go.sh test/integration
 # leave etcd running if integration tests fail
 trap "echo etcd still running" EXIT