Refactor test retry to only affect CI (#4436)

* Refactor test retry to only affect CI

* Move test install deps out of the retry loop

* Add internal targets to PHONY too

.travis.yml

@@ -15,6 +15,6 @@ matrix:
     - env: GOTEST_PKGS_EXCLUDE="./api|./agent|./agent/consul"
 
 script:
-  - GOTEST_FLAGS="-p 3 -parallel 1" make test
+  - make test-ci
 
 sudo: true

GNUmakefile

@@ -150,21 +150,20 @@ cov:
 	gocov test $(GOFILES) | gocov-html > /tmp/coverage.html
 	open /tmp/coverage.html
 
-test: other-consul dev-build vet
+test: other-consul dev-build vet test-install-deps test-internal
+
+test-install-deps:
+	go test -tags '$(GOTAGS)' -i $(GOTEST_PKGS)
+
+test-internal:
 	@echo "--> Running go test"
 	@rm -f test.log exit-code
-	go test -tags '$(GOTAGS)' -i $(GOTEST_PKGS)
 	@# Dump verbose output to test.log so we can surface test names on failure but
 	@# hide it from travis as it exceeds their log limits and causes job to be
 	@# terminated (over 4MB and over 10k lines in the UI). We need to output
 	@# _something_ to stop them terminating us due to inactivity...
 	{ go test $(GOTEST_FLAGS) -tags '$(GOTAGS)' -timeout 7m $(GOTEST_PKGS) 2>&1 ; echo $$? > exit-code ; } | tee test.log | egrep '^(ok|FAIL)\s*github.com/hashicorp/consul'
-	@echo "Exit code: $$(cat exit-code)" >> test.log
+	@echo "Exit code: $$(cat exit-code)"
-	@if [ "0" != "$$(cat exit-code)" ]; then \
-	  echo Retrying to avoid flacky tests >> test.log;\
-	  echo Retrying tests once...;\
-	  go test -p 5 -parallel 1 -tags '$(GOTAGS)' -timeout 5m $(GOTEST_PKGS) 2>&1; echo $$? > exit-code;\
-	fi
 	@# This prints all the race report between ====== lines
 	@awk '/^WARNING: DATA RACE/ {do_print=1; print "=================="} do_print==1 {print} /^={10,}/ {do_print=0}' test.log || true
 	@grep -A10 'panic: ' test.log || true
@@ -181,6 +180,15 @@ test: other-consul dev-build vet
 test-race:
 	$(MAKE) GOTEST_FLAGS=-race
 
+# Run tests with config for CI so `make test` can still be local-dev friendly.
+test-ci: other-consul dev-build vet test-install-deps
+	@ if ! GOTEST_FLAGS="-p 3 -parallel 1" make test-internal; then \
+	    echo "    ============"; \
+			echo "      Retrying"; \
+	    echo "    ============"; \
+			GOTEST_FLAGS="-p 5 -parallel 1" make test-internal; \
+		fi
+
 other-consul:
 	@echo "--> Checking for other consul instances"
 	@if ps -ef | grep 'consul agent' | grep -v grep ; then \
@@ -256,5 +264,5 @@ ui-legacy-docker: ui-legacy-build-image
 	@$(SHELL) $(CURDIR)/build-support/scripts/build-docker.sh ui-legacy
 	
 	
-.PHONY: all ci bin dev dist cov test cover format vet ui static-assets tools vendorfmt 
+.PHONY: all ci bin dev dist cov test test-ci test-internal test-install-deps cover format vet ui static-assets tools vendorfmt
 .PHONY: docker-images go-build-image ui-build-image ui-legacy-build-image static-assets-docker consul-docker ui-docker ui-legacy-docker version