Merge pull request #6655 from yujuhong/parallel_tests

Limit the number of concurrent tests in integration.go

.travis.yml

@@ -24,7 +24,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"

cmd/integration/integration.go

@@ -67,6 +67,8 @@ var (
 	fakeDocker1, fakeDocker2 dockertools.FakeDockerClient
 	// API version that should be used by the client to talk to the server.
 	apiVersion string
+	// Limit the number of concurrent tests.
+	maxConcurrency int
 )
 
 type fakeKubeletClient struct{}
@@ -947,6 +949,8 @@ type testFunc func(*client.Client)
 
 func addFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&apiVersion, "apiVersion", latest.Version, "API version that should be used by the client for communicating with the server")
+	fs.IntVar(
+		&maxConcurrency, "maxConcurrency", -1, "Maximum number of tests to be run simultaneously. Unlimited if set to negative.")
 }
 
 func main() {
@@ -994,16 +998,26 @@ func main() {
 		},
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(len(testFuncs))
-	for i := range testFuncs {
-		f := testFuncs[i]
-		go func() {
-			f(kubeClient)
-			wg.Done()
-		}()
+	// Only run at most maxConcurrency tests in parallel.
+	numFinishedTests := 0
+	for numFinishedTests < len(testFuncs) {
+		numTestsToRun := len(testFuncs) - numFinishedTests
+		if maxConcurrency > 0 && numTestsToRun > maxConcurrency {
+			numTestsToRun = maxConcurrency
+		}
+		glog.Infof("Running %d tests in parallel.", numTestsToRun)
+		var wg sync.WaitGroup
+		wg.Add(numTestsToRun)
+		for i := 0; i < numTestsToRun; i++ {
+			f := testFuncs[i+numFinishedTests]
+			go func() {
+				f(kubeClient)
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		numFinishedTests += numTestsToRun
 	}
-	wg.Wait()
 
 	// Check that kubelet tried to make the containers.
 	// Using a set to list unique creation attempts. Our fake is

hack/test-integration.sh

@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 # Comma separated list of API Versions that should be tested.
 KUBE_TEST_API_VERSIONS=${KUBE_TEST_API_VERSIONS:-"v1beta1,v1beta3"}
-
+KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY:-"-1"}
 
 cleanup() {
   kube::etcd::cleanup
@@ -44,7 +44,8 @@ runTests() {
 
   kube::log::status "Running integration test scenario"
 
-  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1"
+  "${KUBE_OUTPUT_HOSTBIN}/integration" --v=2 --apiVersion="$1" \
+  --maxConcurrency="${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY}"
 
   cleanup
 }

shippable.yml

@@ -33,7 +33,7 @@ install:
 script:
   - KUBE_RACE="-race" KUBE_COVER="y" KUBE_GOVERALLS_BIN="$HOME/gopath/bin/goveralls" KUBE_TIMEOUT='-timeout 300s' KUBE_COVERPROCS=8 KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-go.sh -- -p=2
   - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH ./hack/test-cmd.sh
-  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS ./hack/test-integration.sh
+  - PATH=$HOME/gopath/bin:./third_party/etcd:$PATH KUBE_TEST_API_VERSIONS=$KUBE_TEST_API_VERSIONS KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=4 ./hack/test-integration.sh
 
 notifications:
   irc: "chat.freenode.net#google-containers"