Merge pull request #3615 from zmerlynn/random_shuffle_go

cmd/e2e: Port --orderseed and --times to cmd/e2e from hack/e2e

cmd/e2e/e2e.go

@@ -32,6 +32,8 @@ var (
 	host       = flag.String("host", "", "The host to connect to")
 	repoRoot   = flag.String("repo_root", "./", "Root directory of kubernetes repository, for finding test files. Default assumes working directory is repository root")
 	provider   = flag.String("provider", "", "The name of the Kubernetes provider")
+	orderseed  = flag.Int64("orderseed", 0, "If non-zero, seed of random test shuffle order. (Otherwise random.)")
+	times      = flag.Int("times", 1, "Number of times each test is eligible to be run. Individual order is determined by shuffling --times instances of each test using --orderseed (like a multi-deck shoe of cards).")
 	testList   util.StringList
 )
 
@@ -46,5 +48,9 @@ func main() {
 		glog.Error("e2e needs the have the --provider flag set")
 		os.Exit(1)
 	}
-	e2e.RunE2ETests(*authConfig, *certDir, *host, *repoRoot, *provider, testList)
+	if *times <= 0 {
+		glog.Error("Invalid --times (negative or no testing requested)!")
+		os.Exit(1)
+	}
+	e2e.RunE2ETests(*authConfig, *certDir, *host, *repoRoot, *provider, *orderseed, *times, testList)
 }

test/e2e/driver.go

@@ -17,6 +17,7 @@ limitations under the License.
 package e2e
 
 import (
+	"math/rand"
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
@@ -29,8 +30,6 @@ type testSpec struct {
 	test func(c *client.Client) bool
 	// The human readable name of this test
 	name string
-	// The id for this test.  It should be constant for the life of the test.
-	id int
 }
 
 type testInfo struct {
@@ -42,18 +41,26 @@ type testInfo struct {
 // See http://testanything.org/ for more info
 func outputTAPSummary(infoList []testInfo) {
 	glog.Infof("1..%d", len(infoList))
-	for _, info := range infoList {
+	for i, info := range infoList {
 		if info.passed {
-			glog.Infof("ok %d - %s", info.spec.id, info.spec.name)
+			glog.Infof("ok %d - %s", i+1, info.spec.name)
 		} else {
-			glog.Infof("not ok %d - %s", info.spec.id, info.spec.name)
+			glog.Infof("not ok %d - %s", i+1, info.spec.name)
 		}
 	}
 }
 
+// Fisher-Yates shuffle using the given RNG r
+func shuffleTests(tests []testSpec, r *rand.Rand) {
+	for i := len(tests) - 1; i > 0; i-- {
+		j := r.Intn(i + 1)
+		tests[i], tests[j] = tests[j], tests[i]
+	}
+}
+
 // Run each Go end-to-end-test. This function assumes the
 // creation of a test cluster.
-func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, testList []string) {
+func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, orderseed int64, times int, testList []string) {
 	testContext = testContextType{authConfig, certDir, host, repoRoot, provider}
 	util.ReallyCrash = true
 	util.InitLogs()
@@ -69,23 +76,22 @@ func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, testList
 	c := loadClientOrDie()
 
 	tests := []testSpec{
-		{TestKubernetesROService, "TestKubernetesROService", 1},
-		{TestKubeletSendsEvent, "TestKubeletSendsEvent", 2},
-		{TestImportantURLs, "TestImportantURLs", 3},
-		{TestPodUpdate, "TestPodUpdate", 4},
-		{TestNetwork, "TestNetwork", 5},
-		{TestClusterDNS, "TestClusterDNS", 6},
-		{TestPodHasServiceEnvVars, "TestPodHasServiceEnvVars", 7},
-		{TestBasic, "TestBasic", 8},
-		{TestPrivate, "TestPrivate", 9},
+		{TestKubernetesROService, "TestKubernetesROService"},
+		{TestKubeletSendsEvent, "TestKubeletSendsEvent"},
+		{TestImportantURLs, "TestImportantURLs"},
+		{TestPodUpdate, "TestPodUpdate"},
+		{TestNetwork, "TestNetwork"},
+		{TestClusterDNS, "TestClusterDNS"},
+		{TestPodHasServiceEnvVars, "TestPodHasServiceEnvVars"},
+		{TestBasic, "TestBasic"},
+		{TestPrivate, "TestPrivate"},
 	}
 
+	// Check testList for non-existent tests and populate a StringSet with tests to run.
 	validTestNames := util.NewStringSet()
 	for _, test := range tests {
 		validTestNames.Insert(test.name)
 	}
-
-	// Check testList for non-existent tests and populate a StringSet with tests to run.
 	runTestNames := util.NewStringSet()
 	for _, testName := range testList {
 		if validTestNames.Has(testName) {
@@ -95,15 +101,39 @@ func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, testList
 		}
 	}
 
+	// if testList was specified, filter down now before we expand and shuffle
+	if len(testList) > 0 {
+		newTests := make([]testSpec, 0)
+		for i, test := range tests {
+			// Check if this test is supposed to run, either if listed explicitly in
+			// a --test flag or if no --test flags were supplied.
+			if !runTestNames.Has(test.name) {
+				glog.Infof("Skipping test %d %s", i+1, test.name)
+				continue
+			}
+			newTests = append(newTests, test)
+		}
+		tests = newTests
+	}
+	if times != 1 {
+		newTests := make([]testSpec, 0, times*len(tests))
+		for i := 0; i < times; i++ {
+			newTests = append(newTests, tests...)
+		}
+		tests = newTests
+	}
+	if orderseed == 0 {
+		// Use low order bits of NanoTime as the default seed. (Using
+		// all the bits makes for a long, very similar looking seed
+		// between runs.)
+		orderseed = time.Now().UnixNano() & (1<<32 - 1)
+	}
+	shuffleTests(tests, rand.New(rand.NewSource(orderseed)))
+	glog.Infof("Tests shuffled with orderseed %#x\n", orderseed)
+
 	info := []testInfo{}
 	passed := true
 	for i, test := range tests {
-		// Check if this test is supposed to run, either if listed explicitly in
-		// a --test flag or if no --test flags were supplied.
-		if len(testList) > 0 && !runTestNames.Has(test.name) {
-			glog.Infof("Skipping test %d %s", i+1, test.name)
-			continue
-		}
 		glog.Infof("Running test %d %s", i+1, test.name)
 		testPassed := test.test(c)
 		if !testPassed {