Merge pull request #37866 from fejta/log

Automatic merge from submit-queue

Add optional timeout logic to e2e.go

ref https://github.com/kubernetes/test-infra/issues/1250

Add a --timeout=5 flag.

If unset this does not change behavior.
If set this first sends an interrupt signal and then 15m later starts terminating child processes.

Will test out this change in another PR using the .use_head_e2e trick

hack/e2e.go

@@ -36,6 +36,8 @@ import (
 )
 
 var (
+	interrupt = time.NewTimer(time.Duration(0)) // interrupt testing at this time.
+	terminate = time.NewTimer(time.Duration(0)) // terminate testing at this time.
 	// TODO(fejta): change all these _ flags to -
 	build            = flag.Bool("build", false, "If true, build a new release. Otherwise, use whatever is there.")
 	checkVersionSkew = flag.Bool("check_version_skew", true, ""+
@@ -50,6 +52,7 @@ var (
 	skewTests            = flag.Bool("skew", false, "If true, run tests in another version at ../kubernetes/hack/e2e.go")
 	testArgs             = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
 	test                 = flag.Bool("test", false, "Run Ginkgo tests.")
+	timeout              = flag.Duration("timeout", time.Duration(0), "Terminate testing after the timeout duration (s/m/h)")
 	up                   = flag.Bool("up", false, "If true, start the the e2e cluster. If cluster is already up, recreate it.")
 	upgradeArgs          = flag.String("upgrade_args", "", "If set, run upgrade tests before other tests")
 	verbose              = flag.Bool("v", false, "If true, print all command output.")
@@ -184,6 +187,18 @@ func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 	flag.Parse()
 
+	if !terminate.Stop() {
+		<-terminate.C // Drain the value if necessary.
+	}
+	if !interrupt.Stop() {
+		<-interrupt.C // Drain value
+	}
+
+	if *timeout > 0 {
+		log.Printf("Limiting testing to %s", *timeout)
+		interrupt.Reset(*timeout)
+	}
+
 	if err := validWorkingDirectory(); err != nil {
 		log.Fatalf("Called from invalid working directory: %v", err)
 	}
@@ -921,8 +936,28 @@ func finishRunning(stepName string, cmd *exec.Cmd) error {
 		log.Printf("Step '%s' finished in %s", stepName, time.Since(start))
 	}(time.Now())
 
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("error running %v: %v", stepName, err)
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("error starting %v: %v", stepName, err)
+	}
+
+	finished := make(chan error)
+
+	go func() {
+		finished <- cmd.Wait()
+	}()
+
+	for {
+		select {
+		case <-terminate.C:
+			terminate.Reset(time.Duration(0)) // Kill subsequent processes immediately.
+			cmd.Process.Kill()
+			return fmt.Errorf("Terminate testing after 15m after %s timeout during %s", *timeout, stepName)
+		case <-interrupt.C:
+			log.Printf("Interrupt testing after %s timeout. Will terminate in another 15m", *timeout)
+			terminate.Reset(15 * time.Minute)
+			cmd.Process.Signal(os.Interrupt)
+		case err := <-finished:
+			return err
+		}
 	}
-	return nil
 }