Merge pull request #27898 from lavalamp/fixv

Automatic merge from submit-queue

Detect flakes in PR builder e2e runs

Won't be mergable until https://github.com/onsi/ginkgo/pull/261 is agreed upon and merged.

Tossing a PR here to get the e2e test to run on it.

hack/ginkgo-e2e.sh

@@ -30,6 +30,9 @@ e2e_test=$(kube::util::find-binary "e2e.test")
 
 GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel
 
+# If 'y', will rerun failed tests once to give them a second chance.
+GINKGO_TOLERATE_FLAKES=${GINKGO_TOLERATE_FLAKES:-n}
+
 # The number of tests that can run in parallel depends on what tests
 # are running and on the size of the cluster. Too many, and tests will
 # fail due to resource contention. 25 is a reasonable default for a
@@ -101,6 +104,11 @@ elif [[ ${GINKGO_PARALLEL} =~ ^[yY]$ ]]; then
   ginkgo_args+=("--nodes=25")
 fi
 
+FLAKE_ATTEMPTS=1
+if [[ "${GINKGO_TOLERATE_FLAKES}" == "y" ]]; then
+  FLAKE_ATTEMPTS=2
+fi
+
 # The --host setting is used only when providing --auth_config
 # If --kubeconfig is used, the host to use is retrieved from the .kubeconfig
 # file and the one provided with --host is ignored.
@@ -108,6 +116,7 @@ fi
 export PATH=$(dirname "${e2e_test}"):"${PATH}"
 "${ginkgo}" "${ginkgo_args[@]:+${ginkgo_args[@]}}" "${e2e_test}" -- \
   "${auth_config[@]:+${auth_config[@]}}" \
+  --ginkgo.flakeAttempts="${FLAKE_ATTEMPTS}" \
   --host="${KUBE_MASTER_URL}" \
   --provider="${KUBERNETES_PROVIDER}" \
   --gce-project="${PROJECT:-}" \

hack/jenkins/upload-to-gcs.sh

@@ -46,13 +46,19 @@ fi
 
 if [[ ${JOB_NAME} =~ -pull- ]]; then
   : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/pull/${ghprbPullId:-unknown}"}
+  : ${JENKINS_GCS_LATEST_PATH:="gs://kubernetes-jenkins/pr-logs/directory"}
+  : ${JENKINS_GCS_LOGS_INDIRECT:="gs://kubernetes-jenkins/pr-logs/directory/${JOB_NAME}"}
 else
   : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"}
+  : ${JENKINS_GCS_LATEST_PATH:="gs://kubernetes-jenkins/logs"}
+  : ${JENKINS_GCS_LOGS_INDIRECT:=""}
 fi
 
 readonly artifacts_path="${WORKSPACE}/_artifacts"
 readonly gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}"
 readonly gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}"
+readonly gcs_latest_path="${JENKINS_GCS_LATEST_PATH}/${JOB_NAME}"
+readonly gcs_indirect_path="${JENKINS_GCS_LOGS_INDIRECT}"
 readonly gcs_acl="public-read"
 readonly results_url=${gcs_build_path//"gs:/"/"https://console.cloud.google.com/storage/browser"}
 readonly timestamp=$(date +%s)
@@ -115,11 +121,25 @@ function upload_artifacts_and_build_result() {
       echo "Uploading build log"
       gsutil -q cp -Z -a "${gcs_acl}" "${WORKSPACE}/build-log.txt" "${gcs_build_path}"
     fi
+
+    # For pull jobs, keep a canonical ordering for tools that want to examine
+    # the output.
+    if [[ "${gcs_indirect_path}" != "" ]]; then
+      echo "Writing ${gcs_build_path} to ${gcs_indirect_path}/${BUILD_NUMBER}.txt"
+      echo "${gcs_build_path}" | \
+        gsutil -q -h "Content-Type:text/plain" \
+          cp -a "${gcs_acl}" - "${gcs_indirect_path}/${BUILD_NUMBER}.txt" || continue
+      echo "Marking build ${BUILD_NUMBER} as the latest completed build for this PR"
+      echo "${BUILD_NUMBER}" | \
+        gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
+          cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
+    fi
+
     # Mark this build as the latest completed.
     echo "Marking build ${BUILD_NUMBER} as the latest completed build"
     echo "${BUILD_NUMBER}" | \
       gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform" \
-        cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue
+        cp -a "${gcs_acl}" - "${gcs_latest_path}/latest-build.txt" || continue
     break  # all uploads succeeded if we hit this point
   done
 

vendor/github.com/onsi/ginkgo/config/config.go

@@ -31,6 +31,7 @@ type GinkgoConfigType struct {
 	SkipMeasurements   bool
 	FailOnPending      bool
 	FailFast           bool
+	FlakeAttempts      int
 	EmitSpecProgress   bool
 	DryRun             bool
 
@@ -75,6 +76,8 @@ func Flags(flagSet *flag.FlagSet, prefix string, includeParallelFlags bool) {
 
 	flagSet.BoolVar(&(GinkgoConfig.RegexScansFilePath), prefix+"regexScansFilePath", false, "If set, ginkgo regex matching also will look at the file path (code location).")
 
+	flagSet.IntVar(&(GinkgoConfig.FlakeAttempts), prefix+"flakeAttempts", 1, "Make up to this many attempts to run each spec. Please note that if any of the attempts succeed, the suite will not be failed. But any failures will still be recorded.")
+
 	flagSet.BoolVar(&(GinkgoConfig.EmitSpecProgress), prefix+"progress", false, "If set, ginkgo will emit progress information as each spec runs to the GinkgoWriter.")
 
 	if includeParallelFlags {
@@ -128,6 +131,10 @@ func BuildFlagArgs(prefix string, ginkgo GinkgoConfigType, reporter DefaultRepor
 		result = append(result, fmt.Sprintf("--%sskip=%s", prefix, ginkgo.SkipString))
 	}
 
+	if ginkgo.FlakeAttempts > 1 {
+		result = append(result, fmt.Sprintf("--%sflakeAttempts=%d", prefix, ginkgo.FlakeAttempts))
+	}
+
 	if ginkgo.EmitSpecProgress {
 		result = append(result, fmt.Sprintf("--%sprogress", prefix))
 	}

vendor/github.com/onsi/ginkgo/ginkgo/help_command.go

@@ -9,7 +9,7 @@ func BuildHelpCommand() *Command {
 	return &Command{
 		Name:         "help",
 		FlagSet:      flag.NewFlagSet("help", flag.ExitOnError),
-		UsageCommand: "ginkgo help <COMAND>",
+		UsageCommand: "ginkgo help <COMMAND>",
 		Usage: []string{
 			"Print usage information.  If a command is passed in, print usage information just for that command.",
 		},

vendor/github.com/onsi/ginkgo/ginkgo/interrupthandler/sigquit_swallower_unix.go

@@ -1,4 +1,4 @@
-// +build freebsd openbsd netbsd dragonfly darwin linux
+// +build freebsd openbsd netbsd dragonfly darwin linux solaris
 
 package interrupthandler
 

vendor/github.com/onsi/ginkgo/ginkgo/testrunner/test_runner.go

@@ -51,7 +51,7 @@ func New(suite testsuite.TestSuite, numCPU int, parallelStream bool, race bool,
 	if !suite.Precompiled {
 		dir, err := ioutil.TempDir("", "ginkgo")
 		if err != nil {
-			panic(fmt.Sprintf("coulnd't create temporary directory... might be time to rm -rf:\n%s", err.Error()))
+			panic(fmt.Sprintf("couldn't create temporary directory... might be time to rm -rf:\n%s", err.Error()))
 		}
 		runner.compilationTargetPath = filepath.Join(dir, suite.PackageName+".test")
 	}

vendor/github.com/onsi/ginkgo/ginkgo_dsl.go

@@ -168,6 +168,8 @@ func CurrentGinkgoTestDescription() GinkgoTestDescription {
 //
 //You use the Time() function to time how long the passed in body function takes to run
 //You use the RecordValue() function to track arbitrary numerical measurements.
+//The RecordValueWithPrecision() function can be used alternatively to provide the unit
+//and resolution of the numeric measurement.
 //The optional info argument is passed to the test reporter and can be used to
 // provide the measurement data to a custom reporter with context.
 //
@@ -175,6 +177,7 @@ func CurrentGinkgoTestDescription() GinkgoTestDescription {
 type Benchmarker interface {
 	Time(name string, body func(), info ...interface{}) (elapsedTime time.Duration)
 	RecordValue(name string, value float64, info ...interface{})
+	RecordValueWithPrecision(name string, value float64, units string, precision int, info ...interface{})
 }
 
 //RunSpecs is the entry point for the Ginkgo test runner.

vendor/github.com/onsi/ginkgo/internal/leafnodes/benchmarker.go

@@ -28,20 +28,27 @@ func (b *benchmarker) Time(name string, body func(), info ...interface{}) (elaps
 
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	measurement := b.getMeasurement(name, "Fastest Time", "Slowest Time", "Average Time", "s", info...)
+	measurement := b.getMeasurement(name, "Fastest Time", "Slowest Time", "Average Time", "s", 3, info...)
 	measurement.Results = append(measurement.Results, elapsedTime.Seconds())
 
 	return
 }
 
 func (b *benchmarker) RecordValue(name string, value float64, info ...interface{}) {
-	measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", "", info...)
+	measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", "", 3, info...)
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	measurement.Results = append(measurement.Results, value)
 }
 
-func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestLabel string, averageLabel string, units string, info ...interface{}) *types.SpecMeasurement {
+func (b *benchmarker) RecordValueWithPrecision(name string, value float64, units string, precision int, info ...interface{}) {
+	measurement := b.getMeasurement(name, "Smallest", " Largest", " Average", units, precision, info...)
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	measurement.Results = append(measurement.Results, value)
+}
+
+func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestLabel string, averageLabel string, units string, precision int, info ...interface{}) *types.SpecMeasurement {
 	measurement, ok := b.measurements[name]
 	if !ok {
 		var computedInfo interface{}
@@ -57,6 +64,7 @@ func (b *benchmarker) getMeasurement(name string, smallestLabel string, largestL
 			LargestLabel:  largestLabel,
 			AverageLabel:  averageLabel,
 			Units:         units,
+			Precision:     precision,
 			Results:       make([]float64, 0),
 		}
 		b.measurements[name] = measurement

vendor/github.com/onsi/ginkgo/internal/remote/output_interceptor_unix.go

@@ -1,4 +1,4 @@
-// +build freebsd openbsd netbsd dragonfly darwin linux
+// +build freebsd openbsd netbsd dragonfly darwin linux solaris
 
 package remote
 

vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_solaris.go

@@ -0,0 +1,9 @@
+// +build solaris
+
+package remote
+
+import "golang.org/x/sys/unix"
+
+func syscallDup(oldfd int, newfd int) (err error) {
+	return unix.Dup2(oldfd, newfd)
+}

vendor/github.com/onsi/ginkgo/internal/remote/syscall_dup_unix.go

@@ -1,5 +1,6 @@
 // +build !linux !arm64
 // +build !windows
+// +build !solaris
 
 package remote
 

vendor/github.com/onsi/ginkgo/internal/spec/spec.go

@@ -20,6 +20,7 @@ type Spec struct {
 	state            types.SpecState
 	runTime          time.Duration
 	failure          types.SpecFailure
+	previousFailures bool
 }
 
 func New(subject leafnodes.SubjectNode, containers []*containernode.ContainerNode, announceProgress bool) *Spec {
@@ -58,6 +59,10 @@ func (spec *Spec) Passed() bool {
 	return spec.state == types.SpecStatePassed
 }
 
+func (spec *Spec) Flaked() bool {
+	return spec.state == types.SpecStatePassed && spec.previousFailures
+}
+
 func (spec *Spec) Pending() bool {
 	return spec.state == types.SpecStatePending
 }
@@ -109,6 +114,10 @@ func (spec *Spec) ConcatenatedString() string {
 }
 
 func (spec *Spec) Run(writer io.Writer) {
+	if spec.state == types.SpecStateFailed {
+		spec.previousFailures = true
+	}
+
 	startTime := time.Now()
 	defer func() {
 		spec.runTime = time.Since(startTime)

vendor/github.com/onsi/ginkgo/internal/specrunner/spec_runner.go

@@ -137,20 +137,19 @@ func (runner *SpecRunner) runSpecs() bool {
 		if skipRemainingSpecs {
 			spec.Skip()
 		}
-		runner.reportSpecWillRun(spec.Summary(runner.suiteID))
 
 		if !spec.Skipped() && !spec.Pending() {
-			runner.runningSpec = spec
-			spec.Run(runner.writer)
-			runner.runningSpec = nil
-			if spec.Failed() {
+			if passed := runner.runSpec(spec); !passed {
 				suiteFailed = true
 			}
 		} else if spec.Pending() && runner.config.FailOnPending {
+			runner.reportSpecWillRun(spec.Summary(runner.suiteID))
 			suiteFailed = true
-		}
-
 			runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed())
+		} else {
+			runner.reportSpecWillRun(spec.Summary(runner.suiteID))
+			runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed())
+		}
 
 		if spec.Failed() && runner.config.FailFast {
 			skipRemainingSpecs = true
@@ -160,6 +159,26 @@ func (runner *SpecRunner) runSpecs() bool {
 	return !suiteFailed
 }
 
+func (runner *SpecRunner) runSpec(spec *spec.Spec) (passed bool) {
+	maxAttempts := 1
+	if runner.config.FlakeAttempts > 0 {
+		// uninitialized configs count as 1
+		maxAttempts = runner.config.FlakeAttempts
+	}
+
+	for i := 0; i < maxAttempts; i++ {
+		runner.reportSpecWillRun(spec.Summary(runner.suiteID))
+		runner.runningSpec = spec
+		spec.Run(runner.writer)
+		runner.runningSpec = nil
+		runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed())
+		if !spec.Failed() {
+			return true
+		}
+	}
+	return false
+}
+
 func (runner *SpecRunner) CurrentSpecSummary() (*types.SpecSummary, bool) {
 	if runner.runningSpec == nil {
 		return nil, false
@@ -300,6 +319,10 @@ func (runner *SpecRunner) summary(success bool) *types.SuiteSummary {
 		return ex.Passed()
 	})
 
+	numberOfFlakedSpecs := runner.countSpecsSatisfying(func(ex *spec.Spec) bool {
+		return ex.Flaked()
+	})
+
 	numberOfFailedSpecs := runner.countSpecsSatisfying(func(ex *spec.Spec) bool {
 		return ex.Failed()
 	})
@@ -320,5 +343,6 @@ func (runner *SpecRunner) summary(success bool) *types.SuiteSummary {
 		NumberOfSkippedSpecs:               numberOfSkippedSpecs,
 		NumberOfPassedSpecs:                numberOfPassedSpecs,
 		NumberOfFailedSpecs:                numberOfFailedSpecs,
+		NumberOfFlakedSpecs:                numberOfFlakedSpecs,
 	}
 }

vendor/github.com/onsi/ginkgo/reporters/stenographer/stenographer.go

@@ -506,15 +506,15 @@ func (s *consoleStenographer) measurementReport(spec *types.SpecSummary, succinc
 			message = append(message, fmt.Sprintf("  %s - %s: %s%s, %s: %s%s ± %s%s, %s: %s%s",
 				s.colorize(boldStyle, "%s", measurement.Name),
 				measurement.SmallestLabel,
-				s.colorize(greenColor, "%.3f", measurement.Smallest),
+				s.colorize(greenColor, measurement.PrecisionFmt(), measurement.Smallest),
 				measurement.Units,
 				measurement.AverageLabel,
-				s.colorize(cyanColor, "%.3f", measurement.Average),
+				s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.Average),
 				measurement.Units,
-				s.colorize(cyanColor, "%.3f", measurement.StdDeviation),
+				s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.StdDeviation),
 				measurement.Units,
 				measurement.LargestLabel,
-				s.colorize(redColor, "%.3f", measurement.Largest),
+				s.colorize(redColor, measurement.PrecisionFmt(), measurement.Largest),
 				measurement.Units,
 			))
 		}
@@ -531,15 +531,15 @@ func (s *consoleStenographer) measurementReport(spec *types.SpecSummary, succinc
 				s.colorize(boldStyle, "%s", measurement.Name),
 				info,
 				measurement.SmallestLabel,
-				s.colorize(greenColor, "%.3f", measurement.Smallest),
+				s.colorize(greenColor, measurement.PrecisionFmt(), measurement.Smallest),
 				measurement.Units,
 				measurement.LargestLabel,
-				s.colorize(redColor, "%.3f", measurement.Largest),
+				s.colorize(redColor, measurement.PrecisionFmt(), measurement.Largest),
 				measurement.Units,
 				measurement.AverageLabel,
-				s.colorize(cyanColor, "%.3f", measurement.Average),
+				s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.Average),
 				measurement.Units,
-				s.colorize(cyanColor, "%.3f", measurement.StdDeviation),
+				s.colorize(cyanColor, measurement.PrecisionFmt(), measurement.StdDeviation),
 				measurement.Units,
 			))
 		}

vendor/github.com/onsi/ginkgo/types/types.go

@@ -1,6 +1,9 @@
 package types
 
-import "time"
+import (
+	"strconv"
+	"time"
+)
 
 const GINKGO_FOCUS_EXIT_CODE = 197
 
@@ -16,6 +19,9 @@ type SuiteSummary struct {
 	NumberOfSkippedSpecs               int
 	NumberOfPassedSpecs                int
 	NumberOfFailedSpecs                int
+	// Flaked specs are those that failed initially, but then passed on a
+	// subsequent try.
+	NumberOfFlakedSpecs int
 	RunTime             time.Duration
 }
 
@@ -100,6 +106,17 @@ type SpecMeasurement struct {
 	LargestLabel  string
 	AverageLabel  string
 	Units         string
+	Precision     int
+}
+
+func (s SpecMeasurement) PrecisionFmt() string {
+	if s.Precision == 0 {
+		return "%f"
+	}
+
+	str := strconv.Itoa(s.Precision)
+
+	return "%." + str + "f"
 }
 
 type SpecState uint