Merge pull request #7362 from wojtek-t/density_events_histogram

Don't fail performance tests on single fail event

test/e2e/density.go

@@ -18,6 +18,7 @@ package e2e
 
 import (
 	"fmt"
+	"math"
 	"strconv"
 	"sync"
 	"time"
@@ -108,7 +109,7 @@ var _ = Describe("Density", func() {
 			nameStr := strconv.Itoa(totalPods) + "-" + string(util.NewUUID())
 			RCName = "my-hostname-density" + nameStr
 
-			// Create a listener for events
+			// Create a listener for events.
 			events := make([](*api.Event), 0)
 			_, controller := framework.NewInformer(
 				&cache.ListWatch{
@@ -130,8 +131,11 @@ var _ = Describe("Density", func() {
 			stop := make(chan struct{})
 			go controller.Run(stop)
 
-			// Start the replication controller
+			// Start the replication controller.
+			startTime := time.Now()
 			expectNoError(RunRC(c, RCName, ns, "gcr.io/google_containers/pause:go", totalPods))
+			e2eStartupTime := time.Now().Sub(startTime)
+			Logf("E2E startup time for %d pods: %v", totalPods, e2eStartupTime)
 
 			By("Waiting for all events to be recorded")
 			last := -1
@@ -148,13 +152,9 @@ var _ = Describe("Density", func() {
 			}
 			Logf("Found %d events", current)
 
-			// Verify there were no pod killings or failures
+			// Tune the threshold for allowed failures.
-			By("Verifying there were no pod killings or failures")
+			badEvents := BadEvents(events)
-			for _, e := range events {
+			Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods)))))
-				for _, s := range []string{"kill", "fail"} {
-					Expect(e.Reason).NotTo(ContainSubstring(s), "event:' %s', reason: '%s', message: '%s', field path: '%s'", e, e.ObjectMeta.Name, e.Message, e.InvolvedObject.FieldPath)
-				}
-			}
 		})
 	}
 

test/e2e/util.go

@@ -599,3 +599,30 @@ func VerifyContainersAreNotFailed(pod api.Pod) error {
 	}
 	return nil
 }
+
+// Prints the histogram of the events and returns the number of bad events.
+func BadEvents(events []*api.Event) int {
+	type histogramKey struct {
+		reason string
+		source string
+	}
+	histogram := make(map[histogramKey]int)
+	for _, e := range events {
+		histogram[histogramKey{reason: e.Reason, source: e.Source.Component}]++
+	}
+	for key, number := range histogram {
+		Logf("- reason: %s, source: %s -> %d", key.reason, key.source, number)
+	}
+	badPatterns := []string{"kill", "fail"}
+	badEvents := 0
+	for key, number := range histogram {
+		for _, s := range badPatterns {
+			if strings.Contains(key.reason, s) {
+				Logf("WARNING %d events from %s with reason: %s", number, key.source, key.reason)
+				badEvents += number
+				break
+			}
+		}
+	}
+	return badEvents
+}