test(tsdb): break up repeated test to avoid timeout (#10671)

On macOS, the TestTombstoneCleanRetentionLimitsRace performs very
poorly. It takes more than a second to write out one block, and as it
writes 400 of them, we run into the 10-minute test timeout frequently.

While this doesn't fix the actual performance issue, breaking each
iteration into a subtest makes the test pass reliably (because each
iteration comfortably finishes in under a minute).

Related report: https://groups.google.com/g/prometheus-developers/c/jxQ6Ayg6VJ4/m/03H_DS9PDAAJ

Signed-off-by: Matthias Rampke <matthias@prometheus.io>

tsdb/db_test.go

@@ -1281,38 +1281,40 @@ func TestTombstoneCleanRetentionLimitsRace(t *testing.T) {
 	//
 	// That is something tricky to trigger, so let's try several times just to make sure.
 	for i := 0; i < 20; i++ {
-		db := openTestDB(t, opts, nil)
+		t.Run(fmt.Sprintf("iteration%d", i), func(t *testing.T) {
-		totalBlocks := 20
+			db := openTestDB(t, opts, nil)
-		dbDir := db.Dir()
+			totalBlocks := 20
-		// Generate some blocks with old mint (near epoch).
+			dbDir := db.Dir()
-		for j := 0; j < totalBlocks; j++ {
+			// Generate some blocks with old mint (near epoch).
-			blockDir := createBlock(t, dbDir, genSeries(10, 1, int64(j), int64(j)+1))
+			for j := 0; j < totalBlocks; j++ {
-			block, err := OpenBlock(nil, blockDir, nil)
+				blockDir := createBlock(t, dbDir, genSeries(10, 1, int64(j), int64(j)+1))
-			require.NoError(t, err)
+				block, err := OpenBlock(nil, blockDir, nil)
-			// Cover block with tombstones so it can be deleted with CleanTombstones() as well.
+				require.NoError(t, err)
-			tomb := tombstones.NewMemTombstones()
+				// Cover block with tombstones so it can be deleted with CleanTombstones() as well.
-			tomb.AddInterval(0, tombstones.Interval{Mint: int64(j), Maxt: int64(j) + 1})
+				tomb := tombstones.NewMemTombstones()
-			block.tombstones = tomb
+				tomb.AddInterval(0, tombstones.Interval{Mint: int64(j), Maxt: int64(j) + 1})
+				block.tombstones = tomb
 
-			db.blocks = append(db.blocks, block)
+				db.blocks = append(db.blocks, block)
-		}
+			}
 
-		wg.Add(2)
+			wg.Add(2)
-		// Run reload and CleanTombstones together, with a small time window randomization
+			// Run reload and CleanTombstones together, with a small time window randomization
-		go func() {
+			go func() {
-			defer wg.Done()
+				defer wg.Done()
-			time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond)))
+				time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond)))
-			require.NoError(t, db.reloadBlocks())
+				require.NoError(t, db.reloadBlocks())
-		}()
+			}()
-		go func() {
+			go func() {
-			defer wg.Done()
+				defer wg.Done()
-			time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond)))
+				time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond)))
-			require.NoError(t, db.CleanTombstones())
+				require.NoError(t, db.CleanTombstones())
-		}()
+			}()
 
-		wg.Wait()
+			wg.Wait()
 
-		require.NoError(t, db.Close())
+			require.NoError(t, db.Close())
+		})
 	}
 }