diff --git a/tsdb/tsdbblockutil.go b/tsdb/tsdbblockutil.go index 732446324..a1dcd9a5f 100644 --- a/tsdb/tsdbblockutil.go +++ b/tsdb/tsdbblockutil.go @@ -44,6 +44,8 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l } }() + sampleCount := 0 + const commitAfter = 10000 ctx := context.Background() app := w.Appender(ctx) @@ -57,10 +59,19 @@ func CreateBlock(series []storage.Series, dir string, chunkRange int64, logger l if err != nil { return "", err } + sampleCount++ } if it.Err() != nil { return "", it.Err() } + // Commit and make a new appender periodically, to avoid building up data in memory. + if sampleCount > commitAfter { + if err = app.Commit(); err != nil { + return "", err + } + app = w.Appender(ctx) + sampleCount = 0 + } } if err = app.Commit(); err != nil {