diff --git a/cmd/prometheus/config.go b/cmd/prometheus/config.go index 1c5e7679a..839b3f4ce 100644 --- a/cmd/prometheus/config.go +++ b/cmd/prometheus/config.go @@ -113,7 +113,7 @@ func init() { "How long to retain samples in the local storage.", ) cfg.fs.IntVar( - &cfg.storage.MaxChunksToPersist, "storage.local.max-chunks-to-persist", 1024*1024, + &cfg.storage.MaxChunksToPersist, "storage.local.max-chunks-to-persist", 512*1024, "How many chunks can be waiting for persistence before sample ingestion will stop. Many chunks waiting to be persisted will increase the checkpoint size.", ) cfg.fs.DurationVar( diff --git a/storage/local/persistence.go b/storage/local/persistence.go index 96a478a04..72b989ccb 100644 --- a/storage/local/persistence.go +++ b/storage/local/persistence.go @@ -861,6 +861,12 @@ func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, chunksToPersist in } } + headChunkClosed := persistWatermark >= numChunkDescs + if !headChunkClosed { + // Head chunk is not ready for persisting yet. + chunksToPersist-- + } + fingerprintToSeries[model.Fingerprint(fp)] = &memorySeries{ metric: model.Metric(metric), chunkDescs: chunkDescs, @@ -869,7 +875,7 @@ func (p *persistence) loadSeriesMapAndHeads() (sm *seriesMap, chunksToPersist in chunkDescsOffset: int(chunkDescsOffset), savedFirstTime: model.Time(savedFirstTime), lastTime: chunkDescs[len(chunkDescs)-1].lastTime(), - headChunkClosed: persistWatermark >= numChunkDescs, + headChunkClosed: headChunkClosed, } } return sm, chunksToPersist, nil