From 08fc6cbd39eb6b2f07faab72541c4bf7fd6457b3 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 3 Apr 2017 17:03:59 +0200 Subject: [PATCH] storage: Evict unused chunk.Descs in crash recovery This is in line with the v1.5 change in paradigm to not keep chunk.Descs without chunks around after a series maintenance. It's mainly motivated by avoiding excessive amounts of RAM usage during crash recovery. The code avoids to create memory time series with zero chunk.Descs as that is prone to trigger weird effects. (Series maintenance would archive series with zero chunk.Descs, but we cannot do that here because the archive indices still have to be checked.) --- storage/local/crashrecovery.go | 10 ++++++++++ storage/local/storage.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/storage/local/crashrecovery.go b/storage/local/crashrecovery.go index 3b29da55e..2d89e6f80 100644 --- a/storage/local/crashrecovery.go +++ b/storage/local/crashrecovery.go @@ -304,6 +304,8 @@ func (p *persistence) sanitizeSeries( } s.persistWatermark = len(cds) s.modTime = modTime + // Finally, evict again all chunk.Descs except the latest one to save memory. + s.evictChunkDescs(len(cds) - 1) return fp, true } // This is the tricky one: We have chunks from heads.db, but @@ -356,6 +358,8 @@ func (p *persistence) sanitizeSeries( atomic.AddInt64(&chunk.NumMemChunks, int64(-len(s.chunkDescs))) s.chunkDescs = cds s.headChunkClosed = true + // Finally, evict again all chunk.Descs except the latest one to save memory. + s.evictChunkDescs(len(cds) - 1) return fp, true } log.Warnf( @@ -364,11 +368,15 @@ func (p *persistence) sanitizeSeries( ) chunk.NumMemDescs.Sub(float64(keepIdx)) atomic.AddInt64(&chunk.NumMemChunks, int64(-keepIdx)) + chunkDescsToEvict := len(cds) if keepIdx == len(s.chunkDescs) { // No chunks from series file left, head chunk is evicted, so declare it closed. s.headChunkClosed = true + chunkDescsToEvict-- // Keep one chunk.Desc in this case to avoid a series with zero chunk.Descs. } s.chunkDescs = append(cds, s.chunkDescs[keepIdx:]...) + // Finally, evict again chunk.Descs without chunk to save memory. + s.evictChunkDescs(chunkDescsToEvict) return fp, true } // This series is supposed to be archived. @@ -458,6 +466,8 @@ func (p *persistence) cleanUpArchiveIndexes( return err } fpToSeries[model.Fingerprint(fp)] = series + // Evict all but one chunk.Desc to save memory. + series.evictChunkDescs(len(cds) - 1) return nil }); err != nil { return err diff --git a/storage/local/storage.go b/storage/local/storage.go index b711f2839..3a58e201d 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -1587,7 +1587,7 @@ func (s *MemorySeriesStorage) maintainMemorySeries( } return } - // If we are here, the series is not archived, so check for Chunk.Desc + // If we are here, the series is not archived, so check for chunk.Desc // eviction next. series.evictChunkDescs(iOldestNotEvicted)