From 9a75b5f84b01b891c7a2c6a003c3be8b83d97bcd Mon Sep 17 00:00:00 2001 From: Krasi Georgiev <8903888+krasi-georgiev@users.noreply.github.com> Date: Mon, 20 Jul 2020 18:23:18 +0300 Subject: [PATCH 1/3] Avoid panic when the headChunk is nil during isolation. Signed-off-by: Krasi Georgiev <8903888+krasi-georgiev@users.noreply.github.com> --- tsdb/head.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tsdb/head.go b/tsdb/head.go index 0a16671ea..536487a2f 100644 --- a/tsdb/head.go +++ b/tsdb/head.go @@ -2186,10 +2186,8 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper * previousSamples += int(d.numSamples) } } - // mmappedChunks does not contain the last chunk. Hence check it separately. - if len(s.mmappedChunks) < ix { - previousSamples += s.headChunk.chunk.NumSamples() - } else { + + if s.headChunk != nil { totalSamples += s.headChunk.chunk.NumSamples() } From 30b2592fb5456954d51f9280f5035e32cd360336 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Tue, 14 Jul 2020 14:59:28 +0530 Subject: [PATCH 2/3] Avoid empty mmap files by using .tmp files to write headers Signed-off-by: Ganesh Vernekar --- tsdb/chunks/chunks.go | 29 +++++++++++++++++++++++------ tsdb/fileutil/fileutil.go | 16 +--------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/tsdb/chunks/chunks.go b/tsdb/chunks/chunks.go index acdcc4a17..ffd3e5302 100644 --- a/tsdb/chunks/chunks.go +++ b/tsdb/chunks/chunks.go @@ -220,19 +220,20 @@ func (w *Writer) cut() error { func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, allocSize int64) (headerSize int, newFile *os.File, seq int, err error) { p, seq, err := nextSequenceFile(dirFile.Name()) if err != nil { - return 0, nil, 0, err + return 0, nil, 0, errors.Wrap(err, "next sequence file") } - f, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0666) + ptmp := p + ".tmp" + f, err := os.OpenFile(ptmp, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { - return 0, nil, 0, err + return 0, nil, 0, errors.Wrap(err, "open temp file") } if allocSize > 0 { if err = fileutil.Preallocate(f, allocSize, true); err != nil { - return 0, nil, 0, err + return 0, nil, 0, errors.Wrap(err, "preallocate") } } if err = dirFile.Sync(); err != nil { - return 0, nil, 0, err + return 0, nil, 0, errors.Wrap(err, "sync directory") } // Write header metadata for new file. @@ -242,7 +243,23 @@ func cutSegmentFile(dirFile *os.File, magicNumber uint32, chunksFormat byte, all n, err := f.Write(metab) if err != nil { - return 0, nil, 0, err + return 0, nil, 0, errors.Wrap(err, "write header") + } + if err := f.Close(); err != nil { + return 0, nil, 0, errors.Wrap(err, "close temp file") + } + + if err := fileutil.Rename(ptmp, p); err != nil { + return 0, nil, 0, errors.Wrap(err, "replace file") + } + + f, err = os.OpenFile(p, os.O_WRONLY, 0666) + if err != nil { + return 0, nil, 0, errors.Wrap(err, "open final file") + } + // Skip the header for the further writes. + if _, err := f.Seek(int64(n), 0); err != nil { + return 0, nil, 0, errors.Wrap(err, "seek in final file") } return n, f, seq, nil } diff --git a/tsdb/fileutil/fileutil.go b/tsdb/fileutil/fileutil.go index be9c6f226..00bf63950 100644 --- a/tsdb/fileutil/fileutil.go +++ b/tsdb/fileutil/fileutil.go @@ -125,19 +125,5 @@ func Replace(from, to string) error { } } - if err := os.Rename(from, to); err != nil { - return err - } - - // Directory was renamed; sync parent dir to persist rename. - pdir, err := OpenDir(filepath.Dir(to)) - if err != nil { - return err - } - - if err = pdir.Sync(); err != nil { - pdir.Close() - return err - } - return pdir.Close() + return Rename(from, to) } From 657ba532e42f1db8d7c77bf802378643da0d3118 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Fri, 24 Jul 2020 17:45:23 +0530 Subject: [PATCH 3/3] Cut v2.19.3 (#7657) Signed-off-by: Ganesh Vernekar --- CHANGELOG.md | 5 +++++ VERSION | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce2e18734..3b13be3af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 2.19.3 / 2020-07-24 + +* [BUGFIX] TSDB: Don't panic on WAL corruptions. #7550 +* [BUGFIX] TSDB: Avoid leaving behind empty files in chunks_head, causing startup failures. #7573 + ## 2.19.2 / 2020-06-26 * [BUGFIX] Remote Write: Fix panic when reloading config with modified queue parameters. #7452 diff --git a/VERSION b/VERSION index 17bdb70fa..6ef6ef5ac 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.19.2 +2.19.3