From bedcd88343b55de07506c3fe5748f07ff9cda77a Mon Sep 17 00:00:00 2001 From: Chris Marchbanks Date: Wed, 17 Feb 2021 06:15:49 -0700 Subject: [PATCH] Compress records before checking segment size (#8501) Right now a new segment might be created unnecessarily if the uncompressed record would not fit, but after compression (typically reducing record size in half) it would. Signed-off-by: Chris Marchbanks --- tsdb/wal/wal.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tsdb/wal/wal.go b/tsdb/wal/wal.go index 3aa87b2c0..7b45e0e91 100644 --- a/tsdb/wal/wal.go +++ b/tsdb/wal/wal.go @@ -613,18 +613,8 @@ func (w *WAL) log(rec []byte, final bool) error { return err } } - // If the record is too big to fit within the active page in the current - // segment, terminate the active segment and advance to the next one. - // This ensures that records do not cross segment boundaries. - left := w.page.remaining() - recordHeaderSize // Free space in the active page. - left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment. - - if len(rec) > left { - if err := w.nextSegment(); err != nil { - return err - } - } + // Compress the record before calculating if a new segment is needed. compressed := false if w.compress && len(rec) > 0 { // The snappy library uses `len` to calculate if we need a new buffer. @@ -638,6 +628,18 @@ func (w *WAL) log(rec []byte, final bool) error { } } + // If the record is too big to fit within the active page in the current + // segment, terminate the active segment and advance to the next one. + // This ensures that records do not cross segment boundaries. + left := w.page.remaining() - recordHeaderSize // Free space in the active page. + left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment. + + if len(rec) > left { + if err := w.nextSegment(); err != nil { + return err + } + } + // Populate as many pages as necessary to fit the record. // Be careful to always do one pass to ensure we write zero-length records. for i := 0; i == 0 || len(rec) > 0; i++ {