Browse Source

Compress records before checking segment size (#8501)

Right now a new segment might be created unnecessarily if the
uncompressed record would not fit, but after compression (typically
reducing record size in half) it would.

Signed-off-by: Chris Marchbanks <csmarchbanks@gmail.com>
pull/8415/head
Chris Marchbanks 4 years ago committed by GitHub
parent
commit
bedcd88343
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 24
      tsdb/wal/wal.go

24
tsdb/wal/wal.go

@ -613,18 +613,8 @@ func (w *WAL) log(rec []byte, final bool) error {
return err return err
} }
} }
// If the record is too big to fit within the active page in the current
// segment, terminate the active segment and advance to the next one.
// This ensures that records do not cross segment boundaries.
left := w.page.remaining() - recordHeaderSize // Free space in the active page.
left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment.
if len(rec) > left {
if err := w.nextSegment(); err != nil {
return err
}
}
// Compress the record before calculating if a new segment is needed.
compressed := false compressed := false
if w.compress && len(rec) > 0 { if w.compress && len(rec) > 0 {
// The snappy library uses `len` to calculate if we need a new buffer. // The snappy library uses `len` to calculate if we need a new buffer.
@ -638,6 +628,18 @@ func (w *WAL) log(rec []byte, final bool) error {
} }
} }
// If the record is too big to fit within the active page in the current
// segment, terminate the active segment and advance to the next one.
// This ensures that records do not cross segment boundaries.
left := w.page.remaining() - recordHeaderSize // Free space in the active page.
left += (pageSize - recordHeaderSize) * (w.pagesPerSegment() - w.donePages - 1) // Free pages in the active segment.
if len(rec) > left {
if err := w.nextSegment(); err != nil {
return err
}
}
// Populate as many pages as necessary to fit the record. // Populate as many pages as necessary to fit the record.
// Be careful to always do one pass to ensure we write zero-length records. // Be careful to always do one pass to ensure we write zero-length records.
for i := 0; i == 0 || len(rec) > 0; i++ { for i := 0; i == 0 || len(rec) > 0; i++ {

Loading…
Cancel
Save