Compute WAL size and use it during retention size checks (#5886)

* Compute WAL size and account for it when applying the retention settings.

Signed-off-by: Dipack P Panjabi <dpanjabi@hudson-trading.com>
pull/6308/head
Dipack P Panjabi 5 years ago committed by Krasimir Georgiev
parent ca60bf298c
commit ce7bab04dd

@ -17,6 +17,8 @@ import (
"context"
"encoding/binary"
"github.com/prometheus/prometheus/tsdb/fileutil"
"errors"
"io/ioutil"
"math/rand"
@ -175,7 +177,8 @@ func TestBlockSize(t *testing.T) {
testutil.Ok(t, blockInit.Close())
}()
expSizeInit = blockInit.Size()
actSizeInit := testutil.DirSize(t, blockInit.Dir())
actSizeInit, err := fileutil.DirSize(blockInit.Dir())
testutil.Ok(t, err)
testutil.Equals(t, expSizeInit, actSizeInit)
}
@ -184,7 +187,7 @@ func TestBlockSize(t *testing.T) {
testutil.Ok(t, blockInit.Delete(1, 10, labels.NewMustRegexpMatcher("", ".*")))
expAfterDelete := blockInit.Size()
testutil.Assert(t, expAfterDelete > expSizeInit, "after a delete the block size should be bigger as the tombstone file should grow %v > %v", expAfterDelete, expSizeInit)
actAfterDelete := testutil.DirSize(t, blockDirInit)
actAfterDelete, err := fileutil.DirSize(blockDirInit)
testutil.Ok(t, err)
testutil.Equals(t, expAfterDelete, actAfterDelete, "after a delete reported block size doesn't match actual disk size")
@ -198,7 +201,8 @@ func TestBlockSize(t *testing.T) {
testutil.Ok(t, blockAfterCompact.Close())
}()
expAfterCompact := blockAfterCompact.Size()
actAfterCompact := testutil.DirSize(t, blockAfterCompact.Dir())
actAfterCompact, err := fileutil.DirSize(blockAfterCompact.Dir())
testutil.Ok(t, err)
testutil.Assert(t, actAfterDelete > actAfterCompact, "after a delete and compaction the block size should be smaller %v,%v", actAfterDelete, actAfterCompact)
testutil.Equals(t, expAfterCompact, actAfterCompact, "after a delete and compaction reported block size doesn't match actual disk size")
}

@ -933,7 +933,11 @@ func (db *DB) beyondSizeRetention(blocks []*Block) (deleteable map[ulid.ULID]*Bl
}
deleteable = make(map[ulid.ULID]*Block)
blocksSize := int64(0)
walSize, _ := db.Head().wal.Size()
// Initializing size counter with WAL size,
// as that is part of the retention strategy.
blocksSize := walSize
for i, block := range blocks {
blocksSize += block.Size()
if blocksSize > db.opts.MaxBytes {

@ -25,6 +25,8 @@ import (
"testing"
"time"
"github.com/prometheus/prometheus/tsdb/fileutil"
"github.com/go-kit/kit/log"
"github.com/oklog/ulid"
"github.com/pkg/errors"
@ -1111,11 +1113,49 @@ func TestSizeRetention(t *testing.T) {
createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime))
}
headBlocks := []*BlockMeta{
{MinTime: 700, MaxTime: 800},
}
// Add some data to the WAL.
headApp := db.Head().Appender()
for _, m := range headBlocks {
series := genSeries(100, 10, m.MinTime, m.MaxTime)
for _, s := range series {
it := s.Iterator()
for it.Next() {
tim, v := it.At()
_, err := headApp.Add(s.Labels(), tim, v)
testutil.Ok(t, err)
}
testutil.Ok(t, it.Err())
}
}
testutil.Ok(t, headApp.Commit())
// Test that registered size matches the actual disk size.
testutil.Ok(t, db.reload()) // Reload the db to register the new db size.
testutil.Equals(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered.
expSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the actual internal metrics.
actSize := testutil.DirSize(t, db.Dir())
testutil.Ok(t, db.reload()) // Reload the db to register the new db size.
testutil.Equals(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered.
blockSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics.
walSize, err := db.Head().wal.Size()
testutil.Ok(t, err)
// Expected size should take into account block size + WAL size
expSize := blockSize + walSize
actSize, err := fileutil.DirSize(db.Dir())
testutil.Ok(t, err)
testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size")
// Create a WAL checkpoint, and compare sizes.
first, last, err := db.Head().wal.Segments()
testutil.Ok(t, err)
_, err = wal.Checkpoint(db.Head().wal, first, last-1, func(x uint64) bool { return false }, 0)
testutil.Ok(t, err)
blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics.
walSize, err = db.Head().wal.Size()
testutil.Ok(t, err)
expSize = blockSize + walSize
actSize, err = fileutil.DirSize(db.Dir())
testutil.Ok(t, err)
testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size")
// Decrease the max bytes limit so that a delete is triggered.
@ -1127,9 +1167,14 @@ func TestSizeRetention(t *testing.T) {
expBlocks := blocks[1:]
actBlocks := db.Blocks()
expSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes))
blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes))
walSize, err = db.Head().wal.Size()
testutil.Ok(t, err)
// Expected size should take into account block size + WAL size
expSize = blockSize + walSize
actRetentCount := int(prom_testutil.ToFloat64(db.metrics.sizeRetentionCount))
actSize = testutil.DirSize(t, db.Dir())
actSize, err = fileutil.DirSize(db.Dir())
testutil.Ok(t, err)
testutil.Equals(t, 1, actRetentCount, "metric retention count mismatch")
testutil.Equals(t, actSize, expSize, "metric db size doesn't match actual disk size")
@ -1137,7 +1182,6 @@ func TestSizeRetention(t *testing.T) {
testutil.Equals(t, len(blocks)-1, len(actBlocks), "new block count should be decreased from:%v to:%v", len(blocks), len(blocks)-1)
testutil.Equals(t, expBlocks[0].MaxTime, actBlocks[0].meta.MaxTime, "maxT mismatch of the first block")
testutil.Equals(t, expBlocks[len(expBlocks)-1].MaxTime, actBlocks[len(actBlocks)-1].meta.MaxTime, "maxT mismatch of the last block")
}
func TestSizeRetentionMetric(t *testing.T) {
@ -2298,7 +2342,8 @@ func TestDBReadOnly(t *testing.T) {
testutil.Ok(t, err)
dbWritable.DisableCompactions()
dbSizeBeforeAppend := testutil.DirSize(t, dbWritable.Dir())
dbSizeBeforeAppend, err := fileutil.DirSize(dbWritable.Dir())
testutil.Ok(t, err)
app := dbWritable.Appender()
_, err = app.Add(labels.FromStrings("foo", "bar"), dbWritable.Head().MaxTime()+1, 0)
testutil.Ok(t, err)
@ -2306,7 +2351,8 @@ func TestDBReadOnly(t *testing.T) {
expSeriesCount++
expBlocks = dbWritable.Blocks()
expDbSize := testutil.DirSize(t, dbWritable.Dir())
expDbSize, err := fileutil.DirSize(dbWritable.Dir())
testutil.Ok(t, err)
testutil.Assert(t, expDbSize > dbSizeBeforeAppend, "db size didn't increase after an append")
q, err := dbWritable.Querier(math.MinInt64, math.MaxInt64)

@ -0,0 +1,33 @@
// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"os"
"path/filepath"
)
func DirSize(dir string) (int64, error) {
var size int64
err := filepath.Walk(dir, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
size += info.Size()
}
return nil
})
return size, err
}

@ -876,3 +876,9 @@ func (r *segmentBufReader) Read(b []byte) (n int, err error) {
r.buf.Reset(r.segs[r.cur])
return n, nil
}
// Computing size of the WAL.
// We do this by adding the sizes of all the files under the WAL dir.
func (w *WAL) Size() (int64, error) {
return fileutil.DirSize(w.Dir())
}

@ -23,6 +23,8 @@ import (
"path/filepath"
"testing"
"github.com/prometheus/prometheus/tsdb/fileutil"
client_testutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/prometheus/util/testutil"
)
@ -408,8 +410,10 @@ func TestCompression(t *testing.T) {
testutil.Ok(t, os.RemoveAll(dirUnCompressed))
}()
uncompressedSize := testutil.DirSize(t, dirUnCompressed)
compressedSize := testutil.DirSize(t, dirCompressed)
uncompressedSize, err := fileutil.DirSize(dirUnCompressed)
testutil.Ok(t, err)
compressedSize, err := fileutil.DirSize(dirCompressed)
testutil.Ok(t, err)
testutil.Assert(t, float64(uncompressedSize)*0.75 > float64(compressedSize), "Compressing zeroes should save at least 25%% space - uncompressedSize: %d, compressedSize: %d", uncompressedSize, compressedSize)
}

@ -133,20 +133,6 @@ func NewTemporaryDirectory(name string, t T) (handler TemporaryDirectory) {
return
}
// DirSize returns the size in bytes of all files in a directory.
func DirSize(t *testing.T, path string) int64 {
var size int64
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
Ok(t, err)
if !info.IsDir() {
size += info.Size()
}
return nil
})
Ok(t, err)
return size
}
// DirHash returns a hash of all files attribites and their content within a directory.
func DirHash(t *testing.T, path string) []byte {
hash := sha256.New()

Loading…
Cancel
Save