diff --git a/tsdb/block_test.go b/tsdb/block_test.go index a7a159483..86f07a198 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -17,6 +17,8 @@ import ( "context" "encoding/binary" + "github.com/prometheus/prometheus/tsdb/fileutil" + "errors" "io/ioutil" "math/rand" @@ -175,7 +177,8 @@ func TestBlockSize(t *testing.T) { testutil.Ok(t, blockInit.Close()) }() expSizeInit = blockInit.Size() - actSizeInit := testutil.DirSize(t, blockInit.Dir()) + actSizeInit, err := fileutil.DirSize(blockInit.Dir()) + testutil.Ok(t, err) testutil.Equals(t, expSizeInit, actSizeInit) } @@ -184,7 +187,7 @@ func TestBlockSize(t *testing.T) { testutil.Ok(t, blockInit.Delete(1, 10, labels.NewMustRegexpMatcher("", ".*"))) expAfterDelete := blockInit.Size() testutil.Assert(t, expAfterDelete > expSizeInit, "after a delete the block size should be bigger as the tombstone file should grow %v > %v", expAfterDelete, expSizeInit) - actAfterDelete := testutil.DirSize(t, blockDirInit) + actAfterDelete, err := fileutil.DirSize(blockDirInit) testutil.Ok(t, err) testutil.Equals(t, expAfterDelete, actAfterDelete, "after a delete reported block size doesn't match actual disk size") @@ -198,7 +201,8 @@ func TestBlockSize(t *testing.T) { testutil.Ok(t, blockAfterCompact.Close()) }() expAfterCompact := blockAfterCompact.Size() - actAfterCompact := testutil.DirSize(t, blockAfterCompact.Dir()) + actAfterCompact, err := fileutil.DirSize(blockAfterCompact.Dir()) + testutil.Ok(t, err) testutil.Assert(t, actAfterDelete > actAfterCompact, "after a delete and compaction the block size should be smaller %v,%v", actAfterDelete, actAfterCompact) testutil.Equals(t, expAfterCompact, actAfterCompact, "after a delete and compaction reported block size doesn't match actual disk size") } diff --git a/tsdb/db.go b/tsdb/db.go index 4fdd3fb14..08d041e4d 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -933,7 +933,11 @@ func (db *DB) beyondSizeRetention(blocks []*Block) (deleteable map[ulid.ULID]*Bl } deleteable = make(map[ulid.ULID]*Block) - blocksSize := int64(0) + + walSize, _ := db.Head().wal.Size() + // Initializing size counter with WAL size, + // as that is part of the retention strategy. + blocksSize := walSize for i, block := range blocks { blocksSize += block.Size() if blocksSize > db.opts.MaxBytes { diff --git a/tsdb/db_test.go b/tsdb/db_test.go index e1a3f4afd..4c44c70f4 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -25,6 +25,8 @@ import ( "testing" "time" + "github.com/prometheus/prometheus/tsdb/fileutil" + "github.com/go-kit/kit/log" "github.com/oklog/ulid" "github.com/pkg/errors" @@ -1111,11 +1113,49 @@ func TestSizeRetention(t *testing.T) { createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime)) } + headBlocks := []*BlockMeta{ + {MinTime: 700, MaxTime: 800}, + } + + // Add some data to the WAL. + headApp := db.Head().Appender() + for _, m := range headBlocks { + series := genSeries(100, 10, m.MinTime, m.MaxTime) + for _, s := range series { + it := s.Iterator() + for it.Next() { + tim, v := it.At() + _, err := headApp.Add(s.Labels(), tim, v) + testutil.Ok(t, err) + } + testutil.Ok(t, it.Err()) + } + } + testutil.Ok(t, headApp.Commit()) + // Test that registered size matches the actual disk size. - testutil.Ok(t, db.reload()) // Reload the db to register the new db size. - testutil.Equals(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered. - expSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the actual internal metrics. - actSize := testutil.DirSize(t, db.Dir()) + testutil.Ok(t, db.reload()) // Reload the db to register the new db size. + testutil.Equals(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered. + blockSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics. + walSize, err := db.Head().wal.Size() + testutil.Ok(t, err) + // Expected size should take into account block size + WAL size + expSize := blockSize + walSize + actSize, err := fileutil.DirSize(db.Dir()) + testutil.Ok(t, err) + testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size") + + // Create a WAL checkpoint, and compare sizes. + first, last, err := db.Head().wal.Segments() + testutil.Ok(t, err) + _, err = wal.Checkpoint(db.Head().wal, first, last-1, func(x uint64) bool { return false }, 0) + testutil.Ok(t, err) + blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics. + walSize, err = db.Head().wal.Size() + testutil.Ok(t, err) + expSize = blockSize + walSize + actSize, err = fileutil.DirSize(db.Dir()) + testutil.Ok(t, err) testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size") // Decrease the max bytes limit so that a delete is triggered. @@ -1127,9 +1167,14 @@ func TestSizeRetention(t *testing.T) { expBlocks := blocks[1:] actBlocks := db.Blocks() - expSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) + blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) + walSize, err = db.Head().wal.Size() + testutil.Ok(t, err) + // Expected size should take into account block size + WAL size + expSize = blockSize + walSize actRetentCount := int(prom_testutil.ToFloat64(db.metrics.sizeRetentionCount)) - actSize = testutil.DirSize(t, db.Dir()) + actSize, err = fileutil.DirSize(db.Dir()) + testutil.Ok(t, err) testutil.Equals(t, 1, actRetentCount, "metric retention count mismatch") testutil.Equals(t, actSize, expSize, "metric db size doesn't match actual disk size") @@ -1137,7 +1182,6 @@ func TestSizeRetention(t *testing.T) { testutil.Equals(t, len(blocks)-1, len(actBlocks), "new block count should be decreased from:%v to:%v", len(blocks), len(blocks)-1) testutil.Equals(t, expBlocks[0].MaxTime, actBlocks[0].meta.MaxTime, "maxT mismatch of the first block") testutil.Equals(t, expBlocks[len(expBlocks)-1].MaxTime, actBlocks[len(actBlocks)-1].meta.MaxTime, "maxT mismatch of the last block") - } func TestSizeRetentionMetric(t *testing.T) { @@ -2298,7 +2342,8 @@ func TestDBReadOnly(t *testing.T) { testutil.Ok(t, err) dbWritable.DisableCompactions() - dbSizeBeforeAppend := testutil.DirSize(t, dbWritable.Dir()) + dbSizeBeforeAppend, err := fileutil.DirSize(dbWritable.Dir()) + testutil.Ok(t, err) app := dbWritable.Appender() _, err = app.Add(labels.FromStrings("foo", "bar"), dbWritable.Head().MaxTime()+1, 0) testutil.Ok(t, err) @@ -2306,7 +2351,8 @@ func TestDBReadOnly(t *testing.T) { expSeriesCount++ expBlocks = dbWritable.Blocks() - expDbSize := testutil.DirSize(t, dbWritable.Dir()) + expDbSize, err := fileutil.DirSize(dbWritable.Dir()) + testutil.Ok(t, err) testutil.Assert(t, expDbSize > dbSizeBeforeAppend, "db size didn't increase after an append") q, err := dbWritable.Querier(math.MinInt64, math.MaxInt64) diff --git a/tsdb/fileutil/dir.go b/tsdb/fileutil/dir.go new file mode 100644 index 000000000..e6ac4ec98 --- /dev/null +++ b/tsdb/fileutil/dir.go @@ -0,0 +1,33 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "path/filepath" +) + +func DirSize(dir string) (int64, error) { + var size int64 + err := filepath.Walk(dir, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} diff --git a/tsdb/wal/wal.go b/tsdb/wal/wal.go index 16f2e8794..b0b693174 100644 --- a/tsdb/wal/wal.go +++ b/tsdb/wal/wal.go @@ -876,3 +876,9 @@ func (r *segmentBufReader) Read(b []byte) (n int, err error) { r.buf.Reset(r.segs[r.cur]) return n, nil } + +// Computing size of the WAL. +// We do this by adding the sizes of all the files under the WAL dir. +func (w *WAL) Size() (int64, error) { + return fileutil.DirSize(w.Dir()) +} diff --git a/tsdb/wal/wal_test.go b/tsdb/wal/wal_test.go index 8a6b23f6f..cc2537a20 100644 --- a/tsdb/wal/wal_test.go +++ b/tsdb/wal/wal_test.go @@ -23,6 +23,8 @@ import ( "path/filepath" "testing" + "github.com/prometheus/prometheus/tsdb/fileutil" + client_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/prometheus/util/testutil" ) @@ -408,8 +410,10 @@ func TestCompression(t *testing.T) { testutil.Ok(t, os.RemoveAll(dirUnCompressed)) }() - uncompressedSize := testutil.DirSize(t, dirUnCompressed) - compressedSize := testutil.DirSize(t, dirCompressed) + uncompressedSize, err := fileutil.DirSize(dirUnCompressed) + testutil.Ok(t, err) + compressedSize, err := fileutil.DirSize(dirCompressed) + testutil.Ok(t, err) testutil.Assert(t, float64(uncompressedSize)*0.75 > float64(compressedSize), "Compressing zeroes should save at least 25%% space - uncompressedSize: %d, compressedSize: %d", uncompressedSize, compressedSize) } diff --git a/util/testutil/directory.go b/util/testutil/directory.go index 5f1c31554..3cc43d220 100644 --- a/util/testutil/directory.go +++ b/util/testutil/directory.go @@ -133,20 +133,6 @@ func NewTemporaryDirectory(name string, t T) (handler TemporaryDirectory) { return } -// DirSize returns the size in bytes of all files in a directory. -func DirSize(t *testing.T, path string) int64 { - var size int64 - err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { - Ok(t, err) - if !info.IsDir() { - size += info.Size() - } - return nil - }) - Ok(t, err) - return size -} - // DirHash returns a hash of all files attribites and their content within a directory. func DirHash(t *testing.T, path string) []byte { hash := sha256.New()