github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'de172049ab'
${ noResults }
prometheus/tsdb/db_test.go

6663 lines
208 KiB
Raw Blame History

// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tsdb
import (
"bufio"
"context"
"encoding/binary"
"flag"
"fmt"
"hash/crc32"
"math"
"math/rand"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/go-kit/log"
"github.com/oklog/ulid"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/model/histogram"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/model/metadata"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/tsdb/chunkenc"
"github.com/prometheus/prometheus/tsdb/chunks"
"github.com/prometheus/prometheus/tsdb/fileutil"
"github.com/prometheus/prometheus/tsdb/index"
"github.com/prometheus/prometheus/tsdb/record"
"github.com/prometheus/prometheus/tsdb/tombstones"
"github.com/prometheus/prometheus/tsdb/tsdbutil"
"github.com/prometheus/prometheus/tsdb/wlog"
"github.com/prometheus/prometheus/util/testutil"
)
func TestMain(m *testing.M) {
var isolationEnabled bool
flag.BoolVar(&isolationEnabled, "test.tsdb-isolation", true, "enable isolation")
flag.Parse()
defaultIsolationDisabled = !isolationEnabled
goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func1"), goleak.IgnoreTopFunction("github.com/prometheus/prometheus/tsdb.(*SegmentWAL).cut.func2"))
}
func openTestDB(t testing.TB, opts *Options, rngs []int64) (db *DB) {
tmpdir := t.TempDir()
var err error
if opts == nil {
opts = DefaultOptions()
}
opts.EnableNativeHistograms = true
if len(rngs) == 0 {
db, err = Open(tmpdir, nil, nil, opts, nil)
} else {
opts, rngs = validateOpts(opts, rngs)
db, err = open(tmpdir, nil, nil, opts, rngs, nil)
}
require.NoError(t, err)
// Do not Close() the test database by default as it will deadlock on test failures.
return db
}
// query runs a matcher query against the querier and fully expands its data.
func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]tsdbutil.Sample {
ss := q.Select(false, nil, matchers...)
defer func() {
require.NoError(t, q.Close())
}()
var it chunkenc.Iterator
result := map[string][]tsdbutil.Sample{}
for ss.Next() {
series := ss.At()
samples := []tsdbutil.Sample{}
it = series.Iterator(it)
for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() {
switch typ {
case chunkenc.ValFloat:
ts, v := it.At()
samples = append(samples, sample{t: ts, f: v})
case chunkenc.ValHistogram:
ts, h := it.AtHistogram()
samples = append(samples, sample{t: ts, h: h})
case chunkenc.ValFloatHistogram:
ts, fh := it.AtFloatHistogram()
samples = append(samples, sample{t: ts, fh: fh})
default:
t.Fatalf("unknown sample type in query %s", typ.String())
}
}
require.NoError(t, it.Err())
if len(samples) == 0 {
continue
}
name := series.Labels().String()
result[name] = samples
}
require.NoError(t, ss.Err())
require.Equal(t, 0, len(ss.Warnings()))
return result
}
// queryAndExpandChunks runs a matcher query against the querier and fully expands its data into samples.
func queryAndExpandChunks(t testing.TB, q storage.ChunkQuerier, matchers ...*labels.Matcher) map[string][][]tsdbutil.Sample {
s := queryChunks(t, q, matchers...)
res := make(map[string][][]tsdbutil.Sample)
for k, v := range s {
var samples [][]tsdbutil.Sample
for _, chk := range v {
sam, err := storage.ExpandSamples(chk.Chunk.Iterator(nil), nil)
require.NoError(t, err)
samples = append(samples, sam)
}
res[k] = samples
}
return res
}
// queryChunks runs a matcher query against the querier and expands its data.
func queryChunks(t testing.TB, q storage.ChunkQuerier, matchers ...*labels.Matcher) map[string][]chunks.Meta {
ss := q.Select(false, nil, matchers...)
defer func() {
require.NoError(t, q.Close())
}()
var it chunks.Iterator
result := map[string][]chunks.Meta{}
for ss.Next() {
series := ss.At()
chks := []chunks.Meta{}
it = series.Iterator(it)
for it.Next() {
chks = append(chks, it.At())
}
require.NoError(t, it.Err())
if len(chks) == 0 {
continue
}
name := series.Labels().String()
result[name] = chks
}
require.NoError(t, ss.Err())
require.Equal(t, 0, len(ss.Warnings()))
return result
}
// Ensure that blocks are held in memory in their time order
// and not in ULID order as they are read from the directory.
func TestDB_reloadOrder(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
metas := []BlockMeta{
{MinTime: 90, MaxTime: 100},
{MinTime: 70, MaxTime: 80},
{MinTime: 100, MaxTime: 110},
}
for _, m := range metas {
createBlock(t, db.Dir(), genSeries(1, 1, m.MinTime, m.MaxTime))
}
require.NoError(t, db.reloadBlocks())
blocks := db.Blocks()
require.Equal(t, 3, len(blocks))
require.Equal(t, metas[1].MinTime, blocks[0].Meta().MinTime)
require.Equal(t, metas[1].MaxTime, blocks[0].Meta().MaxTime)
require.Equal(t, metas[0].MinTime, blocks[1].Meta().MinTime)
require.Equal(t, metas[0].MaxTime, blocks[1].Meta().MaxTime)
require.Equal(t, metas[2].MinTime, blocks[2].Meta().MinTime)
require.Equal(t, metas[2].MaxTime, blocks[2].Meta().MaxTime)
}
func TestDataAvailableOnlyAfterCommit(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0)
require.NoError(t, err)
querier, err := db.Querier(context.TODO(), 0, 1)
require.NoError(t, err)
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
require.Equal(t, map[string][]tsdbutil.Sample{}, seriesSet)
err = app.Commit()
require.NoError(t, err)
querier, err = db.Querier(context.TODO(), 0, 1)
require.NoError(t, err)
defer querier.Close()
seriesSet = query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
require.Equal(t, map[string][]tsdbutil.Sample{`{foo="bar"}`: {sample{t: 0, f: 0}}}, seriesSet)
}
// TestNoPanicAfterWALCorruption ensures that querying the db after a WAL corruption doesn't cause a panic.
// https://github.com/prometheus/prometheus/issues/7548
func TestNoPanicAfterWALCorruption(t *testing.T) {
db := openTestDB(t, &Options{WALSegmentSize: 32 * 1024}, nil)
// Append until the first mmaped head chunk.
// This is to ensure that all samples can be read from the mmaped chunks when the WAL is corrupted.
var expSamples []tsdbutil.Sample
var maxt int64
ctx := context.Background()
{
// Appending 121 samples because on the 121st a new chunk will be created.
for i := 0; i < 121; i++ {
app := db.Appender(ctx)
_, err := app.Append(0, labels.FromStrings("foo", "bar"), maxt, 0)
expSamples = append(expSamples, sample{t: maxt, f: 0})
require.NoError(t, err)
require.NoError(t, app.Commit())
maxt++
}
require.NoError(t, db.Close())
}
// Corrupt the WAL after the first sample of the series so that it has at least one sample and
// it is not garbage collected.
// The repair deletes all WAL records after the corrupted record and these are read from the mmaped chunk.
{
walFiles, err := os.ReadDir(path.Join(db.Dir(), "wal"))
require.NoError(t, err)
f, err := os.OpenFile(path.Join(db.Dir(), "wal", walFiles[0].Name()), os.O_RDWR, 0o666)
require.NoError(t, err)
r := wlog.NewReader(bufio.NewReader(f))
require.True(t, r.Next(), "reading the series record")
require.True(t, r.Next(), "reading the first sample record")
// Write an invalid record header to corrupt everything after the first wal sample.
_, err = f.WriteAt([]byte{99}, r.Offset())
require.NoError(t, err)
f.Close()
}
// Query the data.
{
db, err := Open(db.Dir(), nil, nil, nil, nil)
require.NoError(t, err)
defer func() {
require.NoError(t, db.Close())
}()
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.walCorruptionsTotal), "WAL corruption count mismatch")
querier, err := db.Querier(context.TODO(), 0, maxt)
require.NoError(t, err)
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "", ""))
// The last sample should be missing as it was after the WAL segment corruption.
require.Equal(t, map[string][]tsdbutil.Sample{`{foo="bar"}`: expSamples[0 : len(expSamples)-1]}, seriesSet)
}
}
func TestDataNotAvailableAfterRollback(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
app := db.Appender(context.Background())
_, err := app.Append(0, labels.FromStrings("foo", "bar"), 0, 0)
require.NoError(t, err)
err = app.Rollback()
require.NoError(t, err)
querier, err := db.Querier(context.TODO(), 0, 1)
require.NoError(t, err)
defer querier.Close()
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
require.Equal(t, map[string][]tsdbutil.Sample{}, seriesSet)
}
func TestDBAppenderAddRef(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app1 := db.Appender(ctx)
ref1, err := app1.Append(0, labels.FromStrings("a", "b"), 123, 0)
require.NoError(t, err)
// Reference should already work before commit.
ref2, err := app1.Append(ref1, labels.EmptyLabels(), 124, 1)
require.NoError(t, err)
require.Equal(t, ref1, ref2)
err = app1.Commit()
require.NoError(t, err)
app2 := db.Appender(ctx)
// first ref should already work in next transaction.
ref3, err := app2.Append(ref1, labels.EmptyLabels(), 125, 0)
require.NoError(t, err)
require.Equal(t, ref1, ref3)
ref4, err := app2.Append(ref1, labels.FromStrings("a", "b"), 133, 1)
require.NoError(t, err)
require.Equal(t, ref1, ref4)
// Reference must be valid to add another sample.
ref5, err := app2.Append(ref2, labels.EmptyLabels(), 143, 2)
require.NoError(t, err)
require.Equal(t, ref1, ref5)
// Missing labels & invalid refs should fail.
_, err = app2.Append(9999999, labels.EmptyLabels(), 1, 1)
require.Equal(t, ErrInvalidSample, errors.Cause(err))
require.NoError(t, app2.Commit())
q, err := db.Querier(context.TODO(), 0, 200)
require.NoError(t, err)
res := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
require.Equal(t, map[string][]tsdbutil.Sample{
labels.FromStrings("a", "b").String(): {
sample{t: 123, f: 0},
sample{t: 124, f: 1},
sample{t: 125, f: 0},
sample{t: 133, f: 1},
sample{t: 143, f: 2},
},
}, res)
}
func TestAppendEmptyLabelsIgnored(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app1 := db.Appender(ctx)
ref1, err := app1.Append(0, labels.FromStrings("a", "b"), 123, 0)
require.NoError(t, err)
// Add with empty label.
ref2, err := app1.Append(0, labels.FromStrings("a", "b", "c", ""), 124, 0)
require.NoError(t, err)
// Should be the same series.
require.Equal(t, ref1, ref2)
err = app1.Commit()
require.NoError(t, err)
}
func TestDeleteSimple(t *testing.T) {
numSamples := int64(10)
cases := []struct {
Intervals tombstones.Intervals
remaint []int64
}{
{
Intervals: tombstones.Intervals{{Mint: 0, Maxt: 3}},
remaint: []int64{4, 5, 6, 7, 8, 9},
},
{
Intervals: tombstones.Intervals{{Mint: 1, Maxt: 3}},
remaint: []int64{0, 4, 5, 6, 7, 8, 9},
},
{
Intervals: tombstones.Intervals{{Mint: 1, Maxt: 3}, {Mint: 4, Maxt: 7}},
remaint: []int64{0, 8, 9},
},
{
Intervals: tombstones.Intervals{{Mint: 1, Maxt: 3}, {Mint: 4, Maxt: 700}},
remaint: []int64{0},
},
{ // This case is to ensure that labels and symbols are deleted.
Intervals: tombstones.Intervals{{Mint: 0, Maxt: 9}},
remaint: []int64{},
},
}
Outer:
for _, c := range cases {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
smpls := make([]float64, numSamples)
for i := int64(0); i < numSamples; i++ {
smpls[i] = rand.Float64()
app.Append(0, labels.FromStrings("a", "b"), i, smpls[i])
}
require.NoError(t, app.Commit())
// TODO(gouthamve): Reset the tombstones somehow.
// Delete the ranges.
for _, r := range c.Intervals {
require.NoError(t, db.Delete(r.Mint, r.Maxt, labels.MustNewMatcher(labels.MatchEqual, "a", "b")))
}
// Compare the result.
q, err := db.Querier(context.TODO(), 0, numSamples)
require.NoError(t, err)
res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
expSamples := make([]tsdbutil.Sample, 0, len(c.remaint))
for _, ts := range c.remaint {
expSamples = append(expSamples, sample{ts, smpls[ts], nil, nil})
}
expss := newMockSeriesSet([]storage.Series{
storage.NewListSeries(labels.FromStrings("a", "b"), expSamples),
})
for {
eok, rok := expss.Next(), res.Next()
require.Equal(t, eok, rok)
if !eok {
require.Equal(t, 0, len(res.Warnings()))
continue Outer
}
sexp := expss.At()
sres := res.At()
require.Equal(t, sexp.Labels(), sres.Labels())
smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil)
smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil)
require.Equal(t, errExp, errRes)
require.Equal(t, smplExp, smplRes)
}
}
}
func TestAmendHistogramDatapointCausesError(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, labels.FromStrings("a", "b"), 0, 0)
require.NoError(t, err)
require.NoError(t, app.Commit())
app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), 0, 0)
require.NoError(t, err)
_, err = app.Append(0, labels.FromStrings("a", "b"), 0, 1)
require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err)
require.NoError(t, app.Rollback())
h := histogram.Histogram{
Schema: 3,
Count: 61,
Sum: 2.7,
ZeroThreshold: 0.1,
ZeroCount: 42,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 4},
{Offset: 10, Length: 3},
},
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
}
fh := h.ToFloat()
app = db.Appender(ctx)
_, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil)
require.NoError(t, err)
require.NoError(t, app.Commit())
app = db.Appender(ctx)
_, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil)
require.NoError(t, err)
h.Schema = 2
_, err = app.AppendHistogram(0, labels.FromStrings("a", "c"), 0, h.Copy(), nil)
require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err)
require.NoError(t, app.Rollback())
// Float histogram.
app = db.Appender(ctx)
_, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy())
require.NoError(t, err)
require.NoError(t, app.Commit())
app = db.Appender(ctx)
_, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy())
require.NoError(t, err)
fh.Schema = 2
_, err = app.AppendHistogram(0, labels.FromStrings("a", "d"), 0, nil, fh.Copy())
require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err)
require.NoError(t, app.Rollback())
}
func TestDuplicateNaNDatapointNoAmendError(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, labels.FromStrings("a", "b"), 0, math.NaN())
require.NoError(t, err)
require.NoError(t, app.Commit())
app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), 0, math.NaN())
require.NoError(t, err)
}
func TestNonDuplicateNaNDatapointsCausesAmendError(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, labels.FromStrings("a", "b"), 0, math.Float64frombits(0x7ff0000000000001))
require.NoError(t, err)
require.NoError(t, app.Commit())
app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), 0, math.Float64frombits(0x7ff0000000000002))
require.Equal(t, storage.ErrDuplicateSampleForTimestamp, err)
}
func TestEmptyLabelsetCausesError(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, labels.Labels{}, 0, 0)
require.Error(t, err)
require.Equal(t, "empty labelset: invalid sample", err.Error())
}
func TestSkippingInvalidValuesInSameTxn(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
// Append AmendedValue.
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, labels.FromStrings("a", "b"), 0, 1)
require.NoError(t, err)
_, err = app.Append(0, labels.FromStrings("a", "b"), 0, 2)
require.NoError(t, err)
require.NoError(t, app.Commit())
// Make sure the right value is stored.
q, err := db.Querier(context.TODO(), 0, 10)
require.NoError(t, err)
ssMap := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
require.Equal(t, map[string][]tsdbutil.Sample{
labels.New(labels.Label{Name: "a", Value: "b"}).String(): {sample{0, 1, nil, nil}},
}, ssMap)
// Append Out of Order Value.
app = db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), 10, 3)
require.NoError(t, err)
_, err = app.Append(0, labels.FromStrings("a", "b"), 7, 5)
require.NoError(t, err)
require.NoError(t, app.Commit())
q, err = db.Querier(context.TODO(), 0, 10)
require.NoError(t, err)
ssMap = query(t, q, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
require.Equal(t, map[string][]tsdbutil.Sample{
labels.New(labels.Label{Name: "a", Value: "b"}).String(): {sample{0, 1, nil, nil}, sample{10, 3, nil, nil}},
}, ssMap)
}
func TestDB_Snapshot(t *testing.T) {
db := openTestDB(t, nil, nil)
// append data
ctx := context.Background()
app := db.Appender(ctx)
mint := int64(1414141414000)
for i := 0; i < 1000; i++ {
_, err := app.Append(0, labels.FromStrings("foo", "bar"), mint+int64(i), 1.0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
// create snapshot
snap := t.TempDir()
require.NoError(t, db.Snapshot(snap, true))
require.NoError(t, db.Close())
// reopen DB from snapshot
db, err := Open(snap, nil, nil, nil, nil)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
querier, err := db.Querier(context.TODO(), mint, mint+1000)
require.NoError(t, err)
defer func() { require.NoError(t, querier.Close()) }()
// sum values
seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
var series chunkenc.Iterator
sum := 0.0
for seriesSet.Next() {
series = seriesSet.At().Iterator(series)
for series.Next() == chunkenc.ValFloat {
_, v := series.At()
sum += v
}
require.NoError(t, series.Err())
}
require.NoError(t, seriesSet.Err())
require.Equal(t, 0, len(seriesSet.Warnings()))
require.Equal(t, 1000.0, sum)
}
// TestDB_Snapshot_ChunksOutsideOfCompactedRange ensures that a snapshot removes chunks samples
// that are outside the set block time range.
// See https://github.com/prometheus/prometheus/issues/5105
func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) {
db := openTestDB(t, nil, nil)
ctx := context.Background()
app := db.Appender(ctx)
mint := int64(1414141414000)
for i := 0; i < 1000; i++ {
_, err := app.Append(0, labels.FromStrings("foo", "bar"), mint+int64(i), 1.0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
snap := t.TempDir()
// Hackingly introduce "race", by having lower max time then maxTime in last chunk.
db.head.maxTime.Sub(10)
require.NoError(t, db.Snapshot(snap, true))
require.NoError(t, db.Close())
// Reopen DB from snapshot.
db, err := Open(snap, nil, nil, nil, nil)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
querier, err := db.Querier(context.TODO(), mint, mint+1000)
require.NoError(t, err)
defer func() { require.NoError(t, querier.Close()) }()
// Sum values.
seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
var series chunkenc.Iterator
sum := 0.0
for seriesSet.Next() {
series = seriesSet.At().Iterator(series)
for series.Next() == chunkenc.ValFloat {
_, v := series.At()
sum += v
}
require.NoError(t, series.Err())
}
require.NoError(t, seriesSet.Err())
require.Equal(t, 0, len(seriesSet.Warnings()))
// Since we snapshotted with MaxTime - 10, so expect 10 less samples.
require.Equal(t, 1000.0-10, sum)
}
func TestDB_SnapshotWithDelete(t *testing.T) {
numSamples := int64(10)
db := openTestDB(t, nil, nil)
defer func() { require.NoError(t, db.Close()) }()
ctx := context.Background()
app := db.Appender(ctx)
smpls := make([]float64, numSamples)
for i := int64(0); i < numSamples; i++ {
smpls[i] = rand.Float64()
app.Append(0, labels.FromStrings("a", "b"), i, smpls[i])
}
require.NoError(t, app.Commit())
cases := []struct {
intervals tombstones.Intervals
remaint []int64
}{
{
intervals: tombstones.Intervals{{Mint: 1, Maxt: 3}, {Mint: 4, Maxt: 7}},
remaint: []int64{0, 8, 9},
},
}
Outer:
for _, c := range cases {
// TODO(gouthamve): Reset the tombstones somehow.
// Delete the ranges.
for _, r := range c.intervals {
require.NoError(t, db.Delete(r.Mint, r.Maxt, labels.MustNewMatcher(labels.MatchEqual, "a", "b")))
}
// create snapshot
snap := t.TempDir()
require.NoError(t, db.Snapshot(snap, true))
// reopen DB from snapshot
newDB, err := Open(snap, nil, nil, nil, nil)
require.NoError(t, err)
defer func() { require.NoError(t, newDB.Close()) }()
// Compare the result.
q, err := newDB.Querier(context.TODO(), 0, numSamples)
require.NoError(t, err)
defer func() { require.NoError(t, q.Close()) }()
res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
expSamples := make([]tsdbutil.Sample, 0, len(c.remaint))
for _, ts := range c.remaint {
expSamples = append(expSamples, sample{ts, smpls[ts], nil, nil})
}
expss := newMockSeriesSet([]storage.Series{
storage.NewListSeries(labels.FromStrings("a", "b"), expSamples),
})
if len(expSamples) == 0 {
require.False(t, res.Next())
continue
}
for {
eok, rok := expss.Next(), res.Next()
require.Equal(t, eok, rok)
if !eok {
require.Equal(t, 0, len(res.Warnings()))
continue Outer
}
sexp := expss.At()
sres := res.At()
require.Equal(t, sexp.Labels(), sres.Labels())
smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil)
smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil)
require.Equal(t, errExp, errRes)
require.Equal(t, smplExp, smplRes)
}
}
}
func TestDB_e2e(t *testing.T) {
const (
numDatapoints = 1000
numRanges = 1000
timeInterval = int64(3)
)
// Create 8 series with 1000 data-points of different ranges and run queries.
lbls := [][]labels.Label{
{
{Name: "a", Value: "b"},
{Name: "instance", Value: "localhost:9090"},
{Name: "job", Value: "prometheus"},
},
{
{Name: "a", Value: "b"},
{Name: "instance", Value: "127.0.0.1:9090"},
{Name: "job", Value: "prometheus"},
},
{
{Name: "a", Value: "b"},
{Name: "instance", Value: "127.0.0.1:9090"},
{Name: "job", Value: "prom-k8s"},
},
{
{Name: "a", Value: "b"},
{Name: "instance", Value: "localhost:9090"},
{Name: "job", Value: "prom-k8s"},
},
{
{Name: "a", Value: "c"},
{Name: "instance", Value: "localhost:9090"},
{Name: "job", Value: "prometheus"},
},
{
{Name: "a", Value: "c"},
{Name: "instance", Value: "127.0.0.1:9090"},
{Name: "job", Value: "prometheus"},
},
{
{Name: "a", Value: "c"},
{Name: "instance", Value: "127.0.0.1:9090"},
{Name: "job", Value: "prom-k8s"},
},
{
{Name: "a", Value: "c"},
{Name: "instance", Value: "localhost:9090"},
{Name: "job", Value: "prom-k8s"},
},
}
seriesMap := map[string][]tsdbutil.Sample{}
for _, l := range lbls {
seriesMap[labels.New(l...).String()] = []tsdbutil.Sample{}
}
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
for _, l := range lbls {
lset := labels.New(l...)
series := []tsdbutil.Sample{}
ts := rand.Int63n(300)
for i := 0; i < numDatapoints; i++ {
v := rand.Float64()
series = append(series, sample{ts, v, nil, nil})
_, err := app.Append(0, lset, ts, v)
require.NoError(t, err)
ts += rand.Int63n(timeInterval) + 1
}
seriesMap[lset.String()] = series
}
require.NoError(t, app.Commit())
// Query each selector on 1000 random time-ranges.
queries := []struct {
ms []*labels.Matcher
}{
{
ms: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "a", "b")},
},
{
ms: []*labels.Matcher{
labels.MustNewMatcher(labels.MatchEqual, "a", "b"),
labels.MustNewMatcher(labels.MatchEqual, "job", "prom-k8s"),
},
},
{
ms: []*labels.Matcher{
labels.MustNewMatcher(labels.MatchEqual, "a", "c"),
labels.MustNewMatcher(labels.MatchEqual, "instance", "localhost:9090"),
labels.MustNewMatcher(labels.MatchEqual, "job", "prometheus"),
},
},
// TODO: Add Regexp Matchers.
}
for _, qry := range queries {
matched := labels.Slice{}
for _, l := range lbls {
s := labels.Selector(qry.ms)
ls := labels.New(l...)
if s.Matches(ls) {
matched = append(matched, ls)
}
}
sort.Sort(matched)
for i := 0; i < numRanges; i++ {
mint := rand.Int63n(300)
maxt := mint + rand.Int63n(timeInterval*int64(numDatapoints))
expected := map[string][]tsdbutil.Sample{}
// Build the mockSeriesSet.
for _, m := range matched {
smpls := boundedSamples(seriesMap[m.String()], mint, maxt)
if len(smpls) > 0 {
expected[m.String()] = smpls
}
}
q, err := db.Querier(context.TODO(), mint, maxt)
require.NoError(t, err)
ss := q.Select(false, nil, qry.ms...)
result := map[string][]tsdbutil.Sample{}
for ss.Next() {
x := ss.At()
smpls, err := storage.ExpandSamples(x.Iterator(nil), newSample)
require.NoError(t, err)
if len(smpls) > 0 {
result[x.Labels().String()] = smpls
}
}
require.NoError(t, ss.Err())
require.Equal(t, 0, len(ss.Warnings()))
require.Equal(t, expected, result)
q.Close()
}
}
}
func TestWALFlushedOnDBClose(t *testing.T) {
db := openTestDB(t, nil, nil)
dirDb := db.Dir()
lbls := labels.FromStrings("labelname", "labelvalue")
ctx := context.Background()
app := db.Appender(ctx)
_, err := app.Append(0, lbls, 0, 1)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, db.Close())
db, err = Open(dirDb, nil, nil, nil, nil)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
q, err := db.Querier(context.TODO(), 0, 1)
require.NoError(t, err)
values, ws, err := q.LabelValues("labelname")
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, []string{"labelvalue"}, values)
}
func TestWALSegmentSizeOptions(t *testing.T) {
tests := map[int]func(dbdir string, segmentSize int){
// Default Wal Size.
0: func(dbDir string, segmentSize int) {
filesAndDir, err := os.ReadDir(filepath.Join(dbDir, "wal"))
require.NoError(t, err)
files := []os.FileInfo{}
for _, f := range filesAndDir {
if !f.IsDir() {
fi, err := f.Info()
require.NoError(t, err)
files = append(files, fi)
}
}
// All the full segment files (all but the last) should match the segment size option.
for _, f := range files[:len(files)-1] {
require.Equal(t, int64(DefaultOptions().WALSegmentSize), f.Size(), "WAL file size doesn't match WALSegmentSize option, filename: %v", f.Name())
}
lastFile := files[len(files)-1]
require.Greater(t, int64(DefaultOptions().WALSegmentSize), lastFile.Size(), "last WAL file size is not smaller than the WALSegmentSize option, filename: %v", lastFile.Name())
},
// Custom Wal Size.
2 * 32 * 1024: func(dbDir string, segmentSize int) {
filesAndDir, err := os.ReadDir(filepath.Join(dbDir, "wal"))
require.NoError(t, err)
files := []os.FileInfo{}
for _, f := range filesAndDir {
if !f.IsDir() {
fi, err := f.Info()
require.NoError(t, err)
files = append(files, fi)
}
}
require.Greater(t, len(files), 1, "current WALSegmentSize should result in more than a single WAL file.")
// All the full segment files (all but the last) should match the segment size option.
for _, f := range files[:len(files)-1] {
require.Equal(t, int64(segmentSize), f.Size(), "WAL file size doesn't match WALSegmentSize option, filename: %v", f.Name())
}
lastFile := files[len(files)-1]
require.Greater(t, int64(segmentSize), lastFile.Size(), "last WAL file size is not smaller than the WALSegmentSize option, filename: %v", lastFile.Name())
},
// Wal disabled.
-1: func(dbDir string, segmentSize int) {
// Check that WAL dir is not there.
_, err := os.Stat(filepath.Join(dbDir, "wal"))
require.Error(t, err)
// Check that there is chunks dir.
_, err = os.Stat(mmappedChunksDir(dbDir))
require.NoError(t, err)
},
}
for segmentSize, testFunc := range tests {
t.Run(fmt.Sprintf("WALSegmentSize %d test", segmentSize), func(t *testing.T) {
opts := DefaultOptions()
opts.WALSegmentSize = segmentSize
db := openTestDB(t, opts, nil)
for i := int64(0); i < 155; i++ {
app := db.Appender(context.Background())
ref, err := app.Append(0, labels.FromStrings("wal"+fmt.Sprintf("%d", i), "size"), i, rand.Float64())
require.NoError(t, err)
for j := int64(1); j <= 78; j++ {
_, err := app.Append(ref, labels.EmptyLabels(), i+j, rand.Float64())
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
dbDir := db.Dir()
require.NoError(t, db.Close())
testFunc(dbDir, opts.WALSegmentSize)
})
}
}
// https://github.com/prometheus/prometheus/issues/9846
// https://github.com/prometheus/prometheus/issues/9859
func TestWALReplayRaceOnSamplesLoggedBeforeSeries(t *testing.T) {
const (
numRuns = 1
numSamplesBeforeSeriesCreation = 1000
)
// We test both with few and many samples appended after series creation. If samples are < 120 then there's no
// mmap-ed chunk, otherwise there's at least 1 mmap-ed chunk when replaying the WAL.
for _, numSamplesAfterSeriesCreation := range []int{1, 1000} {
for run := 1; run <= numRuns; run++ {
t.Run(fmt.Sprintf("samples after series creation = %d, run = %d", numSamplesAfterSeriesCreation, run), func(t *testing.T) {
testWALReplayRaceOnSamplesLoggedBeforeSeries(t, numSamplesBeforeSeriesCreation, numSamplesAfterSeriesCreation)
})
}
}
}
func testWALReplayRaceOnSamplesLoggedBeforeSeries(t *testing.T, numSamplesBeforeSeriesCreation, numSamplesAfterSeriesCreation int) {
const numSeries = 1000
db := openTestDB(t, nil, nil)
db.DisableCompactions()
for seriesRef := 1; seriesRef <= numSeries; seriesRef++ {
// Log samples before the series is logged to the WAL.
var enc record.Encoder
var samples []record.RefSample
for ts := 0; ts < numSamplesBeforeSeriesCreation; ts++ {
samples = append(samples, record.RefSample{
Ref: chunks.HeadSeriesRef(uint64(seriesRef)),
T: int64(ts),
V: float64(ts),
})
}
err := db.Head().wal.Log(enc.Samples(samples, nil))
require.NoError(t, err)
// Add samples via appender so that they're logged after the series in the WAL.
app := db.Appender(context.Background())
lbls := labels.FromStrings("series_id", strconv.Itoa(seriesRef))
for ts := numSamplesBeforeSeriesCreation; ts < numSamplesBeforeSeriesCreation+numSamplesAfterSeriesCreation; ts++ {
_, err := app.Append(0, lbls, int64(ts), float64(ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
require.NoError(t, db.Close())
// Reopen the DB, replaying the WAL.
reopenDB, err := Open(db.Dir(), log.NewLogfmtLogger(os.Stderr), nil, nil, nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, reopenDB.Close())
})
// Query back chunks for all series.
q, err := reopenDB.ChunkQuerier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
set := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "series_id", ".+"))
actualSeries := 0
var chunksIt chunks.Iterator
for set.Next() {
actualSeries++
actualChunks := 0
chunksIt = set.At().Iterator(chunksIt)
for chunksIt.Next() {
actualChunks++
}
require.NoError(t, chunksIt.Err())
// We expect 1 chunk every 120 samples after series creation.
require.Equalf(t, (numSamplesAfterSeriesCreation/120)+1, actualChunks, "series: %s", set.At().Labels().String())
}
require.NoError(t, set.Err())
require.Equal(t, numSeries, actualSeries)
}
func TestTombstoneClean(t *testing.T) {
numSamples := int64(10)
db := openTestDB(t, nil, nil)
ctx := context.Background()
app := db.Appender(ctx)
smpls := make([]float64, numSamples)
for i := int64(0); i < numSamples; i++ {
smpls[i] = rand.Float64()
app.Append(0, labels.FromStrings("a", "b"), i, smpls[i])
}
require.NoError(t, app.Commit())
cases := []struct {
intervals tombstones.Intervals
remaint []int64
}{
{
intervals: tombstones.Intervals{{Mint: 1, Maxt: 3}, {Mint: 4, Maxt: 7}},
remaint: []int64{0, 8, 9},
},
}
for _, c := range cases {
// Delete the ranges.
// Create snapshot.
snap := t.TempDir()
require.NoError(t, db.Snapshot(snap, true))
require.NoError(t, db.Close())
// Reopen DB from snapshot.
db, err := Open(snap, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
for _, r := range c.intervals {
require.NoError(t, db.Delete(r.Mint, r.Maxt, labels.MustNewMatcher(labels.MatchEqual, "a", "b")))
}
// All of the setup for THIS line.
require.NoError(t, db.CleanTombstones())
// Compare the result.
q, err := db.Querier(context.TODO(), 0, numSamples)
require.NoError(t, err)
defer q.Close()
res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
expSamples := make([]tsdbutil.Sample, 0, len(c.remaint))
for _, ts := range c.remaint {
expSamples = append(expSamples, sample{ts, smpls[ts], nil, nil})
}
expss := newMockSeriesSet([]storage.Series{
storage.NewListSeries(labels.FromStrings("a", "b"), expSamples),
})
if len(expSamples) == 0 {
require.False(t, res.Next())
continue
}
for {
eok, rok := expss.Next(), res.Next()
require.Equal(t, eok, rok)
if !eok {
break
}
sexp := expss.At()
sres := res.At()
require.Equal(t, sexp.Labels(), sres.Labels())
smplExp, errExp := storage.ExpandSamples(sexp.Iterator(nil), nil)
smplRes, errRes := storage.ExpandSamples(sres.Iterator(nil), nil)
require.Equal(t, errExp, errRes)
require.Equal(t, smplExp, smplRes)
}
require.Equal(t, 0, len(res.Warnings()))
for _, b := range db.Blocks() {
require.Equal(t, tombstones.NewMemTombstones(), b.tombstones)
}
}
}
// TestTombstoneCleanResultEmptyBlock tests that a TombstoneClean that results in empty blocks (no timeseries)
// will also delete the resultant block.
func TestTombstoneCleanResultEmptyBlock(t *testing.T) {
numSamples := int64(10)
db := openTestDB(t, nil, nil)
ctx := context.Background()
app := db.Appender(ctx)
smpls := make([]float64, numSamples)
for i := int64(0); i < numSamples; i++ {
smpls[i] = rand.Float64()
app.Append(0, labels.FromStrings("a", "b"), i, smpls[i])
}
require.NoError(t, app.Commit())
// Interval should cover the whole block.
intervals := tombstones.Intervals{{Mint: 0, Maxt: numSamples}}
// Create snapshot.
snap := t.TempDir()
require.NoError(t, db.Snapshot(snap, true))
require.NoError(t, db.Close())
// Reopen DB from snapshot.
db, err := Open(snap, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
// Create tombstones by deleting all samples.
for _, r := range intervals {
require.NoError(t, db.Delete(r.Mint, r.Maxt, labels.MustNewMatcher(labels.MatchEqual, "a", "b")))
}
require.NoError(t, db.CleanTombstones())
// After cleaning tombstones that covers the entire block, no blocks should be left behind.
actualBlockDirs, err := blockDirs(db.dir)
require.NoError(t, err)
require.Equal(t, 0, len(actualBlockDirs))
}
// TestTombstoneCleanFail tests that a failing TombstoneClean doesn't leave any blocks behind.
// When TombstoneClean errors the original block that should be rebuilt doesn't get deleted so
// if TombstoneClean leaves any blocks behind these will overlap.
func TestTombstoneCleanFail(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
var oldBlockDirs []string
// Create some blocks pending for compaction.
// totalBlocks should be >=2 so we have enough blocks to trigger compaction failure.
totalBlocks := 2
for i := 0; i < totalBlocks; i++ {
blockDir := createBlock(t, db.Dir(), genSeries(1, 1, int64(i), int64(i)+1))
block, err := OpenBlock(nil, blockDir, nil)
require.NoError(t, err)
// Add some fake tombstones to trigger the compaction.
tomb := tombstones.NewMemTombstones()
tomb.AddInterval(0, tombstones.Interval{Mint: int64(i), Maxt: int64(i) + 1})
block.tombstones = tomb
db.blocks = append(db.blocks, block)
oldBlockDirs = append(oldBlockDirs, blockDir)
}
// Initialize the mockCompactorFailing with a room for a single compaction iteration.
// mockCompactorFailing will fail on the second iteration so we can check if the cleanup works as expected.
db.compactor = &mockCompactorFailing{
t: t,
blocks: db.blocks,
max: totalBlocks + 1,
}
// The compactor should trigger a failure here.
require.Error(t, db.CleanTombstones())
// Now check that the CleanTombstones replaced the old block even after a failure.
actualBlockDirs, err := blockDirs(db.dir)
require.NoError(t, err)
// Only one block should have been replaced by a new block.
require.Equal(t, len(oldBlockDirs), len(actualBlockDirs))
require.Equal(t, len(intersection(oldBlockDirs, actualBlockDirs)), len(actualBlockDirs)-1)
}
// TestTombstoneCleanRetentionLimitsRace tests that a CleanTombstones operation
// and retention limit policies, when triggered at the same time,
// won't race against each other.
func TestTombstoneCleanRetentionLimitsRace(t *testing.T) {
if testing.Short() {
t.Skip("skipping test in short mode.")
}
opts := DefaultOptions()
var wg sync.WaitGroup
// We want to make sure that a race doesn't happen when a normal reload and a CleanTombstones()
// reload try to delete the same block. Without the correct lock placement, it can happen if a
// block is marked for deletion due to retention limits and also has tombstones to be cleaned at
// the same time.
//
// That is something tricky to trigger, so let's try several times just to make sure.
for i := 0; i < 20; i++ {
t.Run(fmt.Sprintf("iteration%d", i), func(t *testing.T) {
db := openTestDB(t, opts, nil)
totalBlocks := 20
dbDir := db.Dir()
// Generate some blocks with old mint (near epoch).
for j := 0; j < totalBlocks; j++ {
blockDir := createBlock(t, dbDir, genSeries(10, 1, int64(j), int64(j)+1))
block, err := OpenBlock(nil, blockDir, nil)
require.NoError(t, err)
// Cover block with tombstones so it can be deleted with CleanTombstones() as well.
tomb := tombstones.NewMemTombstones()
tomb.AddInterval(0, tombstones.Interval{Mint: int64(j), Maxt: int64(j) + 1})
block.tombstones = tomb
db.blocks = append(db.blocks, block)
}
wg.Add(2)
// Run reload and CleanTombstones together, with a small time window randomization
go func() {
defer wg.Done()
time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond)))
require.NoError(t, db.reloadBlocks())
}()
go func() {
defer wg.Done()
time.Sleep(time.Duration(rand.Float64() * 100 * float64(time.Millisecond)))
require.NoError(t, db.CleanTombstones())
}()
wg.Wait()
require.NoError(t, db.Close())
})
}
}
func intersection(oldBlocks, actualBlocks []string) (intersection []string) {
hash := make(map[string]bool)
for _, e := range oldBlocks {
hash[e] = true
}
for _, e := range actualBlocks {
// If block present in the hashmap then append intersection list.
if hash[e] {
intersection = append(intersection, e)
}
}
return
}
// mockCompactorFailing creates a new empty block on every write and fails when reached the max allowed total.
// For CompactOOO, it always fails.
type mockCompactorFailing struct {
t *testing.T
blocks []*Block
max int
}
func (*mockCompactorFailing) Plan(string) ([]string, error) {
return nil, nil
}
func (c *mockCompactorFailing) Write(dest string, _ BlockReader, _, _ int64, _ *BlockMeta) (ulid.ULID, error) {
if len(c.blocks) >= c.max {
return ulid.ULID{}, fmt.Errorf("the compactor already did the maximum allowed blocks so it is time to fail")
}
block, err := OpenBlock(nil, createBlock(c.t, dest, genSeries(1, 1, 0, 1)), nil)
require.NoError(c.t, err)
require.NoError(c.t, block.Close()) // Close block as we won't be using anywhere.
c.blocks = append(c.blocks, block)
// Now check that all expected blocks are actually persisted on disk.
// This way we make sure that the we have some blocks that are supposed to be removed.
var expectedBlocks []string
for _, b := range c.blocks {
expectedBlocks = append(expectedBlocks, filepath.Join(dest, b.Meta().ULID.String()))
}
actualBlockDirs, err := blockDirs(dest)
require.NoError(c.t, err)
require.Equal(c.t, expectedBlocks, actualBlockDirs)
return block.Meta().ULID, nil
}
func (*mockCompactorFailing) Compact(string, []string, []*Block) (ulid.ULID, error) {
return ulid.ULID{}, nil
}
func (*mockCompactorFailing) CompactOOO(string, *OOOCompactionHead) (result []ulid.ULID, err error) {
return nil, fmt.Errorf("mock compaction failing CompactOOO")
}
func TestTimeRetention(t *testing.T) {
db := openTestDB(t, nil, []int64{1000})
defer func() {
require.NoError(t, db.Close())
}()
blocks := []*BlockMeta{
{MinTime: 500, MaxTime: 900}, // Oldest block
{MinTime: 1000, MaxTime: 1500},
{MinTime: 1500, MaxTime: 2000}, // Newest Block
}
for _, m := range blocks {
createBlock(t, db.Dir(), genSeries(10, 10, m.MinTime, m.MaxTime))
}
require.NoError(t, db.reloadBlocks()) // Reload the db to register the new blocks.
require.Equal(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered.
db.opts.RetentionDuration = blocks[2].MaxTime - blocks[1].MinTime
require.NoError(t, db.reloadBlocks())
expBlocks := blocks[1:]
actBlocks := db.Blocks()
require.Equal(t, 1, int(prom_testutil.ToFloat64(db.metrics.timeRetentionCount)), "metric retention count mismatch")
require.Equal(t, len(expBlocks), len(actBlocks))
require.Equal(t, expBlocks[0].MaxTime, actBlocks[0].meta.MaxTime)
require.Equal(t, expBlocks[len(expBlocks)-1].MaxTime, actBlocks[len(actBlocks)-1].meta.MaxTime)
}
func TestSizeRetention(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 100
db := openTestDB(t, opts, []int64{100})
defer func() {
require.NoError(t, db.Close())
}()
blocks := []*BlockMeta{
{MinTime: 100, MaxTime: 200}, // Oldest block
{MinTime: 200, MaxTime: 300},
{MinTime: 300, MaxTime: 400},
{MinTime: 400, MaxTime: 500},
{MinTime: 500, MaxTime: 600}, // Newest Block
}
for _, m := range blocks {
createBlock(t, db.Dir(), genSeries(100, 10, m.MinTime, m.MaxTime))
}
headBlocks := []*BlockMeta{
{MinTime: 700, MaxTime: 800},
}
// Add some data to the WAL.
headApp := db.Head().Appender(context.Background())
var aSeries labels.Labels
var it chunkenc.Iterator
for _, m := range headBlocks {
series := genSeries(100, 10, m.MinTime, m.MaxTime+1)
for _, s := range series {
aSeries = s.Labels()
it = s.Iterator(it)
for it.Next() == chunkenc.ValFloat {
tim, v := it.At()
_, err := headApp.Append(0, s.Labels(), tim, v)
require.NoError(t, err)
}
require.NoError(t, it.Err())
}
}
require.NoError(t, headApp.Commit())
db.Head().mmapHeadChunks()
require.Eventually(t, func() bool {
return db.Head().chunkDiskMapper.IsQueueEmpty()
}, 2*time.Second, 100*time.Millisecond)
// Test that registered size matches the actual disk size.
require.NoError(t, db.reloadBlocks()) // Reload the db to register the new db size.
require.Equal(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered.
blockSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the actual internal metrics.
walSize, err := db.Head().wal.Size()
require.NoError(t, err)
cdmSize, err := db.Head().chunkDiskMapper.Size()
require.NoError(t, err)
require.NotZero(t, cdmSize)
// Expected size should take into account block size + WAL size + Head
// chunks size
expSize := blockSize + walSize + cdmSize
actSize, err := fileutil.DirSize(db.Dir())
require.NoError(t, err)
require.Equal(t, expSize, actSize, "registered size doesn't match actual disk size")
// Create a WAL checkpoint, and compare sizes.
first, last, err := wlog.Segments(db.Head().wal.Dir())
require.NoError(t, err)
_, err = wlog.Checkpoint(log.NewNopLogger(), db.Head().wal, first, last-1, func(x chunks.HeadSeriesRef) bool { return false }, 0)
require.NoError(t, err)
blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the actual internal metrics.
walSize, err = db.Head().wal.Size()
require.NoError(t, err)
cdmSize, err = db.Head().chunkDiskMapper.Size()
require.NoError(t, err)
require.NotZero(t, cdmSize)
expSize = blockSize + walSize + cdmSize
actSize, err = fileutil.DirSize(db.Dir())
require.NoError(t, err)
require.Equal(t, expSize, actSize, "registered size doesn't match actual disk size")
// Truncate Chunk Disk Mapper and compare sizes.
require.NoError(t, db.Head().chunkDiskMapper.Truncate(900))
cdmSize, err = db.Head().chunkDiskMapper.Size()
require.NoError(t, err)
require.NotZero(t, cdmSize)
expSize = blockSize + walSize + cdmSize
actSize, err = fileutil.DirSize(db.Dir())
require.NoError(t, err)
require.Equal(t, expSize, actSize, "registered size doesn't match actual disk size")
// Add some out of order samples to check the size of WBL.
headApp = db.Head().Appender(context.Background())
for ts := int64(750); ts < 800; ts++ {
_, err := headApp.Append(0, aSeries, ts, float64(ts))
require.NoError(t, err)
}
require.NoError(t, headApp.Commit())
walSize, err = db.Head().wal.Size()
require.NoError(t, err)
wblSize, err := db.Head().wbl.Size()
require.NoError(t, err)
require.NotZero(t, wblSize)
cdmSize, err = db.Head().chunkDiskMapper.Size()
require.NoError(t, err)
expSize = blockSize + walSize + wblSize + cdmSize
actSize, err = fileutil.DirSize(db.Dir())
require.NoError(t, err)
require.Equal(t, expSize, actSize, "registered size doesn't match actual disk size")
// Decrease the max bytes limit so that a delete is triggered.
// Check total size, total count and check that the oldest block was deleted.
firstBlockSize := db.Blocks()[0].Size()
sizeLimit := actSize - firstBlockSize
db.opts.MaxBytes = sizeLimit // Set the new db size limit one block smaller that the actual size.
require.NoError(t, db.reloadBlocks()) // Reload the db to register the new db size.
expBlocks := blocks[1:]
actBlocks := db.Blocks()
blockSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes))
walSize, err = db.Head().wal.Size()
require.NoError(t, err)
cdmSize, err = db.Head().chunkDiskMapper.Size()
require.NoError(t, err)
require.NotZero(t, cdmSize)
// Expected size should take into account block size + WAL size + WBL size
expSize = blockSize + walSize + wblSize + cdmSize
actRetentionCount := int(prom_testutil.ToFloat64(db.metrics.sizeRetentionCount))
actSize, err = fileutil.DirSize(db.Dir())
require.NoError(t, err)
require.Equal(t, 1, actRetentionCount, "metric retention count mismatch")
require.Equal(t, actSize, expSize, "metric db size doesn't match actual disk size")
require.LessOrEqual(t, expSize, sizeLimit, "actual size (%v) is expected to be less than or equal to limit (%v)", expSize, sizeLimit)
require.Equal(t, len(blocks)-1, len(actBlocks), "new block count should be decreased from:%v to:%v", len(blocks), len(blocks)-1)
require.Equal(t, expBlocks[0].MaxTime, actBlocks[0].meta.MaxTime, "maxT mismatch of the first block")
require.Equal(t, expBlocks[len(expBlocks)-1].MaxTime, actBlocks[len(actBlocks)-1].meta.MaxTime, "maxT mismatch of the last block")
}
func TestSizeRetentionMetric(t *testing.T) {
cases := []struct {
maxBytes int64
expMaxBytes int64
}{
{maxBytes: 1000, expMaxBytes: 1000},
{maxBytes: 0, expMaxBytes: 0},
{maxBytes: -1000, expMaxBytes: 0},
}
for _, c := range cases {
db := openTestDB(t, &Options{
MaxBytes: c.maxBytes,
}, []int64{100})
defer func() {
require.NoError(t, db.Close())
}()
actMaxBytes := int64(prom_testutil.ToFloat64(db.metrics.maxBytes))
require.Equal(t, actMaxBytes, c.expMaxBytes, "metric retention limit bytes mismatch")
}
}
func TestNotMatcherSelectsLabelsUnsetSeries(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
labelpairs := []labels.Labels{
labels.FromStrings("a", "abcd", "b", "abcde"),
labels.FromStrings("labelname", "labelvalue"),
}
ctx := context.Background()
app := db.Appender(ctx)
for _, lbls := range labelpairs {
_, err := app.Append(0, lbls, 0, 1)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
cases := []struct {
selector labels.Selector
series []labels.Labels
}{{
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchNotEqual, "lname", "lvalue"),
},
series: labelpairs,
}, {
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchEqual, "a", "abcd"),
labels.MustNewMatcher(labels.MatchNotEqual, "b", "abcde"),
},
series: []labels.Labels{},
}, {
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchEqual, "a", "abcd"),
labels.MustNewMatcher(labels.MatchNotEqual, "b", "abc"),
},
series: []labels.Labels{labelpairs[0]},
}, {
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchNotRegexp, "a", "abd.*"),
},
series: labelpairs,
}, {
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchNotRegexp, "a", "abc.*"),
},
series: labelpairs[1:],
}, {
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchNotRegexp, "c", "abd.*"),
},
series: labelpairs,
}, {
selector: labels.Selector{
labels.MustNewMatcher(labels.MatchNotRegexp, "labelname", "labelvalue"),
},
series: labelpairs[:1],
}}
q, err := db.Querier(context.TODO(), 0, 10)
require.NoError(t, err)
defer func() { require.NoError(t, q.Close()) }()
for _, c := range cases {
ss := q.Select(false, nil, c.selector...)
lres, _, ws, err := expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, c.series, lres)
}
}
// expandSeriesSet returns the raw labels in the order they are retrieved from
// the series set and the samples keyed by Labels().String().
func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample, storage.Warnings, error) {
resultLabels := []labels.Labels{}
resultSamples := map[string][]sample{}
var it chunkenc.Iterator
for ss.Next() {
series := ss.At()
samples := []sample{}
it = series.Iterator(it)
for it.Next() == chunkenc.ValFloat {
t, v := it.At()
samples = append(samples, sample{t: t, f: v})
}
resultLabels = append(resultLabels, series.Labels())
resultSamples[series.Labels().String()] = samples
}
return resultLabels, resultSamples, ss.Warnings(), ss.Err()
}
func TestOverlappingBlocksDetectsAllOverlaps(t *testing.T) {
// Create 10 blocks that does not overlap (0-10, 10-20, ..., 100-110) but in reverse order to ensure our algorithm
// will handle that.
metas := make([]BlockMeta, 11)
for i := 10; i >= 0; i-- {
metas[i] = BlockMeta{MinTime: int64(i * 10), MaxTime: int64((i + 1) * 10)}
}
require.Equal(t, 0, len(OverlappingBlocks(metas)), "we found unexpected overlaps")
// Add overlapping blocks. We've to establish order again since we aren't interested
// in trivial overlaps caused by unorderedness.
add := func(ms ...BlockMeta) []BlockMeta {
repl := append(append([]BlockMeta{}, metas...), ms...)
sort.Slice(repl, func(i, j int) bool {
return repl[i].MinTime < repl[j].MinTime
})
return repl
}
// o1 overlaps with 10-20.
o1 := BlockMeta{MinTime: 15, MaxTime: 17}
require.Equal(t, Overlaps{
{Min: 15, Max: 17}: {metas[1], o1},
}, OverlappingBlocks(add(o1)))
// o2 overlaps with 20-30 and 30-40.
o2 := BlockMeta{MinTime: 21, MaxTime: 31}
require.Equal(t, Overlaps{
{Min: 21, Max: 30}: {metas[2], o2},
{Min: 30, Max: 31}: {o2, metas[3]},
}, OverlappingBlocks(add(o2)))
// o3a and o3b overlaps with 30-40 and each other.
o3a := BlockMeta{MinTime: 33, MaxTime: 39}
o3b := BlockMeta{MinTime: 34, MaxTime: 36}
require.Equal(t, Overlaps{
{Min: 34, Max: 36}: {metas[3], o3a, o3b},
}, OverlappingBlocks(add(o3a, o3b)))
// o4 is 1:1 overlap with 50-60.
o4 := BlockMeta{MinTime: 50, MaxTime: 60}
require.Equal(t, Overlaps{
{Min: 50, Max: 60}: {metas[5], o4},
}, OverlappingBlocks(add(o4)))
// o5 overlaps with 60-70, 70-80 and 80-90.
o5 := BlockMeta{MinTime: 61, MaxTime: 85}
require.Equal(t, Overlaps{
{Min: 61, Max: 70}: {metas[6], o5},
{Min: 70, Max: 80}: {o5, metas[7]},
{Min: 80, Max: 85}: {o5, metas[8]},
}, OverlappingBlocks(add(o5)))
// o6a overlaps with 90-100, 100-110 and o6b, o6b overlaps with 90-100 and o6a.
o6a := BlockMeta{MinTime: 92, MaxTime: 105}
o6b := BlockMeta{MinTime: 94, MaxTime: 99}
require.Equal(t, Overlaps{
{Min: 94, Max: 99}: {metas[9], o6a, o6b},
{Min: 100, Max: 105}: {o6a, metas[10]},
}, OverlappingBlocks(add(o6a, o6b)))
// All together.
require.Equal(t, Overlaps{
{Min: 15, Max: 17}: {metas[1], o1},
{Min: 21, Max: 30}: {metas[2], o2}, {Min: 30, Max: 31}: {o2, metas[3]},
{Min: 34, Max: 36}: {metas[3], o3a, o3b},
{Min: 50, Max: 60}: {metas[5], o4},
{Min: 61, Max: 70}: {metas[6], o5}, {Min: 70, Max: 80}: {o5, metas[7]}, {Min: 80, Max: 85}: {o5, metas[8]},
{Min: 94, Max: 99}: {metas[9], o6a, o6b}, {Min: 100, Max: 105}: {o6a, metas[10]},
}, OverlappingBlocks(add(o1, o2, o3a, o3b, o4, o5, o6a, o6b)))
// Additional case.
var nc1 []BlockMeta
nc1 = append(nc1, BlockMeta{MinTime: 1, MaxTime: 5})
nc1 = append(nc1, BlockMeta{MinTime: 2, MaxTime: 3})
nc1 = append(nc1, BlockMeta{MinTime: 2, MaxTime: 3})
nc1 = append(nc1, BlockMeta{MinTime: 2, MaxTime: 3})
nc1 = append(nc1, BlockMeta{MinTime: 2, MaxTime: 3})
nc1 = append(nc1, BlockMeta{MinTime: 2, MaxTime: 6})
nc1 = append(nc1, BlockMeta{MinTime: 3, MaxTime: 5})
nc1 = append(nc1, BlockMeta{MinTime: 5, MaxTime: 7})
nc1 = append(nc1, BlockMeta{MinTime: 7, MaxTime: 10})
nc1 = append(nc1, BlockMeta{MinTime: 8, MaxTime: 9})
require.Equal(t, Overlaps{
{Min: 2, Max: 3}: {nc1[0], nc1[1], nc1[2], nc1[3], nc1[4], nc1[5]}, // 1-5, 2-3, 2-3, 2-3, 2-3, 2,6
{Min: 3, Max: 5}: {nc1[0], nc1[5], nc1[6]}, // 1-5, 2-6, 3-5
{Min: 5, Max: 6}: {nc1[5], nc1[7]}, // 2-6, 5-7
{Min: 8, Max: 9}: {nc1[8], nc1[9]}, // 7-10, 8-9
}, OverlappingBlocks(nc1))
}
// Regression test for https://github.com/prometheus/tsdb/issues/347
func TestChunkAtBlockBoundary(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
blockRange := db.compactor.(*LeveledCompactor).ranges[0]
label := labels.FromStrings("foo", "bar")
for i := int64(0); i < 3; i++ {
_, err := app.Append(0, label, i*blockRange, 0)
require.NoError(t, err)
_, err = app.Append(0, label, i*blockRange+1000, 0)
require.NoError(t, err)
}
err := app.Commit()
require.NoError(t, err)
err = db.Compact()
require.NoError(t, err)
var builder labels.ScratchBuilder
for _, block := range db.Blocks() {
r, err := block.Index()
require.NoError(t, err)
defer r.Close()
meta := block.Meta()
k, v := index.AllPostingsKey()
p, err := r.Postings(k, v)
require.NoError(t, err)
var chks []chunks.Meta
chunkCount := 0
for p.Next() {
err = r.Series(p.At(), &builder, &chks)
require.NoError(t, err)
for _, c := range chks {
require.True(t, meta.MinTime <= c.MinTime && c.MaxTime <= meta.MaxTime,
"chunk spans beyond block boundaries: [block.MinTime=%d, block.MaxTime=%d]; [chunk.MinTime=%d, chunk.MaxTime=%d]",
meta.MinTime, meta.MaxTime, c.MinTime, c.MaxTime)
chunkCount++
}
}
require.Equal(t, 1, chunkCount, "expected 1 chunk in block %s, got %d", meta.ULID, chunkCount)
}
}
func TestQuerierWithBoundaryChunks(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
ctx := context.Background()
app := db.Appender(ctx)
blockRange := db.compactor.(*LeveledCompactor).ranges[0]
label := labels.FromStrings("foo", "bar")
for i := int64(0); i < 5; i++ {
_, err := app.Append(0, label, i*blockRange, 0)
require.NoError(t, err)
_, err = app.Append(0, labels.FromStrings("blockID", strconv.FormatInt(i, 10)), i*blockRange, 0)
require.NoError(t, err)
}
err := app.Commit()
require.NoError(t, err)
err = db.Compact()
require.NoError(t, err)
require.GreaterOrEqual(t, len(db.blocks), 3, "invalid test, less than three blocks in DB")
q, err := db.Querier(context.TODO(), blockRange, 2*blockRange)
require.NoError(t, err)
defer q.Close()
// The requested interval covers 2 blocks, so the querier's label values for blockID should give us 2 values, one from each block.
b, ws, err := q.LabelValues("blockID")
require.NoError(t, err)
require.Equal(t, storage.Warnings(nil), ws)
require.Equal(t, []string{"1", "2"}, b)
}
// TestInitializeHeadTimestamp ensures that the h.minTime is set properly.
// - no blocks no WAL: set to the time of the first appended sample
// - no blocks with WAL: set to the smallest sample from the WAL
// - with blocks no WAL: set to the last block maxT
// - with blocks with WAL: same as above
func TestInitializeHeadTimestamp(t *testing.T) {
t.Run("clean", func(t *testing.T) {
dir := t.TempDir()
db, err := Open(dir, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
// Should be set to init values if no WAL or blocks exist so far.
require.Equal(t, int64(math.MaxInt64), db.head.MinTime())
require.Equal(t, int64(math.MinInt64), db.head.MaxTime())
// First added sample initializes the writable range.
ctx := context.Background()
app := db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("a", "b"), 1000, 1)
require.NoError(t, err)
require.Equal(t, int64(1000), db.head.MinTime())
require.Equal(t, int64(1000), db.head.MaxTime())
})
t.Run("wal-only", func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.MkdirAll(path.Join(dir, "wal"), 0o777))
w, err := wlog.New(nil, nil, path.Join(dir, "wal"), wlog.CompressionNone)
require.NoError(t, err)
var enc record.Encoder
err = w.Log(
enc.Series([]record.RefSeries{
{Ref: 123, Labels: labels.FromStrings("a", "1")},
{Ref: 124, Labels: labels.FromStrings("a", "2")},
}, nil),
enc.Samples([]record.RefSample{
{Ref: 123, T: 5000, V: 1},
{Ref: 124, T: 15000, V: 1},
}, nil),
)
require.NoError(t, err)
require.NoError(t, w.Close())
db, err := Open(dir, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
require.Equal(t, int64(5000), db.head.MinTime())
require.Equal(t, int64(15000), db.head.MaxTime())
})
t.Run("existing-block", func(t *testing.T) {
dir := t.TempDir()
createBlock(t, dir, genSeries(1, 1, 1000, 2000))
db, err := Open(dir, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
require.Equal(t, int64(2000), db.head.MinTime())
require.Equal(t, int64(2000), db.head.MaxTime())
})
t.Run("existing-block-and-wal", func(t *testing.T) {
dir := t.TempDir()
createBlock(t, dir, genSeries(1, 1, 1000, 6000))
require.NoError(t, os.MkdirAll(path.Join(dir, "wal"), 0o777))
w, err := wlog.New(nil, nil, path.Join(dir, "wal"), wlog.CompressionNone)
require.NoError(t, err)
var enc record.Encoder
err = w.Log(
enc.Series([]record.RefSeries{
{Ref: 123, Labels: labels.FromStrings("a", "1")},
{Ref: 124, Labels: labels.FromStrings("a", "2")},
}, nil),
enc.Samples([]record.RefSample{
{Ref: 123, T: 5000, V: 1},
{Ref: 124, T: 15000, V: 1},
}, nil),
)
require.NoError(t, err)
require.NoError(t, w.Close())
r := prometheus.NewRegistry()
db, err := Open(dir, nil, r, nil, nil)
require.NoError(t, err)
defer db.Close()
require.Equal(t, int64(6000), db.head.MinTime())
require.Equal(t, int64(15000), db.head.MaxTime())
// Check that old series has been GCed.
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.series))
})
}
func TestNoEmptyBlocks(t *testing.T) {
db := openTestDB(t, nil, []int64{100})
ctx := context.Background()
defer func() {
require.NoError(t, db.Close())
}()
db.DisableCompactions()
rangeToTriggerCompaction := db.compactor.(*LeveledCompactor).ranges[0]/2*3 - 1
defaultLabel := labels.FromStrings("foo", "bar")
defaultMatcher := labels.MustNewMatcher(labels.MatchRegexp, "", ".*")
t.Run("Test no blocks after compact with empty head.", func(t *testing.T) {
require.NoError(t, db.Compact())
actBlocks, err := blockDirs(db.Dir())
require.NoError(t, err)
require.Equal(t, len(db.Blocks()), len(actBlocks))
require.Equal(t, 0, len(actBlocks))
require.Equal(t, 0, int(prom_testutil.ToFloat64(db.compactor.(*LeveledCompactor).metrics.Ran)), "no compaction should be triggered here")
})
t.Run("Test no blocks after deleting all samples from head.", func(t *testing.T) {
app := db.Appender(ctx)
_, err := app.Append(0, defaultLabel, 1, 0)
require.NoError(t, err)
_, err = app.Append(0, defaultLabel, 2, 0)
require.NoError(t, err)
_, err = app.Append(0, defaultLabel, 3+rangeToTriggerCompaction, 0)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, db.Delete(math.MinInt64, math.MaxInt64, defaultMatcher))
require.NoError(t, db.Compact())
require.Equal(t, 1, int(prom_testutil.ToFloat64(db.compactor.(*LeveledCompactor).metrics.Ran)), "compaction should have been triggered here")
actBlocks, err := blockDirs(db.Dir())
require.NoError(t, err)
require.Equal(t, len(db.Blocks()), len(actBlocks))
require.Equal(t, 0, len(actBlocks))
app = db.Appender(ctx)
_, err = app.Append(0, defaultLabel, 1, 0)
require.Equal(t, storage.ErrOutOfBounds, err, "the head should be truncated so no samples in the past should be allowed")
// Adding new blocks.
currentTime := db.Head().MaxTime()
_, err = app.Append(0, defaultLabel, currentTime, 0)
require.NoError(t, err)
_, err = app.Append(0, defaultLabel, currentTime+1, 0)
require.NoError(t, err)
_, err = app.Append(0, defaultLabel, currentTime+rangeToTriggerCompaction, 0)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, db.Compact())
require.Equal(t, 2, int(prom_testutil.ToFloat64(db.compactor.(*LeveledCompactor).metrics.Ran)), "compaction should have been triggered here")
actBlocks, err = blockDirs(db.Dir())
require.NoError(t, err)
require.Equal(t, len(db.Blocks()), len(actBlocks))
require.Equal(t, 1, len(actBlocks), "No blocks created when compacting with >0 samples")
})
t.Run(`When no new block is created from head, and there are some blocks on disk
compaction should not run into infinite loop (was seen during development).`, func(t *testing.T) {
oldBlocks := db.Blocks()
app := db.Appender(ctx)
currentTime := db.Head().MaxTime()
_, err := app.Append(0, defaultLabel, currentTime, 0)
require.NoError(t, err)
_, err = app.Append(0, defaultLabel, currentTime+1, 0)
require.NoError(t, err)
_, err = app.Append(0, defaultLabel, currentTime+rangeToTriggerCompaction, 0)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, db.head.Delete(math.MinInt64, math.MaxInt64, defaultMatcher))
require.NoError(t, db.Compact())
require.Equal(t, 3, int(prom_testutil.ToFloat64(db.compactor.(*LeveledCompactor).metrics.Ran)), "compaction should have been triggered here")
require.Equal(t, oldBlocks, db.Blocks())
})
t.Run("Test no blocks remaining after deleting all samples from disk.", func(t *testing.T) {
currentTime := db.Head().MaxTime()
blocks := []*BlockMeta{
{MinTime: currentTime, MaxTime: currentTime + db.compactor.(*LeveledCompactor).ranges[0]},
{MinTime: currentTime + 100, MaxTime: currentTime + 100 + db.compactor.(*LeveledCompactor).ranges[0]},
}
for _, m := range blocks {
createBlock(t, db.Dir(), genSeries(2, 2, m.MinTime, m.MaxTime))
}
oldBlocks := db.Blocks()
require.NoError(t, db.reloadBlocks()) // Reload the db to register the new blocks.
require.Equal(t, len(blocks)+len(oldBlocks), len(db.Blocks())) // Ensure all blocks are registered.
require.NoError(t, db.Delete(math.MinInt64, math.MaxInt64, defaultMatcher))
require.NoError(t, db.Compact())
require.Equal(t, 5, int(prom_testutil.ToFloat64(db.compactor.(*LeveledCompactor).metrics.Ran)), "compaction should have been triggered here once for each block that have tombstones")
actBlocks, err := blockDirs(db.Dir())
require.NoError(t, err)
require.Equal(t, len(db.Blocks()), len(actBlocks))
require.Equal(t, 1, len(actBlocks), "All samples are deleted. Only the most recent block should remain after compaction.")
})
}
func TestDB_LabelNames(t *testing.T) {
tests := []struct {
// Add 'sampleLabels1' -> Test Head -> Compact -> Test Disk ->
// -> Add 'sampleLabels2' -> Test Head+Disk
sampleLabels1 [][2]string // For checking head and disk separately.
// To test Head+Disk, sampleLabels2 should have
// at least 1 unique label name which is not in sampleLabels1.
sampleLabels2 [][2]string // // For checking head and disk together.
exp1 []string // after adding sampleLabels1.
exp2 []string // after adding sampleLabels1 and sampleLabels2.
}{
{
sampleLabels1: [][2]string{
{"name1", "1"},
{"name3", "3"},
{"name2", "2"},
},
sampleLabels2: [][2]string{
{"name4", "4"},
{"name1", "1"},
},
exp1: []string{"name1", "name2", "name3"},
exp2: []string{"name1", "name2", "name3", "name4"},
},
{
sampleLabels1: [][2]string{
{"name2", "2"},
{"name1", "1"},
{"name2", "2"},
},
sampleLabels2: [][2]string{
{"name6", "6"},
{"name0", "0"},
},
exp1: []string{"name1", "name2"},
exp2: []string{"name0", "name1", "name2", "name6"},
},
}
blockRange := int64(1000)
// Appends samples into the database.
appendSamples := func(db *DB, mint, maxt int64, sampleLabels [][2]string) {
t.Helper()
ctx := context.Background()
app := db.Appender(ctx)
for i := mint; i <= maxt; i++ {
for _, tuple := range sampleLabels {
label := labels.FromStrings(tuple[0], tuple[1])
_, err := app.Append(0, label, i*blockRange, 0)
require.NoError(t, err)
}
}
err := app.Commit()
require.NoError(t, err)
}
for _, tst := range tests {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
appendSamples(db, 0, 4, tst.sampleLabels1)
// Testing head.
headIndexr, err := db.head.Index()
require.NoError(t, err)
labelNames, err := headIndexr.LabelNames()
require.NoError(t, err)
require.Equal(t, tst.exp1, labelNames)
require.NoError(t, headIndexr.Close())
// Testing disk.
err = db.Compact()
require.NoError(t, err)
// All blocks have same label names, hence check them individually.
// No need to aggregate and check.
for _, b := range db.Blocks() {
blockIndexr, err := b.Index()
require.NoError(t, err)
labelNames, err = blockIndexr.LabelNames()
require.NoError(t, err)
require.Equal(t, tst.exp1, labelNames)
require.NoError(t, blockIndexr.Close())
}
// Adding more samples to head with new label names
// so that we can test (head+disk).LabelNames() (the union).
appendSamples(db, 5, 9, tst.sampleLabels2)
// Testing DB (union).
q, err := db.Querier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
var ws storage.Warnings
labelNames, ws, err = q.LabelNames()
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.NoError(t, q.Close())
require.Equal(t, tst.exp2, labelNames)
}
}
func TestCorrectNumTombstones(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
blockRange := db.compactor.(*LeveledCompactor).ranges[0]
name, value := "foo", "bar"
defaultLabel := labels.FromStrings(name, value)
defaultMatcher := labels.MustNewMatcher(labels.MatchEqual, name, value)
ctx := context.Background()
app := db.Appender(ctx)
for i := int64(0); i < 3; i++ {
for j := int64(0); j < 15; j++ {
_, err := app.Append(0, defaultLabel, i*blockRange+j, 0)
require.NoError(t, err)
}
}
require.NoError(t, app.Commit())
err := db.Compact()
require.NoError(t, err)
require.Equal(t, 1, len(db.blocks))
require.NoError(t, db.Delete(0, 1, defaultMatcher))
require.Equal(t, uint64(1), db.blocks[0].meta.Stats.NumTombstones)
// {0, 1} and {2, 3} are merged to form 1 tombstone.
require.NoError(t, db.Delete(2, 3, defaultMatcher))
require.Equal(t, uint64(1), db.blocks[0].meta.Stats.NumTombstones)
require.NoError(t, db.Delete(5, 6, defaultMatcher))
require.Equal(t, uint64(2), db.blocks[0].meta.Stats.NumTombstones)
require.NoError(t, db.Delete(9, 11, defaultMatcher))
require.Equal(t, uint64(3), db.blocks[0].meta.Stats.NumTombstones)
}
// TestBlockRanges checks the following use cases:
// - No samples can be added with timestamps lower than the last block maxt.
// - The compactor doesn't create overlapping blocks
//
// even when the last blocks is not within the default boundaries.
// - Lower boundary is based on the smallest sample in the head and
//
// upper boundary is rounded to the configured block range.
//
// This ensures that a snapshot that includes the head and creates a block with a custom time range
// will not overlap with the first block created by the next compaction.
func TestBlockRanges(t *testing.T) {
logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
ctx := context.Background()
dir := t.TempDir()
// Test that the compactor doesn't create overlapping blocks
// when a non standard block already exists.
firstBlockMaxT := int64(3)
createBlock(t, dir, genSeries(1, 1, 0, firstBlockMaxT))
db, err := open(dir, logger, nil, DefaultOptions(), []int64{10000}, nil)
require.NoError(t, err)
rangeToTriggerCompaction := db.compactor.(*LeveledCompactor).ranges[0]/2*3 + 1
app := db.Appender(ctx)
lbl := labels.FromStrings("a", "b")
_, err = app.Append(0, lbl, firstBlockMaxT-1, rand.Float64())
if err == nil {
t.Fatalf("appending a sample with a timestamp covered by a previous block shouldn't be possible")
}
_, err = app.Append(0, lbl, firstBlockMaxT+1, rand.Float64())
require.NoError(t, err)
_, err = app.Append(0, lbl, firstBlockMaxT+2, rand.Float64())
require.NoError(t, err)
secondBlockMaxt := firstBlockMaxT + rangeToTriggerCompaction
_, err = app.Append(0, lbl, secondBlockMaxt, rand.Float64()) // Add samples to trigger a new compaction
require.NoError(t, err)
require.NoError(t, app.Commit())
for x := 0; x < 100; x++ {
if len(db.Blocks()) == 2 {
break
}
time.Sleep(100 * time.Millisecond)
}
require.Equal(t, 2, len(db.Blocks()), "no new block created after the set timeout")
if db.Blocks()[0].Meta().MaxTime > db.Blocks()[1].Meta().MinTime {
t.Fatalf("new block overlaps old:%v,new:%v", db.Blocks()[0].Meta(), db.Blocks()[1].Meta())
}
// Test that wal records are skipped when an existing block covers the same time ranges
// and compaction doesn't create an overlapping block.
app = db.Appender(ctx)
db.DisableCompactions()
_, err = app.Append(0, lbl, secondBlockMaxt+1, rand.Float64())
require.NoError(t, err)
_, err = app.Append(0, lbl, secondBlockMaxt+2, rand.Float64())
require.NoError(t, err)
_, err = app.Append(0, lbl, secondBlockMaxt+3, rand.Float64())
require.NoError(t, err)
_, err = app.Append(0, lbl, secondBlockMaxt+4, rand.Float64())
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, db.Close())
thirdBlockMaxt := secondBlockMaxt + 2
createBlock(t, dir, genSeries(1, 1, secondBlockMaxt+1, thirdBlockMaxt))
db, err = open(dir, logger, nil, DefaultOptions(), []int64{10000}, nil)
require.NoError(t, err)
defer db.Close()
require.Equal(t, 3, len(db.Blocks()), "db doesn't include expected number of blocks")
require.Equal(t, db.Blocks()[2].Meta().MaxTime, thirdBlockMaxt, "unexpected maxt of the last block")
app = db.Appender(ctx)
_, err = app.Append(0, lbl, thirdBlockMaxt+rangeToTriggerCompaction, rand.Float64()) // Trigger a compaction
require.NoError(t, err)
require.NoError(t, app.Commit())
for x := 0; x < 100; x++ {
if len(db.Blocks()) == 4 {
break
}
time.Sleep(100 * time.Millisecond)
}
require.Equal(t, 4, len(db.Blocks()), "no new block created after the set timeout")
if db.Blocks()[2].Meta().MaxTime > db.Blocks()[3].Meta().MinTime {
t.Fatalf("new block overlaps old:%v,new:%v", db.Blocks()[2].Meta(), db.Blocks()[3].Meta())
}
}
// TestDBReadOnly ensures that opening a DB in readonly mode doesn't modify any files on the disk.
// It also checks that the API calls return equivalent results as a normal db.Open() mode.
func TestDBReadOnly(t *testing.T) {
var (
dbDir string
logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
expBlocks []*Block
expBlock *Block
expSeries map[string][]tsdbutil.Sample
expChunks map[string][][]tsdbutil.Sample
expDBHash []byte
matchAll = labels.MustNewMatcher(labels.MatchEqual, "", "")
err error
)
// Bootstrap the db.
{
dbDir = t.TempDir()
dbBlocks := []*BlockMeta{
// Create three 2-sample blocks.
{MinTime: 10, MaxTime: 12},
{MinTime: 12, MaxTime: 14},
{MinTime: 14, MaxTime: 16},
}
for _, m := range dbBlocks {
_ = createBlock(t, dbDir, genSeries(1, 1, m.MinTime, m.MaxTime))
}
// Add head to test DBReadOnly WAL reading capabilities.
w, err := wlog.New(logger, nil, filepath.Join(dbDir, "wal"), wlog.CompressionSnappy)
require.NoError(t, err)
h := createHead(t, w, genSeries(1, 1, 16, 18), dbDir)
require.NoError(t, h.Close())
}
// Open a normal db to use for a comparison.
{
dbWritable, err := Open(dbDir, logger, nil, nil, nil)
require.NoError(t, err)
dbWritable.DisableCompactions()
dbSizeBeforeAppend, err := fileutil.DirSize(dbWritable.Dir())
require.NoError(t, err)
app := dbWritable.Appender(context.Background())
_, err = app.Append(0, labels.FromStrings("foo", "bar"), dbWritable.Head().MaxTime()+1, 0)
require.NoError(t, err)
require.NoError(t, app.Commit())
expBlocks = dbWritable.Blocks()
expBlock = expBlocks[0]
expDbSize, err := fileutil.DirSize(dbWritable.Dir())
require.NoError(t, err)
require.Greater(t, expDbSize, dbSizeBeforeAppend, "db size didn't increase after an append")
q, err := dbWritable.Querier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
expSeries = query(t, q, matchAll)
cq, err := dbWritable.ChunkQuerier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
expChunks = queryAndExpandChunks(t, cq, matchAll)
require.NoError(t, dbWritable.Close()) // Close here to allow getting the dir hash for windows.
expDBHash = testutil.DirHash(t, dbWritable.Dir())
}
// Open a read only db and ensure that the API returns the same result as the normal DB.
dbReadOnly, err := OpenDBReadOnly(dbDir, logger)
require.NoError(t, err)
defer func() { require.NoError(t, dbReadOnly.Close()) }()
t.Run("blocks", func(t *testing.T) {
blocks, err := dbReadOnly.Blocks()
require.NoError(t, err)
require.Equal(t, len(expBlocks), len(blocks))
for i, expBlock := range expBlocks {
require.Equal(t, expBlock.Meta(), blocks[i].Meta(), "block meta mismatch")
}
})
t.Run("block", func(t *testing.T) {
blockID := expBlock.meta.ULID.String()
block, err := dbReadOnly.Block(blockID)
require.NoError(t, err)
require.Equal(t, expBlock.Meta(), block.Meta(), "block meta mismatch")
})
t.Run("invalid block ID", func(t *testing.T) {
blockID := "01GTDVZZF52NSWB5SXQF0P2PGF"
_, err := dbReadOnly.Block(blockID)
require.Error(t, err)
})
t.Run("last block ID", func(t *testing.T) {
blockID, err := dbReadOnly.LastBlockID()
require.NoError(t, err)
require.Equal(t, expBlocks[2].Meta().ULID.String(), blockID)
})
t.Run("querier", func(t *testing.T) {
// Open a read only db and ensure that the API returns the same result as the normal DB.
q, err := dbReadOnly.Querier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
readOnlySeries := query(t, q, matchAll)
readOnlyDBHash := testutil.DirHash(t, dbDir)
require.Equal(t, len(expSeries), len(readOnlySeries), "total series mismatch")
require.Equal(t, expSeries, readOnlySeries, "series mismatch")
require.Equal(t, expDBHash, readOnlyDBHash, "after all read operations the db hash should remain the same")
})
t.Run("chunk querier", func(t *testing.T) {
cq, err := dbReadOnly.ChunkQuerier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
readOnlySeries := queryAndExpandChunks(t, cq, matchAll)
readOnlyDBHash := testutil.DirHash(t, dbDir)
require.Equal(t, len(expChunks), len(readOnlySeries), "total series mismatch")
require.Equal(t, expChunks, readOnlySeries, "series chunks mismatch")
require.Equal(t, expDBHash, readOnlyDBHash, "after all read operations the db hash should remain the same")
})
}
// TestDBReadOnlyClosing ensures that after closing the db
// all api methods return an ErrClosed.
func TestDBReadOnlyClosing(t *testing.T) {
dbDir := t.TempDir()
db, err := OpenDBReadOnly(dbDir, log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)))
require.NoError(t, err)
require.NoError(t, db.Close())
require.Equal(t, db.Close(), ErrClosed)
_, err = db.Blocks()
require.Equal(t, err, ErrClosed)
_, err = db.Querier(context.TODO(), 0, 1)
require.Equal(t, err, ErrClosed)
}
func TestDBReadOnly_FlushWAL(t *testing.T) {
var (
dbDir string
logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
err error
maxt int
ctx = context.Background()
)
// Bootstrap the db.
{
dbDir = t.TempDir()
// Append data to the WAL.
db, err := Open(dbDir, logger, nil, nil, nil)
require.NoError(t, err)
db.DisableCompactions()
app := db.Appender(ctx)
maxt = 1000
for i := 0; i < maxt; i++ {
_, err := app.Append(0, labels.FromStrings(defaultLabelName, "flush"), int64(i), 1.0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
require.NoError(t, db.Close())
}
// Flush WAL.
db, err := OpenDBReadOnly(dbDir, logger)
require.NoError(t, err)
flush := t.TempDir()
require.NoError(t, db.FlushWAL(flush))
require.NoError(t, db.Close())
// Reopen the DB from the flushed WAL block.
db, err = OpenDBReadOnly(flush, logger)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
blocks, err := db.Blocks()
require.NoError(t, err)
require.Equal(t, len(blocks), 1)
querier, err := db.Querier(context.TODO(), 0, int64(maxt)-1)
require.NoError(t, err)
defer func() { require.NoError(t, querier.Close()) }()
// Sum the values.
seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, defaultLabelName, "flush"))
var series chunkenc.Iterator
sum := 0.0
for seriesSet.Next() {
series = seriesSet.At().Iterator(series)
for series.Next() == chunkenc.ValFloat {
_, v := series.At()
sum += v
}
require.NoError(t, series.Err())
}
require.NoError(t, seriesSet.Err())
require.Equal(t, 0, len(seriesSet.Warnings()))
require.Equal(t, 1000.0, sum)
}
func TestDBCannotSeePartialCommits(t *testing.T) {
if defaultIsolationDisabled {
t.Skip("skipping test since tsdb isolation is disabled")
}
tmpdir := t.TempDir()
db, err := Open(tmpdir, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
stop := make(chan struct{})
firstInsert := make(chan struct{})
ctx := context.Background()
// Insert data in batches.
go func() {
iter := 0
for {
app := db.Appender(ctx)
for j := 0; j < 100; j++ {
_, err := app.Append(0, labels.FromStrings("foo", "bar", "a", strconv.Itoa(j)), int64(iter), float64(iter))
require.NoError(t, err)
}
err = app.Commit()
require.NoError(t, err)
if iter == 0 {
close(firstInsert)
}
iter++
select {
case <-stop:
return
default:
}
}
}()
<-firstInsert
// This is a race condition, so do a few tests to tickle it.
// Usually most will fail.
inconsistencies := 0
for i := 0; i < 10; i++ {
func() {
querier, err := db.Querier(context.Background(), 0, 1000000)
require.NoError(t, err)
defer querier.Close()
ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
_, seriesSet, ws, err := expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
values := map[float64]struct{}{}
for _, series := range seriesSet {
values[series[len(series)-1].f] = struct{}{}
}
if len(values) != 1 {
inconsistencies++
}
}()
}
stop <- struct{}{}
require.Equal(t, 0, inconsistencies, "Some queries saw inconsistent results.")
}
func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) {
if defaultIsolationDisabled {
t.Skip("skipping test since tsdb isolation is disabled")
}
tmpdir := t.TempDir()
db, err := Open(tmpdir, nil, nil, nil, nil)
require.NoError(t, err)
defer db.Close()
querierBeforeAdd, err := db.Querier(context.Background(), 0, 1000000)
require.NoError(t, err)
defer querierBeforeAdd.Close()
ctx := context.Background()
app := db.Appender(ctx)
_, err = app.Append(0, labels.FromStrings("foo", "bar"), 0, 0)
require.NoError(t, err)
querierAfterAddButBeforeCommit, err := db.Querier(context.Background(), 0, 1000000)
require.NoError(t, err)
defer querierAfterAddButBeforeCommit.Close()
// None of the queriers should return anything after the Add but before the commit.
ss := querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
_, seriesSet, ws, err := expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, map[string][]sample{}, seriesSet)
ss = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
_, seriesSet, ws, err = expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, map[string][]sample{}, seriesSet)
// This commit is after the queriers are created, so should not be returned.
err = app.Commit()
require.NoError(t, err)
// Nothing returned for querier created before the Add.
ss = querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
_, seriesSet, ws, err = expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, map[string][]sample{}, seriesSet)
// Series exists but has no samples for querier created after Add.
ss = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
_, seriesSet, ws, err = expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, map[string][]sample{`{foo="bar"}`: {}}, seriesSet)
querierAfterCommit, err := db.Querier(context.Background(), 0, 1000000)
require.NoError(t, err)
defer querierAfterCommit.Close()
// Samples are returned for querier created after Commit.
ss = querierAfterCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
_, seriesSet, ws, err = expandSeriesSet(ss)
require.NoError(t, err)
require.Equal(t, 0, len(ws))
require.Equal(t, map[string][]sample{`{foo="bar"}`: {{t: 0, f: 0}}}, seriesSet)
}
func assureChunkFromSamples(t *testing.T, samples []tsdbutil.Sample) chunks.Meta {
chks, err := tsdbutil.ChunkFromSamples(samples)
require.NoError(t, err)
return chks
}
// TestChunkWriter_ReadAfterWrite ensures that chunk segment are cut at the set segment size and
// that the resulted segments includes the expected chunks data.
func TestChunkWriter_ReadAfterWrite(t *testing.T) {
chk1 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 1, nil, nil}})
chk2 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 2, nil, nil}})
chk3 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 3, nil, nil}})
chk4 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 4, nil, nil}})
chk5 := assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 5, nil, nil}})
chunkSize := len(chk1.Chunk.Bytes()) + chunks.MaxChunkLengthFieldSize + chunks.ChunkEncodingSize + crc32.Size
tests := []struct {
chks [][]chunks.Meta
segmentSize,
expSegmentsCount int
expSegmentSizes []int
}{
// 0:Last chunk ends at the segment boundary so
// all chunks should fit in a single segment.
{
chks: [][]chunks.Meta{
{
chk1,
chk2,
chk3,
},
},
segmentSize: 3 * chunkSize,
expSegmentSizes: []int{3 * chunkSize},
expSegmentsCount: 1,
},
// 1:Two chunks can fit in a single segment so the last one should result in a new segment.
{
chks: [][]chunks.Meta{
{
chk1,
chk2,
chk3,
chk4,
chk5,
},
},
segmentSize: 2 * chunkSize,
expSegmentSizes: []int{2 * chunkSize, 2 * chunkSize, chunkSize},
expSegmentsCount: 3,
},
// 2:When the segment size is smaller than the size of 2 chunks
// the last segment should still create a new segment.
{
chks: [][]chunks.Meta{
{
chk1,
chk2,
chk3,
},
},
segmentSize: 2*chunkSize - 1,
expSegmentSizes: []int{chunkSize, chunkSize, chunkSize},
expSegmentsCount: 3,
},
// 3:When the segment is smaller than a single chunk
// it should still be written by ignoring the max segment size.
{
chks: [][]chunks.Meta{
{
chk1,
},
},
segmentSize: chunkSize - 1,
expSegmentSizes: []int{chunkSize},
expSegmentsCount: 1,
},
// 4:All chunks are bigger than the max segment size, but
// these should still be written even when this will result in bigger segment than the set size.
// Each segment will hold a single chunk.
{
chks: [][]chunks.Meta{
{
chk1,
chk2,
chk3,
},
},
segmentSize: 1,
expSegmentSizes: []int{chunkSize, chunkSize, chunkSize},
expSegmentsCount: 3,
},
// 5:Adding multiple batches of chunks.
{
chks: [][]chunks.Meta{
{
chk1,
chk2,
chk3,
},
{
chk4,
chk5,
},
},
segmentSize: 3 * chunkSize,
expSegmentSizes: []int{3 * chunkSize, 2 * chunkSize},
expSegmentsCount: 2,
},
// 6:Adding multiple batches of chunks.
{
chks: [][]chunks.Meta{
{
chk1,
},
{
chk2,
chk3,
},
{
chk4,
},
},
segmentSize: 2 * chunkSize,
expSegmentSizes: []int{2 * chunkSize, 2 * chunkSize},
expSegmentsCount: 2,
},
}
for i, test := range tests {
t.Run(strconv.Itoa(i), func(t *testing.T) {
tempDir := t.TempDir()
chunkw, err := chunks.NewWriterWithSegSize(tempDir, chunks.SegmentHeaderSize+int64(test.segmentSize))
require.NoError(t, err)
for _, chks := range test.chks {
require.NoError(t, chunkw.WriteChunks(chks...))
}
require.NoError(t, chunkw.Close())
files, err := os.ReadDir(tempDir)
require.NoError(t, err)
require.Equal(t, test.expSegmentsCount, len(files), "expected segments count mismatch")
// Verify that all data is written to the segments.
sizeExp := 0
sizeAct := 0
for _, chks := range test.chks {
for _, chk := range chks {
l := make([]byte, binary.MaxVarintLen32)
sizeExp += binary.PutUvarint(l, uint64(len(chk.Chunk.Bytes()))) // The length field.
sizeExp += chunks.ChunkEncodingSize
sizeExp += len(chk.Chunk.Bytes()) // The data itself.
sizeExp += crc32.Size // The 4 bytes of crc32
}
}
sizeExp += test.expSegmentsCount * chunks.SegmentHeaderSize // The segment header bytes.
for i, f := range files {
fi, err := f.Info()
require.NoError(t, err)
size := int(fi.Size())
// Verify that the segment is the same or smaller than the expected size.
require.GreaterOrEqual(t, chunks.SegmentHeaderSize+test.expSegmentSizes[i], size, "Segment:%v should NOT be bigger than:%v actual:%v", i, chunks.SegmentHeaderSize+test.expSegmentSizes[i], size)
sizeAct += size
}
require.Equal(t, sizeExp, sizeAct)
// Check the content of the chunks.
r, err := chunks.NewDirReader(tempDir, nil)
require.NoError(t, err)
defer func() { require.NoError(t, r.Close()) }()
for _, chks := range test.chks {
for _, chkExp := range chks {
chkAct, err := r.Chunk(chkExp)
require.NoError(t, err)
require.Equal(t, chkExp.Chunk.Bytes(), chkAct.Bytes())
}
}
})
}
}
func TestRangeForTimestamp(t *testing.T) {
type args struct {
t int64
width int64
}
tests := []struct {
args args
expected int64
}{
{args{0, 5}, 5},
{args{1, 5}, 5},
{args{5, 5}, 10},
{args{6, 5}, 10},
{args{13, 5}, 15},
{args{95, 5}, 100},
}
for _, tt := range tests {
got := rangeForTimestamp(tt.args.t, tt.args.width)
require.Equal(t, tt.expected, got)
}
}
// TestChunkReader_ConcurrentReads checks that the chunk result can be read concurrently.
// Regression test for https://github.com/prometheus/prometheus/pull/6514.
func TestChunkReader_ConcurrentReads(t *testing.T) {
chks := []chunks.Meta{
assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 1, nil, nil}}),
assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 2, nil, nil}}),
assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 3, nil, nil}}),
assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 4, nil, nil}}),
assureChunkFromSamples(t, []tsdbutil.Sample{sample{1, 5, nil, nil}}),
}
tempDir := t.TempDir()
chunkw, err := chunks.NewWriter(tempDir)
require.NoError(t, err)
require.NoError(t, chunkw.WriteChunks(chks...))
require.NoError(t, chunkw.Close())
r, err := chunks.NewDirReader(tempDir, nil)
require.NoError(t, err)
var wg sync.WaitGroup
for _, chk := range chks {
for i := 0; i < 100; i++ {
wg.Add(1)
go func(chunk chunks.Meta) {
defer wg.Done()
chkAct, err := r.Chunk(chunk)
require.NoError(t, err)
require.Equal(t, chunk.Chunk.Bytes(), chkAct.Bytes())
}(chk)
}
wg.Wait()
}
require.NoError(t, r.Close())
}
// TestCompactHead ensures that the head compaction
// creates a block that is ready for loading and
// does not cause data loss.
// This test:
// * opens a storage;
// * appends values;
// * compacts the head; and
// * queries the db to ensure the samples are present from the compacted head.
func TestCompactHead(t *testing.T) {
dbDir := t.TempDir()
// Open a DB and append data to the WAL.
tsdbCfg := &Options{
RetentionDuration: int64(time.Hour * 24 * 15 / time.Millisecond),
NoLockfile: true,
MinBlockDuration: int64(time.Hour * 2 / time.Millisecond),
MaxBlockDuration: int64(time.Hour * 2 / time.Millisecond),
WALCompression: wlog.CompressionSnappy,
}
db, err := Open(dbDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg, nil)
require.NoError(t, err)
ctx := context.Background()
app := db.Appender(ctx)
var expSamples []sample
maxt := 100
for i := 0; i < maxt; i++ {
val := rand.Float64()
_, err := app.Append(0, labels.FromStrings("a", "b"), int64(i), val)
require.NoError(t, err)
expSamples = append(expSamples, sample{int64(i), val, nil, nil})
}
require.NoError(t, app.Commit())
// Compact the Head to create a new block.
require.NoError(t, db.CompactHead(NewRangeHead(db.Head(), 0, int64(maxt)-1)))
require.NoError(t, db.Close())
// Delete everything but the new block and
// reopen the db to query it to ensure it includes the head data.
require.NoError(t, deleteNonBlocks(db.Dir()))
db, err = Open(dbDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg, nil)
require.NoError(t, err)
require.Equal(t, 1, len(db.Blocks()))
require.Equal(t, int64(maxt), db.Head().MinTime())
defer func() { require.NoError(t, db.Close()) }()
querier, err := db.Querier(context.Background(), 0, int64(maxt)-1)
require.NoError(t, err)
defer func() { require.NoError(t, querier.Close()) }()
seriesSet := querier.Select(false, nil, &labels.Matcher{Type: labels.MatchEqual, Name: "a", Value: "b"})
var series chunkenc.Iterator
var actSamples []sample
for seriesSet.Next() {
series = seriesSet.At().Iterator(series)
for series.Next() == chunkenc.ValFloat {
time, val := series.At()
actSamples = append(actSamples, sample{time, val, nil, nil})
}
require.NoError(t, series.Err())
}
require.Equal(t, expSamples, actSamples)
require.NoError(t, seriesSet.Err())
}
// TestCompactHeadWithDeletion tests https://github.com/prometheus/prometheus/issues/11585.
func TestCompactHeadWithDeletion(t *testing.T) {
db, err := Open(t.TempDir(), log.NewNopLogger(), prometheus.NewRegistry(), nil, nil)
require.NoError(t, err)
app := db.Appender(context.Background())
_, err = app.Append(0, labels.FromStrings("a", "b"), 10, rand.Float64())
require.NoError(t, err)
require.NoError(t, app.Commit())
err = db.Delete(0, 100, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
require.NoError(t, err)
// This recreates the bug.
require.NoError(t, db.CompactHead(NewRangeHead(db.Head(), 0, 100)))
require.NoError(t, db.Close())
}
func deleteNonBlocks(dbDir string) error {
dirs, err := os.ReadDir(dbDir)
if err != nil {
return err
}
for _, dir := range dirs {
if ok := isBlockDir(dir); !ok {
if err := os.RemoveAll(filepath.Join(dbDir, dir.Name())); err != nil {
return err
}
}
}
dirs, err = os.ReadDir(dbDir)
if err != nil {
return err
}
for _, dir := range dirs {
if ok := isBlockDir(dir); !ok {
return errors.Errorf("root folder:%v still hase non block directory:%v", dbDir, dir.Name())
}
}
return nil
}
func TestOpen_VariousBlockStates(t *testing.T) {
tmpDir := t.TempDir()
var (
expectedLoadedDirs = map[string]struct{}{}
expectedRemovedDirs = map[string]struct{}{}
expectedIgnoredDirs = map[string]struct{}{}
)
{
// Ok blocks; should be loaded.
expectedLoadedDirs[createBlock(t, tmpDir, genSeries(10, 2, 0, 10))] = struct{}{}
expectedLoadedDirs[createBlock(t, tmpDir, genSeries(10, 2, 10, 20))] = struct{}{}
}
{
// Block to repair; should be repaired & loaded.
dbDir := filepath.Join("testdata", "repair_index_version", "01BZJ9WJQPWHGNC2W4J9TA62KC")
outDir := filepath.Join(tmpDir, "01BZJ9WJQPWHGNC2W4J9TA62KC")
expectedLoadedDirs[outDir] = struct{}{}
// Touch chunks dir in block.
require.NoError(t, os.MkdirAll(filepath.Join(dbDir, "chunks"), 0o777))
defer func() {
require.NoError(t, os.RemoveAll(filepath.Join(dbDir, "chunks")))
}()
require.NoError(t, os.Mkdir(outDir, os.ModePerm))
require.NoError(t, fileutil.CopyDirs(dbDir, outDir))
}
{
// Missing meta.json; should be ignored and only logged.
// TODO(bwplotka): Probably add metric.
dir := createBlock(t, tmpDir, genSeries(10, 2, 20, 30))
expectedIgnoredDirs[dir] = struct{}{}
require.NoError(t, os.Remove(filepath.Join(dir, metaFilename)))
}
{
// Tmp blocks during creation; those should be removed on start.
dir := createBlock(t, tmpDir, genSeries(10, 2, 30, 40))
require.NoError(t, fileutil.Replace(dir, dir+tmpForCreationBlockDirSuffix))
expectedRemovedDirs[dir+tmpForCreationBlockDirSuffix] = struct{}{}
// Tmp blocks during deletion; those should be removed on start.
dir = createBlock(t, tmpDir, genSeries(10, 2, 40, 50))
require.NoError(t, fileutil.Replace(dir, dir+tmpForDeletionBlockDirSuffix))
expectedRemovedDirs[dir+tmpForDeletionBlockDirSuffix] = struct{}{}
// Pre-2.21 tmp blocks; those should be removed on start.
dir = createBlock(t, tmpDir, genSeries(10, 2, 50, 60))
require.NoError(t, fileutil.Replace(dir, dir+tmpLegacy))
expectedRemovedDirs[dir+tmpLegacy] = struct{}{}
}
{
// One ok block; but two should be replaced.
dir := createBlock(t, tmpDir, genSeries(10, 2, 50, 60))
expectedLoadedDirs[dir] = struct{}{}
m, _, err := readMetaFile(dir)
require.NoError(t, err)
compacted := createBlock(t, tmpDir, genSeries(10, 2, 50, 55))
expectedRemovedDirs[compacted] = struct{}{}
m.Compaction.Parents = append(m.Compaction.Parents,
BlockDesc{ULID: ulid.MustParse(filepath.Base(compacted))},
BlockDesc{ULID: ulid.MustNew(1, nil)},
BlockDesc{ULID: ulid.MustNew(123, nil)},
)
// Regression test: Already removed parent can be still in list, which was causing Open errors.
m.Compaction.Parents = append(m.Compaction.Parents, BlockDesc{ULID: ulid.MustParse(filepath.Base(compacted))})
m.Compaction.Parents = append(m.Compaction.Parents, BlockDesc{ULID: ulid.MustParse(filepath.Base(compacted))})
_, err = writeMetaFile(log.NewLogfmtLogger(os.Stderr), dir, m)
require.NoError(t, err)
}
tmpCheckpointDir := path.Join(tmpDir, "wal/checkpoint.00000001.tmp")
err := os.MkdirAll(tmpCheckpointDir, 0o777)
require.NoError(t, err)
opts := DefaultOptions()
opts.RetentionDuration = 0
db, err := Open(tmpDir, log.NewLogfmtLogger(os.Stderr), nil, opts, nil)
require.NoError(t, err)
loadedBlocks := db.Blocks()
var loaded int
for _, l := range loadedBlocks {
if _, ok := expectedLoadedDirs[filepath.Join(tmpDir, l.meta.ULID.String())]; !ok {
t.Fatal("unexpected block", l.meta.ULID, "was loaded")
}
loaded++
}
require.Equal(t, len(expectedLoadedDirs), loaded)
require.NoError(t, db.Close())
files, err := os.ReadDir(tmpDir)
require.NoError(t, err)
var ignored int
for _, f := range files {
if _, ok := expectedRemovedDirs[filepath.Join(tmpDir, f.Name())]; ok {
t.Fatal("expected", filepath.Join(tmpDir, f.Name()), "to be removed, but still exists")
}
if _, ok := expectedIgnoredDirs[filepath.Join(tmpDir, f.Name())]; ok {
ignored++
}
}
require.Equal(t, len(expectedIgnoredDirs), ignored)
_, err = os.Stat(tmpCheckpointDir)
require.True(t, os.IsNotExist(err))
}
func TestOneCheckpointPerCompactCall(t *testing.T) {
blockRange := int64(1000)
tsdbCfg := &Options{
RetentionDuration: blockRange * 1000,
NoLockfile: true,
MinBlockDuration: blockRange,
MaxBlockDuration: blockRange,
}
tmpDir := t.TempDir()
db, err := Open(tmpDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg, nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, db.Close())
})
db.DisableCompactions()
// Case 1: Lot's of uncompacted data in Head.
lbls := labels.FromStrings("foo_d", "choco_bar")
// Append samples spanning 59 block ranges.
app := db.Appender(context.Background())
for i := int64(0); i < 60; i++ {
_, err := app.Append(0, lbls, blockRange*i, rand.Float64())
require.NoError(t, err)
_, err = app.Append(0, lbls, (blockRange*i)+blockRange/2, rand.Float64())
require.NoError(t, err)
// Rotate the WAL file so that there is >3 files for checkpoint to happen.
_, err = db.head.wal.NextSegment()
require.NoError(t, err)
}
require.NoError(t, app.Commit())
// Check the existing WAL files.
first, last, err := wlog.Segments(db.head.wal.Dir())
require.NoError(t, err)
require.Equal(t, 0, first)
require.Equal(t, 60, last)
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.checkpointCreationTotal))
require.NoError(t, db.Compact())
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.checkpointCreationTotal))
// As the data spans for 59 blocks, 58 go to disk and 1 remains in Head.
require.Equal(t, 58, len(db.Blocks()))
// Though WAL was truncated only once, head should be truncated after each compaction.
require.Equal(t, 58.0, prom_testutil.ToFloat64(db.head.metrics.headTruncateTotal))
// The compaction should have only truncated first 2/3 of WAL (while also rotating the files).
first, last, err = wlog.Segments(db.head.wal.Dir())
require.NoError(t, err)
require.Equal(t, 40, first)
require.Equal(t, 61, last)
// The first checkpoint would be for first 2/3rd of WAL, hence till 39.
// That should be the last checkpoint.
_, cno, err := wlog.LastCheckpoint(db.head.wal.Dir())
require.NoError(t, err)
require.Equal(t, 39, cno)
// Case 2: Old blocks on disk.
// The above blocks will act as old blocks.
// Creating a block to cover the data in the Head so that
// Head will skip the data during replay and start fresh.
blocks := db.Blocks()
newBlockMint := blocks[len(blocks)-1].Meta().MaxTime
newBlockMaxt := db.Head().MaxTime() + 1
require.NoError(t, db.Close())
createBlock(t, db.dir, genSeries(1, 1, newBlockMint, newBlockMaxt))
db, err = Open(db.dir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg, nil)
require.NoError(t, err)
db.DisableCompactions()
// 1 block more.
require.Equal(t, 59, len(db.Blocks()))
// No series in Head because of this new block.
require.Equal(t, 0, int(db.head.NumSeries()))
// Adding sample way into the future.
app = db.Appender(context.Background())
_, err = app.Append(0, lbls, blockRange*120, rand.Float64())
require.NoError(t, err)
require.NoError(t, app.Commit())
// The mint of head is the last block maxt, that means the gap between mint and maxt
// of Head is too large. This will trigger many compactions.
require.Equal(t, newBlockMaxt, db.head.MinTime())
// Another WAL file was rotated.
first, last, err = wlog.Segments(db.head.wal.Dir())
require.NoError(t, err)
require.Equal(t, 40, first)
require.Equal(t, 62, last)
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.checkpointCreationTotal))
require.NoError(t, db.Compact())
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.checkpointCreationTotal))
// No new blocks should be created as there was not data in between the new samples and the blocks.
require.Equal(t, 59, len(db.Blocks()))
// The compaction should have only truncated first 2/3 of WAL (while also rotating the files).
first, last, err = wlog.Segments(db.head.wal.Dir())
require.NoError(t, err)
require.Equal(t, 55, first)
require.Equal(t, 63, last)
// The first checkpoint would be for first 2/3rd of WAL, hence till 54.
// That should be the last checkpoint.
_, cno, err = wlog.LastCheckpoint(db.head.wal.Dir())
require.NoError(t, err)
require.Equal(t, 54, cno)
}
func TestNoPanicOnTSDBOpenError(t *testing.T) {
tmpdir := t.TempDir()
// Taking the lock will cause a TSDB startup error.
l, err := tsdbutil.NewDirLocker(tmpdir, "tsdb", log.NewNopLogger(), nil)
require.NoError(t, err)
require.NoError(t, l.Lock())
_, err = Open(tmpdir, nil, nil, DefaultOptions(), nil)
require.Error(t, err)
require.NoError(t, l.Release())
}
func TestLockfile(t *testing.T) {
tsdbutil.TestDirLockerUsage(t, func(t *testing.T, data string, createLock bool) (*tsdbutil.DirLocker, testutil.Closer) {
opts := DefaultOptions()
opts.NoLockfile = !createLock
// Create the DB. This should create lockfile and its metrics.
db, err := Open(data, nil, nil, opts, nil)
require.NoError(t, err)
return db.locker, testutil.NewCallbackCloser(func() {
require.NoError(t, db.Close())
})
})
}
func TestQuerier_ShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t *testing.T) {
t.Skip("TODO: investigate why process crash in CI")
const numRuns = 5
for i := 1; i <= numRuns; i++ {
t.Run(strconv.Itoa(i), func(t *testing.T) {
testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t)
})
}
}
func testQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t *testing.T) {
const (
numSeries = 1000
numStressIterations = 10000
minStressAllocationBytes = 128 * 1024
maxStressAllocationBytes = 512 * 1024
)
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
// Disable compactions so we can control it.
db.DisableCompactions()
// Generate the metrics we're going to append.
metrics := make([]labels.Labels, 0, numSeries)
for i := 0; i < numSeries; i++ {
metrics = append(metrics, labels.FromStrings(labels.MetricName, fmt.Sprintf("test_%d", i)))
}
// Push 1 sample every 15s for 2x the block duration period.
ctx := context.Background()
interval := int64(15 * time.Second / time.Millisecond)
ts := int64(0)
for ; ts < 2*DefaultBlockDuration; ts += interval {
app := db.Appender(ctx)
for _, metric := range metrics {
_, err := app.Append(0, metric, ts, float64(ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Compact the TSDB head for the first time. We expect the head chunks file has been cut.
require.NoError(t, db.Compact())
require.Equal(t, float64(1), prom_testutil.ToFloat64(db.Head().metrics.headTruncateTotal))
// Push more samples for another 1x block duration period.
for ; ts < 3*DefaultBlockDuration; ts += interval {
app := db.Appender(ctx)
for _, metric := range metrics {
_, err := app.Append(0, metric, ts, float64(ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// At this point we expect 2 mmap-ed head chunks.
// Get a querier and make sure it's closed only once the test is over.
querier, err := db.Querier(ctx, 0, math.MaxInt64)
require.NoError(t, err)
defer func() {
require.NoError(t, querier.Close())
}()
// Query back all series.
hints := &storage.SelectHints{Start: 0, End: math.MaxInt64, Step: interval}
seriesSet := querier.Select(true, hints, labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, ".+"))
// Fetch samples iterators from all series.
var iterators []chunkenc.Iterator
actualSeries := 0
for seriesSet.Next() {
actualSeries++
// Get the iterator and call Next() so that we're sure the chunk is loaded.
it := seriesSet.At().Iterator(nil)
it.Next()
it.At()
iterators = append(iterators, it)
}
require.NoError(t, seriesSet.Err())
require.Equal(t, actualSeries, numSeries)
// Compact the TSDB head again.
require.NoError(t, db.Compact())
require.Equal(t, float64(2), prom_testutil.ToFloat64(db.Head().metrics.headTruncateTotal))
// At this point we expect 1 head chunk has been deleted.
// Stress the memory and call GC. This is required to increase the chances
// the chunk memory area is released to the kernel.
var buf []byte
for i := 0; i < numStressIterations; i++ {
//nolint:staticcheck
buf = append(buf, make([]byte, minStressAllocationBytes+rand.Int31n(maxStressAllocationBytes-minStressAllocationBytes))...)
if i%1000 == 0 {
buf = nil
}
}
// Iterate samples. Here we're summing it just to make sure no golang compiler
// optimization triggers in case we discard the result of it.At().
var sum float64
var firstErr error
for _, it := range iterators {
for it.Next() == chunkenc.ValFloat {
_, v := it.At()
sum += v
}
if err := it.Err(); err != nil {
firstErr = err
}
}
// After having iterated all samples we also want to be sure no error occurred or
// the "cannot populate chunk XXX: not found" error occurred. This error can occur
// when the iterator tries to fetch an head chunk which has been offloaded because
// of the head compaction in the meanwhile.
if firstErr != nil && !strings.Contains(firstErr.Error(), "cannot populate chunk") {
t.Fatalf("unexpected error: %s", firstErr.Error())
}
}
func TestChunkQuerier_ShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t *testing.T) {
t.Skip("TODO: investigate why process crash in CI")
const numRuns = 5
for i := 1; i <= numRuns; i++ {
t.Run(strconv.Itoa(i), func(t *testing.T) {
testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t)
})
}
}
func testChunkQuerierShouldNotPanicIfHeadChunkIsTruncatedWhileReadingQueriedChunks(t *testing.T) {
const (
numSeries = 1000
numStressIterations = 10000
minStressAllocationBytes = 128 * 1024
maxStressAllocationBytes = 512 * 1024
)
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
// Disable compactions so we can control it.
db.DisableCompactions()
// Generate the metrics we're going to append.
metrics := make([]labels.Labels, 0, numSeries)
for i := 0; i < numSeries; i++ {
metrics = append(metrics, labels.FromStrings(labels.MetricName, fmt.Sprintf("test_%d", i)))
}
// Push 1 sample every 15s for 2x the block duration period.
ctx := context.Background()
interval := int64(15 * time.Second / time.Millisecond)
ts := int64(0)
for ; ts < 2*DefaultBlockDuration; ts += interval {
app := db.Appender(ctx)
for _, metric := range metrics {
_, err := app.Append(0, metric, ts, float64(ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Compact the TSDB head for the first time. We expect the head chunks file has been cut.
require.NoError(t, db.Compact())
require.Equal(t, float64(1), prom_testutil.ToFloat64(db.Head().metrics.headTruncateTotal))
// Push more samples for another 1x block duration period.
for ; ts < 3*DefaultBlockDuration; ts += interval {
app := db.Appender(ctx)
for _, metric := range metrics {
_, err := app.Append(0, metric, ts, float64(ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// At this point we expect 2 mmap-ed head chunks.
// Get a querier and make sure it's closed only once the test is over.
querier, err := db.ChunkQuerier(ctx, 0, math.MaxInt64)
require.NoError(t, err)
defer func() {
require.NoError(t, querier.Close())
}()
// Query back all series.
hints := &storage.SelectHints{Start: 0, End: math.MaxInt64, Step: interval}
seriesSet := querier.Select(true, hints, labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, ".+"))
// Iterate all series and get their chunks.
var it chunks.Iterator
var chunks []chunkenc.Chunk
actualSeries := 0
for seriesSet.Next() {
actualSeries++
it = seriesSet.At().Iterator(it)
for it.Next() {
chunks = append(chunks, it.At().Chunk)
}
}
require.NoError(t, seriesSet.Err())
require.Equal(t, actualSeries, numSeries)
// Compact the TSDB head again.
require.NoError(t, db.Compact())
require.Equal(t, float64(2), prom_testutil.ToFloat64(db.Head().metrics.headTruncateTotal))
// At this point we expect 1 head chunk has been deleted.
// Stress the memory and call GC. This is required to increase the chances
// the chunk memory area is released to the kernel.
var buf []byte
for i := 0; i < numStressIterations; i++ {
//nolint:staticcheck
buf = append(buf, make([]byte, minStressAllocationBytes+rand.Int31n(maxStressAllocationBytes-minStressAllocationBytes))...)
if i%1000 == 0 {
buf = nil
}
}
// Iterate chunks and read their bytes slice. Here we're computing the CRC32
// just to iterate through the bytes slice. We don't really care the reason why
// we read this data, we just need to read it to make sure the memory address
// of the []byte is still valid.
chkCRC32 := newCRC32()
for _, chunk := range chunks {
chkCRC32.Reset()
_, err := chkCRC32.Write(chunk.Bytes())
require.NoError(t, err)
}
}
func newTestDB(t *testing.T) *DB {
dir := t.TempDir()
db, err := Open(dir, nil, nil, DefaultOptions(), nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, db.Close())
})
return db
}
func TestOOOWALWrite(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 2
opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, db.Close())
})
s1, s2 := labels.FromStrings("l", "v1"), labels.FromStrings("l", "v2")
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
appendSample := func(app storage.Appender, l labels.Labels, mins int64) {
_, err = app.Append(0, l, minutes(mins), float64(mins))
require.NoError(t, err)
}
// Ingest sample at 1h.
app := db.Appender(context.Background())
appendSample(app, s1, 60)
appendSample(app, s2, 60)
require.NoError(t, app.Commit())
// OOO for s1.
app = db.Appender(context.Background())
appendSample(app, s1, 40)
require.NoError(t, app.Commit())
// OOO for s2.
app = db.Appender(context.Background())
appendSample(app, s2, 42)
require.NoError(t, app.Commit())
// OOO for both s1 and s2 in the same commit.
app = db.Appender(context.Background())
appendSample(app, s2, 45)
appendSample(app, s1, 35)
appendSample(app, s1, 36) // m-maps.
appendSample(app, s1, 37)
require.NoError(t, app.Commit())
// OOO for s1 but not for s2 in the same commit.
app = db.Appender(context.Background())
appendSample(app, s1, 50) // m-maps.
appendSample(app, s2, 65)
require.NoError(t, app.Commit())
// Single commit has 2 times m-mapping and more samples after m-map.
app = db.Appender(context.Background())
appendSample(app, s2, 50) // m-maps.
appendSample(app, s2, 51)
appendSample(app, s2, 52) // m-maps.
appendSample(app, s2, 53)
require.NoError(t, app.Commit())
// The MmapRef in this are not hand calculated, and instead taken from the test run.
// What is important here is the order of records, and that MmapRef increases for each record.
oooRecords := []interface{}{
[]record.RefMmapMarker{
{Ref: 1},
},
[]record.RefSample{
{Ref: 1, T: minutes(40), V: 40},
},
[]record.RefMmapMarker{
{Ref: 2},
},
[]record.RefSample{
{Ref: 2, T: minutes(42), V: 42},
},
[]record.RefSample{
{Ref: 2, T: minutes(45), V: 45},
{Ref: 1, T: minutes(35), V: 35},
},
[]record.RefMmapMarker{ // 3rd sample, hence m-mapped.
{Ref: 1, MmapRef: 4294967304},
},
[]record.RefSample{
{Ref: 1, T: minutes(36), V: 36},
{Ref: 1, T: minutes(37), V: 37},
},
[]record.RefMmapMarker{ // 3rd sample, hence m-mapped.
{Ref: 1, MmapRef: 4294967354},
},
[]record.RefSample{ // Does not contain the in-order sample here.
{Ref: 1, T: minutes(50), V: 50},
},
// Single commit but multiple OOO records.
[]record.RefMmapMarker{
{Ref: 2, MmapRef: 4294967403},
},
[]record.RefSample{
{Ref: 2, T: minutes(50), V: 50},
{Ref: 2, T: minutes(51), V: 51},
},
[]record.RefMmapMarker{
{Ref: 2, MmapRef: 4294967452},
},
[]record.RefSample{
{Ref: 2, T: minutes(52), V: 52},
{Ref: 2, T: minutes(53), V: 53},
},
}
inOrderRecords := []interface{}{
[]record.RefSeries{
{Ref: 1, Labels: s1},
{Ref: 2, Labels: s2},
},
[]record.RefSample{
{Ref: 1, T: minutes(60), V: 60},
{Ref: 2, T: minutes(60), V: 60},
},
[]record.RefSample{
{Ref: 1, T: minutes(40), V: 40},
},
[]record.RefSample{
{Ref: 2, T: minutes(42), V: 42},
},
[]record.RefSample{
{Ref: 2, T: minutes(45), V: 45},
{Ref: 1, T: minutes(35), V: 35},
{Ref: 1, T: minutes(36), V: 36},
{Ref: 1, T: minutes(37), V: 37},
},
[]record.RefSample{ // Contains both in-order and ooo sample.
{Ref: 1, T: minutes(50), V: 50},
{Ref: 2, T: minutes(65), V: 65},
},
[]record.RefSample{
{Ref: 2, T: minutes(50), V: 50},
{Ref: 2, T: minutes(51), V: 51},
{Ref: 2, T: minutes(52), V: 52},
{Ref: 2, T: minutes(53), V: 53},
},
}
getRecords := func(walDir string) []interface{} {
sr, err := wlog.NewSegmentsReader(walDir)
require.NoError(t, err)
r := wlog.NewReader(sr)
defer func() {
require.NoError(t, sr.Close())
}()
var (
records []interface{}
dec record.Decoder
)
for r.Next() {
rec := r.Record()
switch typ := dec.Type(rec); typ {
case record.Series:
series, err := dec.Series(rec, nil)
require.NoError(t, err)
records = append(records, series)
case record.Samples:
samples, err := dec.Samples(rec, nil)
require.NoError(t, err)
records = append(records, samples)
case record.MmapMarkers:
markers, err := dec.MmapMarkers(rec, nil)
require.NoError(t, err)
records = append(records, markers)
default:
t.Fatalf("got a WAL record that is not series or samples: %v", typ)
}
}
return records
}
// The normal WAL.
actRecs := getRecords(path.Join(dir, "wal"))
require.Equal(t, inOrderRecords, actRecs)
// The OOO WAL.
actRecs = getRecords(path.Join(dir, wlog.WblDirName))
require.Equal(t, oooRecords, actRecs)
}
// Tests https://github.com/prometheus/prometheus/issues/10291#issuecomment-1044373110.
func TestDBPanicOnMmappingHeadChunk(t *testing.T) {
dir := t.TempDir()
db, err := Open(dir, nil, nil, DefaultOptions(), nil)
require.NoError(t, err)
db.DisableCompactions()
// Choosing scrape interval of 45s to have chunk larger than 1h.
itvl := int64(45 * time.Second / time.Millisecond)
lastTs := int64(0)
addSamples := func(numSamples int) {
app := db.Appender(context.Background())
var ref storage.SeriesRef
lbls := labels.FromStrings("__name__", "testing", "foo", "bar")
for i := 0; i < numSamples; i++ {
ref, err = app.Append(ref, lbls, lastTs, float64(lastTs))
require.NoError(t, err)
lastTs += itvl
if i%10 == 0 {
require.NoError(t, app.Commit())
app = db.Appender(context.Background())
}
}
require.NoError(t, app.Commit())
}
// Ingest samples upto 2h50m to make the head "about to compact".
numSamples := int(170*time.Minute/time.Millisecond) / int(itvl)
addSamples(numSamples)
require.Len(t, db.Blocks(), 0)
require.NoError(t, db.Compact())
require.Len(t, db.Blocks(), 0)
// Restarting.
require.NoError(t, db.Close())
db, err = Open(dir, nil, nil, DefaultOptions(), nil)
require.NoError(t, err)
db.DisableCompactions()
// Ingest samples upto 20m more to make the head compact.
numSamples = int(20*time.Minute/time.Millisecond) / int(itvl)
addSamples(numSamples)
require.Len(t, db.Blocks(), 0)
require.NoError(t, db.Compact())
require.Len(t, db.Blocks(), 1)
// More samples to m-map and panic.
numSamples = int(120*time.Minute/time.Millisecond) / int(itvl)
addSamples(numSamples)
require.NoError(t, db.Close())
}
func TestMetadataInWAL(t *testing.T) {
updateMetadata := func(t *testing.T, app storage.Appender, s labels.Labels, m metadata.Metadata) {
_, err := app.UpdateMetadata(0, s, m)
require.NoError(t, err)
}
db := newTestDB(t)
ctx := context.Background()
// Add some series so we can append metadata to them.
app := db.Appender(ctx)
s1 := labels.FromStrings("a", "b")
s2 := labels.FromStrings("c", "d")
s3 := labels.FromStrings("e", "f")
s4 := labels.FromStrings("g", "h")
for _, s := range []labels.Labels{s1, s2, s3, s4} {
_, err := app.Append(0, s, 0, 0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
// Add a first round of metadata to the first three series.
// Re-take the Appender, as the previous Commit will have it closed.
m1 := metadata.Metadata{Type: "gauge", Unit: "unit_1", Help: "help_1"}
m2 := metadata.Metadata{Type: "gauge", Unit: "unit_2", Help: "help_2"}
m3 := metadata.Metadata{Type: "gauge", Unit: "unit_3", Help: "help_3"}
app = db.Appender(ctx)
updateMetadata(t, app, s1, m1)
updateMetadata(t, app, s2, m2)
updateMetadata(t, app, s3, m3)
require.NoError(t, app.Commit())
// Add a replicated metadata entry to the first series,
// a completely new metadata entry for the fourth series,
// and a changed metadata entry to the second series.
m4 := metadata.Metadata{Type: "counter", Unit: "unit_4", Help: "help_4"}
m5 := metadata.Metadata{Type: "counter", Unit: "unit_5", Help: "help_5"}
app = db.Appender(ctx)
updateMetadata(t, app, s1, m1)
updateMetadata(t, app, s4, m4)
updateMetadata(t, app, s2, m5)
require.NoError(t, app.Commit())
// Read the WAL to see if the disk storage format is correct.
recs := readTestWAL(t, path.Join(db.Dir(), "wal"))
var gotMetadataBlocks [][]record.RefMetadata
for _, rec := range recs {
if mr, ok := rec.([]record.RefMetadata); ok {
gotMetadataBlocks = append(gotMetadataBlocks, mr)
}
}
expectedMetadata := []record.RefMetadata{
{Ref: 1, Type: record.GetMetricType(m1.Type), Unit: m1.Unit, Help: m1.Help},
{Ref: 2, Type: record.GetMetricType(m2.Type), Unit: m2.Unit, Help: m2.Help},
{Ref: 3, Type: record.GetMetricType(m3.Type), Unit: m3.Unit, Help: m3.Help},
{Ref: 4, Type: record.GetMetricType(m4.Type), Unit: m4.Unit, Help: m4.Help},
{Ref: 2, Type: record.GetMetricType(m5.Type), Unit: m5.Unit, Help: m5.Help},
}
require.Len(t, gotMetadataBlocks, 2)
require.Equal(t, expectedMetadata[:3], gotMetadataBlocks[0])
require.Equal(t, expectedMetadata[3:], gotMetadataBlocks[1])
}
func TestMetadataCheckpointingOnlyKeepsLatestEntry(t *testing.T) {
updateMetadata := func(t *testing.T, app storage.Appender, s labels.Labels, m metadata.Metadata) {
_, err := app.UpdateMetadata(0, s, m)
require.NoError(t, err)
}
ctx := context.Background()
numSamples := 10000
hb, w := newTestHead(t, int64(numSamples)*10, wlog.CompressionNone, false)
// Add some series so we can append metadata to them.
app := hb.Appender(ctx)
s1 := labels.FromStrings("a", "b")
s2 := labels.FromStrings("c", "d")
s3 := labels.FromStrings("e", "f")
s4 := labels.FromStrings("g", "h")
for _, s := range []labels.Labels{s1, s2, s3, s4} {
_, err := app.Append(0, s, 0, 0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
// Add a first round of metadata to the first three series.
// Re-take the Appender, as the previous Commit will have it closed.
m1 := metadata.Metadata{Type: "gauge", Unit: "unit_1", Help: "help_1"}
m2 := metadata.Metadata{Type: "gauge", Unit: "unit_2", Help: "help_2"}
m3 := metadata.Metadata{Type: "gauge", Unit: "unit_3", Help: "help_3"}
m4 := metadata.Metadata{Type: "gauge", Unit: "unit_4", Help: "help_4"}
app = hb.Appender(ctx)
updateMetadata(t, app, s1, m1)
updateMetadata(t, app, s2, m2)
updateMetadata(t, app, s3, m3)
updateMetadata(t, app, s4, m4)
require.NoError(t, app.Commit())
// Update metadata for first series.
m5 := metadata.Metadata{Type: "counter", Unit: "unit_5", Help: "help_5"}
app = hb.Appender(ctx)
updateMetadata(t, app, s1, m5)
require.NoError(t, app.Commit())
// Switch back-and-forth metadata for second series.
// Since it ended on a new metadata record, we expect a single new entry.
m6 := metadata.Metadata{Type: "counter", Unit: "unit_6", Help: "help_6"}
app = hb.Appender(ctx)
updateMetadata(t, app, s2, m6)
require.NoError(t, app.Commit())
app = hb.Appender(ctx)
updateMetadata(t, app, s2, m2)
require.NoError(t, app.Commit())
app = hb.Appender(ctx)
updateMetadata(t, app, s2, m6)
require.NoError(t, app.Commit())
app = hb.Appender(ctx)
updateMetadata(t, app, s2, m2)
require.NoError(t, app.Commit())
app = hb.Appender(ctx)
updateMetadata(t, app, s2, m6)
require.NoError(t, app.Commit())
// Let's create a checkpoint.
first, last, err := wlog.Segments(w.Dir())
require.NoError(t, err)
keep := func(id chunks.HeadSeriesRef) bool {
return id != 3
}
_, err = wlog.Checkpoint(log.NewNopLogger(), w, first, last-1, keep, 0)
require.NoError(t, err)
// Confirm there's been a checkpoint.
cdir, _, err := wlog.LastCheckpoint(w.Dir())
require.NoError(t, err)
// Read in checkpoint and WAL.
recs := readTestWAL(t, cdir)
var gotMetadataBlocks [][]record.RefMetadata
for _, rec := range recs {
if mr, ok := rec.([]record.RefMetadata); ok {
gotMetadataBlocks = append(gotMetadataBlocks, mr)
}
}
// There should only be 1 metadata block present, with only the latest
// metadata kept around.
wantMetadata := []record.RefMetadata{
{Ref: 1, Type: record.GetMetricType(m5.Type), Unit: m5.Unit, Help: m5.Help},
{Ref: 2, Type: record.GetMetricType(m6.Type), Unit: m6.Unit, Help: m6.Help},
{Ref: 4, Type: record.GetMetricType(m4.Type), Unit: m4.Unit, Help: m4.Help},
}
require.Len(t, gotMetadataBlocks, 1)
require.Len(t, gotMetadataBlocks[0], 3)
gotMetadataBlock := gotMetadataBlocks[0]
sort.Slice(gotMetadataBlock, func(i, j int) bool { return gotMetadataBlock[i].Ref < gotMetadataBlock[j].Ref })
require.Equal(t, wantMetadata, gotMetadataBlock)
require.NoError(t, hb.Close())
}
func TestMetadataAssertInMemoryData(t *testing.T) {
updateMetadata := func(t *testing.T, app storage.Appender, s labels.Labels, m metadata.Metadata) {
_, err := app.UpdateMetadata(0, s, m)
require.NoError(t, err)
}
db := openTestDB(t, nil, nil)
ctx := context.Background()
// Add some series so we can append metadata to them.
app := db.Appender(ctx)
s1 := labels.FromStrings("a", "b")
s2 := labels.FromStrings("c", "d")
s3 := labels.FromStrings("e", "f")
s4 := labels.FromStrings("g", "h")
for _, s := range []labels.Labels{s1, s2, s3, s4} {
_, err := app.Append(0, s, 0, 0)
require.NoError(t, err)
}
require.NoError(t, app.Commit())
// Add a first round of metadata to the first three series.
// The in-memory data held in the db Head should hold the metadata.
m1 := metadata.Metadata{Type: "gauge", Unit: "unit_1", Help: "help_1"}
m2 := metadata.Metadata{Type: "gauge", Unit: "unit_2", Help: "help_2"}
m3 := metadata.Metadata{Type: "gauge", Unit: "unit_3", Help: "help_3"}
app = db.Appender(ctx)
updateMetadata(t, app, s1, m1)
updateMetadata(t, app, s2, m2)
updateMetadata(t, app, s3, m3)
require.NoError(t, app.Commit())
series1 := db.head.series.getByHash(s1.Hash(), s1)
series2 := db.head.series.getByHash(s2.Hash(), s2)
series3 := db.head.series.getByHash(s3.Hash(), s3)
series4 := db.head.series.getByHash(s4.Hash(), s4)
require.Equal(t, *series1.meta, m1)
require.Equal(t, *series2.meta, m2)
require.Equal(t, *series3.meta, m3)
require.Nil(t, series4.meta)
// Add a replicated metadata entry to the first series,
// a changed metadata entry to the second series,
// and a completely new metadata entry for the fourth series.
// The in-memory data held in the db Head should be correctly updated.
m4 := metadata.Metadata{Type: "counter", Unit: "unit_4", Help: "help_4"}
m5 := metadata.Metadata{Type: "counter", Unit: "unit_5", Help: "help_5"}
app = db.Appender(ctx)
updateMetadata(t, app, s1, m1)
updateMetadata(t, app, s4, m4)
updateMetadata(t, app, s2, m5)
require.NoError(t, app.Commit())
series1 = db.head.series.getByHash(s1.Hash(), s1)
series2 = db.head.series.getByHash(s2.Hash(), s2)
series3 = db.head.series.getByHash(s3.Hash(), s3)
series4 = db.head.series.getByHash(s4.Hash(), s4)
require.Equal(t, *series1.meta, m1)
require.Equal(t, *series2.meta, m5)
require.Equal(t, *series3.meta, m3)
require.Equal(t, *series4.meta, m4)
require.NoError(t, db.Close())
// Reopen the DB, replaying the WAL. The Head must have been replayed
// correctly in memory.
reopenDB, err := Open(db.Dir(), nil, nil, nil, nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, reopenDB.Close())
})
_, err = reopenDB.head.wal.Size()
require.NoError(t, err)
require.Equal(t, *reopenDB.head.series.getByHash(s1.Hash(), s1).meta, m1)
require.Equal(t, *reopenDB.head.series.getByHash(s2.Hash(), s2).meta, m5)
require.Equal(t, *reopenDB.head.series.getByHash(s3.Hash(), s3).meta, m3)
require.Equal(t, *reopenDB.head.series.getByHash(s4.Hash(), s4).meta, m4)
}
// TODO(codesome): test more samples incoming once compaction has started. To verify new samples after the start
//
// are not included in this compaction.
func TestOOOCompaction(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions() // We want to manually call it.
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
series2 := labels.FromStrings("foo", "bar2")
addSample := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
_, err = app.Append(0, series2, ts, float64(2*ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSample(250, 350)
// Verify that the in-memory ooo chunk is empty.
checkEmptyOOOChunk := func(lbls labels.Labels) {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Nil(t, ms.ooo)
}
checkEmptyOOOChunk(series1)
checkEmptyOOOChunk(series2)
// Add ooo samples that creates multiple chunks.
// 90 to 300 spans across 3 block ranges: [0, 120), [120, 240), [240, 360)
addSample(90, 310)
// Adding same samples to create overlapping chunks.
// Since the active chunk won't start at 90 again, all the new
// chunks will have different time ranges than the previous chunks.
addSample(90, 310)
verifyDBSamples := func() {
var series1Samples, series2Samples []tsdbutil.Sample
for _, r := range [][2]int64{{90, 119}, {120, 239}, {240, 350}} {
fromMins, toMins := r[0], r[1]
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
}
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): series1Samples,
series2.String(): series2Samples,
}
q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
verifyDBSamples() // Before any compaction.
// Verify that the in-memory ooo chunk is not empty.
checkNonEmptyOOOChunk := func(lbls labels.Labels) {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Greater(t, ms.ooo.oooHeadChunk.chunk.NumSamples(), 0)
require.Equal(t, 14, len(ms.ooo.oooMmappedChunks)) // 7 original, 7 duplicate.
}
checkNonEmptyOOOChunk(series1)
checkNonEmptyOOOChunk(series2)
// No blocks before compaction.
require.Equal(t, len(db.Blocks()), 0)
// There is a 0th WBL file.
require.NoError(t, db.head.wbl.Sync()) // syncing to make sure wbl is flushed in windows
files, err := os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 1)
require.Equal(t, "00000000", files[0].Name())
f, err := files[0].Info()
require.NoError(t, err)
require.Greater(t, f.Size(), int64(100))
// OOO compaction happens here.
require.NoError(t, db.CompactOOOHead())
// 3 blocks exist now. [0, 120), [120, 240), [240, 360)
require.Equal(t, len(db.Blocks()), 3)
verifyDBSamples() // Blocks created out of OOO head now.
// 0th WBL file will be deleted and 1st will be the only present.
files, err = os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 1)
require.Equal(t, "00000001", files[0].Name())
f, err = files[0].Info()
require.NoError(t, err)
require.Equal(t, int64(0), f.Size())
// OOO stuff should not be present in the Head now.
checkEmptyOOOChunk(series1)
checkEmptyOOOChunk(series2)
verifySamples := func(block *Block, fromMins, toMins int64) {
series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): series1Samples,
series2.String(): series2Samples,
}
q, err := NewBlockQuerier(block, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
// Checking for expected data in the blocks.
verifySamples(db.Blocks()[0], 90, 119)
verifySamples(db.Blocks()[1], 120, 239)
verifySamples(db.Blocks()[2], 240, 310)
// There should be a single m-map file.
mmapDir := mmappedChunksDir(db.head.opts.ChunkDirRoot)
files, err = os.ReadDir(mmapDir)
require.NoError(t, err)
require.Len(t, files, 1)
// Compact the in-order head and expect another block.
// Since this is a forced compaction, this block is not aligned with 2h.
err = db.CompactHead(NewRangeHead(db.head, 250*time.Minute.Milliseconds(), 350*time.Minute.Milliseconds()))
require.NoError(t, err)
require.Equal(t, len(db.Blocks()), 4) // [0, 120), [120, 240), [240, 360), [250, 351)
verifySamples(db.Blocks()[3], 250, 350)
verifyDBSamples() // Blocks created out of normal and OOO head now. But not merged.
// The compaction also clears out the old m-map files. Including
// the file that has ooo chunks.
files, err = os.ReadDir(mmapDir)
require.NoError(t, err)
require.Len(t, files, 1)
require.Equal(t, "000001", files[0].Name())
// This will merge overlapping block.
require.NoError(t, db.Compact())
require.Equal(t, len(db.Blocks()), 3) // [0, 120), [120, 240), [240, 360)
verifySamples(db.Blocks()[0], 90, 119)
verifySamples(db.Blocks()[1], 120, 239)
verifySamples(db.Blocks()[2], 240, 350) // Merged block.
verifyDBSamples() // Final state. Blocks from normal and OOO head are merged.
}
// TestOOOCompactionWithNormalCompaction tests if OOO compaction is performed
// when the normal head's compaction is done.
func TestOOOCompactionWithNormalCompaction(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions() // We want to manually call it.
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
series2 := labels.FromStrings("foo", "bar2")
addSamples := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
_, err = app.Append(0, series2, ts, float64(2*ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSamples(250, 350)
// Add ooo samples that will result into a single block.
addSamples(90, 110)
// Checking that ooo chunk is not empty.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Greater(t, ms.ooo.oooHeadChunk.chunk.NumSamples(), 0)
}
// If the normal Head is not compacted, the OOO head compaction does not take place.
require.NoError(t, db.Compact())
require.Equal(t, len(db.Blocks()), 0)
// Add more in-order samples in future that would trigger the compaction.
addSamples(400, 450)
// No blocks before compaction.
require.Equal(t, len(db.Blocks()), 0)
// Compacts normal and OOO head.
require.NoError(t, db.Compact())
// 2 blocks exist now. [0, 120), [250, 360)
require.Equal(t, len(db.Blocks()), 2)
require.Equal(t, int64(0), db.Blocks()[0].MinTime())
require.Equal(t, 120*time.Minute.Milliseconds(), db.Blocks()[0].MaxTime())
require.Equal(t, 250*time.Minute.Milliseconds(), db.Blocks()[1].MinTime())
require.Equal(t, 360*time.Minute.Milliseconds(), db.Blocks()[1].MaxTime())
// Checking that ooo chunk is empty.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Nil(t, ms.ooo)
}
verifySamples := func(block *Block, fromMins, toMins int64) {
series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): series1Samples,
series2.String(): series2Samples,
}
q, err := NewBlockQuerier(block, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
// Checking for expected data in the blocks.
verifySamples(db.Blocks()[0], 90, 110)
verifySamples(db.Blocks()[1], 250, 350)
}
// TestOOOCompactionWithDisabledWriteLog tests the scenario where the TSDB is
// configured to not have wal and wbl but its able to compact both the in-order
// and out-of-order head
func TestOOOCompactionWithDisabledWriteLog(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
opts.WALSegmentSize = -1 // disabled WAL and WBL
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions() // We want to manually call it.
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
series2 := labels.FromStrings("foo", "bar2")
addSamples := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
_, err = app.Append(0, series2, ts, float64(2*ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSamples(250, 350)
// Add ooo samples that will result into a single block.
addSamples(90, 110)
// Checking that ooo chunk is not empty.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Greater(t, ms.ooo.oooHeadChunk.chunk.NumSamples(), 0)
}
// If the normal Head is not compacted, the OOO head compaction does not take place.
require.NoError(t, db.Compact())
require.Equal(t, len(db.Blocks()), 0)
// Add more in-order samples in future that would trigger the compaction.
addSamples(400, 450)
// No blocks before compaction.
require.Equal(t, len(db.Blocks()), 0)
// Compacts normal and OOO head.
require.NoError(t, db.Compact())
// 2 blocks exist now. [0, 120), [250, 360)
require.Equal(t, len(db.Blocks()), 2)
require.Equal(t, int64(0), db.Blocks()[0].MinTime())
require.Equal(t, 120*time.Minute.Milliseconds(), db.Blocks()[0].MaxTime())
require.Equal(t, 250*time.Minute.Milliseconds(), db.Blocks()[1].MinTime())
require.Equal(t, 360*time.Minute.Milliseconds(), db.Blocks()[1].MaxTime())
// Checking that ooo chunk is empty.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Nil(t, ms.ooo)
}
verifySamples := func(block *Block, fromMins, toMins int64) {
series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): series1Samples,
series2.String(): series2Samples,
}
q, err := NewBlockQuerier(block, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
// Checking for expected data in the blocks.
verifySamples(db.Blocks()[0], 90, 110)
verifySamples(db.Blocks()[1], 250, 350)
}
// TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL tests the scenario where the WBL goes
// missing after a restart while snapshot was enabled, but the query still returns the right
// data from the mmap chunks.
func TestOOOQueryAfterRestartWithSnapshotAndRemovedWBL(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 10
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
opts.EnableMemorySnapshotOnShutdown = true
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions() // We want to manually call it.
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
series2 := labels.FromStrings("foo", "bar2")
addSamples := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
_, err = app.Append(0, series2, ts, float64(2*ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSamples(250, 350)
// Add ooo samples that will result into a single block.
addSamples(90, 110) // The sample 110 will not be in m-map chunks.
// Checking that there are some ooo m-map chunks.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Equal(t, 2, len(ms.ooo.oooMmappedChunks))
require.NotNil(t, ms.ooo.oooHeadChunk)
}
// Restart DB.
require.NoError(t, db.Close())
// For some reason wbl goes missing.
require.NoError(t, os.RemoveAll(path.Join(dir, "wbl")))
db, err = Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions() // We want to manually call it.
// Check ooo m-map chunks again.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Equal(t, 2, len(ms.ooo.oooMmappedChunks))
require.Equal(t, 109*time.Minute.Milliseconds(), ms.ooo.oooMmappedChunks[1].maxTime)
require.Nil(t, ms.ooo.oooHeadChunk) // Because of missing wbl.
}
verifySamples := func(fromMins, toMins int64) {
series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
series2Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
series2Samples = append(series2Samples, sample{ts, float64(2 * ts), nil, nil})
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): series1Samples,
series2.String(): series2Samples,
}
q, err := db.Querier(context.Background(), fromMins*time.Minute.Milliseconds(), toMins*time.Minute.Milliseconds())
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
// Checking for expected ooo data from mmap chunks.
verifySamples(90, 109)
// Compaction should also work fine.
require.Equal(t, len(db.Blocks()), 0)
require.NoError(t, db.CompactOOOHead())
require.Equal(t, len(db.Blocks()), 1) // One block from OOO data.
require.Equal(t, int64(0), db.Blocks()[0].MinTime())
require.Equal(t, 120*time.Minute.Milliseconds(), db.Blocks()[0].MaxTime())
// Checking that ooo chunk is empty in Head.
for _, lbls := range []labels.Labels{series1, series2} {
ms, created, err := db.head.getOrCreate(lbls.Hash(), lbls)
require.NoError(t, err)
require.False(t, created)
require.Nil(t, ms.ooo)
}
verifySamples(90, 109)
}
func Test_Querier_OOOQuery(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 24 * time.Hour.Milliseconds()
opts.AllowOverlappingCompaction = false
series1 := labels.FromStrings("foo", "bar1")
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []tsdbutil.Sample) ([]tsdbutil.Sample, int) {
app := db.Appender(context.Background())
totalAppended := 0
for min := fromMins; min <= toMins; min += time.Minute.Milliseconds() {
_, err := app.Append(0, series1, min, float64(min))
if min >= queryMinT && min <= queryMaxT {
expSamples = append(expSamples, sample{t: min, f: float64(min)})
}
require.NoError(t, err)
totalAppended++
}
require.NoError(t, app.Commit())
return expSamples, totalAppended
}
tests := []struct {
name string
queryMinT int64
queryMaxT int64
inOrderMinT int64
inOrderMaxT int64
oooMinT int64
oooMaxT int64
}{
{
name: "query interval covering ooomint and inordermaxt returns all ingested samples",
queryMinT: minutes(0),
queryMaxT: minutes(200),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
},
{
name: "partial query interval returns only samples within interval",
queryMinT: minutes(20),
queryMaxT: minutes(180),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
},
}
for _, tc := range tests {
t.Run(fmt.Sprintf("name=%s", tc.name), func(t *testing.T) {
db := openTestDB(t, opts, nil)
db.DisableCompactions()
defer func() {
require.NoError(t, db.Close())
}()
var expSamples []tsdbutil.Sample
// Add in-order samples.
expSamples, _ = addSample(db, tc.inOrderMinT, tc.inOrderMaxT, tc.queryMinT, tc.queryMaxT, expSamples)
// Add out-of-order samples.
expSamples, oooSamples := addSample(db, tc.oooMinT, tc.oooMaxT, tc.queryMinT, tc.queryMaxT, expSamples)
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
})
querier, err := db.Querier(context.TODO(), tc.queryMinT, tc.queryMaxT)
require.NoError(t, err)
defer querier.Close()
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1"))
require.NotNil(t, seriesSet[series1.String()])
require.Equal(t, 1, len(seriesSet))
require.Equal(t, expSamples, seriesSet[series1.String()])
require.GreaterOrEqual(t, float64(oooSamples), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch")
})
}
}
func Test_ChunkQuerier_OOOQuery(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 24 * time.Hour.Milliseconds()
opts.AllowOverlappingCompaction = false
series1 := labels.FromStrings("foo", "bar1")
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
addSample := func(db *DB, fromMins, toMins, queryMinT, queryMaxT int64, expSamples []tsdbutil.Sample) ([]tsdbutil.Sample, int) {
app := db.Appender(context.Background())
totalAppended := 0
for min := fromMins; min <= toMins; min += time.Minute.Milliseconds() {
_, err := app.Append(0, series1, min, float64(min))
if min >= queryMinT && min <= queryMaxT {
expSamples = append(expSamples, sample{t: min, f: float64(min)})
}
require.NoError(t, err)
totalAppended++
}
require.NoError(t, app.Commit())
return expSamples, totalAppended
}
tests := []struct {
name string
queryMinT int64
queryMaxT int64
inOrderMinT int64
inOrderMaxT int64
oooMinT int64
oooMaxT int64
}{
{
name: "query interval covering ooomint and inordermaxt returns all ingested samples",
queryMinT: minutes(0),
queryMaxT: minutes(200),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
},
{
name: "partial query interval returns only samples within interval",
queryMinT: minutes(20),
queryMaxT: minutes(180),
inOrderMinT: minutes(100),
inOrderMaxT: minutes(200),
oooMinT: minutes(0),
oooMaxT: minutes(99),
},
}
for _, tc := range tests {
t.Run(fmt.Sprintf("name=%s", tc.name), func(t *testing.T) {
db := openTestDB(t, opts, nil)
db.DisableCompactions()
defer func() {
require.NoError(t, db.Close())
}()
var expSamples []tsdbutil.Sample
// Add in-order samples.
expSamples, _ = addSample(db, tc.inOrderMinT, tc.inOrderMaxT, tc.queryMinT, tc.queryMaxT, expSamples)
// Add out-of-order samples.
expSamples, oooSamples := addSample(db, tc.oooMinT, tc.oooMaxT, tc.queryMinT, tc.queryMaxT, expSamples)
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
})
querier, err := db.ChunkQuerier(context.TODO(), tc.queryMinT, tc.queryMaxT)
require.NoError(t, err)
defer querier.Close()
chks := queryChunks(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1"))
require.NotNil(t, chks[series1.String()])
require.Equal(t, 1, len(chks))
require.Equal(t, float64(oooSamples), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch")
var gotSamples []tsdbutil.Sample
for _, chunk := range chks[series1.String()] {
it := chunk.Chunk.Iterator(nil)
for it.Next() == chunkenc.ValFloat {
ts, v := it.At()
gotSamples = append(gotSamples, sample{t: ts, f: v})
}
}
require.Equal(t, expSamples, gotSamples)
})
}
}
func TestOOOAppendAndQuery(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 4 * time.Hour.Milliseconds()
db := openTestDB(t, opts, nil)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
s1 := labels.FromStrings("foo", "bar1")
s2 := labels.FromStrings("foo", "bar2")
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
appendedSamples := make(map[string][]tsdbutil.Sample)
totalSamples := 0
addSample := func(lbls labels.Labels, fromMins, toMins int64, faceError bool) {
app := db.Appender(context.Background())
key := lbls.String()
from, to := minutes(fromMins), minutes(toMins)
for min := from; min <= to; min += time.Minute.Milliseconds() {
val := rand.Float64()
_, err := app.Append(0, lbls, min, val)
if faceError {
require.Error(t, err)
} else {
require.NoError(t, err)
appendedSamples[key] = append(appendedSamples[key], sample{t: min, f: val})
totalSamples++
}
}
if faceError {
require.NoError(t, app.Rollback())
} else {
require.NoError(t, app.Commit())
}
}
testQuery := func(from, to int64) {
querier, err := db.Querier(context.TODO(), from, to)
require.NoError(t, err)
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar."))
for k, v := range appendedSamples {
sort.Slice(v, func(i, j int) bool {
return v[i].T() < v[j].T()
})
appendedSamples[k] = v
}
expSamples := make(map[string][]tsdbutil.Sample)
for k, samples := range appendedSamples {
for _, s := range samples {
if s.T() < from {
continue
}
if s.T() > to {
continue
}
expSamples[k] = append(expSamples[k], s)
}
}
require.Equal(t, expSamples, seriesSet)
require.Equal(t, float64(totalSamples-2), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch")
}
verifyOOOMinMaxTimes := func(expMin, expMax int64) {
require.Equal(t, minutes(expMin), db.head.MinOOOTime())
require.Equal(t, minutes(expMax), db.head.MaxOOOTime())
}
// In-order samples.
addSample(s1, 300, 300, false)
addSample(s2, 290, 290, false)
require.Equal(t, float64(2), prom_testutil.ToFloat64(db.head.metrics.chunksCreated))
testQuery(math.MinInt64, math.MaxInt64)
// Some ooo samples.
addSample(s1, 250, 260, false)
addSample(s2, 255, 265, false)
verifyOOOMinMaxTimes(250, 265)
testQuery(math.MinInt64, math.MaxInt64)
testQuery(minutes(250), minutes(265)) // Test querying ono data time range
testQuery(minutes(290), minutes(300)) // Test querying in-order data time range
testQuery(minutes(250), minutes(300)) // Test querying the entire range
// Out of time window.
addSample(s1, 59, 59, true)
addSample(s2, 49, 49, true)
verifyOOOMinMaxTimes(250, 265)
testQuery(math.MinInt64, math.MaxInt64)
// At the edge of time window, also it would be "out of bound" without the ooo support.
addSample(s1, 60, 65, false)
verifyOOOMinMaxTimes(60, 265)
testQuery(math.MinInt64, math.MaxInt64)
// This sample is not within the time window w.r.t. the head's maxt, but it is within the window
// w.r.t. the series' maxt. But we consider only head's maxt.
addSample(s2, 59, 59, true)
verifyOOOMinMaxTimes(60, 265)
testQuery(math.MinInt64, math.MaxInt64)
// Now the sample is within time window w.r.t. the head's maxt.
addSample(s2, 60, 65, false)
verifyOOOMinMaxTimes(60, 265)
testQuery(math.MinInt64, math.MaxInt64)
// Out of time window again.
addSample(s1, 59, 59, true)
addSample(s2, 49, 49, true)
testQuery(math.MinInt64, math.MaxInt64)
// Generating some m-map chunks. The m-map chunks here are in such a way
// that when sorted w.r.t. mint, the last chunk's maxt is not the overall maxt
// of the merged chunk. This tests a bug fixed in https://github.com/grafana/mimir-prometheus/pull/238/.
require.Equal(t, float64(4), prom_testutil.ToFloat64(db.head.metrics.chunksCreated))
addSample(s1, 180, 249, false)
require.Equal(t, float64(6), prom_testutil.ToFloat64(db.head.metrics.chunksCreated))
verifyOOOMinMaxTimes(60, 265)
testQuery(math.MinInt64, math.MaxInt64)
}
func TestOOODisabled(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 0
db := openTestDB(t, opts, nil)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
s1 := labels.FromStrings("foo", "bar1")
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
expSamples := make(map[string][]tsdbutil.Sample)
totalSamples := 0
failedSamples := 0
addSample := func(lbls labels.Labels, fromMins, toMins int64, faceError bool) {
app := db.Appender(context.Background())
key := lbls.String()
from, to := minutes(fromMins), minutes(toMins)
for min := from; min <= to; min += time.Minute.Milliseconds() {
val := rand.Float64()
_, err := app.Append(0, lbls, min, val)
if faceError {
require.Error(t, err)
failedSamples++
} else {
require.NoError(t, err)
expSamples[key] = append(expSamples[key], sample{t: min, f: val})
totalSamples++
}
}
if faceError {
require.NoError(t, app.Rollback())
} else {
require.NoError(t, app.Commit())
}
}
addSample(s1, 300, 300, false) // In-order samples.
addSample(s1, 250, 260, true) // Some ooo samples.
addSample(s1, 59, 59, true) // Out of time window.
addSample(s1, 60, 65, true) // At the edge of time window, also it would be "out of bound" without the ooo support.
addSample(s1, 59, 59, true) // Out of time window again.
addSample(s1, 301, 310, false) // More in-order samples.
querier, err := db.Querier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar."))
require.Equal(t, expSamples, seriesSet)
require.Equal(t, float64(0), prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamplesAppended), "number of ooo appended samples mismatch")
require.Equal(t, float64(failedSamples),
prom_testutil.ToFloat64(db.head.metrics.outOfOrderSamples.WithLabelValues(sampleMetricTypeFloat))+prom_testutil.ToFloat64(db.head.metrics.outOfBoundSamples.WithLabelValues(sampleMetricTypeFloat)),
"number of ooo/oob samples mismatch")
// Verifying that no OOO artifacts were generated.
_, err = os.ReadDir(path.Join(db.Dir(), wlog.WblDirName))
require.True(t, os.IsNotExist(err))
ms, created, err := db.head.getOrCreate(s1.Hash(), s1)
require.NoError(t, err)
require.False(t, created)
require.NotNil(t, ms)
require.Nil(t, ms.ooo)
}
func TestWBLAndMmapReplay(t *testing.T) {
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 4 * time.Hour.Milliseconds()
db := openTestDB(t, opts, nil)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
s1 := labels.FromStrings("foo", "bar1")
minutes := func(m int64) int64 { return m * time.Minute.Milliseconds() }
expSamples := make(map[string][]tsdbutil.Sample)
totalSamples := 0
addSample := func(lbls labels.Labels, fromMins, toMins int64) {
app := db.Appender(context.Background())
key := lbls.String()
from, to := minutes(fromMins), minutes(toMins)
for min := from; min <= to; min += time.Minute.Milliseconds() {
val := rand.Float64()
_, err := app.Append(0, lbls, min, val)
require.NoError(t, err)
expSamples[key] = append(expSamples[key], sample{t: min, f: val})
totalSamples++
}
require.NoError(t, app.Commit())
}
testQuery := func(exp map[string][]tsdbutil.Sample) {
querier, err := db.Querier(context.TODO(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar."))
for k, v := range exp {
sort.Slice(v, func(i, j int) bool {
return v[i].T() < v[j].T()
})
exp[k] = v
}
require.Equal(t, exp, seriesSet)
}
// In-order samples.
addSample(s1, 300, 300)
require.Equal(t, float64(1), prom_testutil.ToFloat64(db.head.metrics.chunksCreated))
// Some ooo samples.
addSample(s1, 250, 260)
addSample(s1, 195, 249) // This creates some m-map chunks.
require.Equal(t, float64(4), prom_testutil.ToFloat64(db.head.metrics.chunksCreated))
testQuery(expSamples)
oooMint, oooMaxt := minutes(195), minutes(260)
// Collect the samples only present in the ooo m-map chunks.
ms, created, err := db.head.getOrCreate(s1.Hash(), s1)
require.False(t, created)
require.NoError(t, err)
var s1MmapSamples []tsdbutil.Sample
for _, mc := range ms.ooo.oooMmappedChunks {
chk, err := db.head.chunkDiskMapper.Chunk(mc.ref)
require.NoError(t, err)
it := chk.Iterator(nil)
for it.Next() == chunkenc.ValFloat {
ts, val := it.At()
s1MmapSamples = append(s1MmapSamples, sample{t: ts, f: val})
}
}
require.Greater(t, len(s1MmapSamples), 0)
require.NoError(t, db.Close())
// Making a copy of original state of WBL and Mmap files to use it later.
mmapDir := mmappedChunksDir(db.head.opts.ChunkDirRoot)
wblDir := db.head.wbl.Dir()
originalWblDir := filepath.Join(t.TempDir(), "original_wbl")
originalMmapDir := filepath.Join(t.TempDir(), "original_mmap")
require.NoError(t, fileutil.CopyDirs(wblDir, originalWblDir))
require.NoError(t, fileutil.CopyDirs(mmapDir, originalMmapDir))
resetWBLToOriginal := func() {
require.NoError(t, os.RemoveAll(wblDir))
require.NoError(t, fileutil.CopyDirs(originalWblDir, wblDir))
}
resetMmapToOriginal := func() {
require.NoError(t, os.RemoveAll(mmapDir))
require.NoError(t, fileutil.CopyDirs(originalMmapDir, mmapDir))
}
t.Run("Restart DB with both WBL and M-map files for ooo data", func(t *testing.T) {
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
require.NoError(t, db.Close())
})
t.Run("Restart DB with only WBL for ooo data", func(t *testing.T) {
require.NoError(t, os.RemoveAll(mmapDir))
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
require.NoError(t, db.Close())
})
t.Run("Restart DB with only M-map files for ooo data", func(t *testing.T) {
require.NoError(t, os.RemoveAll(wblDir))
resetMmapToOriginal()
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
inOrderSample := expSamples[s1.String()][len(expSamples[s1.String()])-1]
testQuery(map[string][]tsdbutil.Sample{
s1.String(): append(s1MmapSamples, inOrderSample),
})
require.NoError(t, db.Close())
})
t.Run("Restart DB with WBL+Mmap while increasing the OOOCapMax", func(t *testing.T) {
resetWBLToOriginal()
resetMmapToOriginal()
opts.OutOfOrderCapMax = 60
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
require.NoError(t, db.Close())
})
t.Run("Restart DB with WBL+Mmap while decreasing the OOOCapMax", func(t *testing.T) {
resetMmapToOriginal() // We need to reset because new duplicate chunks can be written above.
opts.OutOfOrderCapMax = 10
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
require.NoError(t, db.Close())
})
t.Run("Restart DB with WBL+Mmap while having no m-map markers in WBL", func(t *testing.T) {
resetMmapToOriginal() // We neet to reset because new duplicate chunks can be written above.
// Removing m-map markers in WBL by rewriting it.
newWbl, err := wlog.New(log.NewNopLogger(), nil, filepath.Join(t.TempDir(), "new_wbl"), wlog.CompressionNone)
require.NoError(t, err)
sr, err := wlog.NewSegmentsReader(originalWblDir)
require.NoError(t, err)
var dec record.Decoder
r, markers, addedRecs := wlog.NewReader(sr), 0, 0
for r.Next() {
rec := r.Record()
if dec.Type(rec) == record.MmapMarkers {
markers++
continue
}
addedRecs++
require.NoError(t, newWbl.Log(rec))
}
require.Greater(t, markers, 0)
require.Greater(t, addedRecs, 0)
require.NoError(t, newWbl.Close())
require.NoError(t, sr.Close())
require.NoError(t, os.RemoveAll(wblDir))
require.NoError(t, os.Rename(newWbl.Dir(), wblDir))
opts.OutOfOrderCapMax = 30
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, oooMint, db.head.MinOOOTime())
require.Equal(t, oooMaxt, db.head.MaxOOOTime())
testQuery(expSamples)
})
}
func TestOOOCompactionFailure(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions() // We want to manually call it.
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
addSample := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSample(250, 350)
// Add ooo samples that creates multiple chunks.
addSample(90, 310)
// No blocks before compaction.
require.Equal(t, len(db.Blocks()), 0)
// There is a 0th WBL file.
verifyFirstWBLFileIs0 := func(count int) {
require.NoError(t, db.head.wbl.Sync()) // syncing to make sure wbl is flushed in windows
files, err := os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, count)
require.Equal(t, "00000000", files[0].Name())
f, err := files[0].Info()
require.NoError(t, err)
require.Greater(t, f.Size(), int64(100))
}
verifyFirstWBLFileIs0(1)
verifyMmapFiles := func(exp ...string) {
mmapDir := mmappedChunksDir(db.head.opts.ChunkDirRoot)
files, err := os.ReadDir(mmapDir)
require.NoError(t, err)
require.Len(t, files, len(exp))
for i, f := range files {
require.Equal(t, exp[i], f.Name())
}
}
verifyMmapFiles("000001")
// OOO compaction fails 5 times.
originalCompactor := db.compactor
db.compactor = &mockCompactorFailing{t: t}
for i := 0; i < 5; i++ {
require.Error(t, db.CompactOOOHead())
}
require.Equal(t, len(db.Blocks()), 0)
// M-map files don't change after failed compaction.
verifyMmapFiles("000001")
// Because of 5 compaction attempts, there are 6 files now.
verifyFirstWBLFileIs0(6)
db.compactor = originalCompactor
require.NoError(t, db.CompactOOOHead())
oldBlocks := db.Blocks()
require.Equal(t, len(db.Blocks()), 3)
// Check that the ooo chunks were removed.
ms, created, err := db.head.getOrCreate(series1.Hash(), series1)
require.NoError(t, err)
require.False(t, created)
require.Nil(t, ms.ooo)
// The failed compaction should not have left the ooo Head corrupted.
// Hence, expect no new blocks with another OOO compaction call.
require.NoError(t, db.CompactOOOHead())
require.Equal(t, len(db.Blocks()), 3)
require.Equal(t, oldBlocks, db.Blocks())
// There should be a single m-map file
verifyMmapFiles("000001")
// All but last WBL file will be deleted.
// 8 files in total (starting at 0) because of 7 compaction calls.
files, err := os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 1)
require.Equal(t, "00000007", files[0].Name())
f, err := files[0].Info()
require.NoError(t, err)
require.Equal(t, int64(0), f.Size())
verifySamples := func(block *Block, fromMins, toMins int64) {
series1Samples := make([]tsdbutil.Sample, 0, toMins-fromMins+1)
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
series1Samples = append(series1Samples, sample{ts, float64(ts), nil, nil})
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): series1Samples,
}
q, err := NewBlockQuerier(block, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
// Checking for expected data in the blocks.
verifySamples(db.Blocks()[0], 90, 119)
verifySamples(db.Blocks()[1], 120, 239)
verifySamples(db.Blocks()[2], 240, 310)
// Compact the in-order head and expect another block.
// Since this is a forced compaction, this block is not aligned with 2h.
err = db.CompactHead(NewRangeHead(db.head, 250*time.Minute.Milliseconds(), 350*time.Minute.Milliseconds()))
require.NoError(t, err)
require.Equal(t, len(db.Blocks()), 4) // [0, 120), [120, 240), [240, 360), [250, 351)
verifySamples(db.Blocks()[3], 250, 350)
// The compaction also clears out the old m-map files. Including
// the file that has ooo chunks.
verifyMmapFiles("000001")
}
func TestWBLCorruption(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 30
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
var allSamples, expAfterRestart []tsdbutil.Sample
addSamples := func(fromMins, toMins int64, afterRestart bool) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
if afterRestart {
expAfterRestart = append(expAfterRestart, sample{t: ts, f: float64(ts)})
}
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSamples(340, 350, true)
// OOO samples.
addSamples(90, 99, true)
addSamples(100, 119, true)
addSamples(120, 130, true)
// Moving onto the second file.
_, err = db.head.wbl.NextSegment()
require.NoError(t, err)
// More OOO samples.
addSamples(200, 230, true)
addSamples(240, 255, true)
// We corrupt WBL after the sample at 255. So everything added later
// should be deleted after replay.
// Checking where we corrupt it.
require.NoError(t, db.head.wbl.Sync()) // syncing to make sure wbl is flushed in windows
files, err := os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 2)
f1, err := files[1].Info()
require.NoError(t, err)
corruptIndex := f1.Size()
corruptFilePath := path.Join(db.head.wbl.Dir(), files[1].Name())
// Corrupt the WBL by adding a malformed record.
require.NoError(t, db.head.wbl.Log([]byte{byte(record.Samples), 99, 9, 99, 9, 99, 9, 99}))
// More samples after the corruption point.
addSamples(260, 280, false)
addSamples(290, 300, false)
// Another file.
_, err = db.head.wbl.NextSegment()
require.NoError(t, err)
addSamples(310, 320, false)
// Verifying that we have data after corruption point.
require.NoError(t, db.head.wbl.Sync()) // syncing to make sure wbl is flushed in windows
files, err = os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 3)
f1, err = files[1].Info()
require.NoError(t, err)
require.Greater(t, f1.Size(), corruptIndex)
f0, err := files[0].Info()
require.NoError(t, err)
require.Greater(t, f0.Size(), int64(100))
f2, err := files[2].Info()
require.NoError(t, err)
require.Greater(t, f2.Size(), int64(100))
verifySamples := func(expSamples []tsdbutil.Sample) {
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
})
expRes := map[string][]tsdbutil.Sample{
series1.String(): expSamples,
}
q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
verifySamples(allSamples)
require.NoError(t, db.Close())
// We want everything to be replayed from the WBL. So we delete the m-map files.
require.NoError(t, os.RemoveAll(mmappedChunksDir(db.head.opts.ChunkDirRoot)))
// Restart does the replay and repair.
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.walCorruptionsTotal))
require.Less(t, len(expAfterRestart), len(allSamples))
verifySamples(expAfterRestart)
// Verify that it did the repair on disk.
files, err = os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 3)
f0, err = files[0].Info()
require.NoError(t, err)
require.Greater(t, f0.Size(), int64(100))
f2, err = files[2].Info()
require.NoError(t, err)
require.Equal(t, int64(0), f2.Size())
require.Equal(t, corruptFilePath, path.Join(db.head.wbl.Dir(), files[1].Name()))
// Verifying that everything after the corruption point is set to 0.
b, err := os.ReadFile(corruptFilePath)
require.NoError(t, err)
sum := 0
for _, val := range b[corruptIndex:] {
sum += int(val)
}
require.Equal(t, 0, sum)
// Another restart, everything normal with no repair.
require.NoError(t, db.Close())
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.walCorruptionsTotal))
verifySamples(expAfterRestart)
}
func TestOOOMmapCorruption(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderCapMax = 10
opts.OutOfOrderTimeWindow = 300 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
var allSamples, expInMmapChunks []tsdbutil.Sample
addSamples := func(fromMins, toMins int64, inMmapAfterCorruption bool) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
if inMmapAfterCorruption {
expInMmapChunks = append(expInMmapChunks, sample{t: ts, f: float64(ts)})
}
}
require.NoError(t, app.Commit())
}
// Add an in-order samples.
addSamples(340, 350, true)
// OOO samples.
addSamples(90, 99, true)
addSamples(100, 109, true)
// This sample m-maps a chunk. But 120 goes into a new chunk.
addSamples(120, 120, false)
// Second m-map file. We will corrupt this file. Sample 120 goes into this new file.
db.head.chunkDiskMapper.CutNewFile()
// More OOO samples.
addSamples(200, 230, false)
addSamples(240, 255, false)
db.head.chunkDiskMapper.CutNewFile()
addSamples(260, 290, false)
verifySamples := func(expSamples []tsdbutil.Sample) {
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
})
expRes := map[string][]tsdbutil.Sample{
series1.String(): expSamples,
}
q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
verifySamples(allSamples)
// Verifying existing files.
mmapDir := mmappedChunksDir(db.head.opts.ChunkDirRoot)
files, err := os.ReadDir(mmapDir)
require.NoError(t, err)
require.Len(t, files, 3)
// Corrupting the 2nd file.
f, err := os.OpenFile(path.Join(mmapDir, files[1].Name()), os.O_RDWR, 0o666)
require.NoError(t, err)
_, err = f.WriteAt([]byte{99, 9, 99, 9, 99}, 20)
require.NoError(t, err)
require.NoError(t, f.Close())
firstFileName := files[0].Name()
require.NoError(t, db.Close())
// Moving OOO WBL to use it later.
wblDir := db.head.wbl.Dir()
wblDirTmp := path.Join(t.TempDir(), "wbl_tmp")
require.NoError(t, os.Rename(wblDir, wblDirTmp))
// Restart does the replay and repair of m-map files.
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, 1.0, prom_testutil.ToFloat64(db.head.metrics.mmapChunkCorruptionTotal))
require.Less(t, len(expInMmapChunks), len(allSamples))
// Since there is no WBL, only samples from m-map chunks comes in the query.
verifySamples(expInMmapChunks)
// Verify that it did the repair on disk. All files from the point of corruption
// should be deleted.
files, err = os.ReadDir(mmapDir)
require.NoError(t, err)
require.Len(t, files, 1)
f0, err := files[0].Info()
require.NoError(t, err)
require.Greater(t, f0.Size(), int64(100))
require.Equal(t, firstFileName, files[0].Name())
// Another restart, everything normal with no repair.
require.NoError(t, db.Close())
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
require.Equal(t, 0.0, prom_testutil.ToFloat64(db.head.metrics.mmapChunkCorruptionTotal))
verifySamples(expInMmapChunks)
// Restart again with the WBL, all samples should be present now.
require.NoError(t, db.Close())
require.NoError(t, os.RemoveAll(wblDir))
require.NoError(t, os.Rename(wblDirTmp, wblDir))
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
verifySamples(allSamples)
}
func TestOutOfOrderRuntimeConfig(t *testing.T) {
getDB := func(oooTimeWindow int64) *DB {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = oooTimeWindow
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
return db
}
makeConfig := func(oooTimeWindow int) *config.Config {
return &config.Config{
StorageConfig: config.StorageConfig{
TSDBConfig: &config.TSDBConfig{
OutOfOrderTimeWindow: int64(oooTimeWindow) * time.Minute.Milliseconds(),
},
},
}
}
series1 := labels.FromStrings("foo", "bar1")
addSamples := func(t *testing.T, db *DB, fromMins, toMins int64, success bool, allSamples []tsdbutil.Sample) []tsdbutil.Sample {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
if success {
require.NoError(t, err)
allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
} else {
require.Error(t, err)
}
}
require.NoError(t, app.Commit())
return allSamples
}
verifySamples := func(t *testing.T, db *DB, expSamples []tsdbutil.Sample) {
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
})
expRes := map[string][]tsdbutil.Sample{
series1.String(): expSamples,
}
q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
doOOOCompaction := func(t *testing.T, db *DB) {
// WBL is not empty.
size, err := db.head.wbl.Size()
require.NoError(t, err)
require.Greater(t, size, int64(0))
require.Len(t, db.Blocks(), 0)
require.NoError(t, db.compactOOOHead())
require.Greater(t, len(db.Blocks()), 0)
// WBL is empty.
size, err = db.head.wbl.Size()
require.NoError(t, err)
require.Equal(t, int64(0), size)
}
t.Run("increase time window", func(t *testing.T) {
var allSamples []tsdbutil.Sample
db := getDB(30 * time.Minute.Milliseconds())
// In-order.
allSamples = addSamples(t, db, 300, 310, true, allSamples)
// OOO upto 30m old is success.
allSamples = addSamples(t, db, 281, 290, true, allSamples)
// OOO of 59m old fails.
s := addSamples(t, db, 251, 260, false, nil)
require.Len(t, s, 0)
verifySamples(t, db, allSamples)
oldWblPtr := fmt.Sprintf("%p", db.head.wbl)
// Increase time window and try adding again.
err := db.ApplyConfig(makeConfig(60))
require.NoError(t, err)
allSamples = addSamples(t, db, 251, 260, true, allSamples)
// WBL does not change.
newWblPtr := fmt.Sprintf("%p", db.head.wbl)
require.Equal(t, oldWblPtr, newWblPtr)
doOOOCompaction(t, db)
verifySamples(t, db, allSamples)
})
t.Run("decrease time window and increase again", func(t *testing.T) {
var allSamples []tsdbutil.Sample
db := getDB(60 * time.Minute.Milliseconds())
// In-order.
allSamples = addSamples(t, db, 300, 310, true, allSamples)
// OOO upto 59m old is success.
allSamples = addSamples(t, db, 251, 260, true, allSamples)
oldWblPtr := fmt.Sprintf("%p", db.head.wbl)
// Decrease time window.
err := db.ApplyConfig(makeConfig(30))
require.NoError(t, err)
// OOO of 49m old fails.
s := addSamples(t, db, 261, 270, false, nil)
require.Len(t, s, 0)
// WBL does not change.
newWblPtr := fmt.Sprintf("%p", db.head.wbl)
require.Equal(t, oldWblPtr, newWblPtr)
verifySamples(t, db, allSamples)
// Increase time window again and check
err = db.ApplyConfig(makeConfig(60))
require.NoError(t, err)
allSamples = addSamples(t, db, 261, 270, true, allSamples)
verifySamples(t, db, allSamples)
// WBL does not change.
newWblPtr = fmt.Sprintf("%p", db.head.wbl)
require.Equal(t, oldWblPtr, newWblPtr)
doOOOCompaction(t, db)
verifySamples(t, db, allSamples)
})
t.Run("disabled to enabled", func(t *testing.T) {
var allSamples []tsdbutil.Sample
db := getDB(0)
// In-order.
allSamples = addSamples(t, db, 300, 310, true, allSamples)
// OOO fails.
s := addSamples(t, db, 251, 260, false, nil)
require.Len(t, s, 0)
verifySamples(t, db, allSamples)
require.Nil(t, db.head.wbl)
// Increase time window and try adding again.
err := db.ApplyConfig(makeConfig(60))
require.NoError(t, err)
allSamples = addSamples(t, db, 251, 260, true, allSamples)
// WBL gets created.
require.NotNil(t, db.head.wbl)
verifySamples(t, db, allSamples)
// OOO compaction works now.
doOOOCompaction(t, db)
verifySamples(t, db, allSamples)
})
t.Run("enabled to disabled", func(t *testing.T) {
var allSamples []tsdbutil.Sample
db := getDB(60 * time.Minute.Milliseconds())
// In-order.
allSamples = addSamples(t, db, 300, 310, true, allSamples)
// OOO upto 59m old is success.
allSamples = addSamples(t, db, 251, 260, true, allSamples)
oldWblPtr := fmt.Sprintf("%p", db.head.wbl)
// Time Window to 0, hence disabled.
err := db.ApplyConfig(makeConfig(0))
require.NoError(t, err)
// OOO within old time window fails.
s := addSamples(t, db, 290, 309, false, nil)
require.Len(t, s, 0)
// WBL does not change and is not removed.
newWblPtr := fmt.Sprintf("%p", db.head.wbl)
require.Equal(t, oldWblPtr, newWblPtr)
verifySamples(t, db, allSamples)
// Compaction still works after disabling with WBL cleanup.
doOOOCompaction(t, db)
verifySamples(t, db, allSamples)
})
t.Run("disabled to disabled", func(t *testing.T) {
var allSamples []tsdbutil.Sample
db := getDB(0)
// In-order.
allSamples = addSamples(t, db, 300, 310, true, allSamples)
// OOO fails.
s := addSamples(t, db, 290, 309, false, nil)
require.Len(t, s, 0)
verifySamples(t, db, allSamples)
require.Nil(t, db.head.wbl)
// Time window to 0.
err := db.ApplyConfig(makeConfig(0))
require.NoError(t, err)
// OOO still fails.
s = addSamples(t, db, 290, 309, false, nil)
require.Len(t, s, 0)
verifySamples(t, db, allSamples)
require.Nil(t, db.head.wbl)
})
}
func TestNoGapAfterRestartWithOOO(t *testing.T) {
series1 := labels.FromStrings("foo", "bar1")
addSamples := func(t *testing.T, db *DB, fromMins, toMins int64, success bool) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
if success {
require.NoError(t, err)
} else {
require.Error(t, err)
}
}
require.NoError(t, app.Commit())
}
verifySamples := func(t *testing.T, db *DB, fromMins, toMins int64) {
var expSamples []tsdbutil.Sample
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
expSamples = append(expSamples, sample{t: ts, f: float64(ts)})
}
expRes := map[string][]tsdbutil.Sample{
series1.String(): expSamples,
}
q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
cases := []struct {
inOrderMint, inOrderMaxt int64
oooMint, oooMaxt int64
// After compaction.
blockRanges [][2]int64
headMint, headMaxt int64
}{
{
300, 490,
489, 489,
[][2]int64{{300, 360}, {480, 600}},
360, 490,
},
{
300, 490,
479, 479,
[][2]int64{{300, 360}, {360, 480}},
360, 490,
},
}
for i, c := range cases {
t.Run(fmt.Sprintf("case=%d", i), func(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 30 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
// 3h10m=190m worth in-order data.
addSamples(t, db, c.inOrderMint, c.inOrderMaxt, true)
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
// One ooo samples.
addSamples(t, db, c.oooMint, c.oooMaxt, true)
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
// We get 2 blocks. 1 from OOO, 1 from in-order.
require.NoError(t, db.Compact())
verifyBlockRanges := func() {
blocks := db.Blocks()
require.Equal(t, len(c.blockRanges), len(blocks))
for j, br := range c.blockRanges {
require.Equal(t, br[0]*time.Minute.Milliseconds(), blocks[j].MinTime())
require.Equal(t, br[1]*time.Minute.Milliseconds(), blocks[j].MaxTime())
}
}
verifyBlockRanges()
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
// Restart and expect all samples to be present.
require.NoError(t, db.Close())
db, err = Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
verifyBlockRanges()
require.Equal(t, c.headMint*time.Minute.Milliseconds(), db.head.MinTime())
require.Equal(t, c.headMaxt*time.Minute.Milliseconds(), db.head.MaxTime())
verifySamples(t, db, c.inOrderMint, c.inOrderMaxt)
})
}
}
func TestWblReplayAfterOOODisableAndRestart(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 60 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
var allSamples []tsdbutil.Sample
addSamples := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
}
require.NoError(t, app.Commit())
}
// In-order samples.
addSamples(290, 300)
// OOO samples.
addSamples(250, 260)
verifySamples := func(expSamples []tsdbutil.Sample) {
sort.Slice(expSamples, func(i, j int) bool {
return expSamples[i].T() < expSamples[j].T()
})
expRes := map[string][]tsdbutil.Sample{
series1.String(): expSamples,
}
q, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
require.Equal(t, expRes, actRes)
}
verifySamples(allSamples)
// Restart DB with OOO disabled.
require.NoError(t, db.Close())
opts.OutOfOrderTimeWindow = 0
db, err = Open(db.dir, nil, nil, opts, nil)
require.NoError(t, err)
// We can still query OOO samples when OOO is disabled.
verifySamples(allSamples)
}
func TestPanicOnApplyConfig(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 60 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
var allSamples []tsdbutil.Sample
addSamples := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
}
require.NoError(t, app.Commit())
}
// In-order samples.
addSamples(290, 300)
// OOO samples.
addSamples(250, 260)
// Restart DB with OOO disabled.
require.NoError(t, db.Close())
opts.OutOfOrderTimeWindow = 0
db, err = Open(db.dir, nil, prometheus.NewRegistry(), opts, nil)
require.NoError(t, err)
// ApplyConfig with OOO enabled and expect no panic.
err = db.ApplyConfig(&config.Config{
StorageConfig: config.StorageConfig{
TSDBConfig: &config.TSDBConfig{
OutOfOrderTimeWindow: 60 * time.Minute.Milliseconds(),
},
},
})
require.NoError(t, err)
}
func TestDiskFillingUpAfterDisablingOOO(t *testing.T) {
dir := t.TempDir()
opts := DefaultOptions()
opts.OutOfOrderTimeWindow = 60 * time.Minute.Milliseconds()
db, err := Open(dir, nil, nil, opts, nil)
require.NoError(t, err)
db.DisableCompactions()
t.Cleanup(func() {
require.NoError(t, db.Close())
})
series1 := labels.FromStrings("foo", "bar1")
var allSamples []tsdbutil.Sample
addSamples := func(fromMins, toMins int64) {
app := db.Appender(context.Background())
for min := fromMins; min <= toMins; min++ {
ts := min * time.Minute.Milliseconds()
_, err := app.Append(0, series1, ts, float64(ts))
require.NoError(t, err)
allSamples = append(allSamples, sample{t: ts, f: float64(ts)})
}
require.NoError(t, app.Commit())
}
// In-order samples.
addSamples(290, 300)
// OOO samples.
addSamples(250, 299)
// Restart DB with OOO disabled.
require.NoError(t, db.Close())
opts.OutOfOrderTimeWindow = 0
db, err = Open(db.dir, nil, prometheus.NewRegistry(), opts, nil)
require.NoError(t, err)
db.DisableCompactions()
ms := db.head.series.getByHash(series1.Hash(), series1)
require.Greater(t, len(ms.ooo.oooMmappedChunks), 0, "OOO mmap chunk was not replayed")
checkMmapFileContents := func(contains, notContains []string) {
mmapDir := mmappedChunksDir(db.head.opts.ChunkDirRoot)
files, err := os.ReadDir(mmapDir)
require.NoError(t, err)
fnames := make([]string, 0, len(files))
for _, f := range files {
fnames = append(fnames, f.Name())
}
for _, f := range contains {
require.Contains(t, fnames, f)
}
for _, f := range notContains {
require.NotContains(t, fnames, f)
}
}
// Add in-order samples until ready for compaction..
addSamples(301, 500)
// Check that m-map files gets deleted properly after compactions.
db.head.mmapHeadChunks()
checkMmapFileContents([]string{"000001", "000002"}, nil)
require.NoError(t, db.Compact())
checkMmapFileContents([]string{"000002"}, []string{"000001"})
require.Nil(t, ms.ooo, "OOO mmap chunk was not compacted")
addSamples(501, 650)
db.head.mmapHeadChunks()
checkMmapFileContents([]string{"000002", "000003"}, []string{"000001"})
require.NoError(t, db.Compact())
checkMmapFileContents(nil, []string{"000001", "000002", "000003"})
// Verify that WBL is empty.
files, err := os.ReadDir(db.head.wbl.Dir())
require.NoError(t, err)
require.Len(t, files, 1) // Last empty file after compaction.
finfo, err := files[0].Info()
require.NoError(t, err)
require.Equal(t, int64(0), finfo.Size())
}
func TestHistogramAppendAndQuery(t *testing.T) {
t.Run("integer histograms", func(t *testing.T) {
testHistogramAppendAndQueryHelper(t, false)
})
t.Run("float histograms", func(t *testing.T) {
testHistogramAppendAndQueryHelper(t, true)
})
}
func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
t.Helper()
db := openTestDB(t, nil, nil)
minute := func(m int) int64 { return int64(m) * time.Minute.Milliseconds() }
t.Cleanup(func() {
require.NoError(t, db.Close())
})
ctx := context.Background()
appendHistogram := func(
lbls labels.Labels, tsMinute int, h *histogram.Histogram,
exp *[]tsdbutil.Sample, expCRH histogram.CounterResetHint,
) {
t.Helper()
var err error
app := db.Appender(ctx)
if floatHistogram {
_, err = app.AppendHistogram(0, lbls, minute(tsMinute), nil, h.ToFloat())
efh := h.ToFloat()
efh.CounterResetHint = expCRH
*exp = append(*exp, sample{t: minute(tsMinute), fh: efh})
} else {
_, err = app.AppendHistogram(0, lbls, minute(tsMinute), h.Copy(), nil)
eh := h.Copy()
eh.CounterResetHint = expCRH
*exp = append(*exp, sample{t: minute(tsMinute), h: eh})
}
require.NoError(t, err)
require.NoError(t, app.Commit())
}
appendFloat := func(lbls labels.Labels, tsMinute int, val float64, exp *[]tsdbutil.Sample) {
t.Helper()
app := db.Appender(ctx)
_, err := app.Append(0, lbls, minute(tsMinute), val)
require.NoError(t, err)
require.NoError(t, app.Commit())
*exp = append(*exp, sample{t: minute(tsMinute), f: val})
}
testQuery := func(name, value string, exp map[string][]tsdbutil.Sample) {
t.Helper()
q, err := db.Querier(ctx, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
act := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, name, value))
require.Equal(t, exp, act)
}
baseH := &histogram.Histogram{
Count: 11,
ZeroCount: 4,
ZeroThreshold: 0.001,
Sum: 35.5,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
NegativeSpans: []histogram.Span{
{Offset: 0, Length: 1},
{Offset: 1, Length: 2},
},
NegativeBuckets: []int64{1, 2, -1},
}
var (
series1 = labels.FromStrings("foo", "bar1")
series2 = labels.FromStrings("foo", "bar2")
series3 = labels.FromStrings("foo", "bar3")
series4 = labels.FromStrings("foo", "bar4")
exp1, exp2, exp3, exp4 []tsdbutil.Sample
)
// TODO(codesome): test everything for negative buckets as well.
t.Run("series with only histograms", func(t *testing.T) {
h := baseH.Copy() // This is shared across all sub tests.
appendHistogram(series1, 100, h, &exp1, histogram.UnknownCounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
h.PositiveBuckets[0]++
h.NegativeBuckets[0] += 2
h.Count += 10
appendHistogram(series1, 101, h, &exp1, histogram.NotCounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
t.Run("changing schema", func(t *testing.T) {
h.Schema = 2
appendHistogram(series1, 102, h, &exp1, histogram.UnknownCounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
// Schema back to old.
h.Schema = 1
appendHistogram(series1, 103, h, &exp1, histogram.UnknownCounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
})
t.Run("new buckets incoming", func(t *testing.T) {
// In the previous unit test, during the last histogram append, we
// changed the schema and that caused a new chunk creation. Because
// of the next append the layout of the last histogram will change
// because the chunk will be re-encoded. So this forces us to modify
// the last histogram in exp1 so when we query we get the expected
// results.
if floatHistogram {
lh := exp1[len(exp1)-1].FH().Copy()
lh.PositiveSpans[1].Length++
lh.PositiveBuckets = append(lh.PositiveBuckets, 0)
exp1[len(exp1)-1] = sample{t: exp1[len(exp1)-1].T(), fh: lh}
} else {
lh := exp1[len(exp1)-1].H().Copy()
lh.PositiveSpans[1].Length++
lh.PositiveBuckets = append(lh.PositiveBuckets, -2) // -2 makes the last bucket 0.
exp1[len(exp1)-1] = sample{t: exp1[len(exp1)-1].T(), h: lh}
}
// This histogram with new bucket at the end causes the re-encoding of the previous histogram.
// Hence the previous histogram is recoded into this new layout.
// But the query returns the histogram from the in-memory buffer, hence we don't see the recode here yet.
h.PositiveSpans[1].Length++
h.PositiveBuckets = append(h.PositiveBuckets, 1)
h.Count += 3
appendHistogram(series1, 104, h, &exp1, histogram.NotCounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
// Because of the previous two histograms being on the active chunk,
// and the next append is only adding a new bucket, the active chunk
// will be re-encoded to the new layout.
if floatHistogram {
lh := exp1[len(exp1)-2].FH().Copy()
lh.PositiveSpans[0].Length++
lh.PositiveSpans[1].Offset--
lh.PositiveBuckets = []float64{2, 3, 0, 2, 2, 0}
exp1[len(exp1)-2] = sample{t: exp1[len(exp1)-2].T(), fh: lh}
lh = exp1[len(exp1)-1].FH().Copy()
lh.PositiveSpans[0].Length++
lh.PositiveSpans[1].Offset--
lh.PositiveBuckets = []float64{2, 3, 0, 2, 2, 3}
exp1[len(exp1)-1] = sample{t: exp1[len(exp1)-1].T(), fh: lh}
} else {
lh := exp1[len(exp1)-2].H().Copy()
lh.PositiveSpans[0].Length++
lh.PositiveSpans[1].Offset--
lh.PositiveBuckets = []int64{2, 1, -3, 2, 0, -2}
exp1[len(exp1)-2] = sample{t: exp1[len(exp1)-2].T(), h: lh}
lh = exp1[len(exp1)-1].H().Copy()
lh.PositiveSpans[0].Length++
lh.PositiveSpans[1].Offset--
lh.PositiveBuckets = []int64{2, 1, -3, 2, 0, 1}
exp1[len(exp1)-1] = sample{t: exp1[len(exp1)-1].T(), h: lh}
}
// Now we add the new buckets in between. Empty bucket is again not present for the old histogram.
h.PositiveSpans[0].Length++
h.PositiveSpans[1].Offset--
h.Count += 3
// {2, 1, -1, 0, 1} -> {2, 1, 0, -1, 0, 1}
h.PositiveBuckets = append(h.PositiveBuckets[:2], append([]int64{0}, h.PositiveBuckets[2:]...)...)
appendHistogram(series1, 105, h, &exp1, histogram.NotCounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
// We add 4 more histograms to clear out the buffer and see the re-encoded histograms.
appendHistogram(series1, 106, h, &exp1, histogram.NotCounterReset)
appendHistogram(series1, 107, h, &exp1, histogram.NotCounterReset)
appendHistogram(series1, 108, h, &exp1, histogram.NotCounterReset)
appendHistogram(series1, 109, h, &exp1, histogram.NotCounterReset)
// Update the expected histograms to reflect the re-encoding.
if floatHistogram {
l := len(exp1)
h7 := exp1[l-7].FH()
h7.PositiveSpans = exp1[l-1].FH().PositiveSpans
h7.PositiveBuckets = []float64{2, 3, 0, 2, 2, 0}
exp1[l-7] = sample{t: exp1[l-7].T(), fh: h7}
h6 := exp1[l-6].FH()
h6.PositiveSpans = exp1[l-1].FH().PositiveSpans
h6.PositiveBuckets = []float64{2, 3, 0, 2, 2, 3}
exp1[l-6] = sample{t: exp1[l-6].T(), fh: h6}
} else {
l := len(exp1)
h7 := exp1[l-7].H()
h7.PositiveSpans = exp1[l-1].H().PositiveSpans
h7.PositiveBuckets = []int64{2, 1, -3, 2, 0, -2} // -3 and -2 are the empty buckets.
exp1[l-7] = sample{t: exp1[l-7].T(), h: h7}
h6 := exp1[l-6].H()
h6.PositiveSpans = exp1[l-1].H().PositiveSpans
h6.PositiveBuckets = []int64{2, 1, -3, 2, 0, 1} // -3 is the empty bucket.
exp1[l-6] = sample{t: exp1[l-6].T(), h: h6}
}
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
})
t.Run("buckets disappearing", func(t *testing.T) {
h.PositiveSpans[1].Length--
h.PositiveBuckets = h.PositiveBuckets[:len(h.PositiveBuckets)-1]
appendHistogram(series1, 110, h, &exp1, histogram.CounterReset)
testQuery("foo", "bar1", map[string][]tsdbutil.Sample{series1.String(): exp1})
})
})
t.Run("series starting with float and then getting histograms", func(t *testing.T) {
appendFloat(series2, 100, 100, &exp2)
appendFloat(series2, 101, 101, &exp2)
appendFloat(series2, 102, 102, &exp2)
testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2})
h := baseH.Copy()
appendHistogram(series2, 103, h, &exp2, histogram.UnknownCounterReset)
appendHistogram(series2, 104, h, &exp2, histogram.NotCounterReset)
appendHistogram(series2, 105, h, &exp2, histogram.NotCounterReset)
testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2})
// Switching between float and histograms again.
appendFloat(series2, 106, 106, &exp2)
appendFloat(series2, 107, 107, &exp2)
testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2})
appendHistogram(series2, 108, h, &exp2, histogram.UnknownCounterReset)
appendHistogram(series2, 109, h, &exp2, histogram.NotCounterReset)
testQuery("foo", "bar2", map[string][]tsdbutil.Sample{series2.String(): exp2})
})
t.Run("series starting with histogram and then getting float", func(t *testing.T) {
h := baseH.Copy()
appendHistogram(series3, 101, h, &exp3, histogram.UnknownCounterReset)
appendHistogram(series3, 102, h, &exp3, histogram.NotCounterReset)
appendHistogram(series3, 103, h, &exp3, histogram.NotCounterReset)
testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3})
appendFloat(series3, 104, 100, &exp3)
appendFloat(series3, 105, 101, &exp3)
appendFloat(series3, 106, 102, &exp3)
testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3})
// Switching between histogram and float again.
appendHistogram(series3, 107, h, &exp3, histogram.UnknownCounterReset)
appendHistogram(series3, 108, h, &exp3, histogram.NotCounterReset)
testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3})
appendFloat(series3, 109, 106, &exp3)
appendFloat(series3, 110, 107, &exp3)
testQuery("foo", "bar3", map[string][]tsdbutil.Sample{series3.String(): exp3})
})
t.Run("query mix of histogram and float series", func(t *testing.T) {
// A float only series.
appendFloat(series4, 100, 100, &exp4)
appendFloat(series4, 101, 101, &exp4)
appendFloat(series4, 102, 102, &exp4)
testQuery("foo", "bar.*", map[string][]tsdbutil.Sample{
series1.String(): exp1,
series2.String(): exp2,
series3.String(): exp3,
series4.String(): exp4,
})
})
}
func TestQueryHistogramFromBlocksWithCompaction(t *testing.T) {
minute := func(m int) int64 { return int64(m) * time.Minute.Milliseconds() }
testBlockQuerying := func(t *testing.T, blockSeries ...[]storage.Series) {
t.Helper()
opts := DefaultOptions()
opts.AllowOverlappingCompaction = true // TODO(jesusvazquez): This replaced AllowOverlappingBlocks, make sure that works.
db := openTestDB(t, opts, nil)
t.Cleanup(func() {
require.NoError(t, db.Close())
})
ctx := context.Background()
var it chunkenc.Iterator
exp := make(map[string][]tsdbutil.Sample)
for _, series := range blockSeries {
createBlock(t, db.Dir(), series)
for _, s := range series {
key := s.Labels().String()
it = s.Iterator(it)
slice := exp[key]
for typ := it.Next(); typ != chunkenc.ValNone; typ = it.Next() {
switch typ {
case chunkenc.ValFloat:
ts, v := it.At()
slice = append(slice, sample{t: ts, f: v})
case chunkenc.ValHistogram:
ts, h := it.AtHistogram()
slice = append(slice, sample{t: ts, h: h})
case chunkenc.ValFloatHistogram:
ts, h := it.AtFloatHistogram()
slice = append(slice, sample{t: ts, fh: h})
default:
t.Fatalf("unexpected sample value type %d", typ)
}
}
sort.Slice(slice, func(i, j int) bool {
return slice[i].T() < slice[j].T()
})
exp[key] = slice
}
}
require.Len(t, db.Blocks(), 0)
require.NoError(t, db.reload())
require.Len(t, db.Blocks(), len(blockSeries))
q, err := db.Querier(ctx, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
res := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
compareSeries(t, exp, res)
// Compact all the blocks together and query again.
blocks := db.Blocks()
blockDirs := make([]string, 0, len(blocks))
for _, b := range blocks {
blockDirs = append(blockDirs, b.Dir())
}
id, err := db.compactor.Compact(db.Dir(), blockDirs, blocks)
require.NoError(t, err)
require.NotEqual(t, ulid.ULID{}, id)
require.NoError(t, db.reload())
require.Len(t, db.Blocks(), 1)
q, err = db.Querier(ctx, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
res = query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
// After compaction, we do not require "unknown" counter resets
// due to origin from different overlapping chunks anymore.
for _, ss := range exp {
for i, s := range ss[1:] {
if s.H() != nil && ss[i].H() != nil && s.H().CounterResetHint == histogram.UnknownCounterReset {
s.H().CounterResetHint = histogram.NotCounterReset
}
if s.FH() != nil && ss[i].FH() != nil && s.FH().CounterResetHint == histogram.UnknownCounterReset {
s.FH().CounterResetHint = histogram.NotCounterReset
}
}
}
compareSeries(t, exp, res)
}
for _, floatHistogram := range []bool{false, true} {
t.Run(fmt.Sprintf("floatHistogram=%t", floatHistogram), func(t *testing.T) {
t.Run("serial blocks with only histograms", func(t *testing.T) {
testBlockQuerying(t,
genHistogramSeries(10, 5, minute(0), minute(119), minute(1), floatHistogram),
genHistogramSeries(10, 5, minute(120), minute(239), minute(1), floatHistogram),
genHistogramSeries(10, 5, minute(240), minute(359), minute(1), floatHistogram),
)
})
t.Run("serial blocks with either histograms or floats in a block and not both", func(t *testing.T) {
testBlockQuerying(t,
genHistogramSeries(10, 5, minute(0), minute(119), minute(1), floatHistogram),
genSeriesFromSampleGenerator(10, 5, minute(120), minute(239), minute(1), func(ts int64) tsdbutil.Sample {
return sample{t: ts, f: rand.Float64()}
}),
genHistogramSeries(10, 5, minute(240), minute(359), minute(1), floatHistogram),
)
})
t.Run("serial blocks with mix of histograms and float64", func(t *testing.T) {
testBlockQuerying(t,
genHistogramAndFloatSeries(10, 5, minute(0), minute(60), minute(1), floatHistogram),
genHistogramSeries(10, 5, minute(61), minute(120), minute(1), floatHistogram),
genHistogramAndFloatSeries(10, 5, minute(121), minute(180), minute(1), floatHistogram),
genSeriesFromSampleGenerator(10, 5, minute(181), minute(240), minute(1), func(ts int64) tsdbutil.Sample {
return sample{t: ts, f: rand.Float64()}
}),
)
})
t.Run("overlapping blocks with only histograms", func(t *testing.T) {
testBlockQuerying(t,
genHistogramSeries(10, 5, minute(0), minute(120), minute(3), floatHistogram),
genHistogramSeries(10, 5, minute(1), minute(120), minute(3), floatHistogram),
genHistogramSeries(10, 5, minute(2), minute(120), minute(3), floatHistogram),
)
})
t.Run("overlapping blocks with only histograms and only float in a series", func(t *testing.T) {
testBlockQuerying(t,
genHistogramSeries(10, 5, minute(0), minute(120), minute(3), floatHistogram),
genSeriesFromSampleGenerator(10, 5, minute(1), minute(120), minute(3), func(ts int64) tsdbutil.Sample {
return sample{t: ts, f: rand.Float64()}
}),
genHistogramSeries(10, 5, minute(2), minute(120), minute(3), floatHistogram),
)
})
t.Run("overlapping blocks with mix of histograms and float64", func(t *testing.T) {
testBlockQuerying(t,
genHistogramAndFloatSeries(10, 5, minute(0), minute(60), minute(3), floatHistogram),
genHistogramSeries(10, 5, minute(46), minute(100), minute(3), floatHistogram),
genHistogramAndFloatSeries(10, 5, minute(89), minute(140), minute(3), floatHistogram),
genSeriesFromSampleGenerator(10, 5, minute(126), minute(200), minute(3), func(ts int64) tsdbutil.Sample {
return sample{t: ts, f: rand.Float64()}
}),
)
})
})
}
}
func TestNativeHistogramFlag(t *testing.T) {
dir := t.TempDir()
db, err := Open(dir, nil, nil, nil, nil)
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, db.Close())
})
h := &histogram.Histogram{
Count: 6,
ZeroCount: 4,
ZeroThreshold: 0.001,
Sum: 35.5,
Schema: 1,
PositiveSpans: []histogram.Span{
{Offset: 0, Length: 2},
{Offset: 2, Length: 2},
},
PositiveBuckets: []int64{1, 1, -1, 0},
}
l := labels.FromStrings("foo", "bar")
app := db.Appender(context.Background())
// Disabled by default.
_, err = app.AppendHistogram(0, l, 100, h, nil)
require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
_, err = app.AppendHistogram(0, l, 105, nil, h.ToFloat())
require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
// Enable and append.
db.EnableNativeHistograms()
_, err = app.AppendHistogram(0, l, 200, h, nil)
require.NoError(t, err)
_, err = app.AppendHistogram(0, l, 205, nil, h.ToFloat())
require.NoError(t, err)
db.DisableNativeHistograms()
_, err = app.AppendHistogram(0, l, 300, h, nil)
require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
_, err = app.AppendHistogram(0, l, 305, nil, h.ToFloat())
require.Equal(t, storage.ErrNativeHistogramsDisabled, err)
require.NoError(t, app.Commit())
q, err := db.Querier(context.Background(), math.MinInt, math.MaxInt64)
require.NoError(t, err)
act := query(t, q, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
require.Equal(t, map[string][]tsdbutil.Sample{
l.String(): {sample{t: 200, h: h}, sample{t: 205, fh: h.ToFloat()}},
}, act)
}
// compareSeries essentially replaces `require.Equal(t, expected, actual) in
// situations where the actual series might contain more counter reset hints
// "unknown" than the expected series. This can easily happen for long series
// that trigger new chunks. This function therefore tolerates counter reset
// hints "CounterReset" and "NotCounterReset" in an expected series where the
// actual series contains a counter reset hint "UnknownCounterReset".
// "GaugeType" hints are still strictly checked, and any "UnknownCounterReset"
// in an expected series has to be matched precisely by the actual series.
func compareSeries(t require.TestingT, expected, actual map[string][]tsdbutil.Sample) {
if len(expected) != len(actual) {
// The reason for the difference is not the counter reset hints
// (alone), so let's use the pretty diffing by the require
// package.
require.Equal(t, expected, actual, "number of series differs")
}
for key, eSamples := range expected {
aSamples, ok := actual[key]
if !ok {
require.Equal(t, expected, actual, "expected series %q not found", key)
}
if len(eSamples) != len(aSamples) {
require.Equal(t, eSamples, aSamples, "number of samples for series %q differs", key)
}
for i, eS := range eSamples {
aS := aSamples[i]
aH, eH := aS.H(), eS.H()
aFH, eFH := aS.FH(), eS.FH()
switch {
case aH != nil && eH != nil && aH.CounterResetHint == histogram.UnknownCounterReset && eH.CounterResetHint != histogram.GaugeType:
eH = eH.Copy()
eH.CounterResetHint = histogram.UnknownCounterReset
eS = sample{t: eS.T(), h: eH}
case aFH != nil && eFH != nil && aFH.CounterResetHint == histogram.UnknownCounterReset && eFH.CounterResetHint != histogram.GaugeType:
eFH = eFH.Copy()
eFH.CounterResetHint = histogram.UnknownCounterReset
eS = sample{t: eS.T(), fh: eFH}
}
require.Equal(t, eS, aS, "sample %d in series %q differs", i, key)
}
}
}
// TestChunkQuerierReadWriteRace looks for any possible race between appending
// samples and reading chunks because the head chunk that is being appended to
// can be read in parallel and we should be able to make a copy of the chunk without
// worrying about the parallel write.
func TestChunkQuerierReadWriteRace(t *testing.T) {
db := openTestDB(t, nil, nil)
defer func() {
require.NoError(t, db.Close())
}()
lbls := labels.FromStrings("foo", "bar")
writer := func() error {
<-time.After(5 * time.Millisecond) // Initial pause while readers start.
ts := 0
for i := 0; i < 500; i++ {
app := db.Appender(context.Background())
for j := 0; j < 10; j++ {
ts++
_, err := app.Append(0, lbls, int64(ts), float64(ts*100))
if err != nil {
return err
}
}
err := app.Commit()
if err != nil {
return err
}
<-time.After(time.Millisecond)
}
return nil
}
reader := func() {
querier, err := db.ChunkQuerier(context.Background(), math.MinInt64, math.MaxInt64)
require.NoError(t, err)
defer func(q storage.ChunkQuerier) {
require.NoError(t, q.Close())
}(querier)
ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
for ss.Next() {
cs := ss.At()
it := cs.Iterator(nil)
for it.Next() {
m := it.At()
b := m.Chunk.Bytes()
bb := make([]byte, len(b))
copy(bb, b) // This copying of chunk bytes detects any race.
}
}
require.NoError(t, ss.Err())
}
ch := make(chan struct{})
var writerErr error
go func() {
defer close(ch)
writerErr = writer()
}()
Outer:
for {
reader()
select {
case <-ch:
break Outer
default:
}
}
require.NoError(t, writerErr)
}
Reference in new issue View Git Blame Copy Permalink