From 6383994f3e4bc82092536e49906b5b719567cae8 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Mon, 29 Aug 2022 16:21:32 +0530 Subject: [PATCH] Improve WAL/mmap chunks test for histograms (#11208) Signed-off-by: Ganesh Vernekar Signed-off-by: Ganesh Vernekar --- tsdb/head_test.go | 94 ++++++++++++++++++++++++++------------ tsdb/record/record.go | 4 +- tsdb/record/record_test.go | 92 +++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 32 deletions(-) diff --git a/tsdb/head_test.go b/tsdb/head_test.go index a709a7ca9..db14ffb25 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -2855,29 +2855,71 @@ func TestAppendHistogram(t *testing.T) { } } -func TestHistogramInWAL(t *testing.T) { - l := labels.Labels{{Name: "a", Value: "b"}} - numHistograms := 10 +func TestHistogramInWALAndMmapChunk(t *testing.T) { head, _ := newTestHead(t, 1000, false) t.Cleanup(func() { require.NoError(t, head.Close()) }) - require.NoError(t, head.Init(0)) - app := head.Appender(context.Background()) - type timedHistogram struct { - t int64 - h *histogram.Histogram - } - expHistograms := make([]timedHistogram, 0, numHistograms) + // Series with only histograms. + s1 := labels.Labels{{Name: "a", Value: "b1"}} + k1 := s1.String() + numHistograms := 450 + exp := map[string][]tsdbutil.Sample{} + app := head.Appender(context.Background()) for i, h := range GenerateTestHistograms(numHistograms) { h.Count = h.Count * 2 h.NegativeSpans = h.PositiveSpans h.NegativeBuckets = h.PositiveBuckets - _, err := app.AppendHistogram(0, l, int64(i), h) + _, err := app.AppendHistogram(0, s1, int64(i), h) require.NoError(t, err) - expHistograms = append(expHistograms, timedHistogram{int64(i), h}) + exp[k1] = append(exp[k1], sample{t: int64(i), h: h.Copy()}) + if i%5 == 0 { + require.NoError(t, app.Commit()) + app = head.Appender(context.Background()) + } + } + require.NoError(t, app.Commit()) + + // There should be 3 mmap chunks in s1. + ms := head.series.getByHash(s1.Hash(), s1) + require.Len(t, ms.mmappedChunks, 3) + expMmapChunks := make([]*mmappedChunk, 0, 3) + for _, mmap := range ms.mmappedChunks { + require.Greater(t, mmap.numSamples, uint16(0)) + cpy := *mmap + expMmapChunks = append(expMmapChunks, &cpy) + } + expHeadChunkSamples := ms.headChunk.chunk.NumSamples() + require.Greater(t, expHeadChunkSamples, 0) + + // Series with mix of histograms and float. + s2 := labels.Labels{{Name: "a", Value: "b2"}} + k2 := s2.String() + app = head.Appender(context.Background()) + ts := 0 + for _, h := range GenerateTestHistograms(200) { + ts++ + h.Count = h.Count * 2 + h.NegativeSpans = h.PositiveSpans + h.NegativeBuckets = h.PositiveBuckets + _, err := app.AppendHistogram(0, s2, int64(ts), h) + require.NoError(t, err) + exp[k2] = append(exp[k2], sample{t: int64(ts), h: h.Copy()}) + if ts%20 == 0 { + require.NoError(t, app.Commit()) + app = head.Appender(context.Background()) + // Add some float. + for i := 0; i < 10; i++ { + ts++ + _, err := app.Append(0, s2, int64(ts), float64(ts)) + require.NoError(t, err) + exp[k2] = append(exp[k2], sample{t: int64(ts), v: float64(ts)}) + } + require.NoError(t, app.Commit()) + app = head.Appender(context.Background()) + } } require.NoError(t, app.Commit()) @@ -2889,26 +2931,18 @@ func TestHistogramInWAL(t *testing.T) { require.NoError(t, err) require.NoError(t, head.Init(0)) - q, err := NewBlockQuerier(head, head.MinTime(), head.MaxTime()) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, q.Close()) - }) - - ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - - require.True(t, ss.Next()) - s := ss.At() - require.False(t, ss.Next()) - - it := s.Iterator() - actHistograms := make([]timedHistogram, 0, len(expHistograms)) - for it.Next() == chunkenc.ValHistogram { - t, h := it.AtHistogram() - actHistograms = append(actHistograms, timedHistogram{t, h}) + // Checking contents of s1. + ms = head.series.getByHash(s1.Hash(), s1) + require.Equal(t, expMmapChunks, ms.mmappedChunks) + for _, mmap := range ms.mmappedChunks { + require.Greater(t, mmap.numSamples, uint16(0)) } + require.Equal(t, expHeadChunkSamples, ms.headChunk.chunk.NumSamples()) - require.Equal(t, expHistograms, actHistograms) + q, err := NewBlockQuerier(head, head.MinTime(), head.MaxTime()) + require.NoError(t, err) + act := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "a", "b.*")) + require.Equal(t, exp, act) } func TestChunkSnapshot(t *testing.T) { diff --git a/tsdb/record/record.go b/tsdb/record/record.go index ebf3713a0..c6d8781e1 100644 --- a/tsdb/record/record.go +++ b/tsdb/record/record.go @@ -44,10 +44,10 @@ const ( Tombstones Type = 3 // Exemplars is used to match WAL records of type Exemplars. Exemplars Type = 4 - // Histograms is used to match WAL records of type Histograms. - Histograms Type = 5 // Metadata is used to match WAL records of type Metadata. Metadata Type = 6 + // Histograms is used to match WAL records of type Histograms. + Histograms Type = 7 ) func (rt Type) String() string { diff --git a/tsdb/record/record_test.go b/tsdb/record/record_test.go index 7a02eff3f..2f48a0f2f 100644 --- a/tsdb/record/record_test.go +++ b/tsdb/record/record_test.go @@ -15,11 +15,13 @@ package record import ( + "math/rand" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/require" + "github.com/prometheus/prometheus/model/histogram" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/tsdb/encoding" "github.com/prometheus/prometheus/tsdb/tombstones" @@ -107,6 +109,50 @@ func TestRecord_EncodeDecode(t *testing.T) { decExemplars, err := dec.Exemplars(enc.Exemplars(exemplars, nil), nil) require.NoError(t, err) require.Equal(t, exemplars, decExemplars) + + histograms := []RefHistogram{ + { + Ref: 56, + T: 1234, + H: &histogram.Histogram{ + Count: 5, + ZeroCount: 2, + ZeroThreshold: 0.001, + Sum: 18.4 * rand.Float64(), + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + }, + }, + { + Ref: 42, + T: 5678, + H: &histogram.Histogram{ + Count: 11, + ZeroCount: 4, + ZeroThreshold: 0.001, + Sum: 35.5, + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 2, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + NegativeSpans: []histogram.Span{ + {Offset: 0, Length: 1}, + {Offset: 1, Length: 2}, + }, + NegativeBuckets: []int64{1, 2, -1}, + }, + }, + } + + decHistograms, err := dec.Histograms(enc.Histograms(histograms, nil), nil) + require.NoError(t, err) + require.Equal(t, histograms, decHistograms) } // TestRecord_Corrupted ensures that corrupted records return the correct error. @@ -170,6 +216,31 @@ func TestRecord_Corrupted(t *testing.T) { _, err := dec.Metadata(corrupted, nil) require.Equal(t, errors.Cause(err), encoding.ErrInvalidSize) }) + + t.Run("Test corrupted histogram record", func(t *testing.T) { + histograms := []RefHistogram{ + { + Ref: 56, + T: 1234, + H: &histogram.Histogram{ + Count: 5, + ZeroCount: 2, + ZeroThreshold: 0.001, + Sum: 18.4 * rand.Float64(), + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + }, + }, + } + + corrupted := enc.Histograms(histograms, nil)[:8] + _, err := dec.Histograms(corrupted, nil) + require.Equal(t, errors.Cause(err), encoding.ErrInvalidSize) + }) } func TestRecord_Type(t *testing.T) { @@ -192,6 +263,27 @@ func TestRecord_Type(t *testing.T) { recordType = dec.Type(enc.Metadata(metadata, nil)) require.Equal(t, Metadata, recordType) + histograms := []RefHistogram{ + { + Ref: 56, + T: 1234, + H: &histogram.Histogram{ + Count: 5, + ZeroCount: 2, + ZeroThreshold: 0.001, + Sum: 18.4 * rand.Float64(), + Schema: 1, + PositiveSpans: []histogram.Span{ + {Offset: 0, Length: 2}, + {Offset: 1, Length: 2}, + }, + PositiveBuckets: []int64{1, 1, -1, 0}, + }, + }, + } + recordType = dec.Type(enc.Histograms(histograms, nil)) + require.Equal(t, Histograms, recordType) + recordType = dec.Type(nil) require.Equal(t, Unknown, recordType)