mirror of https://github.com/prometheus/prometheus
Always return unknown hint for first sample in non-gauge histogram chunk (#15343)
Always return unknown hint for first sample in non-gauge histogram chunk --------- Signed-off-by: Fiona Liao <fiona.liao@grafana.com> Co-authored-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>pull/15395/head
Fiona Liao
GitHub
parent
b4c0a5b394
commit
c599d37668
|
|
@ -31,22 +31,27 @@ func TestFirstFloatHistogramExplicitCounterReset(t *testing.T) {
|
|||
tests := map[string]struct {
|
||||
hint histogram.CounterResetHint
|
||||
expHeader CounterResetHeader
|
||||
expHint histogram.CounterResetHint
|
||||
}{
|
||||
"CounterReset": {
|
||||
hint: histogram.CounterReset,
|
||||
expHeader: CounterReset,
|
||||
expHint: histogram.UnknownCounterReset,
|
||||
},
|
||||
"NotCounterReset": {
|
||||
hint: histogram.NotCounterReset,
|
||||
expHeader: UnknownCounterReset,
|
||||
expHint: histogram.UnknownCounterReset,
|
||||
},
|
||||
"UnknownCounterReset": {
|
||||
hint: histogram.UnknownCounterReset,
|
||||
expHeader: UnknownCounterReset,
|
||||
expHint: histogram.UnknownCounterReset,
|
||||
},
|
||||
"Gauge": {
|
||||
hint: histogram.GaugeType,
|
||||
expHeader: GaugeType,
|
||||
expHint: histogram.GaugeType,
|
||||
},
|
||||
}
|
||||
for name, test := range tests {
|
||||
|
|
@ -63,6 +68,7 @@ func TestFirstFloatHistogramExplicitCounterReset(t *testing.T) {
|
|||
require.False(t, recoded)
|
||||
require.Equal(t, app, newApp)
|
||||
require.Equal(t, test.expHeader, chk.GetCounterResetHeader())
|
||||
assertFirstFloatHistogramSampleHint(t, chk, test.expHint)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -338,7 +344,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Zero threshold change.
|
||||
|
|
@ -348,7 +354,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has more buckets.
|
||||
|
|
@ -397,7 +403,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has buckets missing but the buckets missing were empty.
|
||||
|
|
@ -480,7 +486,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has a counter reset while new buckets were added.
|
||||
|
|
@ -503,7 +509,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{
|
||||
|
|
@ -532,7 +538,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has an explicit counter reset.
|
||||
|
|
@ -540,7 +546,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
h2 := h1.Copy()
|
||||
h2.CounterResetHint = histogram.CounterReset
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
|
||||
|
|
@ -557,6 +563,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, app, newApp)
|
||||
assertSampleCount(t, nextChunk, 1, ValFloatHistogram)
|
||||
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
|
||||
assertFirstFloatHistogramSampleHint(t, nextChunk, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
|
||||
|
|
@ -574,6 +581,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, app, newApp)
|
||||
assertSampleCount(t, nextChunk, 1, ValFloatHistogram)
|
||||
require.Equal(t, CounterReset, nextChunk.GetCounterResetHeader())
|
||||
assertFirstFloatHistogramSampleHint(t, nextChunk, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
|
||||
|
|
@ -600,6 +608,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, app, newApp)
|
||||
assertSampleCount(t, nextChunk, 1, ValFloatHistogram)
|
||||
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
|
||||
assertFirstFloatHistogramSampleHint(t, nextChunk, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{
|
||||
|
|
@ -664,7 +673,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Custom buckets, change only in custom bounds.
|
||||
|
|
@ -674,7 +683,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Custom buckets, with more buckets.
|
||||
|
|
@ -705,7 +714,7 @@ func TestFloatHistogramChunkAppendable(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
|
||||
func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader, expectHint histogram.CounterResetHint) {
|
||||
oldChunkBytes := oldChunk.Bytes()
|
||||
newChunk, recoded, newAppender, err := hApp.AppendFloatHistogram(nil, ts, h, false)
|
||||
require.Equal(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched.
|
||||
|
|
@ -717,6 +726,7 @@ func assertNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Fl
|
|||
require.NotNil(t, newAppender)
|
||||
require.NotEqual(t, hApp, newAppender)
|
||||
assertSampleCount(t, newChunk, 1, ValFloatHistogram)
|
||||
assertFirstFloatHistogramSampleHint(t, newChunk, expectHint)
|
||||
}
|
||||
|
||||
func assertNoNewFloatHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *FloatHistogramAppender, ts int64, h *histogram.FloatHistogram, expectHeader CounterResetHeader) {
|
||||
|
|
@ -1023,7 +1033,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
|
|||
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType, histogram.GaugeType)
|
||||
}
|
||||
|
||||
{ // Zero threshold change.
|
||||
|
|
@ -1033,7 +1043,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
|
|||
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType, histogram.GaugeType)
|
||||
}
|
||||
|
||||
{ // New histogram that has more buckets.
|
||||
|
|
@ -1208,7 +1218,7 @@ func TestFloatHistogramChunkAppendableGauge(t *testing.T) {
|
|||
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
|
||||
assertNewFloatHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType, histogram.GaugeType)
|
||||
}
|
||||
|
||||
{ // Custom buckets, with more buckets.
|
||||
|
|
@ -1357,3 +1367,10 @@ func TestFloatHistogramUniqueSpansAfterNext(t *testing.T) {
|
|||
require.NotSame(t, &rh1.PositiveSpans[0], &rh2.PositiveSpans[0], "PositiveSpans should be unique between histograms")
|
||||
require.NotSame(t, &rh1.NegativeSpans[0], &rh2.NegativeSpans[0], "NegativeSpans should be unique between histograms")
|
||||
}
|
||||
|
||||
func assertFirstFloatHistogramSampleHint(t *testing.T, chunk Chunk, expected histogram.CounterResetHint) {
|
||||
it := chunk.Iterator(nil)
|
||||
require.Equal(t, ValFloatHistogram, it.Next())
|
||||
_, v := it.AtFloatHistogram(nil)
|
||||
require.Equal(t, expected, v.CounterResetHint)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -558,26 +558,16 @@ func counterResetHint(crh CounterResetHeader, numRead uint16) histogram.CounterR
|
|||
// In a counter histogram chunk, there will not be any counter
|
||||
// resets after the first histogram.
|
||||
return histogram.NotCounterReset
|
||||
case crh == CounterReset:
|
||||
// If the chunk was started because of a counter reset, we can
|
||||
// safely return that hint. This histogram always has to be
|
||||
// treated as a counter reset.
|
||||
return histogram.CounterReset
|
||||
default:
|
||||
// Sadly, we have to return "unknown" as the hint for all other
|
||||
// cases, even if we know that the chunk was started without a
|
||||
// cases, even if we know that the chunk was started with or without a
|
||||
// counter reset. But we cannot be sure that the previous chunk
|
||||
// still exists in the TSDB, so we conservatively return
|
||||
// "unknown". On the bright side, this case should be relatively
|
||||
// rare.
|
||||
// still exists in the TSDB, or if the previous chunk was added later
|
||||
// by out of order or backfill, so we conservatively return "unknown".
|
||||
//
|
||||
// TODO(beorn7): Nevertheless, if the current chunk is in the
|
||||
// middle of a block (not the first chunk in the block for this
|
||||
// series), it's probably safe to assume that the previous chunk
|
||||
// will exist in the TSDB for as long as the current chunk
|
||||
// exist, and we could safely return
|
||||
// "histogram.NotCounterReset". This needs some more work and
|
||||
// might not be worth the effort and/or risk. To be vetted...
|
||||
// TODO: If we can detect whether the previous and current chunk are
|
||||
// actually consecutive then we could trust its hint:
|
||||
// https://github.com/prometheus/prometheus/issues/15346.
|
||||
return histogram.UnknownCounterReset
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,22 +32,27 @@ func TestFirstHistogramExplicitCounterReset(t *testing.T) {
|
|||
tests := map[string]struct {
|
||||
hint histogram.CounterResetHint
|
||||
expHeader CounterResetHeader
|
||||
expHint histogram.CounterResetHint
|
||||
}{
|
||||
"CounterReset": {
|
||||
hint: histogram.CounterReset,
|
||||
expHeader: CounterReset,
|
||||
expHint: histogram.UnknownCounterReset,
|
||||
},
|
||||
"NotCounterReset": {
|
||||
hint: histogram.NotCounterReset,
|
||||
expHeader: UnknownCounterReset,
|
||||
expHint: histogram.UnknownCounterReset,
|
||||
},
|
||||
"UnknownCounterReset": {
|
||||
hint: histogram.UnknownCounterReset,
|
||||
expHeader: UnknownCounterReset,
|
||||
expHint: histogram.UnknownCounterReset,
|
||||
},
|
||||
"Gauge": {
|
||||
hint: histogram.GaugeType,
|
||||
expHeader: GaugeType,
|
||||
expHint: histogram.GaugeType,
|
||||
},
|
||||
}
|
||||
for name, test := range tests {
|
||||
|
|
@ -64,6 +69,7 @@ func TestFirstHistogramExplicitCounterReset(t *testing.T) {
|
|||
require.False(t, recoded)
|
||||
require.Equal(t, app, newApp)
|
||||
require.Equal(t, test.expHeader, chk.GetCounterResetHeader())
|
||||
assertFirstIntHistogramSampleHint(t, chk, test.expHint)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -352,7 +358,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Zero threshold change.
|
||||
|
|
@ -362,7 +368,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, UnknownCounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has more buckets.
|
||||
|
|
@ -413,7 +419,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has buckets missing but the buckets missing were empty.
|
||||
|
|
@ -498,7 +504,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has a counter reset while new buckets were added.
|
||||
|
|
@ -524,7 +530,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{
|
||||
|
|
@ -556,7 +562,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // New histogram that has an explicit counter reset.
|
||||
|
|
@ -564,7 +570,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
h2 := h1.Copy()
|
||||
h2.CounterResetHint = histogram.CounterReset
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Start new chunk explicitly, and append a new histogram that is considered appendable to the previous chunk.
|
||||
|
|
@ -581,6 +587,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, app, newApp)
|
||||
assertSampleCount(t, nextChunk, 1, ValHistogram)
|
||||
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
|
||||
assertFirstIntHistogramSampleHint(t, nextChunk, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Start new chunk explicitly, and append a new histogram that is not considered appendable to the previous chunk.
|
||||
|
|
@ -598,6 +605,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, app, newApp)
|
||||
assertSampleCount(t, nextChunk, 1, ValHistogram)
|
||||
require.Equal(t, CounterReset, nextChunk.GetCounterResetHeader())
|
||||
assertFirstIntHistogramSampleHint(t, nextChunk, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Start new chunk explicitly, and append a new histogram that would need recoding if we added it to the chunk.
|
||||
|
|
@ -627,6 +635,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, app, newApp)
|
||||
assertSampleCount(t, nextChunk, 1, ValHistogram)
|
||||
require.Equal(t, NotCounterReset, nextChunk.GetCounterResetHeader())
|
||||
assertFirstIntHistogramSampleHint(t, nextChunk, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{
|
||||
|
|
@ -691,7 +700,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Custom buckets, change only in custom bounds.
|
||||
|
|
@ -701,7 +710,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
_, _, _, _, ok, _ := hApp.appendable(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, CounterReset, histogram.UnknownCounterReset)
|
||||
}
|
||||
|
||||
{ // Custom buckets, with more buckets.
|
||||
|
|
@ -732,7 +741,7 @@ func TestHistogramChunkAppendable(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
|
||||
func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader, expectHint histogram.CounterResetHint) {
|
||||
oldChunkBytes := oldChunk.Bytes()
|
||||
newChunk, recoded, newAppender, err := hApp.AppendHistogram(nil, ts, h, false)
|
||||
require.Equal(t, oldChunkBytes, oldChunk.Bytes()) // Sanity check that previous chunk is untouched.
|
||||
|
|
@ -744,6 +753,7 @@ func assertNewHistogramChunkOnAppend(t *testing.T, oldChunk Chunk, hApp *Histogr
|
|||
require.NotNil(t, newAppender)
|
||||
require.NotEqual(t, hApp, newAppender)
|
||||
assertSampleCount(t, newChunk, 1, ValHistogram)
|
||||
assertFirstIntHistogramSampleHint(t, newChunk, expectHint)
|
||||
}
|
||||
|
||||
func assertNoNewHistogramChunkOnAppend(t *testing.T, currChunk Chunk, hApp *HistogramAppender, ts int64, h *histogram.Histogram, expectHeader CounterResetHeader) {
|
||||
|
|
@ -1203,7 +1213,7 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
|
|||
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType, histogram.GaugeType)
|
||||
}
|
||||
|
||||
{ // Zero threshold change.
|
||||
|
|
@ -1213,7 +1223,7 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
|
|||
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType, histogram.GaugeType)
|
||||
}
|
||||
|
||||
{ // New histogram that has more buckets.
|
||||
|
|
@ -1388,7 +1398,7 @@ func TestHistogramChunkAppendableGauge(t *testing.T) {
|
|||
_, _, _, _, _, _, ok := hApp.appendableGauge(h2)
|
||||
require.False(t, ok)
|
||||
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType)
|
||||
assertNewHistogramChunkOnAppend(t, c, hApp, ts+1, h2, GaugeType, histogram.GaugeType)
|
||||
}
|
||||
|
||||
{ // Custom buckets, with more buckets.
|
||||
|
|
@ -1635,3 +1645,10 @@ func BenchmarkAppendable(b *testing.B) {
|
|||
b.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
func assertFirstIntHistogramSampleHint(t *testing.T, chunk Chunk, expected histogram.CounterResetHint) {
|
||||
it := chunk.Iterator(nil)
|
||||
require.Equal(t, ValHistogram, it.Next())
|
||||
_, v := it.AtHistogram(nil)
|
||||
require.Equal(t, expected, v.CounterResetHint)
|
||||
}
|
||||
|
|
|
|||
453
tsdb/db_test.go
453
tsdb/db_test.go
|
|
@ -6121,7 +6121,7 @@ func testOOONativeHistogramsWithCounterResets(t *testing.T, scenario sampleTypeS
|
|||
shouldReset: func(v int64) bool {
|
||||
return v == 45
|
||||
},
|
||||
expCounterResetHints: []histogram.CounterResetHint{histogram.UnknownCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.CounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset},
|
||||
expCounterResetHints: []histogram.CounterResetHint{histogram.UnknownCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.UnknownCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset, histogram.NotCounterReset},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -6241,6 +6241,242 @@ func testOOONativeHistogramsWithCounterResets(t *testing.T, scenario sampleTypeS
|
|||
}
|
||||
}
|
||||
|
||||
func TestOOOInterleavedImplicitCounterResets(t *testing.T) {
|
||||
for name, scenario := range sampleTypeScenarios {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
testOOOInterleavedImplicitCounterResets(t, name, scenario)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testOOOInterleavedImplicitCounterResets(t *testing.T, name string, scenario sampleTypeScenario) {
|
||||
var appendFunc func(app storage.Appender, ts, v int64) error
|
||||
|
||||
if scenario.sampleType != sampleMetricTypeHistogram {
|
||||
return
|
||||
}
|
||||
|
||||
switch name {
|
||||
case intHistogram:
|
||||
appendFunc = func(app storage.Appender, ts, v int64) error {
|
||||
h := &histogram.Histogram{
|
||||
Count: uint64(v),
|
||||
Sum: float64(v),
|
||||
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}},
|
||||
PositiveBuckets: []int64{v},
|
||||
}
|
||||
_, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, h, nil)
|
||||
return err
|
||||
}
|
||||
case floatHistogram:
|
||||
appendFunc = func(app storage.Appender, ts, v int64) error {
|
||||
fh := &histogram.FloatHistogram{
|
||||
Count: float64(v),
|
||||
Sum: float64(v),
|
||||
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}},
|
||||
PositiveBuckets: []float64{float64(v)},
|
||||
}
|
||||
_, err := app.AppendHistogram(0, labels.FromStrings("foo", "bar1"), ts, nil, fh)
|
||||
return err
|
||||
}
|
||||
case gaugeIntHistogram, gaugeFloatHistogram:
|
||||
return
|
||||
}
|
||||
|
||||
// Not a sample, we're encoding an integer counter that we convert to a
|
||||
// histogram with a single bucket.
|
||||
type tsValue struct {
|
||||
ts int64
|
||||
v int64
|
||||
}
|
||||
|
||||
type expectedTsValue struct {
|
||||
ts int64
|
||||
v int64
|
||||
hint histogram.CounterResetHint
|
||||
}
|
||||
|
||||
type expectedChunk struct {
|
||||
hint histogram.CounterResetHint
|
||||
size int
|
||||
}
|
||||
|
||||
cases := map[string]struct {
|
||||
samples []tsValue
|
||||
oooCap int64
|
||||
// The expected samples with counter reset.
|
||||
expectedSamples []expectedTsValue
|
||||
// The expected counter reset hint for each chunk.
|
||||
expectedChunks []expectedChunk
|
||||
}{
|
||||
"counter reset in-order cleared by in-memory OOO chunk": {
|
||||
samples: []tsValue{
|
||||
{1, 40}, // New in In-order. I1.
|
||||
{4, 30}, // In-order counter reset. I2.
|
||||
{2, 40}, // New in OOO. O1.
|
||||
{3, 10}, // OOO counter reset. O2.
|
||||
},
|
||||
oooCap: 30,
|
||||
// Expect all to be set to UnknownCounterReset because we switch between
|
||||
// in-order and out-of-order samples.
|
||||
expectedSamples: []expectedTsValue{
|
||||
{1, 40, histogram.UnknownCounterReset}, // I1.
|
||||
{2, 40, histogram.UnknownCounterReset}, // O1.
|
||||
{3, 10, histogram.UnknownCounterReset}, // O2.
|
||||
{4, 30, histogram.UnknownCounterReset}, // I2. Counter reset cleared by iterator change.
|
||||
},
|
||||
expectedChunks: []expectedChunk{
|
||||
{histogram.UnknownCounterReset, 1}, // I1.
|
||||
{histogram.UnknownCounterReset, 1}, // O1.
|
||||
{histogram.UnknownCounterReset, 1}, // O2.
|
||||
{histogram.UnknownCounterReset, 1}, // I2.
|
||||
},
|
||||
},
|
||||
"counter reset in OOO mmapped chunk cleared by in-memory ooo chunk": {
|
||||
samples: []tsValue{
|
||||
{8, 30}, // In-order, new chunk. I1.
|
||||
{1, 10}, // OOO, new chunk (will be mmapped). MO1.
|
||||
{2, 20}, // OOO, no reset (will be mmapped). MO1.
|
||||
{3, 30}, // OOO, no reset (will be mmapped). MO1.
|
||||
{5, 20}, // OOO, reset (will be mmapped). MO2.
|
||||
{6, 10}, // OOO, reset (will be mmapped). MO3.
|
||||
{7, 20}, // OOO, no reset (will be mmapped). MO3.
|
||||
{4, 10}, // OOO, inserted into memory, triggers mmap. O1.
|
||||
},
|
||||
oooCap: 6,
|
||||
expectedSamples: []expectedTsValue{
|
||||
{1, 10, histogram.UnknownCounterReset}, // MO1.
|
||||
{2, 20, histogram.NotCounterReset}, // MO1.
|
||||
{3, 30, histogram.NotCounterReset}, // MO1.
|
||||
{4, 10, histogram.UnknownCounterReset}, // O1. Counter reset cleared by iterator change.
|
||||
{5, 20, histogram.UnknownCounterReset}, // MO2.
|
||||
{6, 10, histogram.UnknownCounterReset}, // MO3.
|
||||
{7, 20, histogram.NotCounterReset}, // MO3.
|
||||
{8, 30, histogram.UnknownCounterReset}, // I1.
|
||||
},
|
||||
expectedChunks: []expectedChunk{
|
||||
{histogram.UnknownCounterReset, 3}, // MO1.
|
||||
{histogram.UnknownCounterReset, 1}, // O1.
|
||||
{histogram.UnknownCounterReset, 1}, // MO2.
|
||||
{histogram.UnknownCounterReset, 2}, // MO3.
|
||||
{histogram.UnknownCounterReset, 1}, // I1.
|
||||
},
|
||||
},
|
||||
"counter reset in OOO mmapped chunk cleared by another OOO mmapped chunk": {
|
||||
samples: []tsValue{
|
||||
{8, 100}, // In-order, new chunk. I1.
|
||||
{1, 50}, // OOO, new chunk (will be mmapped). MO1.
|
||||
{5, 40}, // OOO, reset (will be mmapped). MO2.
|
||||
{6, 50}, // OOO, no reset (will be mmapped). MO2.
|
||||
{2, 10}, // OOO, new chunk no reset (will be mmapped). MO3.
|
||||
{3, 20}, // OOO, no reset (will be mmapped). MO3.
|
||||
{4, 30}, // OOO, no reset (will be mmapped). MO3.
|
||||
{7, 60}, // OOO, no reset in memory. O1.
|
||||
},
|
||||
oooCap: 3,
|
||||
expectedSamples: []expectedTsValue{
|
||||
{1, 50, histogram.UnknownCounterReset}, // MO1.
|
||||
{2, 10, histogram.UnknownCounterReset}, // MO3.
|
||||
{3, 20, histogram.NotCounterReset}, // MO3.
|
||||
{4, 30, histogram.NotCounterReset}, // MO3.
|
||||
{5, 40, histogram.UnknownCounterReset}, // MO2.
|
||||
{6, 50, histogram.NotCounterReset}, // MO2.
|
||||
{7, 60, histogram.UnknownCounterReset}, // O1.
|
||||
{8, 100, histogram.UnknownCounterReset}, // I1.
|
||||
},
|
||||
expectedChunks: []expectedChunk{
|
||||
{histogram.UnknownCounterReset, 1}, // MO1.
|
||||
{histogram.UnknownCounterReset, 3}, // MO3.
|
||||
{histogram.UnknownCounterReset, 2}, // MO2.
|
||||
{histogram.UnknownCounterReset, 1}, // O1.
|
||||
{histogram.UnknownCounterReset, 1}, // I1.
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for tcName, tc := range cases {
|
||||
t.Run(tcName, func(t *testing.T) {
|
||||
opts := DefaultOptions()
|
||||
opts.OutOfOrderCapMax = tc.oooCap
|
||||
opts.OutOfOrderTimeWindow = 24 * time.Hour.Milliseconds()
|
||||
|
||||
db := openTestDB(t, opts, nil)
|
||||
db.DisableCompactions()
|
||||
db.EnableOOONativeHistograms()
|
||||
defer func() {
|
||||
require.NoError(t, db.Close())
|
||||
}()
|
||||
|
||||
app := db.Appender(context.Background())
|
||||
for _, s := range tc.samples {
|
||||
require.NoError(t, appendFunc(app, s.ts, s.v))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
t.Run("querier", func(t *testing.T) {
|
||||
querier, err := db.Querier(0, 10)
|
||||
require.NoError(t, err)
|
||||
defer querier.Close()
|
||||
|
||||
seriesSet := query(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1"))
|
||||
require.Len(t, seriesSet, 1)
|
||||
samples, ok := seriesSet["{foo=\"bar1\"}"]
|
||||
require.True(t, ok)
|
||||
require.Len(t, samples, len(tc.samples))
|
||||
require.Len(t, samples, len(tc.expectedSamples))
|
||||
|
||||
// We expect all unknown counter resets because we clear the counter reset
|
||||
// hint when we switch between in-order and out-of-order samples.
|
||||
for i, s := range samples {
|
||||
switch name {
|
||||
case intHistogram:
|
||||
require.Equal(t, tc.expectedSamples[i].hint, s.H().CounterResetHint, "sample %d", i)
|
||||
require.Equal(t, tc.expectedSamples[i].v, int64(s.H().Count), "sample %d", i)
|
||||
case floatHistogram:
|
||||
require.Equal(t, tc.expectedSamples[i].hint, s.FH().CounterResetHint, "sample %d", i)
|
||||
require.Equal(t, tc.expectedSamples[i].v, int64(s.FH().Count), "sample %d", i)
|
||||
default:
|
||||
t.Fatalf("unexpected sample type %s", name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("chunk-querier", func(t *testing.T) {
|
||||
querier, err := db.ChunkQuerier(0, 10)
|
||||
require.NoError(t, err)
|
||||
defer querier.Close()
|
||||
|
||||
chunkSet := queryAndExpandChunks(t, querier, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar1"))
|
||||
require.Len(t, chunkSet, 1)
|
||||
chunks, ok := chunkSet["{foo=\"bar1\"}"]
|
||||
require.True(t, ok)
|
||||
require.Len(t, chunks, len(tc.expectedChunks))
|
||||
idx := 0
|
||||
for i, samples := range chunks {
|
||||
require.Len(t, samples, tc.expectedChunks[i].size)
|
||||
for j, s := range samples {
|
||||
expectHint := tc.expectedChunks[i].hint
|
||||
if j > 0 {
|
||||
expectHint = histogram.NotCounterReset
|
||||
}
|
||||
switch name {
|
||||
case intHistogram:
|
||||
require.Equal(t, expectHint, s.H().CounterResetHint, "sample %d", idx)
|
||||
require.Equal(t, tc.expectedSamples[idx].v, int64(s.H().Count), "sample %d", idx)
|
||||
case floatHistogram:
|
||||
require.Equal(t, expectHint, s.FH().CounterResetHint, "sample %d", idx)
|
||||
require.Equal(t, tc.expectedSamples[idx].v, int64(s.FH().Count), "sample %d", idx)
|
||||
default:
|
||||
t.Fatalf("unexpected sample type %s", name)
|
||||
}
|
||||
idx++
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOOAppendAndQuery(t *testing.T) {
|
||||
for name, scenario := range sampleTypeScenarios {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
|
|
@ -6746,7 +6982,7 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) {
|
|||
s = addSample(int64(i), series1, 100000+i, histogram.UnknownCounterReset)
|
||||
// The samples with timestamp less than 410 overlap with the samples from chunk 2, so before compaction,
|
||||
// they're all UnknownCounterReset. Samples greater than or equal to 410 don't overlap with other chunks
|
||||
// so they're always detected as NotCounterReset pre and post compaction/
|
||||
// so they're always detected as NotCounterReset pre and post compaction.
|
||||
if i >= 410 {
|
||||
s = copyWithCounterReset(s, histogram.NotCounterReset)
|
||||
}
|
||||
|
|
@ -6772,8 +7008,10 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) {
|
|||
s = addSample(165, series1, 100000, histogram.UnknownCounterReset)
|
||||
// Before compaction, sample has an UnknownCounterReset header due to the chainSampleIterator.
|
||||
series1ExpSamplesPreCompact = append(series1ExpSamplesPreCompact, s)
|
||||
// After compaction, the sample's counter reset is properly detected.
|
||||
series1ExpSamplesPostCompact = append(series1ExpSamplesPostCompact, copyWithCounterReset(s, histogram.CounterReset))
|
||||
// After compaction, the sample's counter reset is still UnknownCounterReset as we cannot trust CounterReset
|
||||
// headers in chunks at the moment, so when reading the first sample in a chunk, its hint is set to
|
||||
// UnknownCounterReset.
|
||||
series1ExpSamplesPostCompact = append(series1ExpSamplesPostCompact, s)
|
||||
|
||||
// Add 23 more samples to complete a chunk.
|
||||
for i := 175; i < 405; i += 10 {
|
||||
|
|
@ -6810,7 +7048,6 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) {
|
|||
}
|
||||
// Counter reset.
|
||||
s = addSample(int64(490), series1, 100000, histogram.UnknownCounterReset)
|
||||
s = copyWithCounterReset(s, histogram.CounterReset)
|
||||
series1ExpSamplesPreCompact = append(series1ExpSamplesPreCompact, s)
|
||||
series1ExpSamplesPostCompact = append(series1ExpSamplesPostCompact, s)
|
||||
// Add some more samples after the counter reset.
|
||||
|
|
@ -6835,7 +7072,6 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) {
|
|||
}
|
||||
// Counter reset.
|
||||
s = addSample(int64(300), series2, 100000, histogram.UnknownCounterReset)
|
||||
s = copyWithCounterReset(s, histogram.CounterReset)
|
||||
series2ExpSamplesPreCompact = append(series2ExpSamplesPreCompact, s)
|
||||
series2ExpSamplesPostCompact = append(series2ExpSamplesPostCompact, s)
|
||||
// Add some more samples after the counter reset.
|
||||
|
|
@ -7007,6 +7243,103 @@ func TestOOOHistogramCompactionWithCounterResets(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestInterleavedInOrderAndOOOHistogramCompactionWithCounterResets(t *testing.T) {
|
||||
for _, floatHistogram := range []bool{false, true} {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
opts := DefaultOptions()
|
||||
opts.OutOfOrderCapMax = 30
|
||||
opts.OutOfOrderTimeWindow = 500 * time.Minute.Milliseconds()
|
||||
|
||||
db, err := Open(dir, nil, nil, opts, nil)
|
||||
require.NoError(t, err)
|
||||
db.DisableCompactions() // We want to manually call it.
|
||||
db.EnableNativeHistograms()
|
||||
db.EnableOOONativeHistograms()
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, db.Close())
|
||||
})
|
||||
|
||||
series1 := labels.FromStrings("foo", "bar1")
|
||||
|
||||
addSample := func(ts int64, l labels.Labels, val int) sample {
|
||||
app := db.Appender(context.Background())
|
||||
tsMs := ts
|
||||
if floatHistogram {
|
||||
h := tsdbutil.GenerateTestFloatHistogram(val)
|
||||
_, err = app.AppendHistogram(0, l, tsMs, nil, h)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
return sample{t: tsMs, fh: h.Copy()}
|
||||
}
|
||||
|
||||
h := tsdbutil.GenerateTestHistogram(val)
|
||||
_, err = app.AppendHistogram(0, l, tsMs, h, nil)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
return sample{t: tsMs, h: h.Copy()}
|
||||
}
|
||||
|
||||
var expSamples []chunks.Sample
|
||||
|
||||
s := addSample(0, series1, 0)
|
||||
expSamples = append(expSamples, s)
|
||||
s = addSample(1, series1, 10)
|
||||
expSamples = append(expSamples, copyWithCounterReset(s, histogram.NotCounterReset))
|
||||
s = addSample(3, series1, 3)
|
||||
expSamples = append(expSamples, copyWithCounterReset(s, histogram.UnknownCounterReset))
|
||||
s = addSample(2, series1, 0)
|
||||
expSamples = append(expSamples, copyWithCounterReset(s, histogram.UnknownCounterReset))
|
||||
|
||||
// Sort samples (as OOO samples not added in time-order).
|
||||
sort.Slice(expSamples, func(i, j int) bool {
|
||||
return expSamples[i].T() < expSamples[j].T()
|
||||
})
|
||||
|
||||
verifyDBSamples := func(s1Samples []chunks.Sample) {
|
||||
t.Helper()
|
||||
expRes := map[string][]chunks.Sample{
|
||||
series1.String(): s1Samples,
|
||||
}
|
||||
|
||||
q, err := db.Querier(math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
|
||||
requireEqualSeries(t, expRes, actRes, false)
|
||||
}
|
||||
|
||||
// Verify DB samples before compaction.
|
||||
verifyDBSamples(expSamples)
|
||||
|
||||
require.NoError(t, db.CompactOOOHead(ctx))
|
||||
|
||||
// Check samples after OOO compaction.
|
||||
verifyDBSamples(expSamples)
|
||||
|
||||
// Checking for expected data in the blocks.
|
||||
// Check that blocks are created after compaction.
|
||||
require.Len(t, db.Blocks(), 1)
|
||||
|
||||
// Compact the in-order head and expect another block.
|
||||
// Since this is a forced compaction, this block is not aligned with 2h.
|
||||
err = db.CompactHead(NewRangeHead(db.head, 0, 3))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, db.Blocks(), 2)
|
||||
|
||||
// Blocks created out of normal and OOO head now. But not merged.
|
||||
verifyDBSamples(expSamples)
|
||||
|
||||
// This will merge overlapping block.
|
||||
require.NoError(t, db.Compact(ctx))
|
||||
|
||||
require.Len(t, db.Blocks(), 1)
|
||||
|
||||
// Final state. Blocks from normal and OOO head are merged.
|
||||
verifyDBSamples(expSamples)
|
||||
}
|
||||
}
|
||||
|
||||
func copyWithCounterReset(s sample, hint histogram.CounterResetHint) sample {
|
||||
if s.h != nil {
|
||||
h := s.h.Copy()
|
||||
|
|
@ -8045,7 +8378,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
})
|
||||
|
||||
ctx := context.Background()
|
||||
appendHistogram := func(
|
||||
appendHistogram := func(t *testing.T,
|
||||
lbls labels.Labels, tsMinute int, h *histogram.Histogram,
|
||||
exp *[]chunks.Sample, expCRH histogram.CounterResetHint,
|
||||
) {
|
||||
|
|
@ -8066,7 +8399,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
}
|
||||
appendFloat := func(lbls labels.Labels, tsMinute int, val float64, exp *[]chunks.Sample) {
|
||||
appendFloat := func(t *testing.T, lbls labels.Labels, tsMinute int, val float64, exp *[]chunks.Sample) {
|
||||
t.Helper()
|
||||
app := db.Appender(ctx)
|
||||
_, err := app.Append(0, lbls, minute(tsMinute), val)
|
||||
|
|
@ -8075,7 +8408,7 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
*exp = append(*exp, sample{t: minute(tsMinute), f: val})
|
||||
}
|
||||
|
||||
testQuery := func(name, value string, exp map[string][]chunks.Sample) {
|
||||
testQuery := func(t *testing.T, name, value string, exp map[string][]chunks.Sample) {
|
||||
t.Helper()
|
||||
q, err := db.Querier(math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
|
|
@ -8113,24 +8446,24 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
t.Run("series with only histograms", func(t *testing.T) {
|
||||
h := baseH.Copy() // This is shared across all sub tests.
|
||||
|
||||
appendHistogram(series1, 100, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 100, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
|
||||
h.PositiveBuckets[0]++
|
||||
h.NegativeBuckets[0] += 2
|
||||
h.Count += 10
|
||||
appendHistogram(series1, 101, h, &exp1, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 101, h, &exp1, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
|
||||
t.Run("changing schema", func(t *testing.T) {
|
||||
h.Schema = 2
|
||||
appendHistogram(series1, 102, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 102, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
|
||||
// Schema back to old.
|
||||
h.Schema = 1
|
||||
appendHistogram(series1, 103, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 103, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
})
|
||||
|
||||
t.Run("new buckets incoming", func(t *testing.T) {
|
||||
|
|
@ -8158,8 +8491,8 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
h.PositiveSpans[1].Length++
|
||||
h.PositiveBuckets = append(h.PositiveBuckets, 1)
|
||||
h.Count += 3
|
||||
appendHistogram(series1, 104, h, &exp1, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 104, h, &exp1, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
|
||||
// Because of the previous two histograms being on the active chunk,
|
||||
// and the next append is only adding a new bucket, the active chunk
|
||||
|
|
@ -8196,14 +8529,14 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
h.Count += 3
|
||||
// {2, 1, -1, 0, 1} -> {2, 1, 0, -1, 0, 1}
|
||||
h.PositiveBuckets = append(h.PositiveBuckets[:2], append([]int64{0}, h.PositiveBuckets[2:]...)...)
|
||||
appendHistogram(series1, 105, h, &exp1, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 105, h, &exp1, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
|
||||
// We add 4 more histograms to clear out the buffer and see the re-encoded histograms.
|
||||
appendHistogram(series1, 106, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(series1, 107, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(series1, 108, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(series1, 109, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(t, series1, 106, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(t, series1, 107, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(t, series1, 108, h, &exp1, histogram.NotCounterReset)
|
||||
appendHistogram(t, series1, 109, h, &exp1, histogram.NotCounterReset)
|
||||
|
||||
// Update the expected histograms to reflect the re-encoding.
|
||||
if floatHistogram {
|
||||
|
|
@ -8230,69 +8563,69 @@ func testHistogramAppendAndQueryHelper(t *testing.T, floatHistogram bool) {
|
|||
exp1[l-6] = sample{t: exp1[l-6].T(), h: h6}
|
||||
}
|
||||
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
})
|
||||
|
||||
t.Run("buckets disappearing", func(t *testing.T) {
|
||||
h.PositiveSpans[1].Length--
|
||||
h.PositiveBuckets = h.PositiveBuckets[:len(h.PositiveBuckets)-1]
|
||||
h.Count -= 3
|
||||
appendHistogram(series1, 110, h, &exp1, histogram.CounterReset)
|
||||
testQuery("foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
appendHistogram(t, series1, 110, h, &exp1, histogram.UnknownCounterReset)
|
||||
testQuery(t, "foo", "bar1", map[string][]chunks.Sample{series1.String(): exp1})
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("series starting with float and then getting histograms", func(t *testing.T) {
|
||||
appendFloat(series2, 100, 100, &exp2)
|
||||
appendFloat(series2, 101, 101, &exp2)
|
||||
appendFloat(series2, 102, 102, &exp2)
|
||||
testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
appendFloat(t, series2, 100, 100, &exp2)
|
||||
appendFloat(t, series2, 101, 101, &exp2)
|
||||
appendFloat(t, series2, 102, 102, &exp2)
|
||||
testQuery(t, "foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
|
||||
h := baseH.Copy()
|
||||
appendHistogram(series2, 103, h, &exp2, histogram.UnknownCounterReset)
|
||||
appendHistogram(series2, 104, h, &exp2, histogram.NotCounterReset)
|
||||
appendHistogram(series2, 105, h, &exp2, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
appendHistogram(t, series2, 103, h, &exp2, histogram.UnknownCounterReset)
|
||||
appendHistogram(t, series2, 104, h, &exp2, histogram.NotCounterReset)
|
||||
appendHistogram(t, series2, 105, h, &exp2, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
|
||||
// Switching between float and histograms again.
|
||||
appendFloat(series2, 106, 106, &exp2)
|
||||
appendFloat(series2, 107, 107, &exp2)
|
||||
testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
appendFloat(t, series2, 106, 106, &exp2)
|
||||
appendFloat(t, series2, 107, 107, &exp2)
|
||||
testQuery(t, "foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
|
||||
appendHistogram(series2, 108, h, &exp2, histogram.UnknownCounterReset)
|
||||
appendHistogram(series2, 109, h, &exp2, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
appendHistogram(t, series2, 108, h, &exp2, histogram.UnknownCounterReset)
|
||||
appendHistogram(t, series2, 109, h, &exp2, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar2", map[string][]chunks.Sample{series2.String(): exp2})
|
||||
})
|
||||
|
||||
t.Run("series starting with histogram and then getting float", func(t *testing.T) {
|
||||
h := baseH.Copy()
|
||||
appendHistogram(series3, 101, h, &exp3, histogram.UnknownCounterReset)
|
||||
appendHistogram(series3, 102, h, &exp3, histogram.NotCounterReset)
|
||||
appendHistogram(series3, 103, h, &exp3, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
appendHistogram(t, series3, 101, h, &exp3, histogram.UnknownCounterReset)
|
||||
appendHistogram(t, series3, 102, h, &exp3, histogram.NotCounterReset)
|
||||
appendHistogram(t, series3, 103, h, &exp3, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
|
||||
appendFloat(series3, 104, 100, &exp3)
|
||||
appendFloat(series3, 105, 101, &exp3)
|
||||
appendFloat(series3, 106, 102, &exp3)
|
||||
testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
appendFloat(t, series3, 104, 100, &exp3)
|
||||
appendFloat(t, series3, 105, 101, &exp3)
|
||||
appendFloat(t, series3, 106, 102, &exp3)
|
||||
testQuery(t, "foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
|
||||
// Switching between histogram and float again.
|
||||
appendHistogram(series3, 107, h, &exp3, histogram.UnknownCounterReset)
|
||||
appendHistogram(series3, 108, h, &exp3, histogram.NotCounterReset)
|
||||
testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
appendHistogram(t, series3, 107, h, &exp3, histogram.UnknownCounterReset)
|
||||
appendHistogram(t, series3, 108, h, &exp3, histogram.NotCounterReset)
|
||||
testQuery(t, "foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
|
||||
appendFloat(series3, 109, 106, &exp3)
|
||||
appendFloat(series3, 110, 107, &exp3)
|
||||
testQuery("foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
appendFloat(t, series3, 109, 106, &exp3)
|
||||
appendFloat(t, series3, 110, 107, &exp3)
|
||||
testQuery(t, "foo", "bar3", map[string][]chunks.Sample{series3.String(): exp3})
|
||||
})
|
||||
|
||||
t.Run("query mix of histogram and float series", func(t *testing.T) {
|
||||
// A float only series.
|
||||
appendFloat(series4, 100, 100, &exp4)
|
||||
appendFloat(series4, 101, 101, &exp4)
|
||||
appendFloat(series4, 102, 102, &exp4)
|
||||
appendFloat(t, series4, 100, 100, &exp4)
|
||||
appendFloat(t, series4, 101, 101, &exp4)
|
||||
appendFloat(t, series4, 102, 102, &exp4)
|
||||
|
||||
testQuery("foo", "bar.*", map[string][]chunks.Sample{
|
||||
testQuery(t, "foo", "bar.*", map[string][]chunks.Sample{
|
||||
series1.String(): exp1,
|
||||
series2.String(): exp2,
|
||||
series3.String(): exp3,
|
||||
|
|
|
|||
|
|
@ -6405,7 +6405,7 @@ func TestHeadAppender_AppendCT(t *testing.T) {
|
|||
expectedSamples: []chunks.Sample{
|
||||
sample{t: 1, h: &histogram.Histogram{}},
|
||||
sample{t: 100, h: testHistogram},
|
||||
sample{t: 101, h: &histogram.Histogram{CounterResetHint: histogram.CounterReset}},
|
||||
sample{t: 101, h: &histogram.Histogram{CounterResetHint: histogram.UnknownCounterReset}},
|
||||
sample{t: 102, h: testHistogram},
|
||||
},
|
||||
},
|
||||
|
|
@ -6418,7 +6418,7 @@ func TestHeadAppender_AppendCT(t *testing.T) {
|
|||
expectedSamples: []chunks.Sample{
|
||||
sample{t: 1, fh: &histogram.FloatHistogram{}},
|
||||
sample{t: 100, fh: testFloatHistogram},
|
||||
sample{t: 101, fh: &histogram.FloatHistogram{CounterResetHint: histogram.CounterReset}},
|
||||
sample{t: 101, fh: &histogram.FloatHistogram{CounterResetHint: histogram.UnknownCounterReset}},
|
||||
sample{t: 102, fh: testFloatHistogram},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -166,6 +166,8 @@ func TestOOOChunks_ToEncodedChunks(t *testing.T) {
|
|||
h2 := h1.Copy()
|
||||
h2.PositiveSpans = append(h2.PositiveSpans, histogram.Span{Offset: 1, Length: 1})
|
||||
h2.PositiveBuckets = append(h2.PositiveBuckets, 12)
|
||||
h2explicit := h2.Copy()
|
||||
h2explicit.CounterResetHint = histogram.CounterReset
|
||||
|
||||
testCases := map[string]struct {
|
||||
samples []sample
|
||||
|
|
@ -198,12 +200,32 @@ func TestOOOChunks_ToEncodedChunks(t *testing.T) {
|
|||
{encoding: chunkenc.EncXOR, minTime: 1200, maxTime: 1200},
|
||||
},
|
||||
},
|
||||
"has a counter reset": {
|
||||
"has an implicit counter reset": {
|
||||
samples: []sample{
|
||||
{t: 1000, h: h2},
|
||||
{t: 1100, h: h1},
|
||||
},
|
||||
expectedCounterResets: []histogram.CounterResetHint{histogram.UnknownCounterReset, histogram.CounterReset},
|
||||
expectedCounterResets: []histogram.CounterResetHint{histogram.UnknownCounterReset, histogram.UnknownCounterReset},
|
||||
expectedChunks: []chunkVerify{
|
||||
{encoding: chunkenc.EncHistogram, minTime: 1000, maxTime: 1000},
|
||||
{encoding: chunkenc.EncHistogram, minTime: 1100, maxTime: 1100},
|
||||
},
|
||||
},
|
||||
"has an explicit counter reset": {
|
||||
samples: []sample{
|
||||
{t: 1100, h: h2explicit},
|
||||
},
|
||||
expectedCounterResets: []histogram.CounterResetHint{histogram.UnknownCounterReset},
|
||||
expectedChunks: []chunkVerify{
|
||||
{encoding: chunkenc.EncHistogram, minTime: 1100, maxTime: 1100},
|
||||
},
|
||||
},
|
||||
"has an explicit counter reset inside": {
|
||||
samples: []sample{
|
||||
{t: 1000, h: h1},
|
||||
{t: 1100, h: h2explicit},
|
||||
},
|
||||
expectedCounterResets: []histogram.CounterResetHint{histogram.UnknownCounterReset, histogram.UnknownCounterReset},
|
||||
expectedChunks: []chunkVerify{
|
||||
{encoding: chunkenc.EncHistogram, minTime: 1000, maxTime: 1000},
|
||||
{encoding: chunkenc.EncHistogram, minTime: 1100, maxTime: 1100},
|
||||
|
|
|
|||
|
|
@ -111,7 +111,11 @@ func requireEqualSeries(t *testing.T, expected, actual map[string][]chunks.Sampl
|
|||
for name, expectedItem := range expected {
|
||||
actualItem, ok := actual[name]
|
||||
require.True(t, ok, "Expected series %s not found", name)
|
||||
if ignoreCounterResets {
|
||||
requireEqualSamples(t, name, expectedItem, actualItem, requireEqualSamplesIgnoreCounterResets)
|
||||
} else {
|
||||
requireEqualSamples(t, name, expectedItem, actualItem)
|
||||
}
|
||||
}
|
||||
for name := range actual {
|
||||
_, ok := expected[name]
|
||||
|
|
|
|||
Loading…
Reference in New Issue