mirror of https://github.com/prometheus/prometheus
448 lines
12 KiB
Go
448 lines
12 KiB
Go
// Copyright 2017 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package storage
|
|
|
|
import (
|
|
"math/rand"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/prometheus/prometheus/model/histogram"
|
|
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
|
"github.com/prometheus/prometheus/tsdb/tsdbutil"
|
|
)
|
|
|
|
func TestSampleRing(t *testing.T) {
|
|
cases := []struct {
|
|
input []int64
|
|
delta int64
|
|
size int
|
|
}{
|
|
{
|
|
input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
|
delta: 2,
|
|
size: 1,
|
|
},
|
|
{
|
|
input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
|
delta: 2,
|
|
size: 2,
|
|
},
|
|
{
|
|
input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
|
delta: 7,
|
|
size: 3,
|
|
},
|
|
{
|
|
input: []int64{1, 2, 3, 4, 5, 16, 17, 18, 19, 20},
|
|
delta: 7,
|
|
size: 1,
|
|
},
|
|
{
|
|
input: []int64{1, 2, 3, 4, 6},
|
|
delta: 4,
|
|
size: 4,
|
|
},
|
|
}
|
|
for _, c := range cases {
|
|
r := newSampleRing(c.delta, c.size, chunkenc.ValFloat)
|
|
|
|
input := []fSample{}
|
|
for _, t := range c.input {
|
|
input = append(input, fSample{
|
|
t: t,
|
|
f: float64(rand.Intn(100)),
|
|
})
|
|
}
|
|
|
|
for i, s := range input {
|
|
r.add(s)
|
|
buffered := r.samples()
|
|
|
|
for _, sold := range input[:i] {
|
|
found := false
|
|
for _, bs := range buffered {
|
|
if bs.T() == sold.t && bs.F() == sold.f {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if found {
|
|
require.GreaterOrEqual(t, sold.t, s.t-c.delta, "%d: unexpected sample %d in buffer; buffer %v", i, sold.t, buffered)
|
|
} else {
|
|
require.Less(t, sold.t, s.t-c.delta, "%d: expected sample %d to be in buffer but was not; buffer %v", i, sold.t, buffered)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSampleRingMixed(t *testing.T) {
|
|
h1 := tsdbutil.GenerateTestHistogram(1)
|
|
h2 := tsdbutil.GenerateTestHistogram(2)
|
|
|
|
// With ValNone as the preferred type, nothing should be initialized.
|
|
r := newSampleRing(10, 2, chunkenc.ValNone)
|
|
require.Empty(t, r.fBuf)
|
|
require.Empty(t, r.hBuf)
|
|
require.Empty(t, r.fhBuf)
|
|
require.Empty(t, r.iBuf)
|
|
|
|
// But then mixed adds should work as expected.
|
|
r.addF(fSample{t: 1, f: 3.14})
|
|
r.addH(hSample{t: 2, h: h1})
|
|
|
|
it := r.iterator()
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Next())
|
|
ts, f := it.At()
|
|
require.Equal(t, int64(1), ts)
|
|
require.Equal(t, 3.14, f)
|
|
require.Equal(t, chunkenc.ValHistogram, it.Next())
|
|
var h *histogram.Histogram
|
|
ts, h = it.AtHistogram()
|
|
require.Equal(t, int64(2), ts)
|
|
require.Equal(t, h1, h)
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
|
|
r.reset()
|
|
it = r.iterator()
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
|
|
r.addF(fSample{t: 3, f: 4.2})
|
|
r.addH(hSample{t: 4, h: h2})
|
|
|
|
it = r.iterator()
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Next())
|
|
ts, f = it.At()
|
|
require.Equal(t, int64(3), ts)
|
|
require.Equal(t, 4.2, f)
|
|
require.Equal(t, chunkenc.ValHistogram, it.Next())
|
|
ts, h = it.AtHistogram()
|
|
require.Equal(t, int64(4), ts)
|
|
require.Equal(t, h2, h)
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
}
|
|
|
|
func TestSampleRingAtFloatHistogram(t *testing.T) {
|
|
fh1 := tsdbutil.GenerateTestFloatHistogram(1)
|
|
fh2 := tsdbutil.GenerateTestFloatHistogram(2)
|
|
h1 := tsdbutil.GenerateTestHistogram(3)
|
|
h2 := tsdbutil.GenerateTestHistogram(4)
|
|
|
|
// With ValNone as the preferred type, nothing should be initialized.
|
|
r := newSampleRing(10, 2, chunkenc.ValNone)
|
|
require.Empty(t, r.fBuf)
|
|
require.Empty(t, r.hBuf)
|
|
require.Empty(t, r.fhBuf)
|
|
require.Empty(t, r.iBuf)
|
|
|
|
var (
|
|
h *histogram.Histogram
|
|
fh *histogram.FloatHistogram
|
|
ts int64
|
|
)
|
|
|
|
it := r.iterator()
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
|
|
r.addFH(fhSample{t: 1, fh: fh1})
|
|
r.addFH(fhSample{t: 2, fh: fh2})
|
|
|
|
it = r.iterator()
|
|
|
|
require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
|
|
ts, fh = it.AtFloatHistogram(fh)
|
|
require.Equal(t, int64(1), ts)
|
|
require.Equal(t, fh1, fh)
|
|
require.Equal(t, chunkenc.ValFloatHistogram, it.Next())
|
|
ts, fh = it.AtFloatHistogram(fh)
|
|
require.Equal(t, int64(2), ts)
|
|
require.Equal(t, fh2, fh)
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
|
|
r.reset()
|
|
it = r.iterator()
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
|
|
r.addH(hSample{t: 3, h: h1})
|
|
r.addH(hSample{t: 4, h: h2})
|
|
|
|
it = r.iterator()
|
|
|
|
require.Equal(t, chunkenc.ValHistogram, it.Next())
|
|
ts, h = it.AtHistogram()
|
|
require.Equal(t, int64(3), ts)
|
|
require.Equal(t, h1, h)
|
|
ts, fh = it.AtFloatHistogram(fh)
|
|
require.Equal(t, int64(3), ts)
|
|
require.Equal(t, h1.ToFloat(nil), fh)
|
|
require.Equal(t, chunkenc.ValHistogram, it.Next())
|
|
ts, h = it.AtHistogram()
|
|
require.Equal(t, int64(4), ts)
|
|
require.Equal(t, h2, h)
|
|
ts, fh = it.AtFloatHistogram(fh)
|
|
require.Equal(t, int64(4), ts)
|
|
require.Equal(t, h2.ToFloat(nil), fh)
|
|
require.Equal(t, chunkenc.ValNone, it.Next())
|
|
}
|
|
|
|
func TestBufferedSeriesIterator(t *testing.T) {
|
|
var it *BufferedSeriesIterator
|
|
|
|
bufferEq := func(exp []fSample) {
|
|
var b []fSample
|
|
bit := it.Buffer()
|
|
for bit.Next() == chunkenc.ValFloat {
|
|
t, f := bit.At()
|
|
b = append(b, fSample{t: t, f: f})
|
|
}
|
|
require.Equal(t, exp, b, "buffer mismatch")
|
|
}
|
|
sampleEq := func(ets int64, ev float64) {
|
|
ts, v := it.At()
|
|
require.Equal(t, ets, ts, "timestamp mismatch")
|
|
require.Equal(t, ev, v, "value mismatch")
|
|
}
|
|
prevSampleEq := func(ets int64, ev float64, eok bool) {
|
|
s, ok := it.PeekBack(1)
|
|
require.Equal(t, eok, ok, "exist mismatch")
|
|
require.Equal(t, ets, s.T(), "timestamp mismatch")
|
|
require.Equal(t, ev, s.F(), "value mismatch")
|
|
}
|
|
|
|
it = NewBufferIterator(NewListSeriesIterator(samples{
|
|
fSample{t: 1, f: 2},
|
|
fSample{t: 2, f: 3},
|
|
fSample{t: 3, f: 4},
|
|
fSample{t: 4, f: 5},
|
|
fSample{t: 5, f: 6},
|
|
fSample{t: 99, f: 8},
|
|
fSample{t: 100, f: 9},
|
|
fSample{t: 101, f: 10},
|
|
}), 2)
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Seek(-123), "seek failed")
|
|
sampleEq(1, 2)
|
|
prevSampleEq(0, 0, false)
|
|
bufferEq(nil)
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
|
|
sampleEq(2, 3)
|
|
prevSampleEq(1, 2, true)
|
|
bufferEq([]fSample{{t: 1, f: 2}})
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
|
|
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
|
|
require.Equal(t, chunkenc.ValFloat, it.Next(), "next failed")
|
|
sampleEq(5, 6)
|
|
prevSampleEq(4, 5, true)
|
|
bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Seek(5), "seek failed")
|
|
sampleEq(5, 6)
|
|
prevSampleEq(4, 5, true)
|
|
bufferEq([]fSample{{t: 2, f: 3}, {t: 3, f: 4}, {t: 4, f: 5}})
|
|
|
|
require.Equal(t, chunkenc.ValFloat, it.Seek(101), "seek failed")
|
|
sampleEq(101, 10)
|
|
prevSampleEq(100, 9, true)
|
|
bufferEq([]fSample{{t: 99, f: 8}, {t: 100, f: 9}})
|
|
|
|
require.Equal(t, chunkenc.ValNone, it.Next(), "next succeeded unexpectedly")
|
|
require.Equal(t, chunkenc.ValNone, it.Seek(1024), "seek succeeded unexpectedly")
|
|
}
|
|
|
|
// At() should not be called once Next() returns false.
|
|
func TestBufferedSeriesIteratorNoBadAt(t *testing.T) {
|
|
done := false
|
|
|
|
m := &mockSeriesIterator{
|
|
seek: func(int64) chunkenc.ValueType { return chunkenc.ValNone },
|
|
at: func() (int64, float64) {
|
|
require.False(t, done, "unexpectedly done")
|
|
done = true
|
|
return 0, 0
|
|
},
|
|
next: func() chunkenc.ValueType {
|
|
if done {
|
|
return chunkenc.ValNone
|
|
}
|
|
return chunkenc.ValFloat
|
|
},
|
|
err: func() error { return nil },
|
|
}
|
|
|
|
it := NewBufferIterator(m, 60)
|
|
it.Next()
|
|
it.Next()
|
|
}
|
|
|
|
func TestBufferedSeriesIteratorMixedHistograms(t *testing.T) {
|
|
histograms := tsdbutil.GenerateTestHistograms(2)
|
|
|
|
it := NewBufferIterator(NewListSeriesIterator(samples{
|
|
fhSample{t: 1, fh: histograms[0].ToFloat(nil)},
|
|
hSample{t: 2, h: histograms[1]},
|
|
}), 2)
|
|
|
|
require.Equal(t, chunkenc.ValNone, it.Seek(3))
|
|
require.NoError(t, it.Err())
|
|
|
|
buf := it.Buffer()
|
|
|
|
require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
|
|
_, fh := buf.AtFloatHistogram(nil)
|
|
require.Equal(t, histograms[0].ToFloat(nil), fh)
|
|
|
|
require.Equal(t, chunkenc.ValHistogram, buf.Next())
|
|
_, fh = buf.AtFloatHistogram(nil)
|
|
require.Equal(t, histograms[1].ToFloat(nil), fh)
|
|
}
|
|
|
|
func TestBufferedSeriesIteratorMixedFloatsAndHistograms(t *testing.T) {
|
|
histograms := tsdbutil.GenerateTestHistograms(5)
|
|
|
|
it := NewBufferIterator(NewListSeriesIteratorWithCopy(samples{
|
|
hSample{t: 1, h: histograms[0].Copy()},
|
|
fSample{t: 2, f: 2},
|
|
hSample{t: 3, h: histograms[1].Copy()},
|
|
hSample{t: 4, h: histograms[2].Copy()},
|
|
fhSample{t: 3, fh: histograms[3].ToFloat(nil)},
|
|
fhSample{t: 4, fh: histograms[4].ToFloat(nil)},
|
|
}), 6)
|
|
|
|
require.Equal(t, chunkenc.ValNone, it.Seek(7))
|
|
require.NoError(t, it.Err())
|
|
|
|
buf := it.Buffer()
|
|
|
|
require.Equal(t, chunkenc.ValHistogram, buf.Next())
|
|
_, h0 := buf.AtHistogram()
|
|
require.Equal(t, histograms[0], h0)
|
|
|
|
require.Equal(t, chunkenc.ValFloat, buf.Next())
|
|
_, v := buf.At()
|
|
require.Equal(t, 2.0, v)
|
|
|
|
require.Equal(t, chunkenc.ValHistogram, buf.Next())
|
|
_, h1 := buf.AtHistogram()
|
|
require.Equal(t, histograms[1], h1)
|
|
|
|
require.Equal(t, chunkenc.ValHistogram, buf.Next())
|
|
_, h2 := buf.AtHistogram()
|
|
require.Equal(t, histograms[2], h2)
|
|
|
|
require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
|
|
_, h3 := buf.AtFloatHistogram(nil)
|
|
require.Equal(t, histograms[3].ToFloat(nil), h3)
|
|
|
|
require.Equal(t, chunkenc.ValFloatHistogram, buf.Next())
|
|
_, h4 := buf.AtFloatHistogram(nil)
|
|
require.Equal(t, histograms[4].ToFloat(nil), h4)
|
|
|
|
// Test for overwrite bug where the buffered histogram was reused
|
|
// between items in the buffer.
|
|
require.Equal(t, histograms[0], h0)
|
|
require.Equal(t, histograms[1], h1)
|
|
require.Equal(t, histograms[2], h2)
|
|
require.Equal(t, histograms[3].ToFloat(nil), h3)
|
|
require.Equal(t, histograms[4].ToFloat(nil), h4)
|
|
}
|
|
|
|
func BenchmarkBufferedSeriesIterator(b *testing.B) {
|
|
// Simulate a 5 minute rate.
|
|
it := NewBufferIterator(newFakeSeriesIterator(int64(b.N), 30), 5*60)
|
|
|
|
b.SetBytes(16)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
|
|
for it.Next() != chunkenc.ValNone {
|
|
// Scan everything.
|
|
}
|
|
require.NoError(b, it.Err())
|
|
}
|
|
|
|
type mockSeriesIterator struct {
|
|
seek func(int64) chunkenc.ValueType
|
|
at func() (int64, float64)
|
|
next func() chunkenc.ValueType
|
|
err func() error
|
|
}
|
|
|
|
func (m *mockSeriesIterator) Seek(t int64) chunkenc.ValueType { return m.seek(t) }
|
|
func (m *mockSeriesIterator) At() (int64, float64) { return m.at() }
|
|
func (m *mockSeriesIterator) Next() chunkenc.ValueType { return m.next() }
|
|
func (m *mockSeriesIterator) Err() error { return m.err() }
|
|
|
|
func (m *mockSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
|
|
return 0, nil // Not really mocked.
|
|
}
|
|
|
|
func (m *mockSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
|
|
return 0, nil // Not really mocked.
|
|
}
|
|
|
|
func (m *mockSeriesIterator) AtT() int64 {
|
|
return 0 // Not really mocked.
|
|
}
|
|
|
|
type fakeSeriesIterator struct {
|
|
nsamples int64
|
|
step int64
|
|
idx int64
|
|
}
|
|
|
|
func newFakeSeriesIterator(nsamples, step int64) *fakeSeriesIterator {
|
|
return &fakeSeriesIterator{nsamples: nsamples, step: step, idx: -1}
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) At() (int64, float64) {
|
|
return it.idx * it.step, 123 // Value doesn't matter.
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
|
|
return it.idx * it.step, &histogram.Histogram{} // Value doesn't matter.
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
|
|
return it.idx * it.step, &histogram.FloatHistogram{} // Value doesn't matter.
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) AtT() int64 {
|
|
return it.idx * it.step
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) Next() chunkenc.ValueType {
|
|
it.idx++
|
|
if it.idx >= it.nsamples {
|
|
return chunkenc.ValNone
|
|
}
|
|
return chunkenc.ValFloat
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) Seek(t int64) chunkenc.ValueType {
|
|
it.idx = t / it.step
|
|
if it.idx >= it.nsamples {
|
|
return chunkenc.ValNone
|
|
}
|
|
return chunkenc.ValFloat
|
|
}
|
|
|
|
func (it *fakeSeriesIterator) Err() error { return nil }
|