github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make head Postings only return series in time range 

benchmark                                                old ns/op      new ns/op      delta
BenchmarkQuerierSelect/Head/1of1000000-8                 405805161      120436132      -70.32%
BenchmarkQuerierSelect/Head/10of1000000-8                403079620      120624292      -70.07%
BenchmarkQuerierSelect/Head/100of1000000-8               404678647      120923522      -70.12%
BenchmarkQuerierSelect/Head/1000of1000000-8              403145813      118636563      -70.57%
BenchmarkQuerierSelect/Head/10000of1000000-8             405020046      125716206      -68.96%
BenchmarkQuerierSelect/Head/100000of1000000-8            426305002      175808499      -58.76%
BenchmarkQuerierSelect/Head/1000000of1000000-8           619002108      567013003      -8.40%
BenchmarkQuerierSelect/SortedHead/1of1000000-8           1276316086     120281094      -90.58%
BenchmarkQuerierSelect/SortedHead/10of1000000-8          1282631170     121836526      -90.50%
BenchmarkQuerierSelect/SortedHead/100of1000000-8         1325824787     121174967      -90.86%
BenchmarkQuerierSelect/SortedHead/1000of1000000-8        1271386268     121025117      -90.48%
BenchmarkQuerierSelect/SortedHead/10000of1000000-8       1280223345     130838948      -89.78%
BenchmarkQuerierSelect/SortedHead/100000of1000000-8      1271401620     243635515      -80.84%
BenchmarkQuerierSelect/SortedHead/1000000of1000000-8     1360256090     1307744674     -3.86%
BenchmarkQuerierSelect/Block/1of1000000-8                748183120      707888498      -5.39%
BenchmarkQuerierSelect/Block/10of1000000-8               741084129      716317249      -3.34%
BenchmarkQuerierSelect/Block/100of1000000-8              722157273      735624256      +1.86%
BenchmarkQuerierSelect/Block/1000of1000000-8             727587744      731981838      +0.60%
BenchmarkQuerierSelect/Block/10000of1000000-8            727518578      726860308      -0.09%
BenchmarkQuerierSelect/Block/100000of1000000-8           765577046      757382386      -1.07%
BenchmarkQuerierSelect/Block/1000000of1000000-8          1126722881     1084779083     -3.72%

benchmark                                                old allocs     new allocs     delta
BenchmarkQuerierSelect/Head/1of1000000-8                 4000018        24             -100.00%
BenchmarkQuerierSelect/Head/10of1000000-8                4000036        82             -100.00%
BenchmarkQuerierSelect/Head/100of1000000-8               4000216        625            -99.98%
BenchmarkQuerierSelect/Head/1000of1000000-8              4002016        6028           -99.85%
BenchmarkQuerierSelect/Head/10000of1000000-8             4020016        60037          -98.51%
BenchmarkQuerierSelect/Head/100000of1000000-8            4200016        600047         -85.71%
BenchmarkQuerierSelect/Head/1000000of1000000-8           6000016        6000016        +0.00%
BenchmarkQuerierSelect/SortedHead/1of1000000-8           4000055        28             -100.00%
BenchmarkQuerierSelect/SortedHead/10of1000000-8          4000073        87             -100.00%
BenchmarkQuerierSelect/SortedHead/100of1000000-8         4000253        630            -99.98%
BenchmarkQuerierSelect/SortedHead/1000of1000000-8        4002053        6036           -99.85%
BenchmarkQuerierSelect/SortedHead/10000of1000000-8       4020053        60054          -98.51%
BenchmarkQuerierSelect/SortedHead/100000of1000000-8      4200053        600074         -85.71%
BenchmarkQuerierSelect/SortedHead/1000000of1000000-8     6000053        6000053        +0.00%
BenchmarkQuerierSelect/Block/1of1000000-8                6000021        6000021        +0.00%
BenchmarkQuerierSelect/Block/10of1000000-8               6000057        6000057        +0.00%
BenchmarkQuerierSelect/Block/100of1000000-8              6000417        6000417        +0.00%
BenchmarkQuerierSelect/Block/1000of1000000-8             6004017        6004017        +0.00%
BenchmarkQuerierSelect/Block/10000of1000000-8            6040017        6040017        +0.00%
BenchmarkQuerierSelect/Block/100000of1000000-8           6400017        6400017        +0.00%
BenchmarkQuerierSelect/Block/1000000of1000000-8          10000018       10000018       +0.00%

benchmark                                                old bytes     new bytes     delta
BenchmarkQuerierSelect/Head/1of1000000-8                 176001177     1392          -100.00%
BenchmarkQuerierSelect/Head/10of1000000-8                176002329     4368          -100.00%
BenchmarkQuerierSelect/Head/100of1000000-8               176013849     33520         -99.98%
BenchmarkQuerierSelect/Head/1000of1000000-8              176129056     321456        -99.82%
BenchmarkQuerierSelect/Head/10000of1000000-8             177281049     3427376       -98.07%
BenchmarkQuerierSelect/Head/100000of1000000-8            188801049     35055408      -81.43%
BenchmarkQuerierSelect/Head/1000000of1000000-8           304001059     304001049     -0.00%
BenchmarkQuerierSelect/SortedHead/1of1000000-8           229192188     2488          -100.00%
BenchmarkQuerierSelect/SortedHead/10of1000000-8          229193340     5568          -100.00%
BenchmarkQuerierSelect/SortedHead/100of1000000-8         229204860     35536         -99.98%
BenchmarkQuerierSelect/SortedHead/1000of1000000-8        229320060     345104        -99.85%
BenchmarkQuerierSelect/SortedHead/10000of1000000-8       230472060     3894672       -98.31%
BenchmarkQuerierSelect/SortedHead/100000of1000000-8      241992060     40511632      -83.26%
BenchmarkQuerierSelect/SortedHead/1000000of1000000-8     357192060     357192060     +0.00%
BenchmarkQuerierSelect/Block/1of1000000-8                227201516     227201506     -0.00%
BenchmarkQuerierSelect/Block/10of1000000-8               227203057     227203041     -0.00%
BenchmarkQuerierSelect/Block/100of1000000-8              227217161     227217165     +0.00%
BenchmarkQuerierSelect/Block/1000of1000000-8             227358279     227358289     +0.00%
BenchmarkQuerierSelect/Block/10000of1000000-8            228769485     228769475     -0.00%
BenchmarkQuerierSelect/Block/100000of1000000-8           242881487     242881477     -0.00%
BenchmarkQuerierSelect/Block/1000000of1000000-8          384001705     384001705     +0.00%

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>
pull/6676/head
Julien Pivotto 2020-01-21 20:30:20 +01:00
parent cebe36c7d5
commit 52630ad0c7
12 changed files with 127 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tsdb/block.go
View File

@ -111,7 +111,8 @@ type ChunkReader interface {
// BlockReader provides reading access to a data block. // BlockReader provides reading access to a data block.
type BlockReader interface { type BlockReader interface {
// Index returns an IndexReader over the block's data. // Index returns an IndexReader over the block's data within the specified
// timeframe.
Index(mint, maxt int64) (IndexReader, error) Index(mint, maxt int64) (IndexReader, error)
// Chunks returns a ChunkReader over the block's data. // Chunks returns a ChunkReader over the block's data.

2
tsdb/cmd/tsdb/main.go
View File

@ -470,7 +470,7 @@ func analyzeBlock(b tsdb.BlockReader, limit int) error {
// Presume 1ms resolution that Prometheus uses. // Presume 1ms resolution that Prometheus uses.
fmt.Printf("Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String()) fmt.Printf("Duration: %s\n", (time.Duration(meta.MaxTime-meta.MinTime) * 1e6).String())
fmt.Printf("Series: %d\n", meta.Stats.NumSeries) fmt.Printf("Series: %d\n", meta.Stats.NumSeries)
ir, err := b.Index() ir, err := b.Index(math.MinInt64, math.MaxInt64)
if err != nil { if err != nil {
return err return err
} }

2
tsdb/compact.go
View File

@ -683,7 +683,7 @@ func (c *LeveledCompactor) populateBlock(blocks []BlockReader, meta *BlockMeta,
} }
} }
indexr, err := b.Index(math.MinInt64, math.MaxInt64) indexr, err := b.Index(math.MinInt64, globalMaxt)
if err != nil { if err != nil {
return errors.Wrapf(err, "open index reader for block %s", b) return errors.Wrapf(err, "open index reader for block %s", b)
} }

6
tsdb/compact_test.go
View File

@ -456,7 +456,7 @@ func metaRange(name string, mint, maxt int64, stats *BlockStats) dirMeta {
type erringBReader struct{} type erringBReader struct{}
func (erringBReader) Index() (IndexReader, error) { return nil, errors.New("index") } func (erringBReader) Index(int64, int64) (IndexReader, error) { return nil, errors.New("index") }
func (erringBReader) Chunks() (ChunkReader, error) { return nil, errors.New("chunks") } func (erringBReader) Chunks() (ChunkReader, error) { return nil, errors.New("chunks") }
func (erringBReader) Tombstones() (tombstones.Reader, error) { return nil, errors.New("tombstones") } func (erringBReader) Tombstones() (tombstones.Reader, error) { return nil, errors.New("tombstones") }
func (erringBReader) Meta() BlockMeta { return BlockMeta{} } func (erringBReader) Meta() BlockMeta { return BlockMeta{} }
@ -652,7 +652,7 @@ func TestCompaction_populateBlock(t *testing.T) {
expErr: errors.New("found chunk with minTime: 10 maxTime: 30 outside of compacted minTime: 0 maxTime: 20"), expErr: errors.New("found chunk with minTime: 10 maxTime: 30 outside of compacted minTime: 0 maxTime: 20"),
}, },
{ {
// Introduced by https://github.com/prometheus/prometheus/tsdb/issues/347. // Introduced by https://github.com/prometheus/tsdb/issues/347.
title: "Populate from single block containing extra chunk", title: "Populate from single block containing extra chunk",
inputSeriesSamples: [][]seriesSamples{ inputSeriesSamples: [][]seriesSamples{
{ {
@ -692,7 +692,7 @@ func TestCompaction_populateBlock(t *testing.T) {
}, },
}, },
{ {
// Introduced by https://github.com/prometheus/prometheus/tsdb/pull/539. // Introduced by https://github.com/prometheus/tsdb/pull/539.
title: "Populate from three blocks that the last two are overlapping.", title: "Populate from three blocks that the last two are overlapping.",
inputSeriesSamples: [][]seriesSamples{ inputSeriesSamples: [][]seriesSamples{
{ {

8
tsdb/db_test.go
View File

@ -1384,7 +1384,7 @@ func TestOverlappingBlocksDetectsAllOverlaps(t *testing.T) {
}, OverlappingBlocks(nc1)) }, OverlappingBlocks(nc1))
} }
// Regression test for https://github.com/prometheus/prometheus/tsdb/issues/347 // Regression test for https://github.com/prometheus/tsdb/issues/347
func TestChunkAtBlockBoundary(t *testing.T) { func TestChunkAtBlockBoundary(t *testing.T) {
db, closeFn := openTestDB(t, nil, nil) db, closeFn := openTestDB(t, nil, nil)
defer func() { defer func() {
@ -1411,7 +1411,7 @@ func TestChunkAtBlockBoundary(t *testing.T) {
testutil.Ok(t, err) testutil.Ok(t, err)
for _, block := range db.Blocks() { for _, block := range db.Blocks() {
r, err := block.Index() r, err := block.Index(math.MinInt64, math.MaxInt64)
testutil.Ok(t, err) testutil.Ok(t, err)
defer r.Close() defer r.Close()
@ -1761,7 +1761,7 @@ func TestDB_LabelNames(t *testing.T) {
appendSamples(db, 0, 4, tst.sampleLabels1) appendSamples(db, 0, 4, tst.sampleLabels1)
// Testing head. // Testing head.
headIndexr, err := db.head.Index() headIndexr, err := db.head.Index(math.MinInt64, math.MaxInt64)
testutil.Ok(t, err) testutil.Ok(t, err)
labelNames, err := headIndexr.LabelNames() labelNames, err := headIndexr.LabelNames()
testutil.Ok(t, err) testutil.Ok(t, err)
@ -1774,7 +1774,7 @@ func TestDB_LabelNames(t *testing.T) {
// All blocks have same label names, hence check them individually. // All blocks have same label names, hence check them individually.
// No need to aggregate and check. // No need to aggregate and check.
for _, b := range db.Blocks() { for _, b := range db.Blocks() {
blockIndexr, err := b.Index() blockIndexr, err := b.Index(math.MinInt64, math.MaxInt64)
testutil.Ok(t, err) testutil.Ok(t, err)
labelNames, err = blockIndexr.LabelNames() labelNames, err = blockIndexr.LabelNames()
testutil.Ok(t, err) testutil.Ok(t, err)

19
tsdb/head.go
View File

@ -754,7 +754,7 @@ type RangeHead struct {
mint, maxt int64 mint, maxt int64
} }
// NewRangeHead returns a *rangeHead. // NewRangeHead returns a *RangeHead.
func NewRangeHead(head *Head, mint, maxt int64) *RangeHead { func NewRangeHead(head *Head, mint, maxt int64) *RangeHead {
return &RangeHead{ return &RangeHead{
head: head, head: head,
@ -764,7 +764,14 @@ func NewRangeHead(head *Head, mint, maxt int64) *RangeHead {
} }
func (h *RangeHead) Index(mint, maxt int64) (IndexReader, error) { func (h *RangeHead) Index(mint, maxt int64) (IndexReader, error) {
return h.head.indexRange(h.mint, h.maxt), nil // rangeHead guarantees that the series returned are within its range.
if mint < h.mint {
mint = h.mint
}
if maxt > h.maxt {
maxt = h.maxt
}
return h.head.indexRange(mint, maxt), nil
} }
func (h *RangeHead) Chunks() (ChunkReader, error) { func (h *RangeHead) Chunks() (ChunkReader, error) {
@ -1347,9 +1354,17 @@ func (h *headIndexReader) LabelNames() ([]string, error) {
// Postings returns the postings list iterator for the label pairs. // Postings returns the postings list iterator for the label pairs.
func (h *headIndexReader) Postings(name string, values ...string) (index.Postings, error) { func (h *headIndexReader) Postings(name string, values ...string) (index.Postings, error) {
fullRange := h.mint <= h.head.MinTime() && h.maxt >= h.head.MaxTime()
res := make([]index.Postings, 0, len(values)) res := make([]index.Postings, 0, len(values))
for _, value := range values { for _, value := range values {
p := h.head.postings.Get(name, value) p := h.head.postings.Get(name, value)
if fullRange {
// The head timerange covers the full index reader timerange.
// All the series can the be appended without filtering.
res = append(res, p)
continue
}
// Filter out series not in the time range, to avoid // Filter out series not in the time range, to avoid
// later on building up all the chunk metadata just to // later on building up all the chunk metadata just to
// discard it. // discard it.

33
tsdb/head_bench_test.go
View File

@ -14,7 +14,6 @@
package tsdb package tsdb
import ( import (
"fmt"
"strconv" "strconv"
"sync/atomic" "sync/atomic"
"testing" "testing"
@ -49,35 +48,3 @@ func BenchmarkHeadStripeSeriesCreateParallel(b *testing.B) {
} }
}) })
} }
func BenchmarkHeadSeries(b *testing.B) {
h, err := NewHead(nil, nil, nil, 1000)
testutil.Ok(b, err)
defer h.Close()
app := h.Appender()
numSeries := 1000000
for i := 0; i < numSeries; i++ {
app.Add(labels.FromStrings("foo", "bar", "i", strconv.Itoa(i)), int64(i), 0)
}
testutil.Ok(b, app.Commit())
matcher := labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")
for s := 1; s <= numSeries; s *= 10 {
b.Run(fmt.Sprintf("%dof%d", s, numSeries), func(b *testing.B) {
q, err := NewBlockQuerier(h, 0, int64(s-1))
testutil.Ok(b, err)
b.ResetTimer()
for i := 0; i < b.N; i++ {
ss, err := q.Select(matcher)
testutil.Ok(b, err)
for ss.Next() {
}
testutil.Ok(b, ss.Err())
}
q.Close()
})
}
}

88
tsdb/head_test.go
View File

@ -1322,3 +1322,91 @@ func TestAddDuplicateLabelName(t *testing.T) {
add(labels.Labels{{Name: "a", Value: "c"}, {Name: "a", Value: "c"}}, "a") add(labels.Labels{{Name: "a", Value: "c"}, {Name: "a", Value: "c"}}, "a")
add(labels.Labels{{Name: "__name__", Value: "up"}, {Name: "job", Value: "prometheus"}, {Name: "le", Value: "500"}, {Name: "le", Value: "400"}, {Name: "unit", Value: "s"}}, "le") add(labels.Labels{{Name: "__name__", Value: "up"}, {Name: "job", Value: "prometheus"}, {Name: "le", Value: "500"}, {Name: "le", Value: "400"}, {Name: "unit", Value: "s"}}, "le")
} }
func TestHeadSeriesWithTimeBoundaries(t *testing.T) {
h, err := NewHead(nil, nil, nil, 15, DefaultStripeSize)
testutil.Ok(t, err)
defer h.Close()
app := h.Appender()
s1, err := app.Add(labels.FromStrings("foo1", "bar"), 2, 0)
testutil.Ok(t, err)
for ts := int64(3); ts < 13; ts++ {
err = app.AddFast(s1, ts, 0)
testutil.Ok(t, err)
}
s2, err := app.Add(labels.FromStrings("foo2", "bar"), 5, 0)
testutil.Ok(t, err)
for ts := int64(6); ts < 11; ts++ {
err = app.AddFast(s2, ts, 0)
testutil.Ok(t, err)
}
s3, err := app.Add(labels.FromStrings("foo3", "bar"), 5, 0)
testutil.Ok(t, err)
err = app.AddFast(s3, 6, 0)
testutil.Ok(t, err)
_, err = app.Add(labels.FromStrings("foo4", "bar"), 9, 0)
testutil.Ok(t, err)
testutil.Ok(t, app.Commit())
cases := []struct {
mint int64
maxt int64
seriesCount int
samplesCount int
}{
// foo1 ..00000000000..
// foo2 .....000000....
// foo3 .....00........
// foo4 .........0.....
{mint: 0, maxt: 0, seriesCount: 0, samplesCount: 0},
{mint: 0, maxt: 1, seriesCount: 0, samplesCount: 0},
{mint: 0, maxt: 2, seriesCount: 1, samplesCount: 1},
{mint: 2, maxt: 2, seriesCount: 1, samplesCount: 1},
{mint: 0, maxt: 4, seriesCount: 1, samplesCount: 3},
{mint: 0, maxt: 5, seriesCount: 3, samplesCount: 6},
{mint: 0, maxt: 6, seriesCount: 3, samplesCount: 9},
{mint: 0, maxt: 7, seriesCount: 3, samplesCount: 11},
{mint: 0, maxt: 8, seriesCount: 3, samplesCount: 13},
{mint: 0, maxt: 9, seriesCount: 4, samplesCount: 16},
{mint: 0, maxt: 10, seriesCount: 4, samplesCount: 18},
{mint: 0, maxt: 11, seriesCount: 4, samplesCount: 19},
{mint: 0, maxt: 12, seriesCount: 4, samplesCount: 20},
{mint: 0, maxt: 13, seriesCount: 4, samplesCount: 20},
{mint: 0, maxt: 14, seriesCount: 4, samplesCount: 20},
{mint: 2, maxt: 14, seriesCount: 4, samplesCount: 20},
{mint: 3, maxt: 14, seriesCount: 4, samplesCount: 19},
{mint: 4, maxt: 14, seriesCount: 4, samplesCount: 18},
{mint: 8, maxt: 9, seriesCount: 3, samplesCount: 5},
{mint: 9, maxt: 9, seriesCount: 3, samplesCount: 3},
{mint: 6, maxt: 9, seriesCount: 4, samplesCount: 10},
{mint: 11, maxt: 11, seriesCount: 1, samplesCount: 1},
{mint: 11, maxt: 12, seriesCount: 1, samplesCount: 2},
{mint: 11, maxt: 14, seriesCount: 1, samplesCount: 2},
{mint: 12, maxt: 14, seriesCount: 1, samplesCount: 1},
}
for i, c := range cases {
matcher := labels.MustNewMatcher(labels.MatchEqual, "", "")
q, err := NewBlockQuerier(h, c.mint, c.maxt)
testutil.Ok(t, err)
seriesCount := 0
samplesCount := 0
ss, _, err := q.Select(nil, matcher)
testutil.Ok(t, err)
for ss.Next() {
i := ss.At().Iterator()
for i.Next() {
samplesCount++
}
seriesCount++
}
testutil.Ok(t, ss.Err())
testutil.Equals(t, c.seriesCount, seriesCount, "test series %d", i)
testutil.Equals(t, c.samplesCount, samplesCount, "test samples %d", i)
q.Close()
}
}

2
tsdb/mocks_test.go
View File

@ -71,7 +71,7 @@ type mockBReader struct {
maxt int64 maxt int64
} }
func (r *mockBReader) Index() (IndexReader, error) { return r.ir, nil } func (r *mockBReader) Index(mint, maxt int64) (IndexReader, error) { return r.ir, nil }
func (r *mockBReader) Chunks() (ChunkReader, error) { return r.cr, nil } func (r *mockBReader) Chunks() (ChunkReader, error) { return r.cr, nil }
func (r *mockBReader) Tombstones() (tombstones.Reader, error) { func (r *mockBReader) Tombstones() (tombstones.Reader, error) {
return tombstones.NewMemTombstones(), nil return tombstones.NewMemTombstones(), nil

5
tsdb/querier_bench_test.go
View File

@ -16,6 +16,7 @@ package tsdb
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"math"
"os" "os"
"strconv" "strconv"
"testing" "testing"
@ -54,7 +55,7 @@ func BenchmarkPostingsForMatchers(b *testing.B) {
} }
testutil.Ok(b, app.Commit()) testutil.Ok(b, app.Commit())
ir, err := h.Index() ir, err := h.Index(math.MinInt64, math.MaxInt64)
testutil.Ok(b, err) testutil.Ok(b, err)
b.Run("Head", func(b *testing.B) { b.Run("Head", func(b *testing.B) {
benchmarkPostingsForMatchers(b, ir) benchmarkPostingsForMatchers(b, ir)
@ -72,7 +73,7 @@ func BenchmarkPostingsForMatchers(b *testing.B) {
defer func() { defer func() {
testutil.Ok(b, block.Close()) testutil.Ok(b, block.Close())
}() }()
ir, err = block.Index() ir, err = block.Index(math.MinInt64, math.MaxInt64)
testutil.Ok(b, err) testutil.Ok(b, err)
defer ir.Close() defer ir.Close()
b.Run("Block", func(b *testing.B) { b.Run("Block", func(b *testing.B) {

4
tsdb/querier_test.go
View File

@ -1044,7 +1044,7 @@ func TestSeriesIterator(t *testing.T) {
}) })
} }
// Regression for: https://github.com/prometheus/prometheus/tsdb/pull/97 // Regression for: https://github.com/prometheus/tsdb/pull/97
func TestChunkSeriesIterator_DoubleSeek(t *testing.T) { func TestChunkSeriesIterator_DoubleSeek(t *testing.T) {
chkMetas := []chunks.Meta{ chkMetas := []chunks.Meta{
tsdbutil.ChunkFromSamples([]tsdbutil.Sample{}), tsdbutil.ChunkFromSamples([]tsdbutil.Sample{}),
@ -2137,7 +2137,7 @@ func TestPostingsForMatchers(t *testing.T) {
}, },
} }
ir, err := h.Index() ir, err := h.Index(math.MinInt64, math.MaxInt64)
testutil.Ok(t, err) testutil.Ok(t, err)
for _, c := range cases { for _, c := range cases {

2
tsdb/wal/wal_test.go
View File

@ -49,7 +49,7 @@ func TestWALRepair_ReadingError(t *testing.T) {
}, },
// Ensures that the page buffer is big enough to fit // Ensures that the page buffer is big enough to fit
// an entire page size without panicking. // an entire page size without panicking.
// https://github.com/prometheus/prometheus/tsdb/pull/414 // https://github.com/prometheus/tsdb/pull/414
"bad_header": { "bad_header": {
1, 1,
func(f *os.File) { func(f *os.File) {