mirror of https://github.com/prometheus/prometheus
Merge pull request #9481 from prometheus/beorn7/cleanup
Style cleanup of all the changes in sparsehistogram so farpull/9490/head
Björn Rabenstein
GitHub
commit
12486b1250
|
|
@ -58,9 +58,9 @@ import (
|
|||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/discovery"
|
||||
_ "github.com/prometheus/prometheus/discovery/install" // Register service discovery implementations.
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/notifier"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/logging"
|
||||
"github.com/prometheus/prometheus/pkg/relabel"
|
||||
|
|
@ -1190,7 +1190,7 @@ func (n notReadyAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar
|
|||
return 0, tsdb.ErrNotReady
|
||||
}
|
||||
|
||||
func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
func (n notReadyAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.Histogram) (uint64, error) {
|
||||
return 0, tsdb.ErrNotReady
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -405,7 +405,6 @@ func readQueryLog(t *testing.T, path string) []queryLogLine {
|
|||
}
|
||||
|
||||
func TestQueryLog(t *testing.T) {
|
||||
t.Skip()
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode.")
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -62,7 +62,7 @@ require (
|
|||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
|
||||
golang.org/x/tools v0.1.5
|
||||
golang.org/x/tools v0.1.7
|
||||
google.golang.org/api v0.56.0
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83
|
||||
google.golang.org/protobuf v1.27.1
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -1308,6 +1308,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
|||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
|
||||
github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
|
||||
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
|
||||
|
|
@ -1497,6 +1498,7 @@ golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qx
|
|||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f h1:w6wWR0H+nyVpbSAQbzVEIACVyr/h8l/BEkY6Sokc7Eg=
|
||||
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
|
@ -1634,6 +1636,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34 h1:GkvMjFtXUmahfDtashnc1mnrCtuBVcwse5QV2lUk/tI=
|
||||
golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
@ -1739,8 +1742,9 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
|||
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
|
||||
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
|
||||
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
|
|
|||
|
|
@ -17,55 +17,90 @@ import (
|
|||
"math"
|
||||
)
|
||||
|
||||
// SparseHistogram encodes a sparse histogram
|
||||
// full details: https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit#
|
||||
// the most tricky bit is how bucket indices represent real bucket boundaries
|
||||
// Histogram encodes a sparse, high-resolution histogram. See the design
|
||||
// document for full details:
|
||||
// https://docs.google.com/document/d/1cLNv3aufPZb3fNfaJgdaRBZsInZKKIHo9E6HinJVbpM/edit#
|
||||
//
|
||||
// an example for schema 0 (which doubles the size of consecutive buckets):
|
||||
// The most tricky bit is how bucket indices represent real bucket boundaries.
|
||||
// An example for schema 0 (by which each bucket is twice as wide as the
|
||||
// previous bucket):
|
||||
//
|
||||
// buckets syntax (LE,GE) (-2,-1) (-1,-0.5) (-0.5,-0.25) ... (-0.001-0.001) ... (0.25-0.5)(0.5-1) (1-2) ....
|
||||
// ^
|
||||
// zero bucket (here width a width of 0.001) ZB
|
||||
// pos bucket idx ... -1 0 1 2 3
|
||||
// neg bucket idx 3 2 1 0 -1 ...
|
||||
// actively used bucket indices themselves are represented by the spans
|
||||
type SparseHistogram struct {
|
||||
Schema int32
|
||||
ZeroThreshold float64
|
||||
ZeroCount, Count uint64
|
||||
Sum float64
|
||||
PositiveSpans, NegativeSpans []Span
|
||||
// Bucket boundaries → [-2,-1) [-1,-0.5) [-0.5,-0.25) ... [-0.001,0.001] ... (0.25,0.5] (0.5,1] (1,2] ....
|
||||
// ↑ ↑ ↑ ↑ ↑ ↑ ↑
|
||||
// Zero bucket (width e.g. 0.001) → | | | ZB | | |
|
||||
// Positive bucket indices → | | | ... -1 0 1 2 3
|
||||
// Negative bucket indices → 3 2 1 0 -1 ...
|
||||
//
|
||||
// Wich bucket indices are actually used is determined by the spans.
|
||||
type Histogram struct {
|
||||
// Currently valid schema numbers are -4 <= n <= 8. They are all for
|
||||
// base-2 bucket schemas, where 1 is a bucket boundary in each case, and
|
||||
// then each power of two is divided into 2^n logarithmic buckets. Or
|
||||
// in other words, each bucket boundary is the previous boundary times
|
||||
// 2^(2^-n).
|
||||
Schema int32
|
||||
// Width of the zero bucket.
|
||||
ZeroThreshold float64
|
||||
// Observations falling into the zero bucket.
|
||||
ZeroCount uint64
|
||||
// Total number of observations.
|
||||
Count uint64
|
||||
// Sum of observations.
|
||||
Sum float64
|
||||
// Spans for positive and negative buckets (see Span below).
|
||||
PositiveSpans, NegativeSpans []Span
|
||||
// Observation counts in buckets. The first element is an absolute
|
||||
// count. All following ones are deltas relative to the previous
|
||||
// element.
|
||||
PositiveBuckets, NegativeBuckets []int64
|
||||
}
|
||||
|
||||
// A Span defines a continuous sequence of buckets.
|
||||
type Span struct {
|
||||
// Gap to previous span (always positive), or starting index for the 1st
|
||||
// span (which can be negative).
|
||||
Offset int32
|
||||
// Length of the span.
|
||||
Length uint32
|
||||
}
|
||||
|
||||
func (s SparseHistogram) Copy() SparseHistogram {
|
||||
c := s
|
||||
// Copy returns a deep copy of the Histogram.
|
||||
func (h Histogram) Copy() Histogram {
|
||||
c := h
|
||||
|
||||
if s.PositiveSpans != nil {
|
||||
c.PositiveSpans = make([]Span, len(s.PositiveSpans))
|
||||
copy(c.PositiveSpans, s.PositiveSpans)
|
||||
if h.PositiveSpans != nil {
|
||||
c.PositiveSpans = make([]Span, len(h.PositiveSpans))
|
||||
copy(c.PositiveSpans, h.PositiveSpans)
|
||||
}
|
||||
if s.NegativeSpans != nil {
|
||||
c.NegativeSpans = make([]Span, len(s.NegativeSpans))
|
||||
copy(c.NegativeSpans, s.NegativeSpans)
|
||||
if h.NegativeSpans != nil {
|
||||
c.NegativeSpans = make([]Span, len(h.NegativeSpans))
|
||||
copy(c.NegativeSpans, h.NegativeSpans)
|
||||
}
|
||||
if s.PositiveBuckets != nil {
|
||||
c.PositiveBuckets = make([]int64, len(s.PositiveBuckets))
|
||||
copy(c.PositiveBuckets, s.PositiveBuckets)
|
||||
if h.PositiveBuckets != nil {
|
||||
c.PositiveBuckets = make([]int64, len(h.PositiveBuckets))
|
||||
copy(c.PositiveBuckets, h.PositiveBuckets)
|
||||
}
|
||||
if s.NegativeBuckets != nil {
|
||||
c.NegativeBuckets = make([]int64, len(s.NegativeBuckets))
|
||||
copy(c.NegativeBuckets, s.NegativeBuckets)
|
||||
if h.NegativeBuckets != nil {
|
||||
c.NegativeBuckets = make([]int64, len(h.NegativeBuckets))
|
||||
copy(c.NegativeBuckets, h.NegativeBuckets)
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
// CumulativeBucketIterator returns a BucketIterator to iterate over a
|
||||
// cumulative view of the buckets. This method currently only supports
|
||||
// Histograms without negative buckets and panics if the Histogram has negative
|
||||
// buckets. It is currently only used for testing.
|
||||
func (h Histogram) CumulativeBucketIterator() BucketIterator {
|
||||
if len(h.NegativeBuckets) > 0 {
|
||||
panic("CumulativeIterator called on Histogram with negative buckets")
|
||||
}
|
||||
return &cumulativeBucketIterator{h: h, posSpansIdx: -1}
|
||||
}
|
||||
|
||||
// BucketIterator iterates over the buckets of a Histogram, returning decoded
|
||||
// buckets.
|
||||
type BucketIterator interface {
|
||||
// Next advances the iterator by one.
|
||||
Next() bool
|
||||
|
|
@ -76,28 +111,23 @@ type BucketIterator interface {
|
|||
Err() error
|
||||
}
|
||||
|
||||
// Bucket represents a bucket (currently only a cumulative one with an upper
|
||||
// inclusive bound and a cumulative count).
|
||||
type Bucket struct {
|
||||
Le float64
|
||||
Upper float64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
// CumulativeExpandSparseHistogram expands the given histogram to produce cumulative buckets.
|
||||
// It assumes that the total length of spans matches the number of buckets for pos and neg respectively.
|
||||
// TODO: supports only positive buckets, also do for negative.
|
||||
func CumulativeExpandSparseHistogram(h SparseHistogram) BucketIterator {
|
||||
return &cumulativeBucketIterator{h: h, posSpansIdx: -1}
|
||||
}
|
||||
|
||||
type cumulativeBucketIterator struct {
|
||||
h SparseHistogram
|
||||
h Histogram
|
||||
|
||||
posSpansIdx int // Index in h.PositiveSpans we are in. -1 means 0 bucket.
|
||||
posBucketsIdx int // Index in h.PositiveBuckets
|
||||
posBucketsIdx int // Index in h.PositiveBuckets.
|
||||
idxInSpan uint32 // Index in the current span. 0 <= idxInSpan < span.Length.
|
||||
|
||||
initialised bool
|
||||
currIdx int32 // The actual bucket index after decoding from spans.
|
||||
currLe float64 // The upper boundary of the current bucket.
|
||||
currUpper float64 // The upper boundary of the current bucket.
|
||||
currCount int64 // Current non-cumulative count for the current bucket. Does not apply for empty bucket.
|
||||
currCumulativeCount uint64 // Current "cumulative" count for the current bucket.
|
||||
|
||||
|
|
@ -116,7 +146,7 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
return c.Next()
|
||||
}
|
||||
|
||||
c.currLe = c.h.ZeroThreshold
|
||||
c.currUpper = c.h.ZeroThreshold
|
||||
c.currCount = int64(c.h.ZeroCount)
|
||||
c.currCumulativeCount = uint64(c.currCount)
|
||||
return true
|
||||
|
|
@ -128,7 +158,7 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
|
||||
if c.emptyBucketCount > 0 {
|
||||
// We are traversing through empty buckets at the moment.
|
||||
c.currLe = getLe(c.currIdx, c.h.Schema)
|
||||
c.currUpper = getUpper(c.currIdx, c.h.Schema)
|
||||
c.currIdx++
|
||||
c.emptyBucketCount--
|
||||
return true
|
||||
|
|
@ -145,7 +175,7 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
|
||||
c.currCount += c.h.PositiveBuckets[c.posBucketsIdx]
|
||||
c.currCumulativeCount += uint64(c.currCount)
|
||||
c.currLe = getLe(c.currIdx, c.h.Schema)
|
||||
c.currUpper = getUpper(c.currIdx, c.h.Schema)
|
||||
|
||||
c.posBucketsIdx++
|
||||
c.idxInSpan++
|
||||
|
|
@ -163,24 +193,26 @@ func (c *cumulativeBucketIterator) Next() bool {
|
|||
}
|
||||
func (c *cumulativeBucketIterator) At() Bucket {
|
||||
return Bucket{
|
||||
Le: c.currLe,
|
||||
Upper: c.currUpper,
|
||||
Count: c.currCumulativeCount,
|
||||
}
|
||||
}
|
||||
func (c *cumulativeBucketIterator) Err() error { return nil }
|
||||
|
||||
func getLe(idx, schema int32) float64 {
|
||||
func getUpper(idx, schema int32) float64 {
|
||||
if schema < 0 {
|
||||
return math.Ldexp(1, int(idx)<<(-schema))
|
||||
}
|
||||
|
||||
fracIdx := idx & ((1 << schema) - 1)
|
||||
frac := sparseBounds[schema][fracIdx]
|
||||
frac := exponentialBounds[schema][fracIdx]
|
||||
exp := (int(idx) >> schema) + 1
|
||||
return math.Ldexp(frac, exp)
|
||||
}
|
||||
|
||||
var sparseBounds = [][]float64{
|
||||
// exponentialBounds is a precalculated table of bucket bounds in the interval
|
||||
// [0.5,1) in schema 0 to 8.
|
||||
var exponentialBounds = [][]float64{
|
||||
// Schema "0":
|
||||
{0.5},
|
||||
// Schema 1:
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package histogram
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCumulativeBucketIterator(t *testing.T) {
|
||||
cases := []struct {
|
||||
histogram Histogram
|
||||
expectedCumulativeBuckets []Bucket
|
||||
}{
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 2},
|
||||
{Offset: 1, Length: 2},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 1, -1, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 1, Count: 1},
|
||||
{Upper: 2, Count: 3},
|
||||
|
||||
{Upper: 4, Count: 3},
|
||||
|
||||
{Upper: 8, Count: 4},
|
||||
{Upper: 16, Count: 5},
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 5},
|
||||
{Offset: 1, Length: 1},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 1, Count: 1},
|
||||
{Upper: 2, Count: 4},
|
||||
{Upper: 4, Count: 5},
|
||||
{Upper: 8, Count: 7},
|
||||
|
||||
{Upper: 16, Count: 8},
|
||||
|
||||
{Upper: 32, Count: 8},
|
||||
{Upper: 64, Count: 9},
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 7},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 1, Count: 1},
|
||||
{Upper: 2, Count: 4},
|
||||
{Upper: 4, Count: 5},
|
||||
{Upper: 8, Count: 7},
|
||||
{Upper: 16, Count: 8},
|
||||
{Upper: 32, Count: 9},
|
||||
{Upper: 64, Count: 10},
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: 3,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -5, Length: 2}, // -5 -4
|
||||
{Offset: 2, Length: 3}, // -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 0.6484197773255048, Count: 1}, // -5
|
||||
{Upper: 0.7071067811865475, Count: 4}, // -4
|
||||
|
||||
{Upper: 0.7711054127039704, Count: 4}, // -3
|
||||
{Upper: 0.8408964152537144, Count: 4}, // -2
|
||||
|
||||
{Upper: 0.9170040432046711, Count: 5}, // -1
|
||||
{Upper: 1, Count: 7}, // 1
|
||||
{Upper: 1.0905077326652577, Count: 8}, // 0
|
||||
|
||||
{Upper: 1.189207115002721, Count: 8}, // 1
|
||||
{Upper: 1.2968395546510096, Count: 8}, // 2
|
||||
|
||||
{Upper: 1.414213562373095, Count: 9}, // 3
|
||||
{Upper: 1.5422108254079407, Count: 13}, // 4
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: -2,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 4}, // -2 -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 0.00390625, Count: 1}, // -2
|
||||
{Upper: 0.0625, Count: 4}, // -1
|
||||
{Upper: 1, Count: 5}, // 0
|
||||
{Upper: 16, Count: 7}, // 1
|
||||
|
||||
{Upper: 256, Count: 7}, // 2
|
||||
{Upper: 4096, Count: 7}, // 3
|
||||
|
||||
{Upper: 65536, Count: 8}, // 4
|
||||
{Upper: 1048576, Count: 9}, // 5
|
||||
},
|
||||
},
|
||||
{
|
||||
histogram: Histogram{
|
||||
Schema: -1,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 5}, // -2 -1 0 1 2
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1},
|
||||
},
|
||||
expectedCumulativeBuckets: []Bucket{
|
||||
{Upper: 0.0625, Count: 1}, // -2
|
||||
{Upper: 0.25, Count: 4}, // -1
|
||||
{Upper: 1, Count: 5}, // 0
|
||||
{Upper: 4, Count: 7}, // 1
|
||||
{Upper: 16, Count: 8}, // 2
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
||||
it := c.histogram.CumulativeBucketIterator()
|
||||
actualBuckets := make([]Bucket, 0, len(c.expectedCumulativeBuckets))
|
||||
for it.Next() {
|
||||
actualBuckets = append(actualBuckets, it.At())
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
require.Equal(t, c.expectedCumulativeBuckets, actualBuckets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package histogram
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCumulativeExpandSparseHistogram(t *testing.T) {
|
||||
cases := []struct {
|
||||
hist SparseHistogram
|
||||
expBuckets []Bucket
|
||||
}{
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 2},
|
||||
{Offset: 1, Length: 2},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 1, -1, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 1, Count: 1},
|
||||
{Le: 2, Count: 3},
|
||||
|
||||
{Le: 4, Count: 3},
|
||||
|
||||
{Le: 8, Count: 4},
|
||||
{Le: 16, Count: 5},
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 5},
|
||||
{Offset: 1, Length: 1},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 1, Count: 1},
|
||||
{Le: 2, Count: 4},
|
||||
{Le: 4, Count: 5},
|
||||
{Le: 8, Count: 7},
|
||||
|
||||
{Le: 16, Count: 8},
|
||||
|
||||
{Le: 32, Count: 8},
|
||||
{Le: 64, Count: 9},
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 0,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: 0, Length: 7},
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 1, Count: 1},
|
||||
{Le: 2, Count: 4},
|
||||
{Le: 4, Count: 5},
|
||||
{Le: 8, Count: 7},
|
||||
{Le: 16, Count: 8},
|
||||
{Le: 32, Count: 9},
|
||||
{Le: 64, Count: 10},
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: 3,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -5, Length: 2}, // -5 -4
|
||||
{Offset: 2, Length: 3}, // -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0, 3},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 0.6484197773255048, Count: 1}, // -5
|
||||
{Le: 0.7071067811865475, Count: 4}, // -4
|
||||
|
||||
{Le: 0.7711054127039704, Count: 4}, // -3
|
||||
{Le: 0.8408964152537144, Count: 4}, // -2
|
||||
|
||||
{Le: 0.9170040432046711, Count: 5}, // -1
|
||||
{Le: 1, Count: 7}, // 1
|
||||
{Le: 1.0905077326652577, Count: 8}, // 0
|
||||
|
||||
{Le: 1.189207115002721, Count: 8}, // 1
|
||||
{Le: 1.2968395546510096, Count: 8}, // 2
|
||||
|
||||
{Le: 1.414213562373095, Count: 9}, // 3
|
||||
{Le: 1.5422108254079407, Count: 13}, // 4
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: -2,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 4}, // -2 -1 0 1
|
||||
{Offset: 2, Length: 2}, // 4 5
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1, 0},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 0.00390625, Count: 1}, // -2
|
||||
{Le: 0.0625, Count: 4}, // -1
|
||||
{Le: 1, Count: 5}, // 0
|
||||
{Le: 16, Count: 7}, // 1
|
||||
|
||||
{Le: 256, Count: 7}, // 2
|
||||
{Le: 4096, Count: 7}, // 3
|
||||
|
||||
{Le: 65536, Count: 8}, // 4
|
||||
{Le: 1048576, Count: 9}, // 5
|
||||
},
|
||||
},
|
||||
{
|
||||
hist: SparseHistogram{
|
||||
Schema: -1,
|
||||
PositiveSpans: []Span{
|
||||
{Offset: -2, Length: 5}, // -2 -1 0 1 2
|
||||
},
|
||||
PositiveBuckets: []int64{1, 2, -2, 1, -1},
|
||||
},
|
||||
expBuckets: []Bucket{
|
||||
{Le: 0.0625, Count: 1}, // -2
|
||||
{Le: 0.25, Count: 4}, // -1
|
||||
{Le: 1, Count: 5}, // 0
|
||||
{Le: 4, Count: 7}, // 1
|
||||
{Le: 16, Count: 8}, // 2
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
|
||||
it := CumulativeExpandSparseHistogram(c.hist)
|
||||
actBuckets := make([]Bucket, 0, len(c.expBuckets))
|
||||
for it.Next() {
|
||||
actBuckets = append(actBuckets, it.At())
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
require.Equal(t, c.expBuckets, actBuckets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -16,8 +16,8 @@ package textparse
|
|||
import (
|
||||
"mime"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
)
|
||||
|
||||
|
|
@ -29,9 +29,8 @@ type Parser interface {
|
|||
Series() ([]byte, *int64, float64)
|
||||
|
||||
// Histogram returns the bytes of a series with a sparse histogram as a
|
||||
// value, the timestamp if set, and the sparse histogram in the current
|
||||
// sample.
|
||||
Histogram() ([]byte, *int64, histogram.SparseHistogram)
|
||||
// value, the timestamp if set, and the histogram in the current sample.
|
||||
Histogram() ([]byte, *int64, histogram.Histogram)
|
||||
|
||||
// Help returns the metric name and help text in the current entry.
|
||||
// Must only be called after Next returned a help entry.
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
|
@ -114,10 +114,10 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
|
|||
return p.series, nil, p.val
|
||||
}
|
||||
|
||||
// Histogram always returns (nil, nil, SparseHistogram{}) because OpenMetrics
|
||||
// does not support sparse histograms.
|
||||
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) {
|
||||
return nil, nil, histogram.SparseHistogram{}
|
||||
// Histogram always returns (nil, nil, histogram.Histogram{}) because
|
||||
// OpenMetrics does not support sparse histograms.
|
||||
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, histogram.Histogram) {
|
||||
return nil, nil, histogram.Histogram{}
|
||||
}
|
||||
|
||||
// Help returns the metric name and help text in the current entry.
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
|
@ -169,10 +169,10 @@ func (p *PromParser) Series() ([]byte, *int64, float64) {
|
|||
return p.series, nil, p.val
|
||||
}
|
||||
|
||||
// Histogram always returns (nil, nil, SparseHistogram{}) because the Prometheus
|
||||
// text format does not support sparse histograms.
|
||||
func (p *PromParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) {
|
||||
return nil, nil, histogram.SparseHistogram{}
|
||||
// Histogram always returns (nil, nil, histogram.Histogram{}) because the
|
||||
// Prometheus text format does not support sparse histograms.
|
||||
func (p *PromParser) Histogram() ([]byte, *int64, histogram.Histogram) {
|
||||
return nil, nil, histogram.Histogram{}
|
||||
}
|
||||
|
||||
// Help returns the metric name and help text in the current entry.
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
|
||||
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
|
||||
|
|
@ -65,6 +65,7 @@ type ProtobufParser struct {
|
|||
metricBytes *bytes.Buffer // A somewhat fluid representation of the current metric.
|
||||
}
|
||||
|
||||
// NewProtobufParser returns a parser for the payload in the byte slice.
|
||||
func NewProtobufParser(b []byte) Parser {
|
||||
return &ProtobufParser{
|
||||
in: b,
|
||||
|
|
@ -134,13 +135,13 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
|
|||
// Histogram returns the bytes of a series with a sparse histogram as a
|
||||
// value, the timestamp if set, and the sparse histogram in the current
|
||||
// sample.
|
||||
func (p *ProtobufParser) Histogram() ([]byte, *int64, histogram.SparseHistogram) {
|
||||
func (p *ProtobufParser) Histogram() ([]byte, *int64, histogram.Histogram) {
|
||||
var (
|
||||
m = p.mf.GetMetric()[p.metricPos]
|
||||
ts = m.GetTimestampMs()
|
||||
h = m.GetHistogram()
|
||||
)
|
||||
sh := histogram.SparseHistogram{
|
||||
sh := histogram.Histogram{
|
||||
Count: h.GetSampleCount(),
|
||||
Sum: h.GetSampleSum(),
|
||||
ZeroThreshold: h.GetSbZeroThreshold(),
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ import (
|
|||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
|
||||
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
|
||||
|
|
@ -266,7 +266,7 @@ metric: <
|
|||
help string
|
||||
unit string
|
||||
comment string
|
||||
shs histogram.SparseHistogram
|
||||
shs histogram.Histogram
|
||||
e []exemplar.Exemplar
|
||||
}{
|
||||
{
|
||||
|
|
@ -332,7 +332,7 @@ metric: <
|
|||
{
|
||||
m: "test_histogram",
|
||||
t: 1234568,
|
||||
shs: histogram.SparseHistogram{
|
||||
shs: histogram.Histogram{
|
||||
Count: 175,
|
||||
ZeroCount: 2,
|
||||
Sum: 0.0008280461746287094,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -296,8 +296,11 @@ func (ssi *storageSeriesIterator) At() (t int64, v float64) {
|
|||
return p.T, p.V
|
||||
}
|
||||
|
||||
func (ssi *storageSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
// TODO(beorn7): Fix that for histogram support in PromQL.
|
||||
func (ssi *storageSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (ssi *storageSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ import (
|
|||
"context"
|
||||
"math/rand"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
)
|
||||
|
|
@ -35,7 +35,7 @@ func (a nopAppender) Append(uint64, labels.Labels, int64, float64) (uint64, erro
|
|||
func (a nopAppender) AppendExemplar(uint64, labels.Labels, exemplar.Exemplar) (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (a nopAppender) AppendHistogram(uint64, labels.Labels, int64, histogram.SparseHistogram) (uint64, error) {
|
||||
func (a nopAppender) AppendHistogram(uint64, labels.Labels, int64, histogram.Histogram) (uint64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
func (a nopAppender) Commit() error { return nil }
|
||||
|
|
@ -47,9 +47,9 @@ type sample struct {
|
|||
v float64
|
||||
}
|
||||
|
||||
type hist struct {
|
||||
h histogram.SparseHistogram
|
||||
type histogramSample struct {
|
||||
t int64
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
// collectResultAppender records all samples that were added through the appender.
|
||||
|
|
@ -61,9 +61,9 @@ type collectResultAppender struct {
|
|||
rolledbackResult []sample
|
||||
pendingExemplars []exemplar.Exemplar
|
||||
resultExemplars []exemplar.Exemplar
|
||||
resultHistograms []hist
|
||||
pendingHistograms []hist
|
||||
rolledbackHistograms []hist
|
||||
resultHistograms []histogramSample
|
||||
pendingHistograms []histogramSample
|
||||
rolledbackHistograms []histogramSample
|
||||
}
|
||||
|
||||
func (a *collectResultAppender) Append(ref uint64, lset labels.Labels, t int64, v float64) (uint64, error) {
|
||||
|
|
@ -96,13 +96,13 @@ func (a *collectResultAppender) AppendExemplar(ref uint64, l labels.Labels, e ex
|
|||
return a.next.AppendExemplar(ref, l, e)
|
||||
}
|
||||
|
||||
func (a *collectResultAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
a.pendingHistograms = append(a.pendingHistograms, hist{h: sh, t: t})
|
||||
func (a *collectResultAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
a.pendingHistograms = append(a.pendingHistograms, histogramSample{h: h, t: t})
|
||||
if a.next == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return a.next.AppendHistogram(ref, l, t, sh)
|
||||
return a.next.AppendHistogram(ref, l, t, h)
|
||||
}
|
||||
|
||||
func (a *collectResultAppender) Commit() error {
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ import (
|
|||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/pool"
|
||||
"github.com/prometheus/prometheus/pkg/relabel"
|
||||
|
|
@ -1411,7 +1411,7 @@ loop:
|
|||
met []byte
|
||||
parsedTimestamp *int64
|
||||
val float64
|
||||
his histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
)
|
||||
if et, err = p.Next(); err != nil {
|
||||
if err == io.EOF {
|
||||
|
|
@ -1439,7 +1439,7 @@ loop:
|
|||
|
||||
t := defTime
|
||||
if isHistogram {
|
||||
met, parsedTimestamp, his = p.Histogram()
|
||||
met, parsedTimestamp, h = p.Histogram()
|
||||
} else {
|
||||
met, parsedTimestamp, val = p.Series()
|
||||
}
|
||||
|
|
@ -1491,7 +1491,7 @@ loop:
|
|||
}
|
||||
|
||||
if isHistogram {
|
||||
ref, err = app.AppendHistogram(ref, lset, t, his)
|
||||
ref, err = app.AppendHistogram(ref, lset, t, h)
|
||||
} else {
|
||||
ref, err = app.Append(ref, lset, t, val)
|
||||
}
|
||||
|
|
@ -1554,7 +1554,6 @@ loop:
|
|||
if err == nil {
|
||||
sl.cache.forEachStale(func(lset labels.Labels) bool {
|
||||
// Series no longer exposed, mark it stale.
|
||||
// TODO(beorn7): Appending staleness markers breaks horribly for histograms.
|
||||
_, err = app.Append(0, lset, defTime, math.Float64frombits(value.StaleNaN))
|
||||
switch errors.Cause(err) {
|
||||
case storage.ErrOutOfOrderSample, storage.ErrDuplicateSampleForTimestamp:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ package storage
|
|||
import (
|
||||
"math"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
)
|
||||
|
||||
|
|
@ -198,8 +198,11 @@ func (it *sampleRingIterator) At() (int64, float64) {
|
|||
return it.r.at(it.i)
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
// TODO(beorn7): Fix that for histogram support in PromQL.
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import (
|
|||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -196,8 +196,8 @@ type mockSeriesIterator struct {
|
|||
|
||||
func (m *mockSeriesIterator) Seek(t int64) bool { return m.seek(t) }
|
||||
func (m *mockSeriesIterator) At() (int64, float64) { return m.at() }
|
||||
func (m *mockSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
func (m *mockSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
func (m *mockSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
return chunkenc.EncXOR
|
||||
|
|
@ -219,8 +219,8 @@ func (it *fakeSeriesIterator) At() (int64, float64) {
|
|||
return it.idx * it.step, 123 // value doesn't matter
|
||||
}
|
||||
|
||||
func (it *fakeSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return it.idx * it.step, histogram.SparseHistogram{} // value doesn't matter
|
||||
func (it *fakeSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return it.idx * it.step, histogram.Histogram{} // value doesn't matter
|
||||
}
|
||||
|
||||
func (it *fakeSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ import (
|
|||
"github.com/go-kit/log/level"
|
||||
"github.com/prometheus/common/model"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
||||
)
|
||||
|
|
@ -173,14 +173,14 @@ func (f *fanoutAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.
|
|||
return ref, nil
|
||||
}
|
||||
|
||||
func (f *fanoutAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
ref, err := f.primary.AppendHistogram(ref, l, t, sh)
|
||||
func (f *fanoutAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
ref, err := f.primary.AppendHistogram(ref, l, t, h)
|
||||
if err != nil {
|
||||
return ref, err
|
||||
}
|
||||
|
||||
for _, appender := range f.secondaries {
|
||||
if _, err := appender.AppendHistogram(ref, l, t, sh); err != nil {
|
||||
if _, err := appender.AppendHistogram(ref, l, t, h); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/chunks"
|
||||
|
|
@ -213,17 +213,16 @@ type ExemplarAppender interface {
|
|||
AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error)
|
||||
}
|
||||
|
||||
// HistogramAppender provides an interface for adding sparse histogram to the Prometheus.
|
||||
// HistogramAppender provides an interface for appending histograms to the storage.
|
||||
type HistogramAppender interface {
|
||||
// AppendHistogram adds a sparse histogram for the given series labels.
|
||||
// An optional reference number can be provided to accelerate calls.
|
||||
// A reference number is returned which can be used to add further
|
||||
// histograms in the same or later transactions.
|
||||
// Returned reference numbers are ephemeral and may be rejected in calls
|
||||
// to Append() at any point. Adding the sample via Append() returns a new
|
||||
// reference number.
|
||||
// AppendHistogram adds a histogram for the given series labels. An
|
||||
// optional reference number can be provided to accelerate calls. A
|
||||
// reference number is returned which can be used to add further
|
||||
// histograms in the same or later transactions. Returned reference
|
||||
// numbers are ephemeral and may be rejected in calls to Append() at any
|
||||
// point. Adding the sample via Append() returns a new reference number.
|
||||
// If the reference is 0 it must not be used for caching.
|
||||
AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error)
|
||||
AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error)
|
||||
}
|
||||
|
||||
// SeriesSet contains a set of series.
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/chunks"
|
||||
|
|
@ -465,7 +465,7 @@ func (c *chainSampleIterator) Seek(t int64) bool {
|
|||
}
|
||||
if len(c.h) > 0 {
|
||||
c.curr = heap.Pop(&c.h).(chunkenc.Iterator)
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
c.lastt, _ = c.curr.AtHistogram()
|
||||
} else {
|
||||
c.lastt, _ = c.curr.At()
|
||||
|
|
@ -483,7 +483,7 @@ func (c *chainSampleIterator) At() (t int64, v float64) {
|
|||
return c.curr.At()
|
||||
}
|
||||
|
||||
func (c *chainSampleIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (c *chainSampleIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
if c.curr == nil {
|
||||
panic("chainSampleIterator.AtHistogram() called before first .Next() or after .Next() returned false.")
|
||||
}
|
||||
|
|
@ -517,7 +517,7 @@ func (c *chainSampleIterator) Next() bool {
|
|||
var currt int64
|
||||
for {
|
||||
if c.curr.Next() {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
currt, _ = c.curr.AtHistogram()
|
||||
} else {
|
||||
currt, _ = c.curr.At()
|
||||
|
|
@ -534,7 +534,7 @@ func (c *chainSampleIterator) Next() bool {
|
|||
|
||||
// Check current iterator with the top of the heap.
|
||||
var nextt int64
|
||||
if c.h[0].ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.h[0].ChunkEncoding() == chunkenc.EncHistogram {
|
||||
nextt, _ = c.h[0].AtHistogram()
|
||||
} else {
|
||||
nextt, _ = c.h[0].At()
|
||||
|
|
@ -552,7 +552,7 @@ func (c *chainSampleIterator) Next() bool {
|
|||
}
|
||||
|
||||
c.curr = heap.Pop(&c.h).(chunkenc.Iterator)
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if c.curr.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
currt, _ = c.curr.AtHistogram()
|
||||
} else {
|
||||
currt, _ = c.curr.At()
|
||||
|
|
@ -581,12 +581,12 @@ func (h samplesIteratorHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
|||
|
||||
func (h samplesIteratorHeap) Less(i, j int) bool {
|
||||
var at, bt int64
|
||||
if h[i].ChunkEncoding() == chunkenc.EncSHS {
|
||||
if h[i].ChunkEncoding() == chunkenc.EncHistogram {
|
||||
at, _ = h[i].AtHistogram()
|
||||
} else {
|
||||
at, _ = h[i].At()
|
||||
}
|
||||
if h[j].ChunkEncoding() == chunkenc.EncSHS {
|
||||
if h[j].ChunkEncoding() == chunkenc.EncHistogram {
|
||||
bt, _ = h[j].AtHistogram()
|
||||
} else {
|
||||
bt, _ = h[j].At()
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ import (
|
|||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/common/model"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/textparse"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
|
|
@ -370,8 +370,11 @@ func (c *concreteSeriesIterator) At() (t int64, v float64) {
|
|||
return s.Timestamp, s.Value
|
||||
}
|
||||
|
||||
func (c *concreteSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
// TODO(beorn7): Fix that for histogram support in remote storage.
|
||||
func (c *concreteSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (c *concreteSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ import (
|
|||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/wal"
|
||||
|
|
@ -241,7 +241,7 @@ func (t *timestampTracker) AppendExemplar(_ uint64, _ labels.Labels, _ exemplar.
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
func (t *timestampTracker) AppendHistogram(_ uint64, _ labels.Labels, ts int64, _ histogram.SparseHistogram) (uint64, error) {
|
||||
func (t *timestampTracker) AppendHistogram(_ uint64, _ labels.Labels, ts int64, _ histogram.Histogram) (uint64, error) {
|
||||
t.histograms++
|
||||
if ts > t.highestTimestamp {
|
||||
t.highestTimestamp = ts
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ import (
|
|||
"github.com/go-kit/log"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/prompb"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -188,7 +188,7 @@ func (m *mockAppendable) AppendExemplar(_ uint64, l labels.Labels, e exemplar.Ex
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
func (*mockAppendable) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
// noop until we implement sparse histograms over remote write
|
||||
func (*mockAppendable) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
// TODO(beorn7): Noop until we implement sparse histograms over remote write.
|
||||
return 0, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import (
|
|||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/chunks"
|
||||
|
|
@ -91,8 +91,10 @@ func (it *listSeriesIterator) At() (int64, float64) {
|
|||
return s.T(), s.V()
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
// AtHistogram always returns (0, histogram.Histogram{}) because there is no
|
||||
// support for histogram values yet.
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
)
|
||||
|
||||
// Encoding is the identifier for a chunk encoding.
|
||||
|
|
@ -30,8 +30,8 @@ func (e Encoding) String() string {
|
|||
return "none"
|
||||
case EncXOR:
|
||||
return "XOR"
|
||||
case EncSHS:
|
||||
return "SHS"
|
||||
case EncHistogram:
|
||||
return "histogram"
|
||||
}
|
||||
return "<unknown>"
|
||||
}
|
||||
|
|
@ -39,7 +39,7 @@ func (e Encoding) String() string {
|
|||
// IsValidEncoding returns true for supported encodings.
|
||||
func IsValidEncoding(e Encoding) bool {
|
||||
switch e {
|
||||
case EncXOR, EncSHS:
|
||||
case EncXOR, EncHistogram:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
|
@ -49,7 +49,7 @@ func IsValidEncoding(e Encoding) bool {
|
|||
const (
|
||||
EncNone Encoding = iota
|
||||
EncXOR
|
||||
EncSHS
|
||||
EncHistogram
|
||||
)
|
||||
|
||||
// Chunk holds a sequence of sample pairs that can be iterated over and appended to.
|
||||
|
|
@ -82,7 +82,7 @@ type Chunk interface {
|
|||
// Appender adds sample pairs to a chunk.
|
||||
type Appender interface {
|
||||
Append(int64, float64)
|
||||
AppendHistogram(t int64, h histogram.SparseHistogram)
|
||||
AppendHistogram(t int64, h histogram.Histogram)
|
||||
}
|
||||
|
||||
// Iterator is a simple iterator that can only get the next value.
|
||||
|
|
@ -100,7 +100,7 @@ type Iterator interface {
|
|||
At() (int64, float64)
|
||||
// AtHistogram returns the current timestamp/histogram pair.
|
||||
// Before the iterator has advanced AtHistogram behaviour is unspecified.
|
||||
AtHistogram() (int64, histogram.SparseHistogram)
|
||||
AtHistogram() (int64, histogram.Histogram)
|
||||
// Err returns the current error. It should be used only after iterator is
|
||||
// exhausted, that is `Next` or `Seek` returns false.
|
||||
Err() error
|
||||
|
|
@ -117,8 +117,8 @@ type nopIterator struct{}
|
|||
|
||||
func (nopIterator) Seek(int64) bool { return false }
|
||||
func (nopIterator) At() (int64, float64) { return math.MinInt64, 0 }
|
||||
func (nopIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return math.MinInt64, histogram.SparseHistogram{}
|
||||
func (nopIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return math.MinInt64, histogram.Histogram{}
|
||||
}
|
||||
func (nopIterator) Next() bool { return false }
|
||||
func (nopIterator) Err() error { return nil }
|
||||
|
|
@ -132,8 +132,8 @@ type Pool interface {
|
|||
|
||||
// pool is a memory pool of chunk objects.
|
||||
type pool struct {
|
||||
xor sync.Pool
|
||||
shs sync.Pool
|
||||
xor sync.Pool
|
||||
histogram sync.Pool
|
||||
}
|
||||
|
||||
// NewPool returns a new pool.
|
||||
|
|
@ -144,9 +144,9 @@ func NewPool() Pool {
|
|||
return &XORChunk{b: bstream{}}
|
||||
},
|
||||
},
|
||||
shs: sync.Pool{
|
||||
histogram: sync.Pool{
|
||||
New: func() interface{} {
|
||||
return &HistoChunk{b: bstream{}}
|
||||
return &HistogramChunk{b: bstream{}}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -159,9 +159,9 @@ func (p *pool) Get(e Encoding, b []byte) (Chunk, error) {
|
|||
c.b.stream = b
|
||||
c.b.count = 0
|
||||
return c, nil
|
||||
case EncSHS:
|
||||
case EncHistogram:
|
||||
// TODO: update metadata
|
||||
c := p.shs.Get().(*HistoChunk)
|
||||
c := p.histogram.Get().(*HistogramChunk)
|
||||
c.b.stream = b
|
||||
c.b.count = 0
|
||||
return c, nil
|
||||
|
|
@ -182,9 +182,9 @@ func (p *pool) Put(c Chunk) error {
|
|||
xc.b.stream = nil
|
||||
xc.b.count = 0
|
||||
p.xor.Put(c)
|
||||
case EncSHS:
|
||||
case EncHistogram:
|
||||
// TODO: update metadata
|
||||
sh, ok := c.(*HistoChunk)
|
||||
sh, ok := c.(*HistogramChunk)
|
||||
// This may happen often with wrapped chunks. Nothing we can really do about
|
||||
// it but returning an error would cause a lot of allocations again. Thus,
|
||||
// we just skip it.
|
||||
|
|
@ -193,7 +193,7 @@ func (p *pool) Put(c Chunk) error {
|
|||
}
|
||||
sh.b.stream = nil
|
||||
sh.b.count = 0
|
||||
p.shs.Put(c)
|
||||
p.histogram.Put(c)
|
||||
default:
|
||||
return errors.Errorf("invalid chunk encoding %q", c.Encoding())
|
||||
}
|
||||
|
|
@ -207,9 +207,9 @@ func FromData(e Encoding, d []byte) (Chunk, error) {
|
|||
switch e {
|
||||
case EncXOR:
|
||||
return &XORChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
case EncSHS:
|
||||
case EncHistogram:
|
||||
// TODO: update metadata
|
||||
return &HistoChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
return &HistogramChunk{b: bstream{count: 0, stream: d}}, nil
|
||||
}
|
||||
return nil, errors.Errorf("invalid chunk encoding %q", e)
|
||||
}
|
||||
|
|
@ -219,8 +219,8 @@ func NewEmptyChunk(e Encoding) (Chunk, error) {
|
|||
switch e {
|
||||
case EncXOR:
|
||||
return NewXORChunk(), nil
|
||||
case EncSHS:
|
||||
return NewHistoChunk(), nil
|
||||
case EncHistogram:
|
||||
return NewHistogramChunk(), nil
|
||||
}
|
||||
return nil, errors.Errorf("invalid chunk encoding %q", e)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,943 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The code in this file was largely written by Damian Gryski as part of
|
||||
// https://github.com/dgryski/go-tsz and published under the license below.
|
||||
// It was modified to accommodate reading from byte slices without modifying
|
||||
// the underlying bytes, which would panic when reading from mmap'd
|
||||
// read-only byte slices.
|
||||
|
||||
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
||||
// All rights reserved.
|
||||
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"math/bits"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
||||
const ()
|
||||
|
||||
// HistoChunk holds sparse histogram encoded sample data.
|
||||
// Appends a histogram sample
|
||||
// * schema defines the resolution (number of buckets per power of 2)
|
||||
// Currently, valid numbers are -4 <= n <= 8.
|
||||
// They are all for base-2 bucket schemas, where 1 is a bucket boundary in each case, and
|
||||
// then each power of two is divided into 2^n logarithmic buckets.
|
||||
// Or in other words, each bucket boundary is the previous boundary times 2^(2^-n).
|
||||
// In the future, more bucket schemas may be added using numbers < -4 or > 8.
|
||||
// The bucket with upper boundary of 1 is always bucket 0.
|
||||
// Then negative numbers for smaller boundaries and positive for uppers.
|
||||
//
|
||||
// fields are stored like so:
|
||||
// field ts count zeroCount sum []posbuckets negbuckets
|
||||
// observation 1 raw raw raw raw []raw []raw
|
||||
// observation 2 delta delta delta xor []delta []delta
|
||||
// observation >2 dod dod dod xor []dod []dod
|
||||
// TODO zerothreshold
|
||||
type HistoChunk struct {
|
||||
b bstream
|
||||
}
|
||||
|
||||
// NewHistoChunk returns a new chunk with Histo encoding of the given size.
|
||||
func NewHistoChunk() *HistoChunk {
|
||||
b := make([]byte, 3, 128)
|
||||
return &HistoChunk{b: bstream{stream: b, count: 0}}
|
||||
}
|
||||
|
||||
// Encoding returns the encoding type.
|
||||
func (c *HistoChunk) Encoding() Encoding {
|
||||
return EncSHS
|
||||
}
|
||||
|
||||
// Bytes returns the underlying byte slice of the chunk.
|
||||
func (c *HistoChunk) Bytes() []byte {
|
||||
return c.b.bytes()
|
||||
}
|
||||
|
||||
// NumSamples returns the number of samples in the chunk.
|
||||
func (c *HistoChunk) NumSamples() int {
|
||||
return int(binary.BigEndian.Uint16(c.Bytes()))
|
||||
}
|
||||
|
||||
// Meta returns the histogram metadata.
|
||||
// callers may only call this on chunks that have at least one sample
|
||||
func (c *HistoChunk) Meta() (int32, float64, []histogram.Span, []histogram.Span, error) {
|
||||
if c.NumSamples() == 0 {
|
||||
panic("HistoChunk.Meta() called on an empty chunk")
|
||||
}
|
||||
b := newBReader(c.Bytes()[2:])
|
||||
return readHistoChunkMeta(&b)
|
||||
}
|
||||
|
||||
// CounterResetHeader defines the first 2 bits of the chunk header.
|
||||
type CounterResetHeader byte
|
||||
|
||||
const (
|
||||
CounterReset CounterResetHeader = 0b10000000
|
||||
NotCounterReset CounterResetHeader = 0b01000000
|
||||
GaugeType CounterResetHeader = 0b11000000
|
||||
UnknownCounterReset CounterResetHeader = 0b00000000
|
||||
)
|
||||
|
||||
// SetCounterResetHeader sets the counter reset header.
|
||||
func (c *HistoChunk) SetCounterResetHeader(h CounterResetHeader) {
|
||||
switch h {
|
||||
case CounterReset, NotCounterReset, GaugeType, UnknownCounterReset:
|
||||
bytes := c.Bytes()
|
||||
bytes[2] = (bytes[2] & 0b00111111) | byte(h)
|
||||
default:
|
||||
panic("invalid CounterResetHeader type")
|
||||
}
|
||||
}
|
||||
|
||||
// GetCounterResetHeader returns the info about the first 2 bits of the chunk header.
|
||||
func (c *HistoChunk) GetCounterResetHeader() CounterResetHeader {
|
||||
return CounterResetHeader(c.Bytes()[2] & 0b11000000)
|
||||
}
|
||||
|
||||
// Compact implements the Chunk interface.
|
||||
func (c *HistoChunk) Compact() {
|
||||
if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold {
|
||||
buf := make([]byte, l)
|
||||
copy(buf, c.b.stream)
|
||||
c.b.stream = buf
|
||||
}
|
||||
}
|
||||
|
||||
// Appender implements the Chunk interface.
|
||||
func (c *HistoChunk) Appender() (Appender, error) {
|
||||
it := c.iterator(nil)
|
||||
|
||||
// To get an appender we must know the state it would have if we had
|
||||
// appended all existing data from scratch.
|
||||
// We iterate through the end and populate via the iterator's state.
|
||||
for it.Next() {
|
||||
}
|
||||
if err := it.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
a := &HistoAppender{
|
||||
b: &c.b,
|
||||
|
||||
schema: it.schema,
|
||||
zeroThreshold: it.zeroThreshold,
|
||||
posSpans: it.posSpans,
|
||||
negSpans: it.negSpans,
|
||||
t: it.t,
|
||||
cnt: it.cnt,
|
||||
zcnt: it.zcnt,
|
||||
tDelta: it.tDelta,
|
||||
cntDelta: it.cntDelta,
|
||||
zcntDelta: it.zcntDelta,
|
||||
posbuckets: it.posbuckets,
|
||||
negbuckets: it.negbuckets,
|
||||
posbucketsDelta: it.posbucketsDelta,
|
||||
negbucketsDelta: it.negbucketsDelta,
|
||||
|
||||
sum: it.sum,
|
||||
leading: it.leading,
|
||||
trailing: it.trailing,
|
||||
|
||||
buf64: make([]byte, binary.MaxVarintLen64),
|
||||
}
|
||||
if binary.BigEndian.Uint16(a.b.bytes()) == 0 {
|
||||
a.leading = 0xff
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func countSpans(spans []histogram.Span) int {
|
||||
var cnt int
|
||||
for _, s := range spans {
|
||||
cnt += int(s.Length)
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
||||
func newHistoIterator(b []byte) *histoIterator {
|
||||
it := &histoIterator{
|
||||
br: newBReader(b),
|
||||
numTotal: binary.BigEndian.Uint16(b),
|
||||
t: math.MinInt64,
|
||||
}
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
_, _ = it.br.readBits(16)
|
||||
return it
|
||||
}
|
||||
|
||||
func (c *HistoChunk) iterator(it Iterator) *histoIterator {
|
||||
// TODO fix this. this is taken from xor.go // dieter not sure what the purpose of this is
|
||||
// Should iterators guarantee to act on a copy of the data so it doesn't lock append?
|
||||
// When using striped locks to guard access to chunks, probably yes.
|
||||
// Could only copy data if the chunk is not completed yet.
|
||||
//if histoIter, ok := it.(*histoIterator); ok {
|
||||
// histoIter.Reset(c.b.bytes())
|
||||
// return histoIter
|
||||
//}
|
||||
return newHistoIterator(c.b.bytes())
|
||||
}
|
||||
|
||||
// Iterator implements the Chunk interface.
|
||||
func (c *HistoChunk) Iterator(it Iterator) Iterator {
|
||||
return c.iterator(it)
|
||||
}
|
||||
|
||||
// HistoAppender is an Appender implementation for sparse histograms.
|
||||
type HistoAppender struct {
|
||||
b *bstream
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zeroThreshold float64
|
||||
posSpans, negSpans []histogram.Span
|
||||
|
||||
// For the fields that are tracked as dod's. Note that we expect to
|
||||
// handle negative deltas (e.g. resets) by creating new chunks, we still
|
||||
// want to support it in general hence signed integer types.
|
||||
t int64
|
||||
cnt, zcnt uint64
|
||||
tDelta, cntDelta, zcntDelta int64
|
||||
|
||||
posbuckets, negbuckets []int64
|
||||
posbucketsDelta, negbucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
buf64 []byte // For working on varint64's.
|
||||
}
|
||||
|
||||
func putVarint(b *bstream, buf []byte, x int64) {
|
||||
for _, byt := range buf[:binary.PutVarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
func putUvarint(b *bstream, buf []byte, x uint64) {
|
||||
for _, byt := range buf[:binary.PutUvarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
// Append implements Appender. This implementation does nothing for now.
|
||||
// TODO(beorn7): Implement in a meaningful way, i.e. we need to support
|
||||
// appending of stale markers, but this should never be used for "real"
|
||||
// samples.
|
||||
func (a *HistoAppender) Append(int64, float64) {}
|
||||
|
||||
// Appendable returns whether the chunk can be appended to, and if so
|
||||
// whether any recoding needs to happen using the provided interjections
|
||||
// (in case of any new buckets, positive or negative range, respectively)
|
||||
// The chunk is not appendable if:
|
||||
// * the schema has changed
|
||||
// * the zerobucket threshold has changed
|
||||
// * any buckets disappeared
|
||||
// * there was a counter reset in the count of observations or in any bucket, including the zero bucket
|
||||
// * the last sample in the chunk was stale while the current sample is not stale
|
||||
// It returns an additional boolean set to true if it is not appendable because of a counter reset.
|
||||
// If the given sample is stale, it will always return true.
|
||||
// If counterReset is true, okToAppend MUST be false.
|
||||
func (a *HistoAppender) Appendable(h histogram.SparseHistogram) (posInterjections []Interjection, negInterjections []Interjection, okToAppend bool, counterReset bool) {
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// This is a stale sample whose buckets and spans don't matter.
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
if value.IsStaleNaN(a.sum) {
|
||||
// If the last sample was stale, then we can only accept stale samples in this chunk.
|
||||
return
|
||||
}
|
||||
|
||||
if h.Count < a.cnt {
|
||||
// There has been a counter reset.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if h.Schema != a.schema || h.ZeroThreshold != a.zeroThreshold {
|
||||
return
|
||||
}
|
||||
|
||||
if h.ZeroCount < a.zcnt {
|
||||
// There has been a counter reset since ZeroThreshold didn't change.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
var ok bool
|
||||
posInterjections, ok = compareSpans(a.posSpans, h.PositiveSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
negInterjections, ok = compareSpans(a.negSpans, h.NegativeSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if counterResetInAnyBucket(a.posbuckets, h.PositiveBuckets, a.posSpans, h.PositiveSpans) ||
|
||||
counterResetInAnyBucket(a.negbuckets, h.NegativeBuckets, a.negSpans, h.NegativeSpans) {
|
||||
counterReset, posInterjections, negInterjections = true, nil, nil
|
||||
return
|
||||
}
|
||||
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
|
||||
// counterResetInAnyBucket returns true if there was a counter reset for any bucket.
|
||||
// This should be called only when buckets are same or new buckets were added,
|
||||
// and does not handle the case of buckets missing.
|
||||
func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool {
|
||||
if len(oldSpans) == 0 || len(oldBuckets) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
oldSpanSliceIdx, newSpanSliceIdx := 0, 0 // Index for the span slices.
|
||||
oldInsideSpanIdx, newInsideSpanIdx := uint32(0), uint32(0) // Index inside a span.
|
||||
oldIdx, newIdx := oldSpans[0].Offset, newSpans[0].Offset
|
||||
|
||||
oldBucketSliceIdx, newBucketSliceIdx := 0, 0 // Index inside bucket slice.
|
||||
oldVal, newVal := oldBuckets[0], newBuckets[0]
|
||||
|
||||
// Since we assume that new spans won't have missing buckets, there will never be a case
|
||||
// where the old index will not find a matching new index.
|
||||
for {
|
||||
if oldIdx == newIdx {
|
||||
if newVal < oldVal {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if oldIdx <= newIdx {
|
||||
// Moving ahead old bucket and span by 1 index.
|
||||
if oldInsideSpanIdx == oldSpans[oldSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
oldSpanSliceIdx++
|
||||
oldInsideSpanIdx = 0
|
||||
if oldSpanSliceIdx >= len(oldSpans) {
|
||||
// All old spans are over.
|
||||
break
|
||||
}
|
||||
oldIdx += 1 + oldSpans[oldSpanSliceIdx].Offset
|
||||
} else {
|
||||
oldInsideSpanIdx++
|
||||
oldIdx++
|
||||
}
|
||||
oldBucketSliceIdx++
|
||||
oldVal += oldBuckets[oldBucketSliceIdx]
|
||||
}
|
||||
|
||||
if oldIdx > newIdx {
|
||||
// Moving ahead new bucket and span by 1 index.
|
||||
if newInsideSpanIdx == newSpans[newSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
newSpanSliceIdx++
|
||||
newInsideSpanIdx = 0
|
||||
if newSpanSliceIdx >= len(newSpans) {
|
||||
// All new spans are over.
|
||||
// This should not happen, old spans above should catch this first.
|
||||
panic("new spans over before old spans in counterReset")
|
||||
}
|
||||
newIdx += 1 + newSpans[newSpanSliceIdx].Offset
|
||||
} else {
|
||||
newInsideSpanIdx++
|
||||
newIdx++
|
||||
}
|
||||
newBucketSliceIdx++
|
||||
newVal += newBuckets[newBucketSliceIdx]
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// AppendHistogram appends a SparseHistogram to the chunk. We assume the
|
||||
// histogram is properly structured. E.g. that the number of pos/neg buckets
|
||||
// used corresponds to the number conveyed by the pos/neg span structures.
|
||||
// callers must call Appendable() first and act accordingly!
|
||||
func (a *HistoAppender) AppendHistogram(t int64, h histogram.SparseHistogram) {
|
||||
var tDelta, cntDelta, zcntDelta int64
|
||||
num := binary.BigEndian.Uint16(a.b.bytes())
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// Emptying out other fields to write no buckets, and an empty meta in case of
|
||||
// first histogram in the chunk.
|
||||
h = histogram.SparseHistogram{Sum: h.Sum}
|
||||
}
|
||||
|
||||
switch num {
|
||||
case 0:
|
||||
// the first append gets the privilege to dictate the metadata
|
||||
// but it's also responsible for encoding it into the chunk!
|
||||
|
||||
writeHistoChunkMeta(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
|
||||
a.schema = h.Schema
|
||||
a.zeroThreshold = h.ZeroThreshold
|
||||
a.posSpans, a.negSpans = h.PositiveSpans, h.NegativeSpans
|
||||
numPosBuckets, numNegBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
|
||||
a.posbuckets = make([]int64, numPosBuckets)
|
||||
a.negbuckets = make([]int64, numNegBuckets)
|
||||
a.posbucketsDelta = make([]int64, numPosBuckets)
|
||||
a.negbucketsDelta = make([]int64, numNegBuckets)
|
||||
|
||||
// now store actual data
|
||||
putVarint(a.b, a.buf64, t)
|
||||
putUvarint(a.b, a.buf64, h.Count)
|
||||
putUvarint(a.b, a.buf64, h.ZeroCount)
|
||||
a.b.writeBits(math.Float64bits(h.Sum), 64)
|
||||
for _, buck := range h.PositiveBuckets {
|
||||
putVarint(a.b, a.buf64, buck)
|
||||
}
|
||||
for _, buck := range h.NegativeBuckets {
|
||||
putVarint(a.b, a.buf64, buck)
|
||||
}
|
||||
case 1:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zcntDelta = int64(h.ZeroCount) - int64(a.zcnt)
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDelta, zcntDelta = 0, 0
|
||||
}
|
||||
|
||||
putVarint(a.b, a.buf64, tDelta)
|
||||
putVarint(a.b, a.buf64, cntDelta)
|
||||
putVarint(a.b, a.buf64, zcntDelta)
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.posbuckets[i]
|
||||
putVarint(a.b, a.buf64, delta)
|
||||
a.posbucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.negbuckets[i]
|
||||
putVarint(a.b, a.buf64, delta)
|
||||
a.negbucketsDelta[i] = delta
|
||||
}
|
||||
|
||||
default:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zcntDelta = int64(h.ZeroCount) - int64(a.zcnt)
|
||||
|
||||
tDod := tDelta - a.tDelta
|
||||
cntDod := cntDelta - a.cntDelta
|
||||
zcntDod := zcntDelta - a.zcntDelta
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDod, zcntDod = 0, 0
|
||||
}
|
||||
|
||||
putInt64VBBucket(a.b, tDod)
|
||||
putInt64VBBucket(a.b, cntDod)
|
||||
putInt64VBBucket(a.b, zcntDod)
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.posbuckets[i]
|
||||
dod := delta - a.posbucketsDelta[i]
|
||||
putInt64VBBucket(a.b, dod)
|
||||
a.posbucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.negbuckets[i]
|
||||
dod := delta - a.negbucketsDelta[i]
|
||||
putInt64VBBucket(a.b, dod)
|
||||
a.negbucketsDelta[i] = delta
|
||||
}
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint16(a.b.bytes(), num+1)
|
||||
|
||||
a.t = t
|
||||
a.cnt = h.Count
|
||||
a.zcnt = h.ZeroCount
|
||||
a.tDelta = tDelta
|
||||
a.cntDelta = cntDelta
|
||||
a.zcntDelta = zcntDelta
|
||||
|
||||
a.posbuckets, a.negbuckets = h.PositiveBuckets, h.NegativeBuckets
|
||||
// note that the bucket deltas were already updated above
|
||||
a.sum = h.Sum
|
||||
}
|
||||
|
||||
// Recode converts the current chunk to accommodate an expansion of the set of
|
||||
// (positive and/or negative) buckets used, according to the provided
|
||||
// interjections, resulting in the honoring of the provided new posSpans and
|
||||
// negSpans.
|
||||
func (a *HistoAppender) Recode(posInterjections, negInterjections []Interjection, posSpans, negSpans []histogram.Span) (Chunk, Appender) {
|
||||
// TODO(beorn7): This currently just decodes everything and then encodes
|
||||
// it again with the new span layout. This can probably be done in-place
|
||||
// by editing the chunk. But let's first see how expensive it is in the
|
||||
// big picture.
|
||||
byts := a.b.bytes()
|
||||
it := newHistoIterator(byts)
|
||||
hc := NewHistoChunk()
|
||||
app, err := hc.Appender()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans)
|
||||
|
||||
for it.Next() {
|
||||
tOld, hOld := it.AtHistogram()
|
||||
|
||||
// We have to newly allocate slices for the modified buckets
|
||||
// here because they are kept by the appender until the next
|
||||
// append.
|
||||
// TODO(beorn7): We might be able to optimize this.
|
||||
posBuckets := make([]int64, numPosBuckets)
|
||||
negBuckets := make([]int64, numNegBuckets)
|
||||
|
||||
// Save the modified histogram to the new chunk.
|
||||
hOld.PositiveSpans, hOld.NegativeSpans = posSpans, negSpans
|
||||
if len(posInterjections) > 0 {
|
||||
hOld.PositiveBuckets = interject(hOld.PositiveBuckets, posBuckets, posInterjections)
|
||||
}
|
||||
if len(negInterjections) > 0 {
|
||||
hOld.NegativeBuckets = interject(hOld.NegativeBuckets, negBuckets, negInterjections)
|
||||
}
|
||||
app.AppendHistogram(tOld, hOld)
|
||||
}
|
||||
|
||||
// Set the flags.
|
||||
hc.SetCounterResetHeader(CounterResetHeader(byts[2] & 0b11000000))
|
||||
return hc, app
|
||||
}
|
||||
|
||||
func (a *HistoAppender) writeSumDelta(v float64) {
|
||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum)
|
||||
|
||||
if vDelta == 0 {
|
||||
a.b.writeBit(zero)
|
||||
return
|
||||
}
|
||||
a.b.writeBit(one)
|
||||
|
||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
||||
|
||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
||||
if leading >= 32 {
|
||||
leading = 31
|
||||
}
|
||||
|
||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
||||
a.b.writeBit(zero)
|
||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
||||
} else {
|
||||
a.leading, a.trailing = leading, trailing
|
||||
|
||||
a.b.writeBit(one)
|
||||
a.b.writeBits(uint64(leading), 5)
|
||||
|
||||
// Note that if leading == trailing == 0, then sigbits == 64. But that value doesn't actually fit into the 6 bits we have.
|
||||
// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
|
||||
// So instead we write out a 0 and adjust it back to 64 on unpacking.
|
||||
sigbits := 64 - leading - trailing
|
||||
a.b.writeBits(uint64(sigbits), 6)
|
||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
||||
}
|
||||
}
|
||||
|
||||
type histoIterator struct {
|
||||
br bstreamReader
|
||||
numTotal uint16
|
||||
numRead uint16
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zeroThreshold float64
|
||||
posSpans, negSpans []histogram.Span
|
||||
|
||||
// For the fields that are tracked as dod's.
|
||||
t int64
|
||||
cnt, zcnt uint64
|
||||
tDelta, cntDelta, zcntDelta int64
|
||||
|
||||
posbuckets, negbuckets []int64
|
||||
posbucketsDelta, negbucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
func (it *histoIterator) Seek(t int64) bool {
|
||||
if it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for t > it.t || it.numRead == 0 {
|
||||
if !it.Next() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histoIterator) At() (int64, float64) {
|
||||
panic("cannot call histoIterator.At().")
|
||||
}
|
||||
|
||||
func (it *histoIterator) ChunkEncoding() Encoding {
|
||||
return EncSHS
|
||||
}
|
||||
|
||||
func (it *histoIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
return it.t, histogram.SparseHistogram{Sum: it.sum}
|
||||
}
|
||||
return it.t, histogram.SparseHistogram{
|
||||
Count: it.cnt,
|
||||
ZeroCount: it.zcnt,
|
||||
Sum: it.sum,
|
||||
ZeroThreshold: it.zeroThreshold,
|
||||
Schema: it.schema,
|
||||
PositiveSpans: it.posSpans,
|
||||
NegativeSpans: it.negSpans,
|
||||
PositiveBuckets: it.posbuckets,
|
||||
NegativeBuckets: it.negbuckets,
|
||||
}
|
||||
}
|
||||
|
||||
func (it *histoIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *histoIterator) Reset(b []byte) {
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
it.br = newBReader(b[2:])
|
||||
it.numTotal = binary.BigEndian.Uint16(b)
|
||||
it.numRead = 0
|
||||
|
||||
it.t, it.cnt, it.zcnt = 0, 0, 0
|
||||
it.tDelta, it.cntDelta, it.zcntDelta = 0, 0, 0
|
||||
|
||||
for i := range it.posbuckets {
|
||||
it.posbuckets[i] = 0
|
||||
it.posbucketsDelta[i] = 0
|
||||
}
|
||||
for i := range it.negbuckets {
|
||||
it.negbuckets[i] = 0
|
||||
it.negbucketsDelta[i] = 0
|
||||
}
|
||||
|
||||
it.sum = 0
|
||||
it.leading = 0
|
||||
it.trailing = 0
|
||||
it.err = nil
|
||||
}
|
||||
|
||||
func (it *histoIterator) Next() bool {
|
||||
if it.err != nil || it.numRead == it.numTotal {
|
||||
return false
|
||||
}
|
||||
|
||||
if it.numRead == 0 {
|
||||
|
||||
// first read is responsible for reading chunk metadata and initializing fields that depend on it
|
||||
// We give counter reset info at chunk level, hence we discard it here.
|
||||
schema, zeroThreshold, posSpans, negSpans, err := readHistoChunkMeta(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.schema = schema
|
||||
it.zeroThreshold = zeroThreshold
|
||||
it.posSpans, it.negSpans = posSpans, negSpans
|
||||
numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans)
|
||||
if numPosBuckets > 0 {
|
||||
it.posbuckets = make([]int64, numPosBuckets)
|
||||
it.posbucketsDelta = make([]int64, numPosBuckets)
|
||||
}
|
||||
if numNegBuckets > 0 {
|
||||
it.negbuckets = make([]int64, numNegBuckets)
|
||||
it.negbucketsDelta = make([]int64, numNegBuckets)
|
||||
}
|
||||
|
||||
// now read actual data
|
||||
|
||||
t, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.t = t
|
||||
|
||||
cnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cnt = cnt
|
||||
|
||||
zcnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zcnt = zcnt
|
||||
|
||||
sum, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.sum = math.Float64frombits(sum)
|
||||
|
||||
for i := range it.posbuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.posbuckets[i] = v
|
||||
}
|
||||
for i := range it.negbuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.negbuckets[i] = v
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
if it.numRead == 1 {
|
||||
tDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = tDelta
|
||||
it.t += int64(it.tDelta)
|
||||
|
||||
cntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = cntDelta
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zcntDelta = zcntDelta
|
||||
it.zcnt = uint64(int64(it.zcnt) + it.zcntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.posbuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.posbucketsDelta[i] = delta
|
||||
it.posbuckets[i] = it.posbuckets[i] + delta
|
||||
}
|
||||
|
||||
for i := range it.negbuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.negbucketsDelta[i] = delta
|
||||
it.negbuckets[i] = it.negbuckets[i] + delta
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
tDod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = it.tDelta + tDod
|
||||
it.t += it.tDelta
|
||||
|
||||
cntDod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = it.cntDelta + cntDod
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zcntDelta = it.zcntDelta + zcntDod
|
||||
it.zcnt = uint64(int64(it.zcnt) + it.zcntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.posbuckets {
|
||||
dod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.posbucketsDelta[i] = it.posbucketsDelta[i] + dod
|
||||
it.posbuckets[i] = it.posbuckets[i] + it.posbucketsDelta[i]
|
||||
}
|
||||
|
||||
for i := range it.negbuckets {
|
||||
dod, err := readInt64VBBucket(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.negbucketsDelta[i] = it.negbucketsDelta[i] + dod
|
||||
it.negbuckets[i] = it.negbuckets[i] + it.negbucketsDelta[i]
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histoIterator) readSum() bool {
|
||||
bit, err := it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if bit == zero {
|
||||
// it.sum = it.sum
|
||||
} else {
|
||||
bit, err := it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bit == zero {
|
||||
// reuse leading/trailing zero bits
|
||||
// it.leading, it.trailing = it.leading, it.trailing
|
||||
} else {
|
||||
bits, err := it.br.readBitsFast(5)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(5)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.leading = uint8(bits)
|
||||
|
||||
bits, err = it.br.readBitsFast(6)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(6)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
mbits := uint8(bits)
|
||||
// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
|
||||
if mbits == 0 {
|
||||
mbits = 64
|
||||
}
|
||||
it.trailing = 64 - it.leading - mbits
|
||||
}
|
||||
|
||||
mbits := 64 - it.leading - it.trailing
|
||||
bits, err := it.br.readBitsFast(mbits)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(mbits)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
vbits := math.Float64bits(it.sum)
|
||||
vbits ^= bits << it.trailing
|
||||
it.sum = math.Float64frombits(vbits)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
@ -1,261 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
)
|
||||
|
||||
func writeHistoChunkMeta(b *bstream, schema int32, zeroThreshold float64, posSpans, negSpans []histogram.Span) {
|
||||
putInt64VBBucket(b, int64(schema))
|
||||
putFloat64VBBucket(b, zeroThreshold)
|
||||
putHistoChunkMetaSpans(b, posSpans)
|
||||
putHistoChunkMetaSpans(b, negSpans)
|
||||
}
|
||||
|
||||
func putHistoChunkMetaSpans(b *bstream, spans []histogram.Span) {
|
||||
putInt64VBBucket(b, int64(len(spans)))
|
||||
for _, s := range spans {
|
||||
putInt64VBBucket(b, int64(s.Length))
|
||||
putInt64VBBucket(b, int64(s.Offset))
|
||||
}
|
||||
}
|
||||
|
||||
func readHistoChunkMeta(b *bstreamReader) (schema int32, zeroThreshold float64, posSpans []histogram.Span, negSpans []histogram.Span, err error) {
|
||||
_, err = b.ReadByte() // The header.
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
v, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
schema = int32(v)
|
||||
|
||||
zeroThreshold, err = readFloat64VBBucket(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
posSpans, err = readHistoChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
negSpans, err = readHistoChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func readHistoChunkMetaSpans(b *bstreamReader) ([]histogram.Span, error) {
|
||||
var spans []histogram.Span
|
||||
num, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := 0; i < int(num); i++ {
|
||||
|
||||
length, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
spans = append(spans, histogram.Span{
|
||||
Length: uint32(length),
|
||||
Offset: int32(offset),
|
||||
})
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
|
||||
type bucketIterator struct {
|
||||
spans []histogram.Span
|
||||
span int // span position of last yielded bucket
|
||||
bucket int // bucket position within span of last yielded bucket
|
||||
idx int // bucket index (globally across all spans) of last yielded bucket
|
||||
}
|
||||
|
||||
func newBucketIterator(spans []histogram.Span) *bucketIterator {
|
||||
b := bucketIterator{
|
||||
spans: spans,
|
||||
span: 0,
|
||||
bucket: -1,
|
||||
idx: -1,
|
||||
}
|
||||
if len(spans) > 0 {
|
||||
b.idx += int(spans[0].Offset)
|
||||
}
|
||||
return &b
|
||||
}
|
||||
|
||||
func (b *bucketIterator) Next() (int, bool) {
|
||||
// we're already out of bounds
|
||||
if b.span >= len(b.spans) {
|
||||
return 0, false
|
||||
}
|
||||
try:
|
||||
if b.bucket < int(b.spans[b.span].Length-1) { // try to move within same span.
|
||||
b.bucket++
|
||||
b.idx++
|
||||
return b.idx, true
|
||||
} else if b.span < len(b.spans)-1 { // try to move from one span to the next
|
||||
b.span++
|
||||
b.idx += int(b.spans[b.span].Offset + 1)
|
||||
b.bucket = 0
|
||||
if b.spans[b.span].Length == 0 {
|
||||
// pathological case that should never happen. We can't use this span, let's try again.
|
||||
goto try
|
||||
}
|
||||
return b.idx, true
|
||||
}
|
||||
// we're out of options
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Interjection describes that num new buckets are introduced before processing the pos'th delta from the original slice
|
||||
type Interjection struct {
|
||||
pos int
|
||||
num int
|
||||
}
|
||||
|
||||
// compareSpans returns the interjections to convert a slice of deltas to a new slice representing an expanded set of buckets, or false if incompatible (e.g. if buckets were removed)
|
||||
// For example:
|
||||
// Let's say the old buckets look like this:
|
||||
// span syntax: [offset, length]
|
||||
// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1]
|
||||
// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15]
|
||||
// raw values 6 3 3 2 4 5 1
|
||||
// deltas 6 -3 0 -1 2 1 -4
|
||||
|
||||
// But now we introduce a new bucket layout. (carefully chosen example where we have a span appended, one unchanged[*], one prepended, and two merge - in that order)
|
||||
// [*] unchanged in terms of which bucket indices they represent. but to achieve that, their offset needs to change if "disrupted" by spans changing ahead of them
|
||||
// \/ this one is "unchanged"
|
||||
// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ]
|
||||
// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15]
|
||||
// raw values 6 3 0 3 0 0 2 4 5 0 1
|
||||
// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1
|
||||
// delta mods: / \ / \ / \
|
||||
// note that whenever any new buckets are introduced, the subsequent "old" bucket needs to readjust its delta to the new base of 0
|
||||
// thus, for the caller, who wants to transform the set of original deltas to a new set of deltas to match a new span layout that adds buckets, we simply
|
||||
// need to generate a list of interjections
|
||||
// note: within compareSpans we don't have to worry about the changes to the spans themselves,
|
||||
// thanks to the iterators, we get to work with the more useful bucket indices (which of course directly correspond to the buckets we have to adjust)
|
||||
func compareSpans(a, b []histogram.Span) ([]Interjection, bool) {
|
||||
ai := newBucketIterator(a)
|
||||
bi := newBucketIterator(b)
|
||||
|
||||
var interjections []Interjection
|
||||
|
||||
// when inter.num becomes > 0, this becomes a valid interjection that should be yielded when we finish a streak of new buckets
|
||||
var inter Interjection
|
||||
|
||||
av, aok := ai.Next()
|
||||
bv, bok := bi.Next()
|
||||
loop:
|
||||
for {
|
||||
switch {
|
||||
case aok && bok:
|
||||
switch {
|
||||
case av == bv: // Both have an identical value. move on!
|
||||
// Finish WIP interjection and reset.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
inter.num = 0
|
||||
av, aok = ai.Next()
|
||||
bv, bok = bi.Next()
|
||||
inter.pos++
|
||||
case av < bv: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case av > bv: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bok = bi.Next()
|
||||
}
|
||||
case aok && !bok: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case !aok && bok: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bok = bi.Next()
|
||||
default: // Both iterators ran out. We're done.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
return interjections, true
|
||||
}
|
||||
|
||||
// interject merges 'in' with the provided interjections and writes them into
|
||||
// 'out', which must already have the appropriate length.
|
||||
func interject(in, out []int64, interjections []Interjection) []int64 {
|
||||
var j int // Position in out.
|
||||
var v int64 // The last value seen.
|
||||
var interj int // The next interjection to process.
|
||||
for i, d := range in {
|
||||
if interj < len(interjections) && i == interjections[interj].pos {
|
||||
|
||||
// We have an interjection!
|
||||
// Add interjection.num new delta values such that their
|
||||
// bucket values equate 0.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
interj++
|
||||
|
||||
// Now save the value from the input. The delta value we
|
||||
// should save is the original delta value + the last
|
||||
// value of the point before the interjection (to undo
|
||||
// the delta that was introduced by the interjection).
|
||||
out[j] = d + v
|
||||
j++
|
||||
v = d + v
|
||||
continue
|
||||
}
|
||||
|
||||
// If there was no interjection, the original delta is still
|
||||
// valid.
|
||||
out[j] = d
|
||||
j++
|
||||
v += d
|
||||
}
|
||||
switch interj {
|
||||
case len(interjections):
|
||||
// All interjections processed. Nothing more to do.
|
||||
case len(interjections) - 1:
|
||||
// One more interjection to process at the end.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
default:
|
||||
panic("unprocessed interjections left")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -0,0 +1,934 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"math/bits"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
)
|
||||
|
||||
const ()
|
||||
|
||||
// HistogramChunk holds encoded sample data for a sparse, high-resolution
|
||||
// histogram.
|
||||
//
|
||||
// TODO(beorn7): Document the layout of chunk metadata.
|
||||
//
|
||||
// Each sample has multiple "fields", stored in the following way (raw = store
|
||||
// number directly, delta = store delta to the previous number, dod = store
|
||||
// delta of the delta to the previous number, xor = what we do for regular
|
||||
// sample values):
|
||||
//
|
||||
// field → ts count zeroCount sum []posbuckets []negbuckets
|
||||
// sample 1 raw raw raw raw []raw []raw
|
||||
// sample 2 delta delta delta xor []delta []delta
|
||||
// sample >2 dod dod dod xor []dod []dod
|
||||
type HistogramChunk struct {
|
||||
b bstream
|
||||
}
|
||||
|
||||
// NewHistogramChunk returns a new chunk with histogram encoding of the given
|
||||
// size.
|
||||
func NewHistogramChunk() *HistogramChunk {
|
||||
b := make([]byte, 3, 128)
|
||||
return &HistogramChunk{b: bstream{stream: b, count: 0}}
|
||||
}
|
||||
|
||||
// Encoding returns the encoding type.
|
||||
func (c *HistogramChunk) Encoding() Encoding {
|
||||
return EncHistogram
|
||||
}
|
||||
|
||||
// Bytes returns the underlying byte slice of the chunk.
|
||||
func (c *HistogramChunk) Bytes() []byte {
|
||||
return c.b.bytes()
|
||||
}
|
||||
|
||||
// NumSamples returns the number of samples in the chunk.
|
||||
func (c *HistogramChunk) NumSamples() int {
|
||||
return int(binary.BigEndian.Uint16(c.Bytes()))
|
||||
}
|
||||
|
||||
// Meta returns the histogram metadata. Only call this on chunks that have at
|
||||
// least one sample.
|
||||
func (c *HistogramChunk) Meta() (
|
||||
schema int32, zeroThreshold float64,
|
||||
negativeSpans, positiveSpans []histogram.Span,
|
||||
err error,
|
||||
) {
|
||||
if c.NumSamples() == 0 {
|
||||
panic("HistoChunk.Meta() called on an empty chunk")
|
||||
}
|
||||
b := newBReader(c.Bytes()[2:])
|
||||
return readHistogramChunkMeta(&b)
|
||||
}
|
||||
|
||||
// CounterResetHeader defines the first 2 bits of the chunk header.
|
||||
type CounterResetHeader byte
|
||||
|
||||
const (
|
||||
CounterReset CounterResetHeader = 0b10000000
|
||||
NotCounterReset CounterResetHeader = 0b01000000
|
||||
GaugeType CounterResetHeader = 0b11000000
|
||||
UnknownCounterReset CounterResetHeader = 0b00000000
|
||||
)
|
||||
|
||||
// SetCounterResetHeader sets the counter reset header.
|
||||
func (c *HistogramChunk) SetCounterResetHeader(h CounterResetHeader) {
|
||||
switch h {
|
||||
case CounterReset, NotCounterReset, GaugeType, UnknownCounterReset:
|
||||
bytes := c.Bytes()
|
||||
bytes[2] = (bytes[2] & 0b00111111) | byte(h)
|
||||
default:
|
||||
panic("invalid CounterResetHeader type")
|
||||
}
|
||||
}
|
||||
|
||||
// GetCounterResetHeader returns the info about the first 2 bits of the chunk
|
||||
// header.
|
||||
func (c *HistogramChunk) GetCounterResetHeader() CounterResetHeader {
|
||||
return CounterResetHeader(c.Bytes()[2] & 0b11000000)
|
||||
}
|
||||
|
||||
// Compact implements the Chunk interface.
|
||||
func (c *HistogramChunk) Compact() {
|
||||
if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold {
|
||||
buf := make([]byte, l)
|
||||
copy(buf, c.b.stream)
|
||||
c.b.stream = buf
|
||||
}
|
||||
}
|
||||
|
||||
// Appender implements the Chunk interface.
|
||||
func (c *HistogramChunk) Appender() (Appender, error) {
|
||||
it := c.iterator(nil)
|
||||
|
||||
// To get an appender, we must know the state it would have if we had
|
||||
// appended all existing data from scratch. We iterate through the end
|
||||
// and populate via the iterator's state.
|
||||
for it.Next() {
|
||||
}
|
||||
if err := it.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
a := &HistogramAppender{
|
||||
b: &c.b,
|
||||
|
||||
schema: it.schema,
|
||||
zThreshold: it.zThreshold,
|
||||
pSpans: it.pSpans,
|
||||
nSpans: it.nSpans,
|
||||
t: it.t,
|
||||
cnt: it.cnt,
|
||||
zCnt: it.zCnt,
|
||||
tDelta: it.tDelta,
|
||||
cntDelta: it.cntDelta,
|
||||
zCntDelta: it.zCntDelta,
|
||||
pBuckets: it.pBuckets,
|
||||
nBuckets: it.nBuckets,
|
||||
pBucketsDelta: it.pBucketsDelta,
|
||||
nBucketsDelta: it.nBucketsDelta,
|
||||
|
||||
sum: it.sum,
|
||||
leading: it.leading,
|
||||
trailing: it.trailing,
|
||||
|
||||
buf64: make([]byte, binary.MaxVarintLen64),
|
||||
}
|
||||
if binary.BigEndian.Uint16(a.b.bytes()) == 0 {
|
||||
a.leading = 0xff
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func countSpans(spans []histogram.Span) int {
|
||||
var cnt int
|
||||
for _, s := range spans {
|
||||
cnt += int(s.Length)
|
||||
}
|
||||
return cnt
|
||||
}
|
||||
|
||||
func newHistogramIterator(b []byte) *histogramIterator {
|
||||
it := &histogramIterator{
|
||||
br: newBReader(b),
|
||||
numTotal: binary.BigEndian.Uint16(b),
|
||||
t: math.MinInt64,
|
||||
}
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
_, _ = it.br.readBits(16)
|
||||
return it
|
||||
}
|
||||
|
||||
func (c *HistogramChunk) iterator(it Iterator) *histogramIterator {
|
||||
// This commet is copied from XORChunk.iterator:
|
||||
// Should iterators guarantee to act on a copy of the data so it doesn't lock append?
|
||||
// When using striped locks to guard access to chunks, probably yes.
|
||||
// Could only copy data if the chunk is not completed yet.
|
||||
if histogramIter, ok := it.(*histogramIterator); ok {
|
||||
histogramIter.Reset(c.b.bytes())
|
||||
return histogramIter
|
||||
}
|
||||
return newHistogramIterator(c.b.bytes())
|
||||
}
|
||||
|
||||
// Iterator implements the Chunk interface.
|
||||
func (c *HistogramChunk) Iterator(it Iterator) Iterator {
|
||||
return c.iterator(it)
|
||||
}
|
||||
|
||||
// HistogramAppender is an Appender implementation for sparse histograms.
|
||||
type HistogramAppender struct {
|
||||
b *bstream
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zThreshold float64
|
||||
pSpans, nSpans []histogram.Span
|
||||
|
||||
// Although we intend to start new chunks on counter resets, we still
|
||||
// have to handle negative deltas for gauge histograms. Therefore, even
|
||||
// deltas are signed types here (even for tDelta to not treat that one
|
||||
// specially).
|
||||
t int64
|
||||
cnt, zCnt uint64
|
||||
tDelta, cntDelta, zCntDelta int64
|
||||
pBuckets, nBuckets []int64
|
||||
pBucketsDelta, nBucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
buf64 []byte // For working on varint64's.
|
||||
}
|
||||
|
||||
func putVarint(b *bstream, buf []byte, x int64) {
|
||||
for _, byt := range buf[:binary.PutVarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
func putUvarint(b *bstream, buf []byte, x uint64) {
|
||||
for _, byt := range buf[:binary.PutUvarint(buf, x)] {
|
||||
b.writeByte(byt)
|
||||
}
|
||||
}
|
||||
|
||||
// Append implements Appender. This implementation panics because normal float
|
||||
// samples must never be appended to a histogram chunk.
|
||||
func (a *HistogramAppender) Append(int64, float64) {
|
||||
panic("appended a float sample to a histogram chunk")
|
||||
}
|
||||
|
||||
// Appendable returns whether the chunk can be appended to, and if so
|
||||
// whether any recoding needs to happen using the provided interjections
|
||||
// (in case of any new buckets, positive or negative range, respectively).
|
||||
//
|
||||
// The chunk is not appendable in the following cases:
|
||||
//
|
||||
// • The schema has changed.
|
||||
//
|
||||
// • The threshold for the zero bucket has changed.
|
||||
//
|
||||
// • Any buckets have disappeared.
|
||||
//
|
||||
// • There was a counter reset in the count of observations or in any bucket,
|
||||
// including the zero bucket.
|
||||
//
|
||||
// • The last sample in the chunk was stale while the current sample is not stale.
|
||||
//
|
||||
// The method returns an additional boolean set to true if it is not appendable
|
||||
// because of a counter reset. If the given sample is stale, it is always ok to
|
||||
// append. If counterReset is true, okToAppend is always false.
|
||||
func (a *HistogramAppender) Appendable(h histogram.Histogram) (
|
||||
positiveInterjections, negativeInterjections []Interjection,
|
||||
okToAppend bool, counterReset bool,
|
||||
) {
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// This is a stale sample whose buckets and spans don't matter.
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
if value.IsStaleNaN(a.sum) {
|
||||
// If the last sample was stale, then we can only accept stale
|
||||
// samples in this chunk.
|
||||
return
|
||||
}
|
||||
|
||||
if h.Count < a.cnt {
|
||||
// There has been a counter reset.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if h.Schema != a.schema || h.ZeroThreshold != a.zThreshold {
|
||||
return
|
||||
}
|
||||
|
||||
if h.ZeroCount < a.zCnt {
|
||||
// There has been a counter reset since ZeroThreshold didn't change.
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
var ok bool
|
||||
positiveInterjections, ok = compareSpans(a.pSpans, h.PositiveSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
negativeInterjections, ok = compareSpans(a.nSpans, h.NegativeSpans)
|
||||
if !ok {
|
||||
counterReset = true
|
||||
return
|
||||
}
|
||||
|
||||
if counterResetInAnyBucket(a.pBuckets, h.PositiveBuckets, a.pSpans, h.PositiveSpans) ||
|
||||
counterResetInAnyBucket(a.nBuckets, h.NegativeBuckets, a.nSpans, h.NegativeSpans) {
|
||||
counterReset, positiveInterjections, negativeInterjections = true, nil, nil
|
||||
return
|
||||
}
|
||||
|
||||
okToAppend = true
|
||||
return
|
||||
}
|
||||
|
||||
// counterResetInAnyBucket returns true if there was a counter reset for any
|
||||
// bucket. This should be called only when the bucket layout is the same or new
|
||||
// buckets were added. It does not handle the case of buckets missing.
|
||||
func counterResetInAnyBucket(oldBuckets, newBuckets []int64, oldSpans, newSpans []histogram.Span) bool {
|
||||
if len(oldSpans) == 0 || len(oldBuckets) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
oldSpanSliceIdx, newSpanSliceIdx := 0, 0 // Index for the span slices.
|
||||
oldInsideSpanIdx, newInsideSpanIdx := uint32(0), uint32(0) // Index inside a span.
|
||||
oldIdx, newIdx := oldSpans[0].Offset, newSpans[0].Offset
|
||||
|
||||
oldBucketSliceIdx, newBucketSliceIdx := 0, 0 // Index inside bucket slice.
|
||||
oldVal, newVal := oldBuckets[0], newBuckets[0]
|
||||
|
||||
// Since we assume that new spans won't have missing buckets, there will never be a case
|
||||
// where the old index will not find a matching new index.
|
||||
for {
|
||||
if oldIdx == newIdx {
|
||||
if newVal < oldVal {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if oldIdx <= newIdx {
|
||||
// Moving ahead old bucket and span by 1 index.
|
||||
if oldInsideSpanIdx == oldSpans[oldSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
oldSpanSliceIdx++
|
||||
oldInsideSpanIdx = 0
|
||||
if oldSpanSliceIdx >= len(oldSpans) {
|
||||
// All old spans are over.
|
||||
break
|
||||
}
|
||||
oldIdx += 1 + oldSpans[oldSpanSliceIdx].Offset
|
||||
} else {
|
||||
oldInsideSpanIdx++
|
||||
oldIdx++
|
||||
}
|
||||
oldBucketSliceIdx++
|
||||
oldVal += oldBuckets[oldBucketSliceIdx]
|
||||
}
|
||||
|
||||
if oldIdx > newIdx {
|
||||
// Moving ahead new bucket and span by 1 index.
|
||||
if newInsideSpanIdx == newSpans[newSpanSliceIdx].Length-1 {
|
||||
// Current span is over.
|
||||
newSpanSliceIdx++
|
||||
newInsideSpanIdx = 0
|
||||
if newSpanSliceIdx >= len(newSpans) {
|
||||
// All new spans are over.
|
||||
// This should not happen, old spans above should catch this first.
|
||||
panic("new spans over before old spans in counterReset")
|
||||
}
|
||||
newIdx += 1 + newSpans[newSpanSliceIdx].Offset
|
||||
} else {
|
||||
newInsideSpanIdx++
|
||||
newIdx++
|
||||
}
|
||||
newBucketSliceIdx++
|
||||
newVal += newBuckets[newBucketSliceIdx]
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// AppendHistogram appends a histogram to the chunk. The caller must ensure that
|
||||
// the histogram is properly structured, e.g. the number of buckets used
|
||||
// corresponds to the number conveyed by the span structures. First call
|
||||
// Appendable() and act accordingly!
|
||||
func (a *HistogramAppender) AppendHistogram(t int64, h histogram.Histogram) {
|
||||
var tDelta, cntDelta, zCntDelta int64
|
||||
num := binary.BigEndian.Uint16(a.b.bytes())
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
// Emptying out other fields to write no buckets, and an empty
|
||||
// meta in case of first histogram in the chunk.
|
||||
h = histogram.Histogram{Sum: h.Sum}
|
||||
}
|
||||
|
||||
switch num {
|
||||
case 0:
|
||||
// The first append gets the privilege to dictate the metadata
|
||||
// but it's also responsible for encoding it into the chunk!
|
||||
writeHistogramChunkMeta(a.b, h.Schema, h.ZeroThreshold, h.PositiveSpans, h.NegativeSpans)
|
||||
a.schema = h.Schema
|
||||
a.zThreshold = h.ZeroThreshold
|
||||
a.pSpans, a.nSpans = h.PositiveSpans, h.NegativeSpans
|
||||
numPBuckets, numNBuckets := countSpans(h.PositiveSpans), countSpans(h.NegativeSpans)
|
||||
a.pBuckets = make([]int64, numPBuckets)
|
||||
a.nBuckets = make([]int64, numNBuckets)
|
||||
a.pBucketsDelta = make([]int64, numPBuckets)
|
||||
a.nBucketsDelta = make([]int64, numNBuckets)
|
||||
|
||||
// Now store the actual data.
|
||||
putVarint(a.b, a.buf64, t)
|
||||
putUvarint(a.b, a.buf64, h.Count) // TODO(beorn7): Use putVarbitInt?
|
||||
putUvarint(a.b, a.buf64, h.ZeroCount) // TODO(beorn7): Use putVarbitInt?
|
||||
a.b.writeBits(math.Float64bits(h.Sum), 64)
|
||||
for _, buck := range h.PositiveBuckets {
|
||||
putVarint(a.b, a.buf64, buck) // TODO(beorn7): Use putVarbitInt?
|
||||
}
|
||||
for _, buck := range h.NegativeBuckets {
|
||||
putVarint(a.b, a.buf64, buck) // TODO(beorn7): Use putVarbitInt?
|
||||
}
|
||||
case 1:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zCntDelta = int64(h.ZeroCount) - int64(a.zCnt)
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDelta, zCntDelta = 0, 0
|
||||
}
|
||||
|
||||
putVarint(a.b, a.buf64, tDelta) // TODO(beorn7): This should probably be putUvarint.
|
||||
putVarint(a.b, a.buf64, cntDelta) // TODO(beorn7): Use putVarbitInt?
|
||||
putVarint(a.b, a.buf64, zCntDelta) // TODO(beorn7): Use putVarbitInt?
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.pBuckets[i]
|
||||
putVarint(a.b, a.buf64, delta) // TODO(beorn7): Use putVarbitInt?
|
||||
a.pBucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.nBuckets[i]
|
||||
putVarint(a.b, a.buf64, delta) // TODO(beorn7): Use putVarbitInt?
|
||||
a.nBucketsDelta[i] = delta
|
||||
}
|
||||
|
||||
default:
|
||||
tDelta = t - a.t
|
||||
cntDelta = int64(h.Count) - int64(a.cnt)
|
||||
zCntDelta = int64(h.ZeroCount) - int64(a.zCnt)
|
||||
|
||||
tDod := tDelta - a.tDelta
|
||||
cntDod := cntDelta - a.cntDelta
|
||||
zCntDod := zCntDelta - a.zCntDelta
|
||||
|
||||
if value.IsStaleNaN(h.Sum) {
|
||||
cntDod, zCntDod = 0, 0
|
||||
}
|
||||
|
||||
putVarbitInt(a.b, tDod)
|
||||
putVarbitInt(a.b, cntDod)
|
||||
putVarbitInt(a.b, zCntDod)
|
||||
|
||||
a.writeSumDelta(h.Sum)
|
||||
|
||||
for i, buck := range h.PositiveBuckets {
|
||||
delta := buck - a.pBuckets[i]
|
||||
dod := delta - a.pBucketsDelta[i]
|
||||
putVarbitInt(a.b, dod)
|
||||
a.pBucketsDelta[i] = delta
|
||||
}
|
||||
for i, buck := range h.NegativeBuckets {
|
||||
delta := buck - a.nBuckets[i]
|
||||
dod := delta - a.nBucketsDelta[i]
|
||||
putVarbitInt(a.b, dod)
|
||||
a.nBucketsDelta[i] = delta
|
||||
}
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint16(a.b.bytes(), num+1)
|
||||
|
||||
a.t = t
|
||||
a.cnt = h.Count
|
||||
a.zCnt = h.ZeroCount
|
||||
a.tDelta = tDelta
|
||||
a.cntDelta = cntDelta
|
||||
a.zCntDelta = zCntDelta
|
||||
|
||||
a.pBuckets, a.nBuckets = h.PositiveBuckets, h.NegativeBuckets
|
||||
// Note that the bucket deltas were already updated above.
|
||||
a.sum = h.Sum
|
||||
}
|
||||
|
||||
// Recode converts the current chunk to accommodate an expansion of the set of
|
||||
// (positive and/or negative) buckets used, according to the provided
|
||||
// interjections, resulting in the honoring of the provided new positive and
|
||||
// negative spans.
|
||||
func (a *HistogramAppender) Recode(
|
||||
positiveInterjections, negativeInterjections []Interjection,
|
||||
positiveSpans, negativeSpans []histogram.Span,
|
||||
) (Chunk, Appender) {
|
||||
// TODO(beorn7): This currently just decodes everything and then encodes
|
||||
// it again with the new span layout. This can probably be done in-place
|
||||
// by editing the chunk. But let's first see how expensive it is in the
|
||||
// big picture.
|
||||
byts := a.b.bytes()
|
||||
it := newHistogramIterator(byts)
|
||||
hc := NewHistogramChunk()
|
||||
app, err := hc.Appender()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
numPositiveBuckets, numNegativeBuckets := countSpans(positiveSpans), countSpans(negativeSpans)
|
||||
|
||||
for it.Next() {
|
||||
tOld, hOld := it.AtHistogram()
|
||||
|
||||
// We have to newly allocate slices for the modified buckets
|
||||
// here because they are kept by the appender until the next
|
||||
// append.
|
||||
// TODO(beorn7): We might be able to optimize this.
|
||||
positiveBuckets := make([]int64, numPositiveBuckets)
|
||||
negativeBuckets := make([]int64, numNegativeBuckets)
|
||||
|
||||
// Save the modified histogram to the new chunk.
|
||||
hOld.PositiveSpans, hOld.NegativeSpans = positiveSpans, negativeSpans
|
||||
if len(positiveInterjections) > 0 {
|
||||
hOld.PositiveBuckets = interject(hOld.PositiveBuckets, positiveBuckets, positiveInterjections)
|
||||
}
|
||||
if len(negativeInterjections) > 0 {
|
||||
hOld.NegativeBuckets = interject(hOld.NegativeBuckets, negativeBuckets, negativeInterjections)
|
||||
}
|
||||
app.AppendHistogram(tOld, hOld)
|
||||
}
|
||||
|
||||
hc.SetCounterResetHeader(CounterResetHeader(byts[2] & 0b11000000))
|
||||
return hc, app
|
||||
}
|
||||
|
||||
func (a *HistogramAppender) writeSumDelta(v float64) {
|
||||
vDelta := math.Float64bits(v) ^ math.Float64bits(a.sum)
|
||||
|
||||
if vDelta == 0 {
|
||||
a.b.writeBit(zero)
|
||||
return
|
||||
}
|
||||
a.b.writeBit(one)
|
||||
|
||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
||||
|
||||
// Clamp number of leading zeros to avoid overflow when encoding.
|
||||
if leading >= 32 {
|
||||
leading = 31
|
||||
}
|
||||
|
||||
if a.leading != 0xff && leading >= a.leading && trailing >= a.trailing {
|
||||
a.b.writeBit(zero)
|
||||
a.b.writeBits(vDelta>>a.trailing, 64-int(a.leading)-int(a.trailing))
|
||||
} else {
|
||||
a.leading, a.trailing = leading, trailing
|
||||
|
||||
a.b.writeBit(one)
|
||||
a.b.writeBits(uint64(leading), 5)
|
||||
|
||||
// Note that if leading == trailing == 0, then sigbits == 64.
|
||||
// But that value doesn't actually fit into the 6 bits we have.
|
||||
// Luckily, we never need to encode 0 significant bits, since
|
||||
// that would put us in the other case (vdelta == 0). So
|
||||
// instead we write out a 0 and adjust it back to 64 on
|
||||
// unpacking.
|
||||
sigbits := 64 - leading - trailing
|
||||
a.b.writeBits(uint64(sigbits), 6)
|
||||
a.b.writeBits(vDelta>>trailing, int(sigbits))
|
||||
}
|
||||
}
|
||||
|
||||
type histogramIterator struct {
|
||||
br bstreamReader
|
||||
numTotal uint16
|
||||
numRead uint16
|
||||
|
||||
// Metadata:
|
||||
schema int32
|
||||
zThreshold float64
|
||||
pSpans, nSpans []histogram.Span
|
||||
|
||||
// For the fields that are tracked as deltas and ultimately dod's.
|
||||
t int64
|
||||
cnt, zCnt uint64
|
||||
tDelta, cntDelta, zCntDelta int64
|
||||
pBuckets, nBuckets []int64
|
||||
pBucketsDelta, nBucketsDelta []int64
|
||||
|
||||
// The sum is Gorilla xor encoded.
|
||||
sum float64
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Seek(t int64) bool {
|
||||
if it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for t > it.t || it.numRead == 0 {
|
||||
if !it.Next() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histogramIterator) At() (int64, float64) {
|
||||
panic("cannot call histogramIterator.At")
|
||||
}
|
||||
|
||||
func (it *histogramIterator) ChunkEncoding() Encoding {
|
||||
return EncHistogram
|
||||
}
|
||||
|
||||
func (it *histogramIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
return it.t, histogram.Histogram{Sum: it.sum}
|
||||
}
|
||||
return it.t, histogram.Histogram{
|
||||
Count: it.cnt,
|
||||
ZeroCount: it.zCnt,
|
||||
Sum: it.sum,
|
||||
ZeroThreshold: it.zThreshold,
|
||||
Schema: it.schema,
|
||||
PositiveSpans: it.pSpans,
|
||||
NegativeSpans: it.nSpans,
|
||||
PositiveBuckets: it.pBuckets,
|
||||
NegativeBuckets: it.nBuckets,
|
||||
}
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Reset(b []byte) {
|
||||
// The first 2 bytes contain chunk headers.
|
||||
// We skip that for actual samples.
|
||||
it.br = newBReader(b[2:])
|
||||
it.numTotal = binary.BigEndian.Uint16(b)
|
||||
it.numRead = 0
|
||||
|
||||
it.t, it.cnt, it.zCnt = 0, 0, 0
|
||||
it.tDelta, it.cntDelta, it.zCntDelta = 0, 0, 0
|
||||
|
||||
// TODO(beorn7): Those will be recreated anyway.
|
||||
// Either delete them here entirely or recycle them
|
||||
// below if big enough.
|
||||
for i := range it.pBuckets {
|
||||
it.pBuckets[i] = 0
|
||||
it.pBucketsDelta[i] = 0
|
||||
}
|
||||
for i := range it.nBuckets {
|
||||
it.nBuckets[i] = 0
|
||||
it.nBucketsDelta[i] = 0
|
||||
}
|
||||
|
||||
it.sum = 0
|
||||
it.leading = 0
|
||||
it.trailing = 0
|
||||
it.err = nil
|
||||
}
|
||||
|
||||
func (it *histogramIterator) Next() bool {
|
||||
if it.err != nil || it.numRead == it.numTotal {
|
||||
return false
|
||||
}
|
||||
|
||||
if it.numRead == 0 {
|
||||
|
||||
// The first read is responsible for reading the chunk metadata
|
||||
// and for initializing fields that depend on it. We give
|
||||
// counter reset info at chunk level, hence we discard it here.
|
||||
schema, zeroThreshold, posSpans, negSpans, err := readHistogramChunkMeta(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.schema = schema
|
||||
it.zThreshold = zeroThreshold
|
||||
it.pSpans, it.nSpans = posSpans, negSpans
|
||||
numPosBuckets, numNegBuckets := countSpans(posSpans), countSpans(negSpans)
|
||||
if numPosBuckets > 0 {
|
||||
it.pBuckets = make([]int64, numPosBuckets)
|
||||
it.pBucketsDelta = make([]int64, numPosBuckets)
|
||||
}
|
||||
if numNegBuckets > 0 {
|
||||
it.nBuckets = make([]int64, numNegBuckets)
|
||||
it.nBucketsDelta = make([]int64, numNegBuckets)
|
||||
}
|
||||
|
||||
// Now read the actual data.
|
||||
t, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.t = t
|
||||
|
||||
cnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cnt = cnt
|
||||
|
||||
zcnt, err := binary.ReadUvarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zCnt = zcnt
|
||||
|
||||
sum, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.sum = math.Float64frombits(sum)
|
||||
|
||||
for i := range it.pBuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.pBuckets[i] = v
|
||||
}
|
||||
for i := range it.nBuckets {
|
||||
v, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.nBuckets[i] = v
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
if it.numRead == 1 {
|
||||
tDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = tDelta
|
||||
it.t += int64(it.tDelta)
|
||||
|
||||
cntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = cntDelta
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDelta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zCntDelta = zcntDelta
|
||||
it.zCnt = uint64(int64(it.zCnt) + it.zCntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.pBuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.pBucketsDelta[i] = delta
|
||||
it.pBuckets[i] = it.pBuckets[i] + delta
|
||||
}
|
||||
|
||||
for i := range it.nBuckets {
|
||||
delta, err := binary.ReadVarint(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.nBucketsDelta[i] = delta
|
||||
it.nBuckets[i] = it.nBuckets[i] + delta
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
tDod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.tDelta = it.tDelta + tDod
|
||||
it.t += it.tDelta
|
||||
|
||||
cntDod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.cntDelta = it.cntDelta + cntDod
|
||||
it.cnt = uint64(int64(it.cnt) + it.cntDelta)
|
||||
|
||||
zcntDod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.zCntDelta = it.zCntDelta + zcntDod
|
||||
it.zCnt = uint64(int64(it.zCnt) + it.zCntDelta)
|
||||
|
||||
ok := it.readSum()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(it.sum) {
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
for i := range it.pBuckets {
|
||||
dod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.pBucketsDelta[i] = it.pBucketsDelta[i] + dod
|
||||
it.pBuckets[i] = it.pBuckets[i] + it.pBucketsDelta[i]
|
||||
}
|
||||
|
||||
for i := range it.nBuckets {
|
||||
dod, err := readVarbitInt(&it.br)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.nBucketsDelta[i] = it.nBucketsDelta[i] + dod
|
||||
it.nBuckets[i] = it.nBuckets[i] + it.nBucketsDelta[i]
|
||||
}
|
||||
|
||||
it.numRead++
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *histogramIterator) readSum() bool {
|
||||
bit, err := it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if bit == zero {
|
||||
return true // it.sum = it.sum
|
||||
}
|
||||
|
||||
bit, err = it.br.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = it.br.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
if bit == zero {
|
||||
// Reuse leading/trailing zero bits.
|
||||
// it.leading, it.trailing = it.leading, it.trailing
|
||||
} else {
|
||||
bits, err := it.br.readBitsFast(5)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(5)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.leading = uint8(bits)
|
||||
|
||||
bits, err = it.br.readBitsFast(6)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(6)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
mbits := uint8(bits)
|
||||
// 0 significant bits here means we overflowed and we actually
|
||||
// need 64; see comment in encoder.
|
||||
if mbits == 0 {
|
||||
mbits = 64
|
||||
}
|
||||
it.trailing = 64 - it.leading - mbits
|
||||
}
|
||||
|
||||
mbits := 64 - it.leading - it.trailing
|
||||
bits, err := it.br.readBitsFast(mbits)
|
||||
if err != nil {
|
||||
bits, err = it.br.readBits(mbits)
|
||||
}
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
vbits := math.Float64bits(it.sum)
|
||||
vbits ^= bits << it.trailing
|
||||
it.sum = math.Float64frombits(vbits)
|
||||
return true
|
||||
}
|
||||
|
|
@ -0,0 +1,286 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
)
|
||||
|
||||
func writeHistogramChunkMeta(b *bstream, schema int32, zeroThreshold float64, positiveSpans, negativeSpans []histogram.Span) {
|
||||
putVarbitInt(b, int64(schema))
|
||||
putVarbitFloat(b, zeroThreshold)
|
||||
putHistogramChunkMetaSpans(b, positiveSpans)
|
||||
putHistogramChunkMetaSpans(b, negativeSpans)
|
||||
}
|
||||
|
||||
func putHistogramChunkMetaSpans(b *bstream, spans []histogram.Span) {
|
||||
putVarbitInt(b, int64(len(spans)))
|
||||
for _, s := range spans {
|
||||
putVarbitInt(b, int64(s.Length))
|
||||
putVarbitInt(b, int64(s.Offset))
|
||||
}
|
||||
}
|
||||
|
||||
func readHistogramChunkMeta(b *bstreamReader) (
|
||||
schema int32, zeroThreshold float64,
|
||||
positiveSpans, negativeSpans []histogram.Span,
|
||||
err error,
|
||||
) {
|
||||
_, err = b.ReadByte() // The header.
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
v, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
schema = int32(v)
|
||||
|
||||
zeroThreshold, err = readVarbitFloat(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
positiveSpans, err = readHistogramChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
negativeSpans, err = readHistogramChunkMetaSpans(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func readHistogramChunkMetaSpans(b *bstreamReader) ([]histogram.Span, error) {
|
||||
var spans []histogram.Span
|
||||
num, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := 0; i < int(num); i++ {
|
||||
|
||||
length, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
offset, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
spans = append(spans, histogram.Span{
|
||||
Length: uint32(length),
|
||||
Offset: int32(offset),
|
||||
})
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
|
||||
type bucketIterator struct {
|
||||
spans []histogram.Span
|
||||
span int // Span position of last yielded bucket.
|
||||
bucket int // Bucket position within span of last yielded bucket.
|
||||
idx int // Bucket index (globally across all spans) of last yielded bucket.
|
||||
}
|
||||
|
||||
func newBucketIterator(spans []histogram.Span) *bucketIterator {
|
||||
b := bucketIterator{
|
||||
spans: spans,
|
||||
span: 0,
|
||||
bucket: -1,
|
||||
idx: -1,
|
||||
}
|
||||
if len(spans) > 0 {
|
||||
b.idx += int(spans[0].Offset)
|
||||
}
|
||||
return &b
|
||||
}
|
||||
|
||||
func (b *bucketIterator) Next() (int, bool) {
|
||||
// We're already out of bounds.
|
||||
if b.span >= len(b.spans) {
|
||||
return 0, false
|
||||
}
|
||||
try:
|
||||
if b.bucket < int(b.spans[b.span].Length-1) { // Try to move within same span.
|
||||
b.bucket++
|
||||
b.idx++
|
||||
return b.idx, true
|
||||
} else if b.span < len(b.spans)-1 { // Try to move from one span to the next.
|
||||
b.span++
|
||||
b.idx += int(b.spans[b.span].Offset + 1)
|
||||
b.bucket = 0
|
||||
if b.spans[b.span].Length == 0 {
|
||||
// Pathological case that should never happen. We can't use this span, let's try again.
|
||||
goto try
|
||||
}
|
||||
return b.idx, true
|
||||
}
|
||||
// We're out of options.
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// An Interjection describes how many new buckets have to be introduced before
|
||||
// processing the pos'th delta from the original slice.
|
||||
type Interjection struct {
|
||||
pos int
|
||||
num int
|
||||
}
|
||||
|
||||
// compareSpans returns the interjections to convert a slice of deltas to a new
|
||||
// slice representing an expanded set of buckets, or false if incompatible
|
||||
// (e.g. if buckets were removed).
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// Let's say the old buckets look like this:
|
||||
//
|
||||
// span syntax: [offset, length]
|
||||
// spans : [ 0 , 2 ] [2,1] [ 3 , 2 ] [3,1] [1,1]
|
||||
// bucket idx : [0] [1] 2 3 [4] 5 6 7 [8] [9] 10 11 12 [13] 14 [15]
|
||||
// raw values 6 3 3 2 4 5 1
|
||||
// deltas 6 -3 0 -1 2 1 -4
|
||||
//
|
||||
// But now we introduce a new bucket layout. (Carefully chosen example where we
|
||||
// have a span appended, one unchanged[*], one prepended, and two merge - in
|
||||
// that order.)
|
||||
//
|
||||
// [*] unchanged in terms of which bucket indices they represent. but to achieve
|
||||
// that, their offset needs to change if "disrupted" by spans changing ahead of
|
||||
// them
|
||||
//
|
||||
// \/ this one is "unchanged"
|
||||
// spans : [ 0 , 3 ] [1,1] [ 1 , 4 ] [ 3 , 3 ]
|
||||
// bucket idx : [0] [1] [2] 3 [4] 5 [6] [7] [8] [9] 10 11 12 [13] [14] [15]
|
||||
// raw values 6 3 0 3 0 0 2 4 5 0 1
|
||||
// deltas 6 -3 -3 3 -3 0 2 2 1 -5 1
|
||||
// delta mods: / \ / \ / \
|
||||
//
|
||||
// Note that whenever any new buckets are introduced, the subsequent "old"
|
||||
// bucket needs to readjust its delta to the new base of 0. Thus, for the caller
|
||||
// who wants to transform the set of original deltas to a new set of deltas to
|
||||
// match a new span layout that adds buckets, we simply need to generate a list
|
||||
// of interjections.
|
||||
//
|
||||
// Note: Within compareSpans we don't have to worry about the changes to the
|
||||
// spans themselves, thanks to the iterators we get to work with the more useful
|
||||
// bucket indices (which of course directly correspond to the buckets we have to
|
||||
// adjust).
|
||||
func compareSpans(a, b []histogram.Span) ([]Interjection, bool) {
|
||||
ai := newBucketIterator(a)
|
||||
bi := newBucketIterator(b)
|
||||
|
||||
var interjections []Interjection
|
||||
|
||||
// When inter.num becomes > 0, this becomes a valid interjection that
|
||||
// should be yielded when we finish a streak of new buckets.
|
||||
var inter Interjection
|
||||
|
||||
av, aOK := ai.Next()
|
||||
bv, bOK := bi.Next()
|
||||
loop:
|
||||
for {
|
||||
switch {
|
||||
case aOK && bOK:
|
||||
switch {
|
||||
case av == bv: // Both have an identical value. move on!
|
||||
// Finish WIP interjection and reset.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
inter.num = 0
|
||||
av, aOK = ai.Next()
|
||||
bv, bOK = bi.Next()
|
||||
inter.pos++
|
||||
case av < bv: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case av > bv: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bOK = bi.Next()
|
||||
}
|
||||
case aOK && !bOK: // b misses a value that is in a.
|
||||
return interjections, false
|
||||
case !aOK && bOK: // a misses a value that is in b. Forward b and recompare.
|
||||
inter.num++
|
||||
bv, bOK = bi.Next()
|
||||
default: // Both iterators ran out. We're done.
|
||||
if inter.num > 0 {
|
||||
interjections = append(interjections, inter)
|
||||
}
|
||||
break loop
|
||||
}
|
||||
}
|
||||
|
||||
return interjections, true
|
||||
}
|
||||
|
||||
// interject merges 'in' with the provided interjections and writes them into
|
||||
// 'out', which must already have the appropriate length.
|
||||
func interject(in, out []int64, interjections []Interjection) []int64 {
|
||||
var (
|
||||
j int // Position in out.
|
||||
v int64 // The last value seen.
|
||||
interj int // The next interjection to process.
|
||||
)
|
||||
for i, d := range in {
|
||||
if interj < len(interjections) && i == interjections[interj].pos {
|
||||
|
||||
// We have an interjection!
|
||||
// Add interjection.num new delta values such that their
|
||||
// bucket values equate 0.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
interj++
|
||||
|
||||
// Now save the value from the input. The delta value we
|
||||
// should save is the original delta value + the last
|
||||
// value of the point before the interjection (to undo
|
||||
// the delta that was introduced by the interjection).
|
||||
out[j] = d + v
|
||||
j++
|
||||
v = d + v
|
||||
continue
|
||||
}
|
||||
|
||||
// If there was no interjection, the original delta is still
|
||||
// valid.
|
||||
out[j] = d
|
||||
j++
|
||||
v += d
|
||||
}
|
||||
switch interj {
|
||||
case len(interjections):
|
||||
// All interjections processed. Nothing more to do.
|
||||
case len(interjections) - 1:
|
||||
// One more interjection to process at the end.
|
||||
out[j] = int64(-v)
|
||||
j++
|
||||
for x := 1; x < interjections[interj].num; x++ {
|
||||
out[j] = 0
|
||||
j++
|
||||
}
|
||||
default:
|
||||
panic("unprocessed interjections left")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -21,13 +21,15 @@ package chunkenc
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// example of a span layout and resulting bucket indices (_idx_ is used in this histogram, others are shown just for context)
|
||||
// spans : [offset: 0, length: 2] [offset 1, length 1]
|
||||
// bucket idx : _0_ _1_ 2 [3] 4 ...
|
||||
// Example of a span layout and resulting bucket indices (_idx_ is used in this
|
||||
// histogram, others are shown just for context):
|
||||
//
|
||||
// spans : [offset: 0, length: 2] [offset 1, length 1]
|
||||
// bucket idx : _0_ _1_ 2 [3] 4 ...
|
||||
|
||||
func TestBucketIterator(t *testing.T) {
|
||||
type test struct {
|
||||
|
|
@ -74,7 +76,7 @@ func TestBucketIterator(t *testing.T) {
|
|||
},
|
||||
idxs: []int{100, 101, 102, 103, 112, 113, 114, 115, 116, 117, 118, 119},
|
||||
},
|
||||
// the below 2 sets ore the ones described in compareSpans's comments
|
||||
// The below 2 sets ore the ones described in compareSpans's comments.
|
||||
{
|
||||
spans: []histogram.Span{
|
||||
{Offset: 0, Length: 2},
|
||||
|
|
@ -16,21 +16,21 @@ package chunkenc
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestHistoChunkSameBuckets(t *testing.T) {
|
||||
c := NewHistoChunk()
|
||||
func TestHistogramChunkSameBuckets(t *testing.T) {
|
||||
c := NewHistogramChunk()
|
||||
var exp []res
|
||||
|
||||
// Create fresh appender and add the first histogram
|
||||
// Create fresh appender and add the first histogram.
|
||||
app, err := c.Appender()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, c.NumSamples())
|
||||
|
||||
ts := int64(1234567890)
|
||||
h := histogram.SparseHistogram{
|
||||
h := histogram.Histogram{
|
||||
Count: 5,
|
||||
ZeroCount: 2,
|
||||
Sum: 18.4,
|
||||
|
|
@ -56,7 +56,7 @@ func TestHistoChunkSameBuckets(t *testing.T) {
|
|||
exp = append(exp, res{t: ts, h: h})
|
||||
require.Equal(t, 2, c.NumSamples())
|
||||
|
||||
// Add update with new appender
|
||||
// Add update with new appender.
|
||||
app, err = c.Appender()
|
||||
require.NoError(t, err)
|
||||
|
||||
|
|
@ -113,12 +113,12 @@ func TestHistoChunkSameBuckets(t *testing.T) {
|
|||
|
||||
type res struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
// mimics the scenario described for compareSpans()
|
||||
func TestHistoChunkBucketChanges(t *testing.T) {
|
||||
c := Chunk(NewHistoChunk())
|
||||
// Mimics the scenario described for compareSpans().
|
||||
func TestHistogramChunkBucketChanges(t *testing.T) {
|
||||
c := Chunk(NewHistogramChunk())
|
||||
|
||||
// Create fresh appender and add the first histogram.
|
||||
app, err := c.Appender()
|
||||
|
|
@ -126,7 +126,7 @@ func TestHistoChunkBucketChanges(t *testing.T) {
|
|||
require.Equal(t, 0, c.NumSamples())
|
||||
|
||||
ts1 := int64(1234567890)
|
||||
h1 := histogram.SparseHistogram{
|
||||
h1 := histogram.Histogram{
|
||||
Count: 5,
|
||||
ZeroCount: 2,
|
||||
Sum: 18.4,
|
||||
|
|
@ -157,24 +157,26 @@ func TestHistoChunkBucketChanges(t *testing.T) {
|
|||
h2.Count += 9
|
||||
h2.ZeroCount++
|
||||
h2.Sum = 30
|
||||
// Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
|
||||
|
||||
// This is how span changes will be handled.
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Greater(t, len(posInterjections), 0)
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.True(t, ok) // Only new buckets came in.
|
||||
require.False(t, cr)
|
||||
c, app = histoApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
|
||||
c, app = hApp.Recode(posInterjections, negInterjections, h2.PositiveSpans, h2.NegativeSpans)
|
||||
app.AppendHistogram(ts2, h2)
|
||||
|
||||
require.Equal(t, 2, c.NumSamples())
|
||||
|
||||
// Because the 2nd histogram has expanded buckets, we should expect all histograms (in particular the first)
|
||||
// to come back using the new spans metadata as well as the expanded buckets.
|
||||
// Because the 2nd histogram has expanded buckets, we should expect all
|
||||
// histograms (in particular the first) to come back using the new spans
|
||||
// metadata as well as the expanded buckets.
|
||||
h1.PositiveSpans = h2.PositiveSpans
|
||||
h1.PositiveBuckets = []int64{6, -3, -3, 3, -3, 0, 2, 2, 1, -5, 1}
|
||||
exp := []res{
|
||||
|
|
@ -192,7 +194,7 @@ func TestHistoChunkBucketChanges(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestHistoChunkAppendable(t *testing.T) {
|
||||
c := Chunk(NewHistoChunk())
|
||||
c := Chunk(NewHistogramChunk())
|
||||
|
||||
// Create fresh appender and add the first histogram.
|
||||
app, err := c.Appender()
|
||||
|
|
@ -200,7 +202,7 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
require.Equal(t, 0, c.NumSamples())
|
||||
|
||||
ts := int64(1234567890)
|
||||
h1 := histogram.SparseHistogram{
|
||||
h1 := histogram.Histogram{
|
||||
Count: 5,
|
||||
ZeroCount: 2,
|
||||
Sum: 18.4,
|
||||
|
|
@ -230,12 +232,13 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
h2.Count += 9
|
||||
h2.ZeroCount++
|
||||
h2.Sum = 30
|
||||
// Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 1} // 7 5 1 3 1 0 2 5 5 0 1 (total 30)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Greater(t, len(posInterjections), 0)
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.True(t, ok) // Only new buckets came in.
|
||||
|
|
@ -253,8 +256,8 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
h2.Sum = 21
|
||||
h2.PositiveBuckets = []int64{6, -3, -1, 2, 1, -4} // counts: 6, 3, 2, 4, 5, 1 (total 21)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
|
|
@ -266,8 +269,8 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
h2.Sum = 23
|
||||
h2.PositiveBuckets = []int64{6, -4, 1, -1, 2, 1, -4} // counts: 6, 2, 3, 2, 4, 5, 1 (total 23)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
|
|
@ -283,20 +286,24 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
{Offset: 3, Length: 3},
|
||||
}
|
||||
h2.Sum = 29
|
||||
// Existing histogram should get values converted from the above to: 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 6 3 0 3 0 0 2 4 5 0 1 (previous values with some new empty buckets in between)
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{7, -2, -4, 2, -2, -1, 2, 3, 0, -5, 0} // 7 5 1 3 1 0 2 5 5 0 0 (total 29)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
require.True(t, cr)
|
||||
}
|
||||
|
||||
{ // New histogram that has a counter reset while new buckets were added before the first bucket and reset on first bucket.
|
||||
// (to catch the edge case where the new bucket should be forwarded ahead until first old bucket at start)
|
||||
{
|
||||
// New histogram that has a counter reset while new buckets were
|
||||
// added before the first bucket and reset on first bucket. (to
|
||||
// catch the edge case where the new bucket should be forwarded
|
||||
// ahead until first old bucket at start)
|
||||
h2 := h1
|
||||
h2.PositiveSpans = []histogram.Span{
|
||||
{Offset: -3, Length: 2},
|
||||
|
|
@ -307,12 +314,13 @@ func TestHistoChunkAppendable(t *testing.T) {
|
|||
{Offset: 1, Length: 1},
|
||||
}
|
||||
h2.Sum = 26
|
||||
// Existing histogram should get values converted from the above to: 0, 0, 6, 3, 3, 2, 4, 5, 1
|
||||
// Existing histogram should get values converted from the above to:
|
||||
// 0, 0, 6, 3, 3, 2, 4, 5, 1
|
||||
// so the new histogram should have new counts >= these per-bucket counts, e.g.:
|
||||
h2.PositiveBuckets = []int64{1, 1, 3, -2, 0, -1, 2, 1, -4} // counts: 1, 2, 5, 3, 3, 2, 4, 5, 1 (total 26)
|
||||
|
||||
histoApp, _ := app.(*HistoAppender)
|
||||
posInterjections, negInterjections, ok, cr := histoApp.Appendable(h2)
|
||||
hApp, _ := app.(*HistogramAppender)
|
||||
posInterjections, negInterjections, ok, cr := hApp.Appendable(h2)
|
||||
require.Equal(t, 0, len(posInterjections))
|
||||
require.Equal(t, 0, len(negInterjections))
|
||||
require.False(t, ok) // Need to cut a new chunk.
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// putVarbitFloat writes a float64 using varbit encoding. It does so by
|
||||
// converting the underlying bits into an int64.
|
||||
func putVarbitFloat(b *bstream, val float64) {
|
||||
// TODO(beorn7): The resulting int64 here will almost never be a small
|
||||
// integer. Thus, the varbit encoding doesn't really make sense
|
||||
// here. This function is only used to encode the zero threshold in
|
||||
// histograms. Based on that, here is an idea to improve the encoding:
|
||||
//
|
||||
// It is recommended to use (usually negative) powers of two as
|
||||
// threshoulds. The default value for the zero threshald is in fact
|
||||
// 2^-128, or 0.5*2^-127, as it is represented by IEEE 754. It is
|
||||
// therefore worth a try to test if the threshold is a power of 2 and
|
||||
// then just store the exponent. 0 is also a commen threshold for those
|
||||
// use cases where only observations of precisely zero should go to the
|
||||
// zero bucket. This results in the following proposal:
|
||||
// - First we store 1 byte.
|
||||
// - Iff that byte is 255 (all bits set), it is followed by a direct
|
||||
// 8byte representation of the float.
|
||||
// - If the byte is 0, the threshold is 0.
|
||||
// - In all other cases, take the number represented by the byte,
|
||||
// subtract 246, and that's the exponent (i.e. between -245 and
|
||||
// +8, covering thresholds that are powers of 2 between 2^-246
|
||||
// to 128).
|
||||
putVarbitInt(b, int64(math.Float64bits(val)))
|
||||
}
|
||||
|
||||
// readVarbitFloat reads a float64 encoded with putVarbitFloat
|
||||
func readVarbitFloat(b *bstreamReader) (float64, error) {
|
||||
val, err := readVarbitInt(b)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return math.Float64frombits(uint64(val)), nil
|
||||
}
|
||||
|
||||
// putVarbitInt writes an int64 using varbit encoding with a bit bucketing
|
||||
// optimized for the dod's observed in histogram buckets.
|
||||
//
|
||||
// TODO(Dieterbe): We could improve this further: Each branch doesn't need to
|
||||
// support any values of any of the prior branches. So we can expand the range
|
||||
// of each branch. Do more with fewer bits. It comes at the price of more
|
||||
// expensive encoding and decoding (cutting out and later adding back that
|
||||
// center-piece we skip).
|
||||
func putVarbitInt(b *bstream, val int64) {
|
||||
switch {
|
||||
case val == 0:
|
||||
b.writeBit(zero)
|
||||
case bitRange(val, 3): // -3 <= val <= 4
|
||||
b.writeBits(0b10, 2)
|
||||
b.writeBits(uint64(val), 3)
|
||||
case bitRange(val, 6): // -31 <= val <= 32
|
||||
b.writeBits(0b110, 3)
|
||||
b.writeBits(uint64(val), 6)
|
||||
case bitRange(val, 9): // -255 <= val <= 256
|
||||
b.writeBits(0b1110, 4)
|
||||
b.writeBits(uint64(val), 9)
|
||||
case bitRange(val, 12): // -2047 <= val <= 2048
|
||||
b.writeBits(0b11110, 5)
|
||||
b.writeBits(uint64(val), 12)
|
||||
default:
|
||||
b.writeBits(0b11111, 5)
|
||||
b.writeBits(uint64(val), 64)
|
||||
}
|
||||
}
|
||||
|
||||
// readVarbitInt reads an int64 encoced with putVarbitInt.
|
||||
func readVarbitInt(b *bstreamReader) (int64, error) {
|
||||
var d byte
|
||||
for i := 0; i < 5; i++ {
|
||||
d <<= 1
|
||||
bit, err := b.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = b.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bit == zero {
|
||||
break
|
||||
}
|
||||
d |= 1
|
||||
}
|
||||
|
||||
var val int64
|
||||
var sz uint8
|
||||
|
||||
switch d {
|
||||
case 0b0:
|
||||
// val == 0
|
||||
case 0b10:
|
||||
sz = 3
|
||||
case 0b110:
|
||||
sz = 6
|
||||
case 0b1110:
|
||||
sz = 9
|
||||
case 0b11110:
|
||||
sz = 12
|
||||
case 0b11111:
|
||||
// Do not use fast because it's very unlikely it will succeed.
|
||||
bits, err := b.readBits(64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
if sz != 0 {
|
||||
bits, err := b.readBitsFast(sz)
|
||||
if err != nil {
|
||||
bits, err = b.readBits(sz)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bits > (1 << (sz - 1)) {
|
||||
// Or something.
|
||||
bits = bits - (1 << sz)
|
||||
}
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
return val, nil
|
||||
}
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The code in this file was largely written by Damian Gryski as part of
|
||||
// https://github.com/dgryski/go-tsz and published under the license below.
|
||||
// It was modified to accommodate reading from byte slices without modifying
|
||||
// the underlying bytes, which would panic when reading from mmap'd
|
||||
// read-only byte slices.
|
||||
|
||||
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
||||
// All rights reserved.
|
||||
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
package chunkenc
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// putFloat64VBBucket writes a float64 using varbit optimized for SHS buckets.
|
||||
// It does so by converting the underlying bits into an int64.
|
||||
func putFloat64VBBucket(b *bstream, val float64) {
|
||||
// TODO: Since this is used for the zero threshold, this almost always goes into the default
|
||||
// bit range (i.e. using 5+64 bits). So we can consider skipping `putInt64VBBucket` and directly
|
||||
// write the float and save 5 bits here.
|
||||
putInt64VBBucket(b, int64(math.Float64bits(val)))
|
||||
}
|
||||
|
||||
// readFloat64VBBucket reads a float64 using varbit optimized for SHS buckets
|
||||
func readFloat64VBBucket(b *bstreamReader) (float64, error) {
|
||||
val, err := readInt64VBBucket(b)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return math.Float64frombits(uint64(val)), nil
|
||||
}
|
||||
|
||||
// putInt64VBBucket writes an int64 using varbit optimized for SHS buckets.
|
||||
//
|
||||
// TODO(Dieterbe): We could improve this further: Each branch doesn't need to
|
||||
// support any values of any of the prior branches. So we can expand the range
|
||||
// of each branch. Do more with fewer bits. It comes at the price of more
|
||||
// expensive encoding and decoding (cutting out and later adding back that
|
||||
// center-piece we skip).
|
||||
func putInt64VBBucket(b *bstream, val int64) {
|
||||
switch {
|
||||
case val == 0:
|
||||
b.writeBit(zero)
|
||||
case bitRange(val, 3): // -3 <= val <= 4
|
||||
b.writeBits(0b10, 2)
|
||||
b.writeBits(uint64(val), 3)
|
||||
case bitRange(val, 6): // -31 <= val <= 32
|
||||
b.writeBits(0b110, 3)
|
||||
b.writeBits(uint64(val), 6)
|
||||
case bitRange(val, 9): // -255 <= val <= 256
|
||||
b.writeBits(0b1110, 4)
|
||||
b.writeBits(uint64(val), 9)
|
||||
case bitRange(val, 12): // -2047 <= val <= 2048
|
||||
b.writeBits(0b11110, 5)
|
||||
b.writeBits(uint64(val), 12)
|
||||
default:
|
||||
b.writeBits(0b11111, 5)
|
||||
b.writeBits(uint64(val), 64)
|
||||
}
|
||||
}
|
||||
|
||||
// readInt64VBBucket reads an int64 using varbit optimized for SHS buckets
|
||||
func readInt64VBBucket(b *bstreamReader) (int64, error) {
|
||||
var d byte
|
||||
for i := 0; i < 5; i++ {
|
||||
d <<= 1
|
||||
bit, err := b.readBitFast()
|
||||
if err != nil {
|
||||
bit, err = b.readBit()
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bit == zero {
|
||||
break
|
||||
}
|
||||
d |= 1
|
||||
}
|
||||
|
||||
var val int64
|
||||
var sz uint8
|
||||
|
||||
switch d {
|
||||
case 0b0:
|
||||
// val == 0
|
||||
case 0b10:
|
||||
sz = 3
|
||||
case 0b110:
|
||||
sz = 6
|
||||
case 0b1110:
|
||||
sz = 9
|
||||
case 0b11110:
|
||||
sz = 12
|
||||
case 0b11111:
|
||||
// Do not use fast because it's very unlikely it will succeed.
|
||||
bits, err := b.readBits(64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
if sz != 0 {
|
||||
bits, err := b.readBitsFast(sz)
|
||||
if err != nil {
|
||||
bits, err = b.readBits(sz)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bits > (1 << (sz - 1)) {
|
||||
// or something
|
||||
bits = bits - (1 << sz)
|
||||
}
|
||||
val = int64(bits)
|
||||
}
|
||||
|
||||
return val, nil
|
||||
}
|
||||
|
|
@ -48,7 +48,7 @@ import (
|
|||
"math"
|
||||
"math/bits"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -150,8 +150,8 @@ type xorAppender struct {
|
|||
trailing uint8
|
||||
}
|
||||
|
||||
func (a *xorAppender) AppendHistogram(t int64, h histogram.SparseHistogram) {
|
||||
//panic("cannot call xorAppender.AppendHistogram().")
|
||||
func (a *xorAppender) AppendHistogram(t int64, h histogram.Histogram) {
|
||||
panic("appended a histogram to an xor chunk")
|
||||
}
|
||||
|
||||
func (a *xorAppender) Append(t int64, v float64) {
|
||||
|
|
@ -181,6 +181,12 @@ func (a *xorAppender) Append(t int64, v float64) {
|
|||
|
||||
// Gorilla has a max resolution of seconds, Prometheus milliseconds.
|
||||
// Thus we use higher value range steps with larger bit size.
|
||||
//
|
||||
// TODO(beorn7): This seems to needlessly jump to large bit
|
||||
// sizes even for very small deviations from zero. Timestamp
|
||||
// compression can probably benefit from some smaller bit
|
||||
// buckets. See also what was done for histogram encoding in
|
||||
// varbit.go.
|
||||
switch {
|
||||
case dod == 0:
|
||||
a.b.writeBit(zero)
|
||||
|
|
@ -278,7 +284,7 @@ func (it *xorIterator) At() (int64, float64) {
|
|||
return it.t, it.val
|
||||
}
|
||||
|
||||
func (it *xorIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (it *xorIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
panic("cannot call xorIterator.AtHistogram().")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import (
|
|||
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -1328,7 +1328,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) {
|
|||
|
||||
type timedHist struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
// Ingest samples.
|
||||
|
|
@ -1384,7 +1384,7 @@ func TestHeadCompactionWithHistograms(t *testing.T) {
|
|||
// Depending on numSeriesPerSchema, it can take few gigs of memory;
|
||||
// the test adds all samples to appender before committing instead of
|
||||
// buffering the writes to make it run faster.
|
||||
func TestSparseHistoSpaceSavings(t *testing.T) {
|
||||
func TestSparseHistogramSpaceSavings(t *testing.T) {
|
||||
t.Skip()
|
||||
|
||||
cases := []struct {
|
||||
|
|
@ -1455,7 +1455,7 @@ func TestSparseHistoSpaceSavings(t *testing.T) {
|
|||
|
||||
var allSparseSeries []struct {
|
||||
baseLabels labels.Labels
|
||||
hists []histogram.SparseHistogram
|
||||
hists []histogram.Histogram
|
||||
}
|
||||
|
||||
for sid, schema := range allSchemas {
|
||||
|
|
@ -1467,7 +1467,7 @@ func TestSparseHistoSpaceSavings(t *testing.T) {
|
|||
}
|
||||
allSparseSeries = append(allSparseSeries, struct {
|
||||
baseLabels labels.Labels
|
||||
hists []histogram.SparseHistogram
|
||||
hists []histogram.Histogram
|
||||
}{baseLabels: lbls, hists: generateCustomHistograms(numHistograms, c.numBuckets, c.numSpans, c.gapBetweenSpans, schema)})
|
||||
}
|
||||
}
|
||||
|
|
@ -1522,13 +1522,13 @@ func TestSparseHistoSpaceSavings(t *testing.T) {
|
|||
h := ah.hists[i]
|
||||
|
||||
numOldSeriesPerHistogram = 0
|
||||
it := histogram.CumulativeExpandSparseHistogram(h)
|
||||
it := h.CumulativeBucketIterator()
|
||||
itIdx := 0
|
||||
var err error
|
||||
for it.Next() {
|
||||
numOldSeriesPerHistogram++
|
||||
b := it.At()
|
||||
lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Le)})
|
||||
lbls := append(ah.baseLabels, labels.Label{Name: "le", Value: fmt.Sprintf("%.16f", b.Upper)})
|
||||
refs[itIdx], err = oldApp.Append(refs[itIdx], lbls, ts, float64(b.Count))
|
||||
require.NoError(t, err)
|
||||
itIdx++
|
||||
|
|
@ -1614,9 +1614,9 @@ Savings: Index=%.2f%%, Chunks=%.2f%%, Total=%.2f%%
|
|||
}
|
||||
}
|
||||
|
||||
func generateCustomHistograms(numHists, numBuckets, numSpans, gapBetweenSpans, schema int) (r []histogram.SparseHistogram) {
|
||||
func generateCustomHistograms(numHists, numBuckets, numSpans, gapBetweenSpans, schema int) (r []histogram.Histogram) {
|
||||
// First histogram with all the settings.
|
||||
h := histogram.SparseHistogram{
|
||||
h := histogram.Histogram{
|
||||
Sum: 1000 * rand.Float64(),
|
||||
Schema: int32(schema),
|
||||
}
|
||||
|
|
@ -1711,7 +1711,7 @@ func TestSparseHistogramCompactionAndQuery(t *testing.T) {
|
|||
|
||||
type timedHist struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make(map[string][]timedHist)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,8 +34,14 @@ in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes).
|
|||
└───────────────┴───────────────────┴──────────────┴────────────────┘
|
||||
```
|
||||
|
||||
## XOR chunk
|
||||
|
||||
TODO(beorn7): Add.
|
||||
|
||||
## Histogram chunk
|
||||
|
||||
TODO(beorn7): This is out of date. Update once settled on the (more or less) final format.
|
||||
|
||||
```
|
||||
┌──────────────┬─────────────────┬──────────────────────────┬──────────────────────────┬──────────────┐
|
||||
│ len <uint16> │ schema <varint> │ pos-spans <span-section> │ neg-spans <span-section> │ data <bytes> │
|
||||
|
|
|
|||
14
tsdb/head.go
14
tsdb/head.go
|
|
@ -29,8 +29,8 @@ import (
|
|||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -614,7 +614,7 @@ func (h *Head) Init(minValidTime int64) error {
|
|||
sparseHistogramSeries := 0
|
||||
for _, m := range h.series.series {
|
||||
for _, ms := range m {
|
||||
if ms.sparseHistogramSeries {
|
||||
if ms.histogramSeries {
|
||||
sparseHistogramSeries++
|
||||
}
|
||||
}
|
||||
|
|
@ -1397,7 +1397,7 @@ func (s *stripeSeries) gc(mint int64) (map[uint64]struct{}, int, int64, int) {
|
|||
s.locks[j].Lock()
|
||||
}
|
||||
|
||||
if series.sparseHistogramSeries {
|
||||
if series.histogramSeries {
|
||||
sparseHistogramSeriesDeleted++
|
||||
}
|
||||
deleted[series.ref] = struct{}{}
|
||||
|
|
@ -1488,9 +1488,9 @@ func (s *stripeSeries) getOrSet(hash uint64, lset labels.Labels, createSeries fu
|
|||
return series, true, nil
|
||||
}
|
||||
|
||||
type hist struct {
|
||||
type histogramSample struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
|
||||
type sample struct {
|
||||
|
|
@ -1517,7 +1517,7 @@ type memSeries struct {
|
|||
|
||||
nextAt int64 // Timestamp at which to cut the next chunk.
|
||||
sampleBuf [4]sample
|
||||
histBuf [4]hist
|
||||
histogramBuf [4]histogramSample
|
||||
pendingCommit bool // Whether there are samples waiting to be committed to this series.
|
||||
|
||||
app chunkenc.Appender // Current appender for the chunk.
|
||||
|
|
@ -1527,7 +1527,7 @@ type memSeries struct {
|
|||
txs *txRing
|
||||
|
||||
// Temporary variable for sparsehistogram experiment.
|
||||
sparseHistogramSeries bool
|
||||
histogramSeries bool
|
||||
}
|
||||
|
||||
func newMemSeries(lset labels.Labels, id uint64, chunkRange int64, memChunkPool *sync.Pool) *memSeries {
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ import (
|
|||
"github.com/go-kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -67,14 +67,14 @@ func (a *initAppender) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Ex
|
|||
return a.app.AppendExemplar(ref, l, e)
|
||||
}
|
||||
|
||||
func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
func (a *initAppender) AppendHistogram(ref uint64, l labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
if a.app != nil {
|
||||
return a.app.AppendHistogram(ref, l, t, sh)
|
||||
return a.app.AppendHistogram(ref, l, t, h)
|
||||
}
|
||||
a.head.initTime(t)
|
||||
a.app = a.head.appender()
|
||||
|
||||
return a.app.AppendHistogram(ref, l, t, sh)
|
||||
return a.app.AppendHistogram(ref, l, t, h)
|
||||
}
|
||||
|
||||
// initTime initializes a head with the first timestamp. This only needs to be called
|
||||
|
|
@ -269,8 +269,8 @@ func (a *headAppender) Append(ref uint64, lset labels.Labels, t int64, v float64
|
|||
}
|
||||
}
|
||||
|
||||
if value.IsStaleNaN(v) && s.sparseHistogramSeries {
|
||||
return a.AppendHistogram(ref, lset, t, histogram.SparseHistogram{Sum: v})
|
||||
if value.IsStaleNaN(v) && s.histogramSeries {
|
||||
return a.AppendHistogram(ref, lset, t, histogram.Histogram{Sum: v})
|
||||
}
|
||||
|
||||
s.Lock()
|
||||
|
|
@ -322,7 +322,7 @@ func (s *memSeries) appendable(t int64, v float64) error {
|
|||
}
|
||||
|
||||
// appendableHistogram checks whether the given sample is valid for appending to the series.
|
||||
func (s *memSeries) appendableHistogram(t int64, sh histogram.SparseHistogram) error {
|
||||
func (s *memSeries) appendableHistogram(t int64, sh histogram.Histogram) error {
|
||||
c := s.head()
|
||||
if c == nil {
|
||||
return nil
|
||||
|
|
@ -372,7 +372,7 @@ func (a *headAppender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Ex
|
|||
return s.ref, nil
|
||||
}
|
||||
|
||||
func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, sh histogram.SparseHistogram) (uint64, error) {
|
||||
func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64, h histogram.Histogram) (uint64, error) {
|
||||
if t < a.minValidTime {
|
||||
a.head.metrics.outOfBoundSamples.Inc()
|
||||
return 0, storage.ErrOutOfBounds
|
||||
|
|
@ -396,7 +396,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64,
|
|||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.sparseHistogramSeries = true
|
||||
s.histogramSeries = true
|
||||
if created {
|
||||
a.head.metrics.sparseHistogramSeries.Inc()
|
||||
a.series = append(a.series, record.RefSeries{
|
||||
|
|
@ -407,7 +407,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64,
|
|||
}
|
||||
|
||||
s.Lock()
|
||||
if err := s.appendableHistogram(t, sh); err != nil {
|
||||
if err := s.appendableHistogram(t, h); err != nil {
|
||||
s.Unlock()
|
||||
if err == storage.ErrOutOfOrderSample {
|
||||
a.head.metrics.outOfOrderSamples.Inc()
|
||||
|
|
@ -427,7 +427,7 @@ func (a *headAppender) AppendHistogram(ref uint64, lset labels.Labels, t int64,
|
|||
a.histograms = append(a.histograms, record.RefHistogram{
|
||||
Ref: s.ref,
|
||||
T: t,
|
||||
H: sh,
|
||||
H: h,
|
||||
})
|
||||
a.histogramSeries = append(a.histogramSeries, s)
|
||||
return s.ref, nil
|
||||
|
|
@ -604,37 +604,39 @@ func (s *memSeries) append(t int64, v float64, appendID uint64, chunkDiskMapper
|
|||
return true, chunkCreated
|
||||
}
|
||||
|
||||
// appendHistogram adds the sparse histogram.
|
||||
// appendHistogram adds the histogram.
|
||||
// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
|
||||
func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) {
|
||||
func (s *memSeries) appendHistogram(t int64, sh histogram.Histogram, appendID uint64, chunkDiskMapper *chunks.ChunkDiskMapper) (sampleInOrder, chunkCreated bool) {
|
||||
// Head controls the execution of recoding, so that we own the proper chunk reference afterwards.
|
||||
// We check for Appendable before appendPreprocessor because in case it ends up creating a new chunk,
|
||||
// we need to know if there was also a counter reset or not to set the meta properly.
|
||||
app, _ := s.app.(*chunkenc.HistoAppender)
|
||||
app, _ := s.app.(*chunkenc.HistogramAppender)
|
||||
var (
|
||||
posInterjections, negInterjections []chunkenc.Interjection
|
||||
okToAppend, counterReset bool
|
||||
positiveInterjections, negativeInterjections []chunkenc.Interjection
|
||||
okToAppend, counterReset bool
|
||||
)
|
||||
if app != nil {
|
||||
posInterjections, negInterjections, okToAppend, counterReset = app.Appendable(sh)
|
||||
positiveInterjections, negativeInterjections, okToAppend, counterReset = app.Appendable(sh)
|
||||
}
|
||||
|
||||
c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncSHS, chunkDiskMapper)
|
||||
c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncHistogram, chunkDiskMapper)
|
||||
if !sampleInOrder {
|
||||
return sampleInOrder, chunkCreated
|
||||
}
|
||||
|
||||
if !chunkCreated {
|
||||
// We have 3 cases here
|
||||
// !okToAppend -> we need to cut a new chunk
|
||||
// okToAppend but we have interjections -> existing chunk needs recoding before we can append our histogram
|
||||
// okToAppend and no interjections -> chunk is ready to support our histogram
|
||||
// - !okToAppend -> We need to cut a new chunk.
|
||||
// - okToAppend but we have interjections -> Existing chunk needs recoding before we can append our histogram.
|
||||
// - okToAppend and no interjections -> Chunk is ready to support our histogram.
|
||||
if !okToAppend || counterReset {
|
||||
c = s.cutNewHeadChunk(t, chunkenc.EncSHS, chunkDiskMapper)
|
||||
c = s.cutNewHeadChunk(t, chunkenc.EncHistogram, chunkDiskMapper)
|
||||
chunkCreated = true
|
||||
} else if len(posInterjections) > 0 || len(negInterjections) > 0 {
|
||||
// new buckets have appeared. we need to recode all prior histograms within the chunk before we can process this one.
|
||||
chunk, app := app.Recode(posInterjections, negInterjections, sh.PositiveSpans, sh.NegativeSpans)
|
||||
} else if len(positiveInterjections) > 0 || len(negativeInterjections) > 0 {
|
||||
// New buckets have appeared. We need to recode all
|
||||
// prior histogram samples within the chunk before we
|
||||
// can process this one.
|
||||
chunk, app := app.Recode(positiveInterjections, negativeInterjections, sh.PositiveSpans, sh.NegativeSpans)
|
||||
s.headChunk = &memChunk{
|
||||
minTime: s.headChunk.minTime,
|
||||
maxTime: s.headChunk.maxTime,
|
||||
|
|
@ -645,7 +647,7 @@ func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appen
|
|||
}
|
||||
|
||||
if chunkCreated {
|
||||
hc := s.headChunk.chunk.(*chunkenc.HistoChunk)
|
||||
hc := s.headChunk.chunk.(*chunkenc.HistogramChunk)
|
||||
header := chunkenc.UnknownCounterReset
|
||||
if counterReset {
|
||||
header = chunkenc.CounterReset
|
||||
|
|
@ -656,14 +658,14 @@ func (s *memSeries) appendHistogram(t int64, sh histogram.SparseHistogram, appen
|
|||
}
|
||||
|
||||
s.app.AppendHistogram(t, sh)
|
||||
s.sparseHistogramSeries = true
|
||||
s.histogramSeries = true
|
||||
|
||||
c.maxTime = t
|
||||
|
||||
s.histBuf[0] = s.histBuf[1]
|
||||
s.histBuf[1] = s.histBuf[2]
|
||||
s.histBuf[2] = s.histBuf[3]
|
||||
s.histBuf[3] = hist{t: t, h: sh}
|
||||
s.histogramBuf[0] = s.histogramBuf[1]
|
||||
s.histogramBuf[1] = s.histogramBuf[2]
|
||||
s.histogramBuf[2] = s.histogramBuf[3]
|
||||
s.histogramBuf[3] = histogramSample{t: t, h: sh}
|
||||
|
||||
if appendID > 0 {
|
||||
s.txs.add(appendID)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
"github.com/go-kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -443,7 +443,7 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper *
|
|||
msIter.total = numSamples
|
||||
msIter.stopAfter = stopAfter
|
||||
msIter.buf = s.sampleBuf
|
||||
msIter.histBuf = s.histBuf
|
||||
msIter.histogramBuf = s.histogramBuf
|
||||
return msIter
|
||||
}
|
||||
return &memSafeIterator{
|
||||
|
|
@ -452,18 +452,18 @@ func (s *memSeries) iterator(id int, isoState *isolationState, chunkDiskMapper *
|
|||
i: -1,
|
||||
stopAfter: stopAfter,
|
||||
},
|
||||
total: numSamples,
|
||||
buf: s.sampleBuf,
|
||||
histBuf: s.histBuf,
|
||||
total: numSamples,
|
||||
buf: s.sampleBuf,
|
||||
histogramBuf: s.histogramBuf,
|
||||
}
|
||||
}
|
||||
|
||||
type memSafeIterator struct {
|
||||
stopIterator
|
||||
|
||||
total int
|
||||
buf [4]sample
|
||||
histBuf [4]hist
|
||||
total int
|
||||
buf [4]sample
|
||||
histogramBuf [4]histogramSample
|
||||
}
|
||||
|
||||
func (it *memSafeIterator) Seek(t int64) bool {
|
||||
|
|
@ -502,11 +502,11 @@ func (it *memSafeIterator) At() (int64, float64) {
|
|||
return s.t, s.v
|
||||
}
|
||||
|
||||
func (it *memSafeIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (it *memSafeIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
if it.total-it.i > 4 {
|
||||
return it.Iterator.AtHistogram()
|
||||
}
|
||||
s := it.histBuf[4-(it.total-it.i)]
|
||||
s := it.histogramBuf[4-(it.total-it.i)]
|
||||
return s.t, s.h
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ import (
|
|||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/prometheus/prometheus/config"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/exemplar"
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/pkg/value"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
|
|
@ -2539,15 +2539,15 @@ func TestAppendHistogram(t *testing.T) {
|
|||
require.NoError(t, head.Init(0))
|
||||
app := head.Appender(context.Background())
|
||||
|
||||
type timedHist struct {
|
||||
type timedHistogram struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make([]timedHist, 0, numHistograms)
|
||||
expHistograms := make([]timedHistogram, 0, numHistograms)
|
||||
for i, h := range generateHistograms(numHistograms) {
|
||||
_, err := app.AppendHistogram(0, l, int64(i), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{int64(i), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{int64(i), h})
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
|
|
@ -2564,13 +2564,13 @@ func TestAppendHistogram(t *testing.T) {
|
|||
require.False(t, ss.Next())
|
||||
|
||||
it := s.Iterator()
|
||||
actHists := make([]timedHist, 0, len(expHists))
|
||||
actHistograms := make([]timedHistogram, 0, len(expHistograms))
|
||||
for it.Next() {
|
||||
t, h := it.AtHistogram()
|
||||
actHists = append(actHists, timedHist{t, h.Copy()})
|
||||
actHistograms = append(actHistograms, timedHistogram{t, h.Copy()})
|
||||
}
|
||||
|
||||
require.Equal(t, expHists, actHists)
|
||||
require.Equal(t, expHistograms, actHistograms)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -2586,17 +2586,17 @@ func TestHistogramInWAL(t *testing.T) {
|
|||
require.NoError(t, head.Init(0))
|
||||
app := head.Appender(context.Background())
|
||||
|
||||
type timedHist struct {
|
||||
type timedHistogram struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make([]timedHist, 0, numHistograms)
|
||||
expHistograms := make([]timedHistogram, 0, numHistograms)
|
||||
for i, h := range generateHistograms(numHistograms) {
|
||||
h.NegativeSpans = h.PositiveSpans
|
||||
h.NegativeBuckets = h.PositiveBuckets
|
||||
_, err := app.AppendHistogram(0, l, int64(i), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{int64(i), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{int64(i), h})
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
|
|
@ -2621,18 +2621,18 @@ func TestHistogramInWAL(t *testing.T) {
|
|||
require.False(t, ss.Next())
|
||||
|
||||
it := s.Iterator()
|
||||
actHists := make([]timedHist, 0, len(expHists))
|
||||
actHistograms := make([]timedHistogram, 0, len(expHistograms))
|
||||
for it.Next() {
|
||||
t, h := it.AtHistogram()
|
||||
actHists = append(actHists, timedHist{t, h.Copy()})
|
||||
actHistograms = append(actHistograms, timedHistogram{t, h.Copy()})
|
||||
}
|
||||
|
||||
require.Equal(t, expHists, actHists)
|
||||
require.Equal(t, expHistograms, actHistograms)
|
||||
}
|
||||
|
||||
func generateHistograms(n int) (r []histogram.SparseHistogram) {
|
||||
func generateHistograms(n int) (r []histogram.Histogram) {
|
||||
for i := 0; i < n; i++ {
|
||||
r = append(r, histogram.SparseHistogram{
|
||||
r = append(r, histogram.Histogram{
|
||||
Count: 5 + uint64(i*4),
|
||||
ZeroCount: 2 + uint64(i),
|
||||
ZeroThreshold: 0.001,
|
||||
|
|
@ -2957,22 +2957,22 @@ func TestSparseHistogramMetrics(t *testing.T) {
|
|||
})
|
||||
require.NoError(t, head.Init(0))
|
||||
|
||||
expHistSeries, expHistSamples := 0, 0
|
||||
expHSeries, expHSamples := 0, 0
|
||||
|
||||
for x := 0; x < 5; x++ {
|
||||
expHistSeries++
|
||||
expHSeries++
|
||||
l := labels.Labels{{Name: "a", Value: fmt.Sprintf("b%d", x)}}
|
||||
for i, h := range generateHistograms(10) {
|
||||
app := head.Appender(context.Background())
|
||||
_, err := app.AppendHistogram(0, l, int64(i), h)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
expHistSamples++
|
||||
expHSamples++
|
||||
}
|
||||
}
|
||||
|
||||
require.Equal(t, float64(expHistSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(expHistSamples), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal))
|
||||
require.Equal(t, float64(expHSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(expHSamples), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal))
|
||||
|
||||
require.NoError(t, head.Close())
|
||||
w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false)
|
||||
|
|
@ -2981,7 +2981,7 @@ func TestSparseHistogramMetrics(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.NoError(t, head.Init(0))
|
||||
|
||||
require.Equal(t, float64(expHistSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(expHSeries), prom_testutil.ToFloat64(head.metrics.sparseHistogramSeries))
|
||||
require.Equal(t, float64(0), prom_testutil.ToFloat64(head.metrics.sparseHistogramSamplesTotal)) // Counter reset.
|
||||
}
|
||||
|
||||
|
|
@ -2994,11 +2994,11 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
})
|
||||
require.NoError(t, head.Init(0))
|
||||
|
||||
type timedHist struct {
|
||||
type timedHistogram struct {
|
||||
t int64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
}
|
||||
expHists := make([]timedHist, 0, numHistograms)
|
||||
expHistograms := make([]timedHistogram, 0, numHistograms)
|
||||
|
||||
testQuery := func(numStale int) {
|
||||
q, err := NewBlockQuerier(head, head.MinTime(), head.MaxTime())
|
||||
|
|
@ -3014,17 +3014,17 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
require.False(t, ss.Next())
|
||||
|
||||
it := s.Iterator()
|
||||
actHists := make([]timedHist, 0, len(expHists))
|
||||
actHistograms := make([]timedHistogram, 0, len(expHistograms))
|
||||
for it.Next() {
|
||||
t, h := it.AtHistogram()
|
||||
actHists = append(actHists, timedHist{t, h.Copy()})
|
||||
actHistograms = append(actHistograms, timedHistogram{t, h.Copy()})
|
||||
}
|
||||
|
||||
// We cannot compare StaleNAN with require.Equal, hence checking each histogram manually.
|
||||
require.Equal(t, len(expHists), len(actHists))
|
||||
require.Equal(t, len(expHistograms), len(actHistograms))
|
||||
actNumStale := 0
|
||||
for i, eh := range expHists {
|
||||
ah := actHists[i]
|
||||
for i, eh := range expHistograms {
|
||||
ah := actHistograms[i]
|
||||
if value.IsStaleNaN(eh.h.Sum) {
|
||||
actNumStale++
|
||||
require.True(t, value.IsStaleNaN(ah.h.Sum))
|
||||
|
|
@ -3040,14 +3040,14 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
// Adding stale in the same appender.
|
||||
app := head.Appender(context.Background())
|
||||
for _, h := range generateHistograms(numHistograms) {
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHists)), h)
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100 * int64(len(expHists)), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{100 * int64(len(expHistograms)), h})
|
||||
}
|
||||
// +1 so that delta-of-delta is not 0.
|
||||
_, err := app.Append(0, l, 100*int64(len(expHists))+1, math.Float64frombits(value.StaleNaN))
|
||||
_, err := app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN))
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100*int64(len(expHists)) + 1, histogram.SparseHistogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
expHistograms = append(expHistograms, timedHistogram{100*int64(len(expHistograms)) + 1, histogram.Histogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Only 1 chunk in the memory, no m-mapped chunk.
|
||||
|
|
@ -3059,17 +3059,17 @@ func TestSparseHistogramStaleSample(t *testing.T) {
|
|||
// Adding stale in different appender and continuing series after a stale sample.
|
||||
app = head.Appender(context.Background())
|
||||
for _, h := range generateHistograms(2 * numHistograms)[numHistograms:] {
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHists)), h)
|
||||
_, err := app.AppendHistogram(0, l, 100*int64(len(expHistograms)), h)
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100 * int64(len(expHists)), h})
|
||||
expHistograms = append(expHistograms, timedHistogram{100 * int64(len(expHistograms)), h})
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
app = head.Appender(context.Background())
|
||||
// +1 so that delta-of-delta is not 0.
|
||||
_, err = app.Append(0, l, 100*int64(len(expHists))+1, math.Float64frombits(value.StaleNaN))
|
||||
_, err = app.Append(0, l, 100*int64(len(expHistograms))+1, math.Float64frombits(value.StaleNaN))
|
||||
require.NoError(t, err)
|
||||
expHists = append(expHists, timedHist{100*int64(len(expHists)) + 1, histogram.SparseHistogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
expHistograms = append(expHistograms, timedHistogram{100*int64(len(expHistograms)) + 1, histogram.Histogram{Sum: math.Float64frombits(value.StaleNaN)}})
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Total 2 chunks, 1 m-mapped.
|
||||
|
|
|
|||
|
|
@ -15,11 +15,6 @@ package tsdb
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/encoding"
|
||||
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
||||
"github.com/prometheus/prometheus/tsdb/fileutil"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
|
|
@ -30,6 +25,12 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/tsdb/encoding"
|
||||
tsdb_errors "github.com/prometheus/prometheus/tsdb/errors"
|
||||
"github.com/prometheus/prometheus/tsdb/fileutil"
|
||||
|
||||
"github.com/go-kit/log/level"
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/atomic"
|
||||
|
|
@ -159,7 +160,7 @@ func (h *Head) loadWAL(r *wal.Reader, multiRef map[uint64]uint64, mmappedChunks
|
|||
|
||||
if ms.head() == nil {
|
||||
// First histogram for the series. Count this in metrics.
|
||||
ms.sparseHistogramSeries = true
|
||||
ms.histogramSeries = true
|
||||
}
|
||||
|
||||
if rh.T < h.minValidTime.Load() {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
|
|
@ -649,7 +649,7 @@ func (p *populateWithDelSeriesIterator) Seek(t int64) bool {
|
|||
}
|
||||
|
||||
func (p *populateWithDelSeriesIterator) At() (int64, float64) { return p.curr.At() }
|
||||
func (p *populateWithDelSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (p *populateWithDelSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return p.curr.AtHistogram()
|
||||
}
|
||||
func (p *populateWithDelSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
@ -688,8 +688,8 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool {
|
|||
app chunkenc.Appender
|
||||
err error
|
||||
)
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncSHS {
|
||||
newChunk = chunkenc.NewHistoChunk()
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
newChunk = chunkenc.NewHistogramChunk()
|
||||
app, err = newChunk.Appender()
|
||||
} else {
|
||||
newChunk = chunkenc.NewXORChunk()
|
||||
|
|
@ -714,11 +714,11 @@ func (p *populateWithDelChunkSeriesIterator) Next() bool {
|
|||
var (
|
||||
t int64
|
||||
v float64
|
||||
h histogram.SparseHistogram
|
||||
h histogram.Histogram
|
||||
)
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if hc, ok := p.currChkMeta.Chunk.(*chunkenc.HistoChunk); ok {
|
||||
newChunk.(*chunkenc.HistoChunk).SetCounterResetHeader(hc.GetCounterResetHeader())
|
||||
if p.currDelIter.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
if hc, ok := p.currChkMeta.Chunk.(*chunkenc.HistogramChunk); ok {
|
||||
newChunk.(*chunkenc.HistogramChunk).SetCounterResetHeader(hc.GetCounterResetHeader())
|
||||
}
|
||||
t, h = p.currDelIter.AtHistogram()
|
||||
p.curr.MinTime = t
|
||||
|
|
@ -870,7 +870,7 @@ func (it *DeletedIterator) At() (int64, float64) {
|
|||
return it.Iter.At()
|
||||
}
|
||||
|
||||
func (it *DeletedIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
func (it *DeletedIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
t, h := it.Iter.AtHistogram()
|
||||
return t, h
|
||||
}
|
||||
|
|
@ -889,7 +889,7 @@ func (it *DeletedIterator) Seek(t int64) bool {
|
|||
|
||||
// Now double check if the entry falls into a deleted interval.
|
||||
var ts int64
|
||||
if it.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if it.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
ts, _ = it.AtHistogram()
|
||||
} else {
|
||||
ts, _ = it.At()
|
||||
|
|
@ -916,7 +916,7 @@ func (it *DeletedIterator) Next() bool {
|
|||
Outer:
|
||||
for it.Iter.Next() {
|
||||
var ts int64
|
||||
if it.ChunkEncoding() == chunkenc.EncSHS {
|
||||
if it.ChunkEncoding() == chunkenc.EncHistogram {
|
||||
ts, _ = it.AtHistogram()
|
||||
} else {
|
||||
ts, _ = it.At()
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import (
|
|||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/tsdb/encoding"
|
||||
"github.com/prometheus/prometheus/tsdb/tombstones"
|
||||
|
|
@ -74,7 +74,7 @@ type RefExemplar struct {
|
|||
type RefHistogram struct {
|
||||
Ref uint64
|
||||
T int64
|
||||
H histogram.SparseHistogram
|
||||
H histogram.Histogram
|
||||
}
|
||||
|
||||
// Decoder decodes series, sample, and tombstone records.
|
||||
|
|
@ -233,14 +233,14 @@ func (d *Decoder) ExemplarsFromBuffer(dec *encoding.Decbuf, exemplars []RefExemp
|
|||
return exemplars, nil
|
||||
}
|
||||
|
||||
func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram, error) {
|
||||
func (d *Decoder) Histograms(rec []byte, histograms []RefHistogram) ([]RefHistogram, error) {
|
||||
dec := encoding.Decbuf{B: rec}
|
||||
t := Type(dec.Byte())
|
||||
if t != Histograms {
|
||||
return nil, errors.New("invalid record type")
|
||||
}
|
||||
if dec.Len() == 0 {
|
||||
return hists, nil
|
||||
return histograms, nil
|
||||
}
|
||||
var (
|
||||
baseRef = dec.Be64()
|
||||
|
|
@ -253,7 +253,7 @@ func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram,
|
|||
rh := RefHistogram{
|
||||
Ref: baseRef + uint64(dref),
|
||||
T: baseTime + dtime,
|
||||
H: histogram.SparseHistogram{
|
||||
H: histogram.Histogram{
|
||||
Schema: 0,
|
||||
ZeroThreshold: 0,
|
||||
ZeroCount: 0,
|
||||
|
|
@ -303,16 +303,16 @@ func (d *Decoder) Histograms(rec []byte, hists []RefHistogram) ([]RefHistogram,
|
|||
rh.H.NegativeBuckets[i] = dec.Varint64()
|
||||
}
|
||||
|
||||
hists = append(hists, rh)
|
||||
histograms = append(histograms, rh)
|
||||
}
|
||||
|
||||
if dec.Err() != nil {
|
||||
return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(hists))
|
||||
return nil, errors.Wrapf(dec.Err(), "decode error after %d histograms", len(histograms))
|
||||
}
|
||||
if len(dec.B) > 0 {
|
||||
return nil, errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
|
||||
}
|
||||
return hists, nil
|
||||
return histograms, nil
|
||||
}
|
||||
|
||||
// Encoder encodes series, sample, and tombstones records.
|
||||
|
|
@ -410,21 +410,21 @@ func (e *Encoder) EncodeExemplarsIntoBuffer(exemplars []RefExemplar, buf *encodi
|
|||
}
|
||||
}
|
||||
|
||||
func (e *Encoder) Histograms(hists []RefHistogram, b []byte) []byte {
|
||||
func (e *Encoder) Histograms(histograms []RefHistogram, b []byte) []byte {
|
||||
buf := encoding.Encbuf{B: b}
|
||||
buf.PutByte(byte(Histograms))
|
||||
|
||||
if len(hists) == 0 {
|
||||
if len(histograms) == 0 {
|
||||
return buf.Get()
|
||||
}
|
||||
|
||||
// Store base timestamp and base reference number of first histogram.
|
||||
// All histograms encode their timestamp and ref as delta to those.
|
||||
first := hists[0]
|
||||
first := histograms[0]
|
||||
buf.PutBE64(first.Ref)
|
||||
buf.PutBE64int64(first.T)
|
||||
|
||||
for _, h := range hists {
|
||||
for _, h := range histograms {
|
||||
buf.PutVarint64(int64(h.Ref) - int64(first.Ref))
|
||||
buf.PutVarint64(h.T - first.T)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ package tsdbutil
|
|||
import (
|
||||
"math"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
)
|
||||
|
||||
|
|
@ -160,8 +160,8 @@ func (it *sampleRingIterator) At() (int64, float64) {
|
|||
return it.r.at(it.i)
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
func (it *sampleRingIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *sampleRingIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/pkg/histogram"
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
|
@ -152,8 +152,8 @@ func (it *listSeriesIterator) At() (int64, float64) {
|
|||
return s.t, s.v
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.SparseHistogram) {
|
||||
return 0, histogram.SparseHistogram{}
|
||||
func (it *listSeriesIterator) AtHistogram() (int64, histogram.Histogram) {
|
||||
return 0, histogram.Histogram{}
|
||||
}
|
||||
|
||||
func (it *listSeriesIterator) ChunkEncoding() chunkenc.Encoding {
|
||||
|
|
|
|||
|
|
@ -296,7 +296,6 @@ var sampleFlagMap = map[string]string{
|
|||
}
|
||||
|
||||
func TestEndpoints(t *testing.T) {
|
||||
t.Skip()
|
||||
suite, err := promql.NewTest(t, `
|
||||
load 1m
|
||||
test_metric1{foo="bar"} 0+100x100
|
||||
|
|
|
|||
Loading…
Reference in New Issue