github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

histograms: parse float histograms from proto definition (#11149) 

* histograms: parse float histograms from proto definition

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Improve comment

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Ignore float buckets

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Refactor Histogram() function

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Fix test_float_histogram

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Update model/textparse/protobufparse.go

Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update protobufparse.go

Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update scrape.go

Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update scrape/scrape.go

Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>

Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
pull/11208/head
Marc Tudurí 2 years ago committed by GitHub
parent
0f4e5196c4
commit
f7df3b86ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 163 additions and 18 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      model/textparse/interface.go
  2. 6
      model/textparse/openmetricsparse.go
  3. 6
      model/textparse/promparse.go
  4. 40
      model/textparse/protobufparse.go
  5. 118
      model/textparse/protobufparse_test.go
  6. 7
      scrape/scrape.go

4
model/textparse/interface.go
Unescape View File

@ -30,7 +30,9 @@ type Parser interface {
// Histogram returns the bytes of a series with a sparse histogram as a
// value, the timestamp if set, and the histogram in the current sample.
Histogram() ([]byte, *int64, *histogram.Histogram)
// Depending on the parsed input, the function returns an (integer) Histogram
// or a FloatHistogram, with the respective other return value being nil.
Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram)
// Help returns the metric name and help text in the current entry.
// Must only be called after Next returned a help entry.

6
model/textparse/openmetricsparse.go
Unescape View File

@ -113,10 +113,10 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
return p.series, nil, p.val
}
// Histogram always returns (nil, nil, nil) because OpenMetrics does not support
// Histogram always returns (nil, nil, nil, nil) because OpenMetrics does not support
// sparse histograms.
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
return nil, nil, nil
func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
return nil, nil, nil, nil
}
// Help returns the metric name and help text in the current entry.

6
model/textparse/promparse.go
Unescape View File

@ -168,10 +168,10 @@ func (p *PromParser) Series() ([]byte, *int64, float64) {
return p.series, nil, p.val
}
// Histogram always returns (nil, nil, nil) because the Prometheus text format
// Histogram always returns (nil, nil, nil, nil) because the Prometheus text format
// does not support sparse histograms.
func (p *PromParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
return nil, nil, nil
func (p *PromParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
return nil, nil, nil, nil
}
// Help returns the metric name and help text in the current entry.

40
model/textparse/protobufparse.go
Unescape View File

@ -135,12 +135,42 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
// Histogram returns the bytes of a series with a native histogram as a
// value, the timestamp if set, and the native histogram in the current
// sample.
func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
var (
m = p.mf.GetMetric()[p.metricPos]
ts = m.GetTimestampMs()
h = m.GetHistogram()
)
if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 {
// It is a float histogram.
fh := histogram.FloatHistogram{
Count: h.GetSampleCountFloat(),
Sum: h.GetSampleSum(),
ZeroThreshold: h.GetZeroThreshold(),
ZeroCount: h.GetZeroCountFloat(),
Schema: h.GetSchema(),
PositiveSpans: make([]histogram.Span, len(h.GetPositiveSpan())),
PositiveBuckets: h.GetPositiveCount(),
NegativeSpans: make([]histogram.Span, len(h.GetNegativeSpan())),
NegativeBuckets: h.GetNegativeCount(),
}
for i, span := range h.GetPositiveSpan() {
fh.PositiveSpans[i].Offset = span.GetOffset()
fh.PositiveSpans[i].Length = span.GetLength()
}
for i, span := range h.GetNegativeSpan() {
fh.NegativeSpans[i].Offset = span.GetOffset()
fh.NegativeSpans[i].Length = span.GetLength()
}
if ts != 0 {
return p.metricBytes.Bytes(), &ts, nil, &fh
}
// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
// general, but proto3 has no distinction between unset and
// default. Need to avoid in the final format.
return p.metricBytes.Bytes(), nil, nil, &fh
}
sh := histogram.Histogram{
Count: h.GetSampleCount(),
Sum: h.GetSampleSum(),
@ -160,13 +190,11 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
sh.NegativeSpans[i].Offset = span.GetOffset()
sh.NegativeSpans[i].Length = span.GetLength()
}
if ts != 0 {
return p.metricBytes.Bytes(), &ts, &sh
return p.metricBytes.Bytes(), &ts, &sh, nil
}
// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
// general, but proto3 has no distinction between unset and
// default. Need to avoid in the final format.
return p.metricBytes.Bytes(), nil, &sh
return p.metricBytes.Bytes(), nil, &sh, nil
}
// Help returns the metric name and help text in the current entry.

118
model/textparse/protobufparse_test.go
Unescape View File

@ -154,6 +154,78 @@ metric: <
timestamp_ms: 1234568
>
`,
`name: "test_float_histogram"
help: "Test float histogram with many buckets removed to keep it manageable in size."
type: HISTOGRAM
metric: <
histogram: <
sample_count: 175
sample_count_float: 175.0
sample_sum: 0.0008280461746287094
bucket: <
cumulative_count_float: 2.0
upper_bound: -0.0004899999999999998
>
bucket: <
cumulative_count_float: 4.0
upper_bound: -0.0003899999999999998
exemplar: <
label: <
name: "dummyID"
value: "59727"
>
value: -0.00039
timestamp: <
seconds: 1625851155
nanos: 146848499
>
>
>
bucket: <
cumulative_count_float: 16
upper_bound: -0.0002899999999999998
exemplar: <
label: <
name: "dummyID"
value: "5617"
>
value: -0.00029
>
>
schema: 3
zero_threshold: 2.938735877055719e-39
zero_count_float: 2.0
negative_span: <
offset: -162
length: 1
>
negative_span: <
offset: 23
length: 4
>
negative_count: 1.0
negative_count: 3.0
negative_count: -2.0
negative_count: -1.0
negative_count: 1.0
positive_span: <
offset: -161
length: 1
>
positive_span: <
offset: 8
length: 3
>
positive_count: 1.0
positive_count: 2.0
positive_count: -1.0
positive_count: -1.0
>
timestamp_ms: 1234568
>
`,
`name: "test_histogram2"
help: "Similar histogram as before but now without sparse buckets."
@ -263,6 +335,7 @@ metric: <
unit string
comment string
shs *histogram.Histogram
fhs *histogram.FloatHistogram
e []exemplar.Exemplar
}{
{
@ -353,6 +426,42 @@ metric: <
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{
m: "test_float_histogram",
help: "Test float histogram with many buckets removed to keep it manageable in size.",
},
{
m: "test_float_histogram",
typ: MetricTypeHistogram,
},
{
m: "test_float_histogram",
t: 1234568,
fhs: &histogram.FloatHistogram{
Count: 175.0,
ZeroCount: 2.0,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0},
NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0},
},
lset: labels.FromStrings(
"__name__", "test_float_histogram",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{
m: "test_histogram2",
help: "Similar histogram as before but now without sparse buckets.",
@ -524,8 +633,7 @@ metric: <
res = res[:0]
case EntryHistogram:
m, ts, shs := p.Histogram()
m, ts, shs, fhs := p.Histogram()
p.Metric(&res)
require.Equal(t, exp[i].m, string(m))
if ts != nil {
@ -536,7 +644,11 @@ metric: <
require.Equal(t, exp[i].lset, res)
res = res[:0]
require.Equal(t, exp[i].m, string(m))
require.Equal(t, exp[i].shs, shs)
if shs != nil {
require.Equal(t, exp[i].shs, shs)
} else {
require.Equal(t, exp[i].fhs, fhs)
}
j := 0
for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ {
require.Equal(t, exp[i].e[j], e)

7
scrape/scrape.go
Unescape View File

@ -1512,7 +1512,8 @@ loop:
t := defTime
if isHistogram {
met, parsedTimestamp, h = p.Histogram()
met, parsedTimestamp, h, _ = p.Histogram()
// TODO: ingest float histograms in tsdb.
} else {
met, parsedTimestamp, val = p.Series()
}
@ -1564,7 +1565,9 @@ loop:
}
if isHistogram {
ref, err = app.AppendHistogram(ref, lset, t, h)
if h != nil {
ref, err = app.AppendHistogram(ref, lset, t, h)
}
} else {
ref, err = app.Append(ref, lset, t, val)
}

Write Preview
Loading…
Cancel
Save