histograms: parse float histograms from proto definition (#11149)

* histograms: parse float histograms from proto definition

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Improve comment

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Ignore float buckets

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Refactor Histogram() function

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Fix test_float_histogram

Signed-off-by: Marc Tuduri <marctc@protonmail.com>

* Update model/textparse/protobufparse.go

Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update protobufparse.go

Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update scrape.go

Signed-off-by: Marc Tudurí <marctc@protonmail.com>

* Update scrape/scrape.go

Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>

Signed-off-by: Marc Tuduri <marctc@protonmail.com>
Signed-off-by: Marc Tudurí <marctc@protonmail.com>
Co-authored-by: Ganesh Vernekar <15064823+codesome@users.noreply.github.com>

model/textparse/interface.go

@@ -30,7 +30,9 @@ type Parser interface {
 
 	// Histogram returns the bytes of a series with a sparse histogram as a
 	// value, the timestamp if set, and the histogram in the current sample.
-	Histogram() ([]byte, *int64, *histogram.Histogram)
+	// Depending on the parsed input, the function returns an (integer) Histogram
+	// or a FloatHistogram, with the respective other return value being nil.
+	Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram)
 
 	// Help returns the metric name and help text in the current entry.
 	// Must only be called after Next returned a help entry.

model/textparse/openmetricsparse.go

@@ -113,10 +113,10 @@ func (p *OpenMetricsParser) Series() ([]byte, *int64, float64) {
 	return p.series, nil, p.val
 }
 
-// Histogram always returns (nil, nil, nil) because OpenMetrics does not support
+// Histogram always returns (nil, nil, nil, nil) because OpenMetrics does not support
 // sparse histograms.
-func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
+func (p *OpenMetricsParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
-	return nil, nil, nil
+	return nil, nil, nil, nil
 }
 
 // Help returns the metric name and help text in the current entry.

model/textparse/promparse.go

@@ -168,10 +168,10 @@ func (p *PromParser) Series() ([]byte, *int64, float64) {
 	return p.series, nil, p.val
 }
 
-// Histogram always returns (nil, nil, nil) because the Prometheus text format
+// Histogram always returns (nil, nil, nil, nil) because the Prometheus text format
 // does not support sparse histograms.
-func (p *PromParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
+func (p *PromParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
-	return nil, nil, nil
+	return nil, nil, nil, nil
 }
 
 // Help returns the metric name and help text in the current entry.

model/textparse/protobufparse.go

@@ -135,12 +135,42 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
 // Histogram returns the bytes of a series with a native histogram as a
 // value, the timestamp if set, and the native histogram in the current
 // sample.
-func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
+func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *histogram.FloatHistogram) {
 	var (
 		m  = p.mf.GetMetric()[p.metricPos]
 		ts = m.GetTimestampMs()
 		h  = m.GetHistogram()
 	)
+	if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 {
+		// It is a float histogram.
+		fh := histogram.FloatHistogram{
+			Count:           h.GetSampleCountFloat(),
+			Sum:             h.GetSampleSum(),
+			ZeroThreshold:   h.GetZeroThreshold(),
+			ZeroCount:       h.GetZeroCountFloat(),
+			Schema:          h.GetSchema(),
+			PositiveSpans:   make([]histogram.Span, len(h.GetPositiveSpan())),
+			PositiveBuckets: h.GetPositiveCount(),
+			NegativeSpans:   make([]histogram.Span, len(h.GetNegativeSpan())),
+			NegativeBuckets: h.GetNegativeCount(),
+		}
+		for i, span := range h.GetPositiveSpan() {
+			fh.PositiveSpans[i].Offset = span.GetOffset()
+			fh.PositiveSpans[i].Length = span.GetLength()
+		}
+		for i, span := range h.GetNegativeSpan() {
+			fh.NegativeSpans[i].Offset = span.GetOffset()
+			fh.NegativeSpans[i].Length = span.GetLength()
+		}
+		if ts != 0 {
+			return p.metricBytes.Bytes(), &ts, nil, &fh
+		}
+		// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
+		// general, but proto3 has no distinction between unset and
+		// default. Need to avoid in the final format.
+		return p.metricBytes.Bytes(), nil, nil, &fh
+	}
+
 	sh := histogram.Histogram{
 		Count:           h.GetSampleCount(),
 		Sum:             h.GetSampleSum(),
@@ -160,13 +190,11 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram) {
 		sh.NegativeSpans[i].Offset = span.GetOffset()
 		sh.NegativeSpans[i].Length = span.GetLength()
 	}
+
 	if ts != 0 {
-		return p.metricBytes.Bytes(), &ts, &sh
+		return p.metricBytes.Bytes(), &ts, &sh, nil
 	}
-	// Nasty hack: Assume that ts==0 means no timestamp. That's not true in
+	return p.metricBytes.Bytes(), nil, &sh, nil
-	// general, but proto3 has no distinction between unset and
-	// default. Need to avoid in the final format.
-	return p.metricBytes.Bytes(), nil, &sh
 }
 
 // Help returns the metric name and help text in the current entry.

model/textparse/protobufparse_test.go

@@ -154,6 +154,78 @@ metric: <
   timestamp_ms: 1234568
 >
 
+`,
+
+		`name: "test_float_histogram"
+help: "Test float histogram with many buckets removed to keep it manageable in size."
+type: HISTOGRAM
+metric: <
+  histogram: <
+    sample_count: 175
+	sample_count_float: 175.0
+    sample_sum: 0.0008280461746287094
+    bucket: <
+      cumulative_count_float: 2.0
+      upper_bound: -0.0004899999999999998
+    >
+    bucket: <
+      cumulative_count_float: 4.0
+      upper_bound: -0.0003899999999999998
+      exemplar: <
+        label: <
+          name: "dummyID"
+          value: "59727"
+        >
+        value: -0.00039
+        timestamp: <
+          seconds: 1625851155
+          nanos: 146848499
+        >
+      >
+    >
+    bucket: <
+      cumulative_count_float: 16
+      upper_bound: -0.0002899999999999998
+      exemplar: <
+        label: <
+          name: "dummyID"
+          value: "5617"
+        >
+        value: -0.00029
+      >
+    >
+    schema: 3
+    zero_threshold: 2.938735877055719e-39
+    zero_count_float: 2.0
+    negative_span: <
+      offset: -162
+      length: 1
+    >
+    negative_span: <
+      offset: 23
+      length: 4
+    >
+    negative_count: 1.0
+    negative_count: 3.0
+    negative_count: -2.0
+    negative_count: -1.0
+    negative_count: 1.0
+    positive_span: <
+      offset: -161
+      length: 1
+    >
+    positive_span: <
+      offset: 8
+      length: 3
+    >
+    positive_count: 1.0
+    positive_count: 2.0
+    positive_count: -1.0
+    positive_count: -1.0
+  >
+  timestamp_ms: 1234568
+>
+
 `,
 		`name: "test_histogram2"
 help: "Similar histogram as before but now without sparse buckets."
@@ -263,6 +335,7 @@ metric: <
 		unit    string
 		comment string
 		shs     *histogram.Histogram
+		fhs     *histogram.FloatHistogram
 		e       []exemplar.Exemplar
 	}{
 		{
@@ -353,6 +426,42 @@ metric: <
 				{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
 			},
 		},
+		{
+			m:    "test_float_histogram",
+			help: "Test float histogram with many buckets removed to keep it manageable in size.",
+		},
+		{
+			m:   "test_float_histogram",
+			typ: MetricTypeHistogram,
+		},
+		{
+			m: "test_float_histogram",
+			t: 1234568,
+			fhs: &histogram.FloatHistogram{
+				Count:         175.0,
+				ZeroCount:     2.0,
+				Sum:           0.0008280461746287094,
+				ZeroThreshold: 2.938735877055719e-39,
+				Schema:        3,
+				PositiveSpans: []histogram.Span{
+					{Offset: -161, Length: 1},
+					{Offset: 8, Length: 3},
+				},
+				NegativeSpans: []histogram.Span{
+					{Offset: -162, Length: 1},
+					{Offset: 23, Length: 4},
+				},
+				PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0},
+				NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0},
+			},
+			lset: labels.FromStrings(
+				"__name__", "test_float_histogram",
+			),
+			e: []exemplar.Exemplar{
+				{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
+				{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
+			},
+		},
 		{
 			m:    "test_histogram2",
 			help: "Similar histogram as before but now without sparse buckets.",
@@ -524,8 +633,7 @@ metric: <
 			res = res[:0]
 
 		case EntryHistogram:
-			m, ts, shs := p.Histogram()
+			m, ts, shs, fhs := p.Histogram()
-
 			p.Metric(&res)
 			require.Equal(t, exp[i].m, string(m))
 			if ts != nil {
@@ -536,7 +644,11 @@ metric: <
 			require.Equal(t, exp[i].lset, res)
 			res = res[:0]
 			require.Equal(t, exp[i].m, string(m))
-			require.Equal(t, exp[i].shs, shs)
+			if shs != nil {
+				require.Equal(t, exp[i].shs, shs)
+			} else {
+				require.Equal(t, exp[i].fhs, fhs)
+			}
 			j := 0
 			for e := (exemplar.Exemplar{}); p.Exemplar(&e); j++ {
 				require.Equal(t, exp[i].e[j], e)

scrape/scrape.go

@@ -1512,7 +1512,8 @@ loop:
 
 		t := defTime
 		if isHistogram {
-			met, parsedTimestamp, h = p.Histogram()
+			met, parsedTimestamp, h, _ = p.Histogram()
+			// TODO: ingest float histograms in tsdb.
 		} else {
 			met, parsedTimestamp, val = p.Series()
 		}
@@ -1564,7 +1565,9 @@ loop:
 		}
 
 		if isHistogram {
-			ref, err = app.AppendHistogram(ref, lset, t, h)
+			if h != nil {
+				ref, err = app.AppendHistogram(ref, lset, t, h)
+			}
 		} else {
 			ref, err = app.Append(ref, lset, t, val)
 		}