OTLP receiver: Convert also metric metadata (#15416)

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## unreleased
 
+* [ENHANCEMENT] OTLP receiver: Convert also metric metadata. #15416
+
 ## 3.0.0 / 2024-11-14
 
 This release includes new features such as a brand new UI and UTF-8 support enabled by default. As this marks the first new major version in seven years, several breaking changes are introduced. The breaking changes are mainly around the removal of deprecated feature flags and CLI arguments, and the full list can be found below. For users that want to upgrade we recommend to read through our [migration guide](https://prometheus.io/docs/prometheus/3.0/migration/).

storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go

@@ -37,7 +37,6 @@ type Settings struct {
 	DisableTargetInfo         bool
 	ExportCreatedMetric       bool
 	AddMetricSuffixes         bool
-	SendMetadata              bool
 	AllowUTF8                 bool
 	PromoteResourceAttributes []string
 }
@@ -47,6 +46,7 @@ type PrometheusConverter struct {
 	unique    map[uint64]*prompb.TimeSeries
 	conflicts map[uint64][]*prompb.TimeSeries
 	everyN    everyNTimes
+	metadata  []prompb.MetricMetadata
 }
 
 func NewPrometheusConverter() *PrometheusConverter {
@@ -60,6 +60,16 @@ func NewPrometheusConverter() *PrometheusConverter {
 func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metrics, settings Settings) (annots annotations.Annotations, errs error) {
 	c.everyN = everyNTimes{n: 128}
 	resourceMetricsSlice := md.ResourceMetrics()
+
+	numMetrics := 0
+	for i := 0; i < resourceMetricsSlice.Len(); i++ {
+		scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics()
+		for j := 0; j < scopeMetricsSlice.Len(); j++ {
+			numMetrics += scopeMetricsSlice.At(j).Metrics().Len()
+		}
+	}
+	c.metadata = make([]prompb.MetricMetadata, 0, numMetrics)
+
 	for i := 0; i < resourceMetricsSlice.Len(); i++ {
 		resourceMetrics := resourceMetricsSlice.At(i)
 		resource := resourceMetrics.Resource()
@@ -86,6 +96,12 @@ func (c *PrometheusConverter) FromMetrics(ctx context.Context, md pmetric.Metric
 				}
 
 				promName := prometheustranslator.BuildCompliantName(metric, settings.Namespace, settings.AddMetricSuffixes, settings.AllowUTF8)
+				c.metadata = append(c.metadata, prompb.MetricMetadata{
+					Type:             otelMetricTypeToPromMetricType(metric),
+					MetricFamilyName: promName,
+					Help:             metric.Description(),
+					Unit:             metric.Unit(),
+				})
 
 				// handle individual metrics based on type
 				//exhaustive:enforce

storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw_test.go

@@ -22,20 +22,46 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.opentelemetry.io/collector/pdata/pmetric/pmetricotlp"
+
+	"github.com/prometheus/prometheus/prompb"
+	prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus"
 )
 
 func TestFromMetrics(t *testing.T) {
 	t.Run("successful", func(t *testing.T) {
 		converter := NewPrometheusConverter()
 		payload := createExportRequest(5, 128, 128, 2, 0)
+		var expMetadata []prompb.MetricMetadata
+		resourceMetricsSlice := payload.Metrics().ResourceMetrics()
+		for i := 0; i < resourceMetricsSlice.Len(); i++ {
+			scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics()
+			for j := 0; j < scopeMetricsSlice.Len(); j++ {
+				metricSlice := scopeMetricsSlice.At(j).Metrics()
+				for k := 0; k < metricSlice.Len(); k++ {
+					metric := metricSlice.At(k)
+					promName := prometheustranslator.BuildCompliantName(metric, "", false, false)
+					expMetadata = append(expMetadata, prompb.MetricMetadata{
+						Type:             otelMetricTypeToPromMetricType(metric),
+						MetricFamilyName: promName,
+						Help:             metric.Description(),
+						Unit:             metric.Unit(),
+					})
+				}
+			}
+		}
 
 		annots, err := converter.FromMetrics(context.Background(), payload.Metrics(), Settings{})
 		require.NoError(t, err)
 		require.Empty(t, annots)
+
+		if diff := cmp.Diff(expMetadata, converter.Metadata()); diff != "" {
+			t.Errorf("mismatch (-want +got):\n%s", diff)
+		}
 	})
 
 	t.Run("context cancellation", func(t *testing.T) {
@@ -115,13 +141,15 @@ func BenchmarkPrometheusConverter_FromMetrics(b *testing.B) {
 									for _, exemplarsPerSeries := range []int{0, 5, 10} {
 										b.Run(fmt.Sprintf("exemplars per series: %v", exemplarsPerSeries), func(b *testing.B) {
 											payload := createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCount, labelsPerMetric, exemplarsPerSeries)
+											b.ResetTimer()
 
-											for i := 0; i < b.N; i++ {
+											for range b.N {
 												converter := NewPrometheusConverter()
 												annots, err := converter.FromMetrics(context.Background(), payload.Metrics(), Settings{})
 												require.NoError(b, err)
 												require.Empty(b, annots)
 												require.NotNil(b, converter.TimeSeries())
+												require.NotNil(b, converter.Metadata())
 											}
 										})
 									}
@@ -148,6 +176,8 @@ func createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCou
 		m := metrics.AppendEmpty()
 		m.SetEmptyHistogram()
 		m.SetName(fmt.Sprintf("histogram-%v", i))
+		m.SetDescription("histogram")
+		m.SetUnit("unit")
 		m.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
 		h := m.Histogram().DataPoints().AppendEmpty()
 		h.SetTimestamp(ts)
@@ -166,6 +196,8 @@ func createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCou
 		m := metrics.AppendEmpty()
 		m.SetEmptySum()
 		m.SetName(fmt.Sprintf("sum-%v", i))
+		m.SetDescription("sum")
+		m.SetUnit("unit")
 		m.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative)
 		point := m.Sum().DataPoints().AppendEmpty()
 		point.SetTimestamp(ts)
@@ -178,6 +210,8 @@ func createExportRequest(resourceAttributeCount, histogramCount, nonHistogramCou
 		m := metrics.AppendEmpty()
 		m.SetEmptyGauge()
 		m.SetName(fmt.Sprintf("gauge-%v", i))
+		m.SetDescription("gauge")
+		m.SetUnit("unit")
 		point := m.Gauge().DataPoints().AppendEmpty()
 		point.SetTimestamp(ts)
 		point.SetDoubleValue(1.23)

storage/remote/otlptranslator/prometheusremotewrite/otlp_to_openmetrics_metadata.go

@@ -20,7 +20,6 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 
 	"github.com/prometheus/prometheus/prompb"
-	prometheustranslator "github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheus"
 )
 
 func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) prompb.MetricMetadata_MetricType {
@@ -42,36 +41,3 @@ func otelMetricTypeToPromMetricType(otelMetric pmetric.Metric) prompb.MetricMeta
 	}
 	return prompb.MetricMetadata_UNKNOWN
 }
-
-func OtelMetricsToMetadata(md pmetric.Metrics, addMetricSuffixes, allowUTF8 bool) []*prompb.MetricMetadata {
-	resourceMetricsSlice := md.ResourceMetrics()
-
-	metadataLength := 0
-	for i := 0; i < resourceMetricsSlice.Len(); i++ {
-		scopeMetricsSlice := resourceMetricsSlice.At(i).ScopeMetrics()
-		for j := 0; j < scopeMetricsSlice.Len(); j++ {
-			metadataLength += scopeMetricsSlice.At(j).Metrics().Len()
-		}
-	}
-
-	var metadata = make([]*prompb.MetricMetadata, 0, metadataLength)
-	for i := 0; i < resourceMetricsSlice.Len(); i++ {
-		resourceMetrics := resourceMetricsSlice.At(i)
-		scopeMetricsSlice := resourceMetrics.ScopeMetrics()
-
-		for j := 0; j < scopeMetricsSlice.Len(); j++ {
-			scopeMetrics := scopeMetricsSlice.At(j)
-			for k := 0; k < scopeMetrics.Metrics().Len(); k++ {
-				metric := scopeMetrics.Metrics().At(k)
-				entry := prompb.MetricMetadata{
-					Type:             otelMetricTypeToPromMetricType(metric),
-					MetricFamilyName: prometheustranslator.BuildCompliantName(metric, "", addMetricSuffixes, allowUTF8),
-					Help:             metric.Description(),
-				}
-				metadata = append(metadata, &entry)
-			}
-		}
-	}
-
-	return metadata
-}

storage/remote/otlptranslator/prometheusremotewrite/timeseries.go

@@ -39,3 +39,8 @@ func (c *PrometheusConverter) TimeSeries() []prompb.TimeSeries {
 
 	return allTS
 }
+
+// Metadata returns a slice of the prompb.Metadata that were converted from OTel format.
+func (c *PrometheusConverter) Metadata() []prompb.MetricMetadata {
+	return c.metadata
+}

storage/remote/write_handler.go

@@ -526,6 +526,7 @@ func (h *otlpWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	err = h.rwHandler.write(r.Context(), &prompb.WriteRequest{
 		Timeseries: converter.TimeSeries(),
+		Metadata:   converter.Metadata(),
 	})
 
 	switch {