From ca6580828aeef078370116b86ff3fbe9759b75ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Gouteroux?= Date: Wed, 31 May 2023 15:17:44 +0200 Subject: [PATCH] feat: support histogram and summary metric types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: François Gouteroux --- util/fmtutil/format.go | 171 +++++++++++++++++++++++++----------- util/fmtutil/format_test.go | 140 ++++++++++++++++++++++++++++- 2 files changed, 256 insertions(+), 55 deletions(-) diff --git a/util/fmtutil/format.go b/util/fmtutil/format.go index b5bb9469c..291308dc2 100644 --- a/util/fmtutil/format.go +++ b/util/fmtutil/format.go @@ -14,6 +14,7 @@ package fmtutil import ( + "errors" "fmt" "io" "sort" @@ -26,6 +27,12 @@ import ( "github.com/prometheus/prometheus/prompb" ) +const ( + sumStr = "_sum" + countStr = "_count" + bucketStr = "_bucket" +) + var MetricMetadataTypeValue = map[string]int32{ "UNKNOWN": 0, "COUNTER": 1, @@ -60,71 +67,129 @@ func FormatMetrics(mf map[string]*dto.MetricFamily, extraLabels map[string]strin wr.Metadata = append(wr.Metadata, metadata) for _, metric := range mf[metricName].Metric { - var timeserie prompb.TimeSeries - - // build labels map - labels := make(map[string]string, len(metric.Label)+len(extraLabels)) - labels[model.MetricNameLabel] = metricName - - // add extra labels - for key, value := range extraLabels { - labels[key] = value - } - - // add metric labels - for _, label := range metric.Label { - labelname := label.GetName() - if labelname == model.JobLabel { - labelname = fmt.Sprintf("%s%s", model.ExportedLabelPrefix, labelname) - } - labels[labelname] = label.GetValue() - } - - // build labels name list - sortedLabelNames := make([]string, 0, len(labels)) - for label := range labels { - sortedLabelNames = append(sortedLabelNames, label) + labels := makeLabelsMap(metric, metricName, extraLabels) + if err := makeTimeseries(wr, labels, metric); err != nil { + return wr, err } - // sort labels name in lexicographical order - sort.Strings(sortedLabelNames) - - for _, label := range sortedLabelNames { - timeserie.Labels = append(timeserie.Labels, prompb.Label{ - Name: label, - Value: labels[label], - }) - } - - timestamp := metric.GetTimestampMs() - if timestamp == 0 { - timestamp = time.Now().UnixNano() / int64(time.Millisecond) - } - - timeserie.Samples = []prompb.Sample{ - { - Timestamp: timestamp, - Value: getMetricsValue(metric), - }, - } - - wr.Timeseries = append(wr.Timeseries, timeserie) } } return wr, nil } -// getMetricsValue return the value of a timeserie without the need to give value type -func getMetricsValue(m *dto.Metric) float64 { +func makeTimeserie(wr *prompb.WriteRequest, labels map[string]string, timestamp int64, value float64) { + var timeserie prompb.TimeSeries + timeserie.Labels = makeLabels(labels) + timeserie.Samples = []prompb.Sample{ + { + Timestamp: timestamp, + Value: value, + }, + } + wr.Timeseries = append(wr.Timeseries, timeserie) +} + +func makeTimeseries(wr *prompb.WriteRequest, labels map[string]string, m *dto.Metric) error { + var err error + + timestamp := m.GetTimestampMs() + if timestamp == 0 { + timestamp = time.Now().UnixNano() / int64(time.Millisecond) + } + switch { case m.Gauge != nil: - return m.GetGauge().GetValue() + makeTimeserie(wr, labels, timestamp, m.GetGauge().GetValue()) case m.Counter != nil: - return m.GetCounter().GetValue() + makeTimeserie(wr, labels, timestamp, m.GetCounter().GetValue()) + case m.Summary != nil: + metricName := labels[model.MetricNameLabel] + // Preserve metric name order with first quantile labels timeseries then sum suffix timeserie and finally count suffix timeserie + // Add Summary quantile timeseries + quantileLabels := make(map[string]string, len(labels)+1) + for key, value := range labels { + quantileLabels[key] = value + } + + for _, q := range m.GetSummary().Quantile { + quantileLabels[model.QuantileLabel] = fmt.Sprint(q.GetQuantile()) + makeTimeserie(wr, quantileLabels, timestamp, q.GetValue()) + } + // Overwrite label model.MetricNameLabel for count and sum metrics + // Add Summary sum timeserie + labels[model.MetricNameLabel] = metricName + sumStr + makeTimeserie(wr, labels, timestamp, m.GetSummary().GetSampleSum()) + // Add Summary count timeserie + labels[model.MetricNameLabel] = metricName + countStr + makeTimeserie(wr, labels, timestamp, float64(m.GetSummary().GetSampleCount())) + + case m.Histogram != nil: + metricName := labels[model.MetricNameLabel] + // Preserve metric name order with first bucket suffix timeseries then sum suffix timeserie and finally count suffix timeserie + // Add Histogram bucket timeseries + bucketLabels := make(map[string]string, len(labels)+1) + for key, value := range labels { + bucketLabels[key] = value + } + for _, b := range m.GetHistogram().Bucket { + bucketLabels[model.MetricNameLabel] = metricName + bucketStr + bucketLabels[model.BucketLabel] = fmt.Sprint(b.GetUpperBound()) + makeTimeserie(wr, bucketLabels, timestamp, float64(b.GetCumulativeCount())) + } + // Overwrite label model.MetricNameLabel for count and sum metrics + // Add Histogram sum timeserie + labels[model.MetricNameLabel] = metricName + sumStr + makeTimeserie(wr, labels, timestamp, m.GetHistogram().GetSampleSum()) + // Add Histogram count timeserie + labels[model.MetricNameLabel] = metricName + countStr + makeTimeserie(wr, labels, timestamp, float64(m.GetHistogram().GetSampleCount())) + case m.Untyped != nil: - return m.GetUntyped().GetValue() + makeTimeserie(wr, labels, timestamp, m.GetUntyped().GetValue()) default: - return 0. + err = errors.New("unsupported metric type") + } + return err +} + +func makeLabels(labelsMap map[string]string) []prompb.Label { + // build labels name list + sortedLabelNames := make([]string, 0, len(labelsMap)) + for label := range labelsMap { + sortedLabelNames = append(sortedLabelNames, label) } + // sort labels name in lexicographical order + sort.Strings(sortedLabelNames) + + var labels []prompb.Label + for _, label := range sortedLabelNames { + labels = append(labels, prompb.Label{ + Name: label, + Value: labelsMap[label], + }) + } + return labels +} + +func makeLabelsMap(m *dto.Metric, metricName string, extraLabels map[string]string) map[string]string { + // build labels map + labels := make(map[string]string, len(m.Label)+len(extraLabels)) + labels[model.MetricNameLabel] = metricName + + // add extra labels + for key, value := range extraLabels { + labels[key] = value + } + + // add metric labels + for _, label := range m.Label { + labelname := label.GetName() + if labelname == model.JobLabel { + labelname = fmt.Sprintf("%s%s", model.ExportedLabelPrefix, labelname) + } + labels[labelname] = label.GetValue() + } + + return labels } // ParseMetricsTextReader consumes an io.Reader and returns the MetricFamily. diff --git a/util/fmtutil/format_test.go b/util/fmtutil/format_test.go index 9deed2de9..e2ac30135 100644 --- a/util/fmtutil/format_test.go +++ b/util/fmtutil/format_test.go @@ -24,13 +24,130 @@ import ( var writeRequestFixture = &prompb.WriteRequest{ Metadata: []prompb.MetricMetadata{ + { + MetricFamilyName: "http_request_duration_seconds", + Type: 3, + Help: "A histogram of the request duration.", + }, + { + MetricFamilyName: "http_requests_total", + Type: 1, + Help: "The total number of HTTP requests.", + }, + { + MetricFamilyName: "rpc_duration_seconds", + Type: 5, + Help: "A summary of the RPC duration in seconds.", + }, { MetricFamilyName: "test_metric1", Type: 2, - Help: "this is a test metric", + Help: "This is a test metric.", }, }, Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_request_duration_seconds_bucket"}, + {Name: "job", Value: "promtool"}, + {Name: "le", Value: "0.1"}, + }, + Samples: []prompb.Sample{{Value: 33444, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_request_duration_seconds_bucket"}, + {Name: "job", Value: "promtool"}, + {Name: "le", Value: "0.5"}, + }, + Samples: []prompb.Sample{{Value: 129389, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_request_duration_seconds_bucket"}, + {Name: "job", Value: "promtool"}, + {Name: "le", Value: "1"}, + }, + Samples: []prompb.Sample{{Value: 133988, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_request_duration_seconds_bucket"}, + {Name: "job", Value: "promtool"}, + {Name: "le", Value: "+Inf"}, + }, + Samples: []prompb.Sample{{Value: 144320, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_request_duration_seconds_sum"}, + {Name: "job", Value: "promtool"}, + }, + Samples: []prompb.Sample{{Value: 53423, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_request_duration_seconds_count"}, + {Name: "job", Value: "promtool"}, + }, + Samples: []prompb.Sample{{Value: 144320, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_requests_total"}, + {Name: "code", Value: "200"}, + {Name: "job", Value: "promtool"}, + {Name: "method", Value: "post"}, + }, + Samples: []prompb.Sample{{Value: 1027, Timestamp: 1395066363000}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "http_requests_total"}, + {Name: "code", Value: "400"}, + {Name: "job", Value: "promtool"}, + {Name: "method", Value: "post"}, + }, + Samples: []prompb.Sample{{Value: 3, Timestamp: 1395066363000}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "job", Value: "promtool"}, + {Name: "quantile", Value: "0.01"}, + }, + Samples: []prompb.Sample{{Value: 3102, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "job", Value: "promtool"}, + {Name: "quantile", Value: "0.5"}, + }, + Samples: []prompb.Sample{{Value: 4773, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds"}, + {Name: "job", Value: "promtool"}, + {Name: "quantile", Value: "0.99"}, + }, + Samples: []prompb.Sample{{Value: 76656, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds_sum"}, + {Name: "job", Value: "promtool"}, + }, + Samples: []prompb.Sample{{Value: 1.7560473e+07, Timestamp: 1}}, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "rpc_duration_seconds_count"}, + {Name: "job", Value: "promtool"}, + }, + Samples: []prompb.Sample{{Value: 2693, Timestamp: 1}}, + }, { Labels: []prompb.Label{ {Name: "__name__", Value: "test_metric1"}, @@ -58,7 +175,26 @@ var writeRequestFixture = &prompb.WriteRequest{ func TestParseMetricsTextAndFormat(t *testing.T) { input := bytes.NewReader([]byte(` - # HELP test_metric1 this is a test metric + # HELP http_request_duration_seconds A histogram of the request duration. + # TYPE http_request_duration_seconds histogram + http_request_duration_seconds_bucket{le="0.1"} 33444 1 + http_request_duration_seconds_bucket{le="0.5"} 129389 1 + http_request_duration_seconds_bucket{le="1"} 133988 1 + http_request_duration_seconds_bucket{le="+Inf"} 144320 1 + http_request_duration_seconds_sum 53423 1 + http_request_duration_seconds_count 144320 1 + # HELP http_requests_total The total number of HTTP requests. + # TYPE http_requests_total counter + http_requests_total{method="post",code="200"} 1027 1395066363000 + http_requests_total{method="post",code="400"} 3 1395066363000 + # HELP rpc_duration_seconds A summary of the RPC duration in seconds. + # TYPE rpc_duration_seconds summary + rpc_duration_seconds{quantile="0.01"} 3102 1 + rpc_duration_seconds{quantile="0.5"} 4773 1 + rpc_duration_seconds{quantile="0.99"} 76656 1 + rpc_duration_seconds_sum 1.7560473e+07 1 + rpc_duration_seconds_count 2693 1 + # HELP test_metric1 This is a test metric. # TYPE test_metric1 gauge test_metric1{b="c",baz="qux",d="e",foo="bar"} 1 1 test_metric1{b="c",baz="qux",d="e",foo="bar"} 2 1