Merge pull request #8271 from coignetp/http-metrics-label

Use method and path as labels for http metrics

agent/config/default.go

@@ -122,6 +122,7 @@ func DefaultSource() Source {
 		telemetry = {
 			metrics_prefix = "consul"
 			filter_default = true
+			prefix_filter = [ "-consul.api.http" ]
 		}
 
 	`,

agent/config/runtime_test.go

@@ -2261,7 +2261,7 @@ func TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases(t *testing.T) {
 			patch: func(rt *RuntimeConfig) {
 				rt.DataDir = dataDir
 				rt.Telemetry.AllowedPrefixes = []string{"foo"}
-				rt.Telemetry.BlockedPrefixes = []string{"bar"}
+				rt.Telemetry.BlockedPrefixes = []string{"consul.api.http", "bar"}
 			},
 			warns: []string{`Filter rule must begin with either '+' or '-': "nix"`},
 		},
@@ -6686,7 +6686,7 @@ func TestFullConfig(t *testing.T) {
 			DogstatsdTags:                      []string{"3N81zSUB", "Xtj8AnXZ"},
 			FilterDefault:                      true,
 			AllowedPrefixes:                    []string{"oJotS8XJ"},
-			BlockedPrefixes:                    []string{"cazlEhGn"},
+			BlockedPrefixes:                    []string{"consul.api.http", "cazlEhGn"},
 			MetricsPrefix:                      "ftO6DySn",
 			PrometheusRetentionTime:            15 * time.Second,
 			StatsdAddr:                         "drce87cy",

agent/http.go

@@ -184,12 +184,23 @@ func (s *HTTPHandlers) handler(enableDebug bool) http.Handler {
 			parts = append(parts, part)
 		}
 
-		// Register the wrapper, which will close over the expensive-to-compute
-		// parts from above.
-		// TODO (kyhavlov): Convert this to utilize metric labels in a major release
+		// Tranform the pattern to a valid label by replacing the '/' by '_'.
+		// Omit the leading slash.
+		// Distinguish thing like /v1/query from /v1/query/<query_id> by having
+		// an extra underscore.
+		path_label := strings.Replace(pattern[1:], "/", "_", -1)
+
+		// Register the wrapper.
 		wrapper := func(resp http.ResponseWriter, req *http.Request) {
 			start := time.Now()
 			handler(resp, req)
+
+			// This new metric is disabled by default with the prefix_filter option.
+			// It will be enabled by default in a future version.
+			labels := []metrics.Label{{Name: "method", Value: req.Method}, {Name: "path", Value: path_label}}
+			metrics.MeasureSinceWithLabels([]string{"api", "http"}, start, labels)
+
+			// Duplicated information. Kept for backward compatibility.
 			key := append([]string{"http", req.Method}, parts...)
 			metrics.MeasureSince(key, start)
 		}

website/pages/docs/agent/telemetry.mdx

@@ -180,7 +180,7 @@ This is a full list of metrics emitted by Consul.
 | `consul.dns.stale_queries`                            | This increments when an agent serves a query within the allowed stale threshold.                                                                                                                                                                                                                                                                                                                                    | queries              | counter |
 | `consul.dns.ptr_query.`                               | This measures the time spent handling a reverse DNS query for the given node.                                                                                                                                                                                                                                                                                                                                       | ms                   | timer   |
 | `consul.dns.domain_query.`                            | This measures the time spent handling a domain query for the given node.                                                                                                                                                                                                                                                                                                                                            | ms                   | timer   |
-| `consul.http..`                                       | This tracks how long it takes to service the given HTTP request for the given verb and path. Paths do not include details like service or key names, for these an underscore will be present as a placeholder (eg. `consul.http.GET.v1.kv._`)                                                                                                                                                                       | ms                   | timer   |
+| `consul.http`                                         | This tracks how long it takes to service the given HTTP request for the given verb and path. Paths do not include details like service or key names, for these an underscore will be present as a placeholder (eg. label `path=v1_kv_`)                                                                                                                                                                             | ms                   | timer   |
 
 ## Server Health