consul: Export some telemetry on service queries

consul/catalog_endpoint.go

@@ -134,7 +134,7 @@ func (c *Catalog) ServiceNodes(args *structs.ServiceSpecificRequest, reply *stru
 
 	// Get the nodes
 	state := c.srv.fsm.State()
-	return c.srv.blockingRPC(&args.BlockingQuery,
+	err := c.srv.blockingRPC(&args.BlockingQuery,
 		state.QueryTables("ServiceNodes"),
 		func() (uint64, error) {
 			if args.TagFilter {
@@ -144,6 +144,18 @@ func (c *Catalog) ServiceNodes(args *structs.ServiceSpecificRequest, reply *stru
 			}
 			return reply.Index, nil
 		})
+
+	// Provide some metrics
+	if err == nil {
+		metrics.IncrCounter([]string{"consul", "catalog", "service", "query", args.ServiceName}, 1)
+		if args.ServiceTag != "" {
+			metrics.IncrCounter([]string{"consul", "catalog", "service", "query-tag", args.ServiceName, args.ServiceTag}, 1)
+		}
+		if len(reply.ServiceNodes) == 0 {
+			metrics.IncrCounter([]string{"consul", "catalog", "service", "not-found", args.ServiceName}, 1)
+		}
+	}
+	return err
 }
 
 // NodeServices returns all the services registered as part of a node

consul/health_endpoint.go

@@ -2,6 +2,7 @@ package consul
 
 import (
 	"fmt"
+	"github.com/armon/go-metrics"
 	"github.com/hashicorp/consul/consul/structs"
 )
 
@@ -80,7 +81,7 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc
 
 	// Get the nodes
 	state := h.srv.fsm.State()
-	return h.srv.blockingRPC(&args.BlockingQuery,
+	err := h.srv.blockingRPC(&args.BlockingQuery,
 		state.QueryTables("CheckServiceNodes"),
 		func() (uint64, error) {
 			if args.TagFilter {
@@ -90,4 +91,16 @@ func (h *Health) ServiceNodes(args *structs.ServiceSpecificRequest, reply *struc
 			}
 			return reply.Index, nil
 		})
+
+	// Provide some metrics
+	if err == nil {
+		metrics.IncrCounter([]string{"consul", "health", "service", "query", args.ServiceName}, 1)
+		if args.ServiceTag != "" {
+			metrics.IncrCounter([]string{"consul", "health", "service", "query-tag", args.ServiceName, args.ServiceTag}, 1)
+		}
+		if len(reply.Nodes) == 0 {
+			metrics.IncrCounter([]string{"consul", "health", "service", "not-found", args.ServiceName}, 1)
+		}
+	}
+	return err
 }