agent/grpc: add connection count metrics

Gauge metrics are great for understanding the current state, but can somtimes hide problems
if there are many disconnect/reconnects.

This commit adds counter metrics for connections and streams to make it easier to see the
count of newly created connections and streams.

agent/grpc/stats.go

@@ -36,7 +36,7 @@ func (c *statsHandler) HandleRPC(_ context.Context, s stats.RPCStats) {
 	}
 	switch s.(type) {
 	case *stats.InHeader:
-		c.metrics.IncrCounter([]string{"grpc", label, "request"}, 1)
+		c.metrics.IncrCounter([]string{"grpc", label, "request", "count"}, 1)
 	}
 }
 
@@ -56,6 +56,7 @@ func (c *statsHandler) HandleConn(_ context.Context, s stats.ConnStats) {
 	switch s.(type) {
 	case *stats.ConnBegin:
 		count = atomic.AddUint64(&c.activeConns, 1)
+		c.metrics.IncrCounter([]string{"grpc", label, "connection", "count"}, 1)
 	case *stats.ConnEnd:
 		// Decrement!
 		count = atomic.AddUint64(&c.activeConns, ^uint64(0))
@@ -80,6 +81,7 @@ func (i *activeStreamCounter) Intercept(
 ) error {
 	count := atomic.AddUint64(&i.count, 1)
 	i.metrics.SetGauge([]string{"grpc", "server", "streams"}, float32(count))
+	i.metrics.IncrCounter([]string{"grpc", "server", "stream", "count"}, 1)
 	defer func() {
 		count := atomic.AddUint64(&i.count, ^uint64(0))
 		i.metrics.SetGauge([]string{"grpc", "server", "streams"}, float32(count))

agent/grpc/stats_test.go

@@ -88,7 +88,9 @@ func TestHandler_EmitsStats(t *testing.T) {
 	assertDeepEqual(t, expectedGauge, sink.gaugeCalls, cmpMetricCalls)
 
 	expectedCounter := []metricCall{
-		{key: []string{"testing", "grpc", "server", "request"}, val: 1},
+		{key: []string{"testing", "grpc", "server", "connection", "count"}, val: 1},
+		{key: []string{"testing", "grpc", "server", "request", "count"}, val: 1},
+		{key: []string{"testing", "grpc", "server", "stream", "count"}, val: 1},
 	}
 	assertDeepEqual(t, expectedCounter, sink.incrCounterCalls, cmpMetricCalls)
 }