Break request metrics in apiserver by client.

pkg/apiserver/apiserver.go

@@ -52,18 +52,18 @@ var (
 	requestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "apiserver_request_count",
-			Help: "Counter of apiserver requests broken out for each request handler, verb, API resource, and HTTP response code.",
+			Help: "Counter of apiserver requests broken out for each verb, API resource, client, and HTTP response code.",
 		},
-		[]string{"handler", "verb", "resource", "code"},
+		[]string{"verb", "resource", "client", "code"},
 	)
 	requestLatencies = prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Name: "apiserver_request_latencies",
-			Help: "Response latency distribution in microseconds for each request handler and verb.",
+			Help: "Response latency distribution in microseconds for each verb, resource and client.",
 			// Use buckets ranging from 125 ms to 8 seconds.
 			Buckets: prometheus.ExponentialBuckets(125000, 2.0, 7),
 		},
-		[]string{"handler", "verb"},
+		[]string{"verb", "resource", "client"},
 	)
 )
 
@@ -74,9 +74,9 @@ func init() {
 
 // monitor is a helper function for each HTTP request handler to use for
 // instrumenting basic request counter and latency metrics.
-func monitor(handler string, verb, resource *string, httpCode *int, reqStart time.Time) {
+func monitor(verb, resource *string, client string, httpCode *int, reqStart time.Time) {
-	requestCounter.WithLabelValues(handler, *verb, *resource, strconv.Itoa(*httpCode)).Inc()
+	requestCounter.WithLabelValues(*verb, *resource, client, strconv.Itoa(*httpCode)).Inc()
-	requestLatencies.WithLabelValues(handler, *verb).Observe(float64((time.Since(reqStart)) / time.Microsecond))
+	requestLatencies.WithLabelValues(*verb, *resource, client).Observe(float64((time.Since(reqStart)) / time.Microsecond))
 }
 
 // monitorFilter creates a filter that reports the metrics for a given resource and action.
@@ -85,7 +85,7 @@ func monitorFilter(action, resource string) restful.FilterFunction {
 		reqStart := time.Now()
 		chain.ProcessFilter(req, res)
 		httpCode := res.StatusCode()
-		monitor("rest", &action, &resource, &httpCode, reqStart)
+		monitor(&action, &resource, util.GetClient(req.Request), &httpCode, reqStart)
 	}
 }
 

pkg/apiserver/proxy.go

@@ -33,6 +33,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/rest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/httplog"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/httpstream"
 	proxyutil "github.com/GoogleCloudPlatform/kubernetes/pkg/util/proxy"
 
@@ -55,7 +56,7 @@ func (r *ProxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	var apiResource string
 	var httpCode int
 	reqStart := time.Now()
-	defer monitor("proxy", &verb, &apiResource, &httpCode, reqStart)
+	defer monitor(&verb, &apiResource, util.GetClient(req), &httpCode, reqStart)
 
 	requestInfo, err := r.apiRequestInfoResolver.GetAPIRequestInfo(req)
 	if err != nil {

pkg/apiserver/redirect.go

@@ -25,6 +25,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/rest"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/httplog"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 )
 
 type RedirectHandler struct {
@@ -39,7 +40,7 @@ func (r *RedirectHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	var apiResource string
 	var httpCode int
 	reqStart := time.Now()
-	defer monitor("redirect", &verb, &apiResource, &httpCode, reqStart)
+	defer monitor(&verb, &apiResource, util.GetClient(req), &httpCode, reqStart)
 
 	requestInfo, err := r.apiRequestInfoResolver.GetAPIRequestInfo(req)
 	if err != nil {

pkg/apiserver/validator.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 )
 
 // TODO: this basic interface is duplicated in N places.  consolidate?
@@ -101,7 +102,7 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	apiResource := ""
 	var httpCode int
 	reqStart := time.Now()
-	defer monitor("validate", &verb, &apiResource, &httpCode, reqStart)
+	defer monitor(&verb, &apiResource, util.GetClient(r), &httpCode, reqStart)
 
 	reply := []ServerStatus{}
 	for name, server := range v.servers() {

pkg/util/util.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"path"
 	"reflect"
@@ -491,3 +492,12 @@ func chooseHostInterfaceFromRoute(inFile io.Reader, nw networkInterfacer) (net.I
 	}
 	return nil, nil
 }
+
+func GetClient(req *http.Request) string {
+	if userAgent, ok := req.Header["User-Agent"]; ok {
+		if len(userAgent) > 0 {
+			return userAgent[0]
+		}
+	}
+	return "unknown"
+}