Merge pull request #6409 from fgrzadkowski/measure_client

Add latency metrics to REST client

pkg/client/metrics/metrics.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2015 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const restClientSubsystem = "rest_client"
+
+var (
+	RequestLatency = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Subsystem: restClientSubsystem,
+			Name:      "request_latency_microseconds",
+			Help:      "Request latency in microseconds. Broken down by verb and URL",
+		},
+		[]string{"verb", "url"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	// Register the metrics.
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(RequestLatency)
+	})
+}
+
+// Gets the time since the specified start in microseconds.
+func SinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
+}

pkg/client/request.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/metrics"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
@@ -125,6 +126,7 @@ type Request struct {
 // NewRequest creates a new request helper object for accessing runtime.Objects on a server.
 func NewRequest(client HTTPClient, verb string, baseURL *url.URL, apiVersion string,
 	codec runtime.Codec, namespaceInQuery bool, preserveResourceCase bool) *Request {
+	metrics.Register()
 	return &Request{
 		client:  client,
 		verb:    verb,
@@ -405,7 +407,10 @@ func (r *Request) finalURL() string {
 		p = path.Join(p, r.resourceName, r.subresource, r.subpath)
 	}
 
-	finalURL := *r.baseURL
+	finalURL := url.URL{}
+	if r.baseURL != nil {
+		finalURL = *r.baseURL
+	}
 	finalURL.Path = p
 
 	query := url.Values{}
@@ -427,6 +432,15 @@ func (r *Request) finalURL() string {
 	return finalURL.String()
 }
 
+// Similar to finalURL(), but if the request contains name of an object
+// (e.g. GET for a specific Pod) it will be substited with "<name>".
+func (r Request) finalURLTemplate() string {
+	if len(r.resourceName) != 0 {
+		r.resourceName = "<name>"
+	}
+	return r.finalURL()
+}
+
 // Watch attempts to begin watching the requested location.
 // Returns a watch.Interface, or an error.
 func (r *Request) Watch() (watch.Interface, error) {
@@ -599,6 +613,10 @@ func (r *Request) DoRaw() ([]byte, error) {
 //  * If the status code and body don't make sense together: *UnexpectedStatusError
 //  * http.Client.Do errors are returned directly.
 func (r *Request) Do() Result {
+	start := time.Now()
+	defer func() {
+		metrics.RequestLatency.WithLabelValues(r.verb, r.finalURLTemplate()).Observe(metrics.SinceInMicroseconds(start))
+	}()
 	body, err := r.DoRaw()
 	if err != nil {
 		return Result{err: err}