diff --git a/cmd/hyperkube/main.go b/cmd/hyperkube/main.go index 9eff6f7254..f2865c4e75 100644 --- a/cmd/hyperkube/main.go +++ b/cmd/hyperkube/main.go @@ -23,6 +23,8 @@ package main import ( "os" + + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration ) func main() { diff --git a/cmd/kube-apiserver/apiserver.go b/cmd/kube-apiserver/apiserver.go index 223e94e1f0..46b5e2ab35 100644 --- a/cmd/kube-apiserver/apiserver.go +++ b/cmd/kube-apiserver/apiserver.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" "k8s.io/kubernetes/pkg/version/verflag" diff --git a/cmd/kube-controller-manager/controller-manager.go b/cmd/kube-controller-manager/controller-manager.go index 4390ae9a3f..fc121503f5 100644 --- a/cmd/kube-controller-manager/controller-manager.go +++ b/cmd/kube-controller-manager/controller-manager.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/cmd/kube-controller-manager/app" "k8s.io/kubernetes/cmd/kube-controller-manager/app/options" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" diff --git a/cmd/kube-dns/dns.go b/cmd/kube-dns/dns.go index e1c59029e3..439337f7b7 100644 --- a/cmd/kube-dns/dns.go +++ b/cmd/kube-dns/dns.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/pflag" "k8s.io/kubernetes/cmd/kube-dns/app" "k8s.io/kubernetes/cmd/kube-dns/app/options" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" "k8s.io/kubernetes/pkg/version/verflag" diff --git a/cmd/kube-proxy/proxy.go b/cmd/kube-proxy/proxy.go index f410154d9b..f9e8945921 100644 --- a/cmd/kube-proxy/proxy.go +++ b/cmd/kube-proxy/proxy.go @@ -22,6 +22,7 @@ import ( "k8s.io/kubernetes/cmd/kube-proxy/app" "k8s.io/kubernetes/cmd/kube-proxy/app/options" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/healthz" "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" diff --git a/cmd/kubectl/app/kubectl.go b/cmd/kubectl/app/kubectl.go index a23e4b9756..da6253360e 100644 --- a/cmd/kubectl/app/kubectl.go +++ b/cmd/kubectl/app/kubectl.go @@ -19,6 +19,7 @@ package app import ( "os" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/kubectl/cmd" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/util/logs" diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 8ab2715edf..334c33a0d6 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -26,6 +26,7 @@ import ( "k8s.io/kubernetes/cmd/kubelet/app" "k8s.io/kubernetes/cmd/kubelet/app/options" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" "k8s.io/kubernetes/pkg/version/verflag" diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 77ab67e952..8382b3881b 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -20,6 +20,7 @@ import ( "fmt" "k8s.io/kubernetes/pkg/api" + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration "k8s.io/kubernetes/pkg/client/record" client "k8s.io/kubernetes/pkg/client/unversioned" clientset "k8s.io/kubernetes/pkg/client/unversioned/adapters/internalclientset" diff --git a/contrib/mesos/cmd/km/km.go b/contrib/mesos/cmd/km/km.go index eb3d4435ad..3637a9afc6 100644 --- a/contrib/mesos/cmd/km/km.go +++ b/contrib/mesos/cmd/km/km.go @@ -19,6 +19,8 @@ package main import ( "os" + + _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration ) func main() { diff --git a/hack/.linted_packages b/hack/.linted_packages index f0f6d0554c..be236de465 100644 --- a/hack/.linted_packages +++ b/hack/.linted_packages @@ -74,6 +74,8 @@ pkg/apis/imagepolicy/install pkg/api/v1 pkg/auth/authenticator pkg/auth/authorizer/union +pkg/client/metrics +pkg/client/metrics/prometheus pkg/client/testing/core pkg/client/unversioned pkg/client/unversioned/adapters/internalclientset diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index 53029b53e9..a01306c65d 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -14,54 +14,48 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package metrics provides utilities for registering client metrics to Prometheus. +// Package metrics provides abstractions for registering which metrics +// to record. package metrics import ( + "net/url" "sync" "time" - - "github.com/prometheus/client_golang/prometheus" -) - -const restClientSubsystem = "rest_client" - -var ( - // RequestLatency is a Prometheus Summary metric type partitioned by - // "verb" and "url" labels. It is used for the rest client latency metrics. - RequestLatency = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Subsystem: restClientSubsystem, - Name: "request_latency_microseconds", - Help: "Request latency in microseconds. Broken down by verb and URL", - MaxAge: time.Hour, - }, - []string{"verb", "url"}, - ) - - RequestResult = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Subsystem: restClientSubsystem, - Name: "request_status_codes", - Help: "Number of http requests, partitioned by metadata", - }, - []string{"code", "method", "host"}, - ) ) var registerMetrics sync.Once -// Register registers all metrics to Prometheus with -// respect to the RequestLatency. -func Register() { - // Register the metrics. +// LatencyMetric observes client latency partitioned by verb and url. +type LatencyMetric interface { + Observe(verb string, u url.URL, latency time.Duration) +} + +// ResultMetric counts response codes partitioned by method and host. +type ResultMetric interface { + Increment(code string, method string, host string) +} + +var ( + // RequestLatency is the latency metric that rest clients will update. + RequestLatency LatencyMetric = noopLatency{} + // RequestResult is the result metric that rest clients will update. + RequestResult ResultMetric = noopResult{} +) + +// Register registers metrics for the rest client to use. This can +// only be called once. +func Register(lm LatencyMetric, rm ResultMetric) { registerMetrics.Do(func() { - prometheus.MustRegister(RequestLatency) - prometheus.MustRegister(RequestResult) + RequestLatency = lm + RequestResult = rm }) } -// Calculates the time since the specified start in microseconds. -func SinceInMicroseconds(start time.Time) float64 { - return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds()) -} +type noopLatency struct{} + +func (noopLatency) Observe(string, url.URL, time.Duration) {} + +type noopResult struct{} + +func (noopResult) Increment(string, string, string) {} diff --git a/pkg/client/metrics/prometheus/prometheus.go b/pkg/client/metrics/prometheus/prometheus.go new file mode 100644 index 0000000000..812b935ef5 --- /dev/null +++ b/pkg/client/metrics/prometheus/prometheus.go @@ -0,0 +1,76 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package prometheus creates and registers prometheus metrics with +// rest clients. To use this package, you just have to import it. +package prometheus + +import ( + "net/url" + "time" + + "k8s.io/kubernetes/pkg/client/metrics" + + "github.com/prometheus/client_golang/prometheus" +) + +const restClientSubsystem = "rest_client" + +var ( + // requestLatency is a Prometheus Summary metric type partitioned by + // "verb" and "url" labels. It is used for the rest client latency metrics. + requestLatency = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: restClientSubsystem, + Name: "request_latency_microseconds", + Help: "Request latency in microseconds. Broken down by verb and URL", + MaxAge: time.Hour, + }, + []string{"verb", "url"}, + ) + + requestResult = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: restClientSubsystem, + Name: "request_status_codes", + Help: "Number of http requests, partitioned by metadata", + }, + []string{"code", "method", "host"}, + ) +) + +func init() { + prometheus.MustRegister(requestLatency) + prometheus.MustRegister(requestResult) + metrics.Register(&latencyAdapter{requestLatency}, &resultAdapter{requestResult}) +} + +type latencyAdapter struct { + m *prometheus.SummaryVec +} + +func (l *latencyAdapter) Observe(verb string, u url.URL, latency time.Duration) { + microseconds := float64(latency) / float64(time.Microsecond) + l.m.WithLabelValues(verb, u.String()).Observe(microseconds) +} + +type resultAdapter struct { + m *prometheus.CounterVec +} + +func (r *resultAdapter) Increment(code, method, host string) { + r.m.WithLabelValues(code, method, host).Inc() +} diff --git a/pkg/client/restclient/request.go b/pkg/client/restclient/request.go index dd1bbb35a5..be10b1a7a2 100644 --- a/pkg/client/restclient/request.go +++ b/pkg/client/restclient/request.go @@ -58,10 +58,6 @@ var ( longThrottleLatency = 50 * time.Millisecond ) -func init() { - metrics.Register() -} - // HTTPClient is an interface for testing a request object. type HTTPClient interface { Do(req *http.Request) (*http.Response, error) @@ -609,7 +605,7 @@ func (r *Request) URL() *url.URL { // underyling object. This means some useful request info (like the types of field // selectors in use) will be lost. // TODO: preserve field selector keys -func (r Request) finalURLTemplate() string { +func (r Request) finalURLTemplate() url.URL { if len(r.resourceName) != 0 { r.resourceName = "{name}" } @@ -622,7 +618,8 @@ func (r Request) finalURLTemplate() string { newParams[k] = v } r.params = newParams - return r.URL().String() + url := r.URL() + return *url } func (r *Request) tryThrottle() { @@ -697,10 +694,10 @@ func updateURLMetrics(req *Request, resp *http.Response, err error) { // If we have an error (i.e. apiserver down) we report that as a metric label. if err != nil { - metrics.RequestResult.WithLabelValues(err.Error(), req.verb, url).Inc() + metrics.RequestResult.Increment(err.Error(), req.verb, url) } else { //Metrics for failure codes - metrics.RequestResult.WithLabelValues(strconv.Itoa(resp.StatusCode), req.verb, url).Inc() + metrics.RequestResult.Increment(strconv.Itoa(resp.StatusCode), req.verb, url) } } @@ -775,7 +772,7 @@ func (r *Request) request(fn func(*http.Request, *http.Response)) error { //Metrics for total request latency start := time.Now() defer func() { - metrics.RequestLatency.WithLabelValues(r.verb, r.finalURLTemplate()).Observe(metrics.SinceInMicroseconds(start)) + metrics.RequestLatency.Observe(r.verb, r.finalURLTemplate(), time.Since(start)) }() if r.err != nil { diff --git a/pkg/client/restclient/request_test.go b/pkg/client/restclient/request_test.go index 29cd90884c..e7051308db 100755 --- a/pkg/client/restclient/request_test.go +++ b/pkg/client/restclient/request_test.go @@ -331,7 +331,8 @@ func TestURLTemplate(t *testing.T) { if full.String() != "http://localhost/pre1/namespaces/ns/r1/nm?p0=v0" { t.Errorf("unexpected initial URL: %s", full) } - actual := r.finalURLTemplate() + actualURL := r.finalURLTemplate() + actual := actualURL.String() expected := "http://localhost/pre1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D?p0=%7Bvalue%7D" if actual != expected { t.Errorf("unexpected URL template: %s %s", actual, expected)