prometheus/discovery/kubernetes/client_metrics.go

// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
	"net/url"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/metrics"
	"k8s.io/client-go/util/workqueue"
)

const (
	cacheMetricsNamespace     = metricsNamespace + "_cache"
	workqueueMetricsNamespace = metricsNamespace + "_workqueue"
)

var (
	// Metrics for client-go's HTTP requests.
	clientGoRequestResultMetricVec = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Name:      "http_request_total",
			Help:      "Total number of HTTP requests to the Kubernetes API by status code.",
		},
		[]string{"status_code"},
	)
	clientGoRequestLatencyMetricVec = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  metricsNamespace,
			Name:       "http_request_duration_seconds",
			Help:       "Summary of latencies for HTTP requests to the Kubernetes API by endpoint.",
			Objectives: map[float64]float64{},
		},
		[]string{"endpoint"},
	)

	// Definition of metrics for client-go cache metrics provider.
	clientGoCacheListTotalMetric = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: cacheMetricsNamespace,
			Name:      "list_total",
			Help:      "Total number of list operations.",
		},
	)
	clientGoCacheListDurationMetric = prometheus.NewSummary(
		prometheus.SummaryOpts{
			Namespace:  cacheMetricsNamespace,
			Name:       "list_duration_seconds",
			Help:       "Duration of a Kubernetes API call in seconds.",
			Objectives: map[float64]float64{},
		},
	)
	clientGoCacheItemsInListCountMetric = prometheus.NewSummary(
		prometheus.SummaryOpts{
			Namespace:  cacheMetricsNamespace,
			Name:       "list_items",
			Help:       "Count of items in a list from the Kubernetes API.",
			Objectives: map[float64]float64{},
		},
	)
	clientGoCacheWatchesCountMetric = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: cacheMetricsNamespace,
			Name:      "watches_total",
			Help:      "Total number of watch operations.",
		},
	)
	clientGoCacheShortWatchesCountMetric = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: cacheMetricsNamespace,
			Name:      "short_watches_total",
			Help:      "Total number of short watch operations.",
		},
	)
	clientGoCacheWatchesDurationMetric = prometheus.NewSummary(
		prometheus.SummaryOpts{
			Namespace:  cacheMetricsNamespace,
			Name:       "watch_duration_seconds",
			Help:       "Duration of watches on the Kubernetes API.",
			Objectives: map[float64]float64{},
		},
	)
	clientGoCacheItemsInWatchesCountMetric = prometheus.NewSummary(
		prometheus.SummaryOpts{
			Namespace:  cacheMetricsNamespace,
			Name:       "watch_events",
			Help:       "Number of items in watches on the Kubernetes API.",
			Objectives: map[float64]float64{},
		},
	)
	clientGoCacheLastResourceVersionMetric = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: cacheMetricsNamespace,
			Name:      "last_resource_version",
			Help:      "Last resource version from the Kubernetes API.",
		},
	)

	// Definition of metrics for client-go workflow metrics provider
	clientGoWorkqueueDepthMetricVec = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: workqueueMetricsNamespace,
			Name:      "depth",
			Help:      "Current depth of the work queue.",
		},
		[]string{"queue_name"},
	)
	clientGoWorkqueueAddsMetricVec = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: workqueueMetricsNamespace,
			Name:      "items_total",
			Help:      "Total number of items added to the work queue.",
		},
		[]string{"queue_name"},
	)
	clientGoWorkqueueLatencyMetricVec = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  workqueueMetricsNamespace,
			Name:       "latency_seconds",
			Help:       "How long an item stays in the work queue.",
			Objectives: map[float64]float64{},
		},
		[]string{"queue_name"},
	)
	clientGoWorkqueueUnfinishedWorkSecondsMetricVec = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: workqueueMetricsNamespace,
			Name:      "unfinished_work_seconds",
			Help:      "How long an item has remained unfinished in the work queue.",
		},
		[]string{"queue_name"},
	)
	clientGoWorkqueueLongestRunningProcessorMetricVec = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: workqueueMetricsNamespace,
			Name:      "longest_running_processor_seconds",
			Help:      "Duration of the longest running processor in the work queue.",
		},
		[]string{"queue_name"},
	)
	clientGoWorkqueueWorkDurationMetricVec = prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace:  workqueueMetricsNamespace,
			Name:       "work_duration_seconds",
			Help:       "How long processing an item from the work queue takes.",
			Objectives: map[float64]float64{},
		},
		[]string{"queue_name"},
	)
)

// gaugeSetFunc is an adapter that allows the use of functions as gauge metric setters.
type gaugeSetFunc func(float64)

func (s gaugeSetFunc) Set(value float64) {
	s(value)
}

// Definition of dummy metric used as a placeholder if we don't want to observe some data.
type noopMetric struct{}

func (noopMetric) Inc()            {}
func (noopMetric) Dec()            {}
func (noopMetric) Observe(float64) {}

// Definition of client-go metrics adapters for HTTP requests observation
type clientGoRequestMetricAdapter struct{}

func (f *clientGoRequestMetricAdapter) Register(registerer prometheus.Registerer) {
	metrics.Register(f, f)
	registerer.MustRegister(clientGoRequestResultMetricVec)
	registerer.MustRegister(clientGoRequestLatencyMetricVec)
}
func (clientGoRequestMetricAdapter) Increment(code string, method string, host string) {
	clientGoRequestResultMetricVec.WithLabelValues(code).Inc()
}
func (clientGoRequestMetricAdapter) Observe(verb string, u url.URL, latency time.Duration) {
	clientGoRequestLatencyMetricVec.WithLabelValues(u.EscapedPath()).Observe(latency.Seconds())
}

// Definition of client-go cache metrics provider definition
type clientGoCacheMetricsProvider struct{}

func (f *clientGoCacheMetricsProvider) Register(registerer prometheus.Registerer) {
	cache.SetReflectorMetricsProvider(f)
	registerer.MustRegister(clientGoCacheWatchesDurationMetric)
	registerer.MustRegister(clientGoCacheWatchesCountMetric)
	registerer.MustRegister(clientGoCacheListDurationMetric)
	registerer.MustRegister(clientGoCacheListTotalMetric)
	registerer.MustRegister(clientGoCacheLastResourceVersionMetric)
	registerer.MustRegister(clientGoCacheShortWatchesCountMetric)
	registerer.MustRegister(clientGoCacheItemsInWatchesCountMetric)
	registerer.MustRegister(clientGoCacheItemsInListCountMetric)

}
func (clientGoCacheMetricsProvider) NewListsMetric(name string) cache.CounterMetric {
	return clientGoCacheListTotalMetric
}
func (clientGoCacheMetricsProvider) NewListDurationMetric(name string) cache.SummaryMetric {
	return clientGoCacheListDurationMetric
}
func (clientGoCacheMetricsProvider) NewItemsInListMetric(name string) cache.SummaryMetric {
	return clientGoCacheItemsInListCountMetric
}
func (clientGoCacheMetricsProvider) NewWatchesMetric(name string) cache.CounterMetric {
	return clientGoCacheWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewShortWatchesMetric(name string) cache.CounterMetric {
	return clientGoCacheShortWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewWatchDurationMetric(name string) cache.SummaryMetric {
	return clientGoCacheWatchesDurationMetric
}
func (clientGoCacheMetricsProvider) NewItemsInWatchMetric(name string) cache.SummaryMetric {
	return clientGoCacheItemsInWatchesCountMetric
}
func (clientGoCacheMetricsProvider) NewLastResourceVersionMetric(name string) cache.GaugeMetric {
	return clientGoCacheLastResourceVersionMetric
}

// Definition of client-go workqueue metrics provider definition
type clientGoWorkqueueMetricsProvider struct{}

func (f *clientGoWorkqueueMetricsProvider) Register(registerer prometheus.Registerer) {
	workqueue.SetProvider(f)
	registerer.MustRegister(clientGoWorkqueueDepthMetricVec)
	registerer.MustRegister(clientGoWorkqueueAddsMetricVec)
	registerer.MustRegister(clientGoWorkqueueLatencyMetricVec)
	registerer.MustRegister(clientGoWorkqueueWorkDurationMetricVec)
	registerer.MustRegister(clientGoWorkqueueUnfinishedWorkSecondsMetricVec)
	registerer.MustRegister(clientGoWorkqueueLongestRunningProcessorMetricVec)
}

func (f *clientGoWorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
	return clientGoWorkqueueDepthMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
	return clientGoWorkqueueAddsMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.SummaryMetric {
	metric := clientGoWorkqueueLatencyMetricVec.WithLabelValues(name)
	// Convert microseconds to seconds for consistency across metrics.
	return prometheus.ObserverFunc(func(v float64) {
		metric.Observe(v / 1e6)
	})
}
func (f *clientGoWorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.SummaryMetric {
	metric := clientGoWorkqueueWorkDurationMetricVec.WithLabelValues(name)
	// Convert microseconds to seconds for consistency across metrics.
	return prometheus.ObserverFunc(func(v float64) {
		metric.Observe(v / 1e6)
	})
}
func (f *clientGoWorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
	return clientGoWorkqueueUnfinishedWorkSecondsMetricVec.WithLabelValues(name)
}
func (f *clientGoWorkqueueMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(name string) workqueue.SettableGaugeMetric {
	metric := clientGoWorkqueueLongestRunningProcessorMetricVec.WithLabelValues(name)
	return gaugeSetFunc(func(v float64) {
		metric.Set(v / 1e6)
	})

}
func (clientGoWorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
	// Retries are not used so the metric is omitted.
	return noopMetric{}
}