github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #72323 from danielqsj/dockershim 

Change docker metrics to conform metrics guidelines
pull/564/head
Kubernetes Prow Robot 2019-02-06 09:30:54 -08:00 committed by GitHub
parent 939f8ffa87 7aef2efe5b
commit ae45068688
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 97 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/kubelet/dockershim/libdocker/instrumented_client.go
View File

@ -42,7 +42,9 @@ func NewInstrumentedInterface(dockerClient Interface) Interface {
// recordOperation records the duration of the operation.
func recordOperation(operation string, start time.Time) {
metrics.DockerOperations.WithLabelValues(operation).Inc()
metrics.DockerOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInMicroseconds(start))
metrics.DeprecatedDockerOperations.WithLabelValues(operation).Inc()
metrics.DockerOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInSeconds(start))
metrics.DeprecatedDockerOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInMicroseconds(start))
}
// recordError records error for metric if an error occurred.
@ -50,9 +52,11 @@ func recordError(operation string, err error) {
if err != nil {
if _, ok := err.(operationTimeout); ok {
metrics.DockerOperationsTimeout.WithLabelValues(operation).Inc()
metrics.DeprecatedDockerOperationsTimeout.WithLabelValues(operation).Inc()
}
// Docker operation timeout error is also a docker error, so we don't add else here.
metrics.DockerOperationsErrors.WithLabelValues(operation).Inc()
metrics.DeprecatedDockerOperationsErrors.WithLabelValues(operation).Inc()
}
}

72
pkg/kubelet/dockershim/metrics/metrics.go
View File

@ -25,13 +25,22 @@ import (
const (
// DockerOperationsKey is the key for docker operation metrics.
DockerOperationsKey = "docker_operations"
DockerOperationsKey = "docker_operations_total"
// DockerOperationsLatencyKey is the key for the operation latency metrics.
DockerOperationsLatencyKey = "docker_operations_latency_microseconds"
DockerOperationsLatencyKey = "docker_operations_latency_seconds"
// DockerOperationsErrorsKey is the key for the operation error metrics.
DockerOperationsErrorsKey = "docker_operations_errors"
DockerOperationsErrorsKey = "docker_operations_errors_total"
// DockerOperationsTimeoutKey is the key for the operation timeout metrics.
DockerOperationsTimeoutKey = "docker_operations_timeout"
DockerOperationsTimeoutKey = "docker_operations_timeout_total"
// DeprecatedDockerOperationsKey is the deprecated key for docker operation metrics.
DeprecatedDockerOperationsKey = "docker_operations"
// DeprecatedDockerOperationsLatencyKey is the deprecated key for the operation latency metrics.
DeprecatedDockerOperationsLatencyKey = "docker_operations_latency_microseconds"
// DeprecatedDockerOperationsErrorsKey is the deprecated key for the operation error metrics.
DeprecatedDockerOperationsErrorsKey = "docker_operations_errors"
// DeprecatedDockerOperationsTimeoutKey is the deprecated key for the operation timeout metrics.
DeprecatedDockerOperationsTimeoutKey = "docker_operations_timeout"
// Keep the "kubelet" subsystem for backward compatibility.
kubeletSubsystem = "kubelet"
@ -40,11 +49,12 @@ const (
var (
// DockerOperationsLatency collects operation latency numbers by operation
// type.
DockerOperationsLatency = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
DockerOperationsLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Subsystem: kubeletSubsystem,
Name: DockerOperationsLatencyKey,
Help: "Latency in microseconds of Docker operations. Broken down by operation type.",
Help: "Latency in seconds of Docker operations. Broken down by operation type.",
Buckets: prometheus.DefBuckets,
},
[]string{"operation_type"},
)
@ -76,6 +86,45 @@ var (
},
[]string{"operation_type"},
)
// DeprecatedDockerOperationsLatency collects operation latency numbers by operation
// type.
DeprecatedDockerOperationsLatency = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Subsystem: kubeletSubsystem,
Name: DeprecatedDockerOperationsLatencyKey,
Help: "(Deprecated) Latency in microseconds of Docker operations. Broken down by operation type.",
},
[]string{"operation_type"},
)
// DeprecatedDockerOperations collects operation counts by operation type.
DeprecatedDockerOperations = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: kubeletSubsystem,
Name: DeprecatedDockerOperationsKey,
Help: "(Deprecated) Cumulative number of Docker operations by operation type.",
},
[]string{"operation_type"},
)
// DeprecatedDockerOperationsErrors collects operation errors by operation
// type.
DeprecatedDockerOperationsErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: kubeletSubsystem,
Name: DeprecatedDockerOperationsErrorsKey,
Help: "(Deprecated) Cumulative number of Docker operation errors by operation type.",
},
[]string{"operation_type"},
)
// DeprecatedDockerOperationsTimeout collects operation timeouts by operation type.
DeprecatedDockerOperationsTimeout = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: kubeletSubsystem,
Name: DeprecatedDockerOperationsTimeoutKey,
Help: "(Deprecated) Cumulative number of Docker operation timeout by operation type.",
},
[]string{"operation_type"},
)
)
var registerMetrics sync.Once
@ -87,6 +136,10 @@ func Register() {
prometheus.MustRegister(DockerOperations)
prometheus.MustRegister(DockerOperationsErrors)
prometheus.MustRegister(DockerOperationsTimeout)
prometheus.MustRegister(DeprecatedDockerOperationsLatency)
prometheus.MustRegister(DeprecatedDockerOperations)
prometheus.MustRegister(DeprecatedDockerOperationsErrors)
prometheus.MustRegister(DeprecatedDockerOperationsTimeout)
})
}
@ -94,3 +147,8 @@ func Register() {
func SinceInMicroseconds(start time.Time) float64 {
return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
}
// SinceInSeconds gets the time since the specified start in seconds.
func SinceInSeconds(start time.Time) float64 {
return time.Since(start).Seconds()
}

28
pkg/kubelet/dockershim/network/metrics/metrics.go
View File

@ -27,7 +27,9 @@ const (
// NetworkPluginOperationsKey is the key for operation count metrics.
NetworkPluginOperationsKey = "network_plugin_operations"
// NetworkPluginOperationsLatencyKey is the key for the operation latency metrics.
NetworkPluginOperationsLatencyKey = "network_plugin_operations_latency_microseconds"
NetworkPluginOperationsLatencyKey = "network_plugin_operations_latency_seconds"
// DeprecatedNetworkPluginOperationsLatencyKey is the deprecated key for the operation latency metrics.
DeprecatedNetworkPluginOperationsLatencyKey = "network_plugin_operations_latency_microseconds"
// Keep the "kubelet" subsystem for backward compatibility.
kubeletSubsystem = "kubelet"
@ -36,11 +38,23 @@ const (
var (
// NetworkPluginOperationsLatency collects operation latency numbers by operation
// type.
NetworkPluginOperationsLatency = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
NetworkPluginOperationsLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Subsystem: kubeletSubsystem,
Name: NetworkPluginOperationsLatencyKey,
Help: "Latency in microseconds of network plugin operations. Broken down by operation type.",
Help: "Latency in seconds of network plugin operations. Broken down by operation type.",
Buckets: prometheus.DefBuckets,
},
[]string{"operation_type"},
)
// DeprecatedNetworkPluginOperationsLatency collects operation latency numbers by operation
// type.
DeprecatedNetworkPluginOperationsLatency = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Subsystem: kubeletSubsystem,
Name: DeprecatedNetworkPluginOperationsLatencyKey,
Help: "(Deprecated) Latency in microseconds of network plugin operations. Broken down by operation type.",
},
[]string{"operation_type"},
)
@ -52,6 +66,7 @@ var registerMetrics sync.Once
func Register() {
registerMetrics.Do(func() {
prometheus.MustRegister(NetworkPluginOperationsLatency)
prometheus.MustRegister(DeprecatedNetworkPluginOperationsLatency)
})
}
@ -59,3 +74,8 @@ func Register() {
func SinceInMicroseconds(start time.Time) float64 {
return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
}
// SinceInSeconds gets the time since the specified start in seconds.
func SinceInSeconds(start time.Time) float64 {
return time.Since(start).Seconds()
}

3
pkg/kubelet/dockershim/network/plugins.go
View File

@ -351,7 +351,8 @@ func (pm *PluginManager) podUnlock(fullPodName string) {
// recordOperation records operation and duration
func recordOperation(operation string, start time.Time) {
metrics.NetworkPluginOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInMicroseconds(start))
metrics.NetworkPluginOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInSeconds(start))
metrics.DeprecatedNetworkPluginOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInMicroseconds(start))
}
func (pm *PluginManager) GetPodNetworkStatus(podNamespace, podName string, id kubecontainer.ContainerID) (*PodNetworkStatus, error) {

2
test/e2e/framework/metrics_util.go
View File

@ -171,7 +171,7 @@ var InterestingControllerManagerMetrics = []string{
var InterestingKubeletMetrics = []string{
"kubelet_container_manager_latency_microseconds",
"kubelet_docker_errors",
"kubelet_docker_operations_latency_microseconds",
"kubelet_docker_operations_latency_seconds",
"kubelet_generate_pod_status_latency_microseconds",
"kubelet_pod_start_latency_microseconds",
"kubelet_pod_worker_latency_microseconds",