mirror of https://github.com/k3s-io/k3s
161 lines
11 KiB
Go
161 lines
11 KiB
Go
/*
|
|
Copyright 2015 The Kubernetes Authors All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"k8s.io/kubernetes/pkg/util/sets"
|
|
|
|
"github.com/prometheus/common/model"
|
|
)
|
|
|
|
var NecessaryKubeletMetrics = map[string][]string{
|
|
"cadvisor_version_info": {"cadvisorRevision", "cadvisorVersion", "dockerVersion", "kernelVersion", "osVersion"},
|
|
"container_cpu_system_seconds_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_cpu_usage_seconds_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name", "cpu"},
|
|
"container_cpu_user_seconds_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_io_current": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_io_time_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_io_time_weighted_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_limit_bytes": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_read_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_reads_merged_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_reads_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_sector_reads_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_sector_writes_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_usage_bytes": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_write_seconds_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_writes_merged_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_fs_writes_total": {"device", "id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_last_seen": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_memory_cache": {},
|
|
"container_memory_rss": {},
|
|
"container_memory_failcnt": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_memory_failures_total": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name", "scope", "type"},
|
|
"container_memory_usage_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_memory_working_set_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_bytes_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_errors_total": {"id", "image", "interface", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_packets_dropped_total": {"id", "image", "interface", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_receive_packets_total": {"id", "image", "interface", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_bytes_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_errors_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_packets_dropped_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_network_transmit_packets_total": {"id", "interface", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_scrape_error": {},
|
|
"container_spec_cpu_shares": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_spec_memory_limit_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_spec_memory_swap_limit_bytes": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_start_time_seconds": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name"},
|
|
"container_tasks_state": {"id", "image", "kubernetes_container_name", "kubernetes_namespace", "kubernetes_pod_name", "name", "state"},
|
|
"kubelet_container_manager_latency_microseconds": {"operation_type", "quantile"},
|
|
"kubelet_container_manager_latency_microseconds_count": {"operation_type"},
|
|
"kubelet_container_manager_latency_microseconds_sum": {"operation_type"},
|
|
"kubelet_containers_per_pod_count": {"quantile"},
|
|
"kubelet_containers_per_pod_count_count": {},
|
|
"kubelet_containers_per_pod_count_sum": {},
|
|
"kubelet_docker_errors": {"operation_type"},
|
|
"kubelet_docker_operations_latency_microseconds": {"operation_type", "quantile"},
|
|
"kubelet_docker_operations_latency_microseconds_count": {"operation_type"},
|
|
"kubelet_docker_operations_latency_microseconds_sum": {"operation_type"},
|
|
"kubelet_generate_pod_status_latency_microseconds": {"quantile"},
|
|
"kubelet_generate_pod_status_latency_microseconds_count": {},
|
|
"kubelet_generate_pod_status_latency_microseconds_sum": {},
|
|
"kubelet_pleg_relist_latency_microseconds": {"quantile"},
|
|
"kubelet_pleg_relist_latency_microseconds_sum": {},
|
|
"kubelet_pleg_relist_latency_microseconds_count": {},
|
|
"kubelet_pleg_relist_interval_microseconds": {"quantile"},
|
|
"kubelet_pleg_relist_interval_microseconds_sum": {},
|
|
"kubelet_pleg_relist_interval_microseconds_count": {},
|
|
"kubelet_pod_start_latency_microseconds": {"quantile"},
|
|
"kubelet_pod_start_latency_microseconds_count": {},
|
|
"kubelet_pod_start_latency_microseconds_sum": {},
|
|
"kubelet_pod_worker_latency_microseconds": {"operation_type", "quantile"},
|
|
"kubelet_pod_worker_latency_microseconds_count": {"operation_type"},
|
|
"kubelet_pod_worker_latency_microseconds_sum": {"operation_type"},
|
|
"kubelet_pod_worker_start_latency_microseconds": {"quantile"},
|
|
"kubelet_pod_worker_start_latency_microseconds_count": {},
|
|
"kubelet_pod_worker_start_latency_microseconds_sum": {},
|
|
"kubelet_running_container_count": {},
|
|
"kubelet_running_pod_count": {},
|
|
"kubelet_sync_pods_latency_microseconds": {"quantile"},
|
|
"kubelet_sync_pods_latency_microseconds_count": {},
|
|
"kubelet_sync_pods_latency_microseconds_sum": {},
|
|
"machine_cpu_cores": {},
|
|
"machine_memory_bytes": {},
|
|
"rest_client_request_latency_microseconds": {"quantile", "url", "verb"},
|
|
"rest_client_request_latency_microseconds_count": {"url", "verb"},
|
|
"rest_client_request_latency_microseconds_sum": {"url", "verb"},
|
|
"rest_client_request_status_codes": {"code", "host", "method"},
|
|
}
|
|
|
|
var KubeletMetricsLabelsToSkip = sets.NewString(
|
|
"kubernetes_namespace",
|
|
"image",
|
|
"name",
|
|
)
|
|
|
|
type KubeletMetrics Metrics
|
|
|
|
func (m *KubeletMetrics) Equal(o KubeletMetrics) bool {
|
|
return (*Metrics)(m).Equal(Metrics(o))
|
|
}
|
|
|
|
func NewKubeletMetrics() KubeletMetrics {
|
|
result := NewMetrics()
|
|
for metric := range NecessaryKubeletMetrics {
|
|
result[metric] = make(model.Samples, 0)
|
|
}
|
|
return KubeletMetrics(result)
|
|
}
|
|
|
|
func parseKubeletMetrics(data string) (KubeletMetrics, error) {
|
|
result := NewKubeletMetrics()
|
|
if err := parseMetrics(data, NecessaryKubeletMetrics, (*Metrics)(&result), nil); err != nil {
|
|
return KubeletMetrics{}, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (g *MetricsGrabber) getMetricsFromNode(nodeName string, kubeletPort int) (string, error) {
|
|
// There's a problem with timing out during proxy. Wrapping this in a goroutine to prevent deadlock.
|
|
// Hanging goroutine will be leaked.
|
|
finished := make(chan struct{})
|
|
var err error
|
|
var rawOutput []byte
|
|
go func() {
|
|
rawOutput, err = g.client.Get().
|
|
Prefix("proxy").
|
|
Resource("nodes").
|
|
Name(fmt.Sprintf("%v:%v", nodeName, kubeletPort)).
|
|
Suffix("metrics").
|
|
Do().Raw()
|
|
finished <- struct{}{}
|
|
}()
|
|
select {
|
|
case <-time.After(ProxyTimeout):
|
|
return "", fmt.Errorf("Timed out when waiting for proxy to gather metrics from %v", nodeName)
|
|
case <-finished:
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(rawOutput), nil
|
|
}
|
|
}
|