github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #63896 from mtaufen/refactor-test-metrics 

Automatic merge from submit-queue (batch tested with PRs 64013, 63896, 64139, 57527, 62102). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Refactor test utils that deal with Kubelet metrics for clarity

I found these functions hard to understand, because the names did not
accurately reflect their behavior. For example, GetKubeletMetrics
assumed that all of the metrics passed in were measuring latency.
The caller of GetKubeletMetrics was implicitly making this assumption,
but it was not obvious at the call site.

```release-note
NONE
```
pull/8/head
Kubernetes Submit Queue 2018-05-23 19:44:15 -07:00 committed by GitHub
parent 195914d02a 83509a092f
commit 10377f6593
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
test/e2e/framework/kubelet_stats.go
View File

@ -97,10 +97,11 @@ func getKubeletMetrics(c clientset.Interface, nodeName string) (metrics.KubeletM
return kubeletMetrics, nil
}
// GetKubeletLatencyMetrics gets all latency related kubelet metrics. Note that the KubeletMetrcis
// passed in should not contain subsystem prefix.
func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
latencyMethods := sets.NewString(
// GetDefaultKubeletLatencyMetrics calls GetKubeletLatencyMetrics with a set of default metricNames
// identifying common latency metrics.
// Note that the KubeletMetrics passed in should not contain subsystem prefix.
func GetDefaultKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
latencyMetricNames := sets.NewString(
kubeletmetrics.PodWorkerLatencyKey,
kubeletmetrics.PodWorkerStartLatencyKey,
kubeletmetrics.PodStartLatencyKey,
@ -109,13 +110,15 @@ func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
kubeletmetrics.PodWorkerStartLatencyKey,
kubeletmetrics.PLEGRelistLatencyKey,
)
return GetKubeletMetrics(ms, latencyMethods)
return GetKubeletLatencyMetrics(ms, latencyMetricNames)
}
func GetKubeletMetrics(ms metrics.KubeletMetrics, methods sets.String) KubeletLatencyMetrics {
// GetKubeletLatencyMetrics filters ms to include only those contained in the metricNames set,
// then constructs a KubeletLatencyMetrics list based on the samples associated with those metrics.
func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics, filterMetricNames sets.String) KubeletLatencyMetrics {
var latencyMetrics KubeletLatencyMetrics
for method, samples := range ms {
if !methods.Has(method) {
for name, samples := range ms {
if !filterMetricNames.Has(name) {
continue
}
for _, sample := range samples {
@ -131,7 +134,7 @@ func GetKubeletMetrics(ms metrics.KubeletMetrics, methods sets.String) KubeletLa
latencyMetrics = append(latencyMetrics, KubeletLatencyMetric{
Operation: operation,
Method: method,
Method: name,
Quantile: quantile,
Latency: time.Duration(int64(latency)) * time.Microsecond,
})
@ -265,7 +268,7 @@ func HighLatencyKubeletOperations(c clientset.Interface, threshold time.Duration
if err != nil {
return KubeletLatencyMetrics{}, err
}
latencyMetrics := GetKubeletLatencyMetrics(ms)
latencyMetrics := GetDefaultKubeletLatencyMetrics(ms)
sort.Sort(latencyMetrics)
var badMetrics KubeletLatencyMetrics
logFunc("\nLatency metrics for node %v", nodeName)

6
test/e2e_node/eviction_test.go
View File

@ -412,7 +412,7 @@ func runEvictionTest(f *framework.Framework, pressureTimeout time.Duration, expe
framework.Logf("Node does NOT have %s", expectedNodeCondition)
}
}
logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
logKubeletLatencyMetrics(kubeletmetrics.EvictionStatsAgeKey)
logFunc()
return verifyEvictionOrdering(f, testSpecs)
}, pressureTimeout, evictionPollInterval).Should(BeNil())
@ -426,7 +426,7 @@ func runEvictionTest(f *framework.Framework, pressureTimeout time.Duration, expe
By(fmt.Sprintf("Waiting for NodeCondition: %s to no longer exist on the node", expectedNodeCondition))
Eventually(func() error {
logFunc()
logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
logKubeletLatencyMetrics(kubeletmetrics.EvictionStatsAgeKey)
if expectedNodeCondition != noPressure && hasNodeCondition(f, expectedNodeCondition) {
return fmt.Errorf("Conditions havent returned to normal, node still has %s", expectedNodeCondition)
}
@ -439,7 +439,7 @@ func runEvictionTest(f *framework.Framework, pressureTimeout time.Duration, expe
return fmt.Errorf("%s dissappeared and then reappeared", expectedNodeCondition)
}
logFunc()
logKubeletMetrics(kubeletmetrics.EvictionStatsAgeKey)
logKubeletLatencyMetrics(kubeletmetrics.EvictionStatsAgeKey)
return verifyEvictionOrdering(f, testSpecs)
}, postTestConditionMonitoringPeriod, evictionPollInterval).Should(BeNil())
})

10
test/e2e_node/util.go
View File

@ -319,17 +319,19 @@ func getLocalNode(f *framework.Framework) *apiv1.Node {
return &nodeList.Items[0]
}
// logs prometheus metrics from the local kubelet.
func logKubeletMetrics(metricKeys ...string) {
// logKubeletLatencyMetrics logs KubeletLatencyMetrics computed from the Prometheus
// metrics exposed on the current node and identified by the metricNames.
// The Kubelet subsystem prefix is automatically prepended to these metric names.
func logKubeletLatencyMetrics(metricNames ...string) {
metricSet := sets.NewString()
for _, key := range metricKeys {
for _, key := range metricNames {
metricSet.Insert(kubeletmetrics.KubeletSubsystem + "_" + key)
}
metric, err := metrics.GrabKubeletMetricsWithoutProxy(framework.TestContext.NodeName + ":10255")
if err != nil {
framework.Logf("Error getting kubelet metrics: %v", err)
} else {
framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletMetrics(metric, metricSet))
framework.Logf("Kubelet Metrics: %+v", framework.GetKubeletLatencyMetrics(metric, metricSet))
}
}