github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Increase api latency threshold for cluster-scoped list calls

pull/6/head
Shyam Jeedigunta 2017-09-19 16:40:11 +02:00
parent 16670f1a95
commit f373645865
1 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
test/e2e/framework/metrics_util.go
View File

@ -52,9 +52,11 @@ const (
apiCallLatencyThreshold time.Duration = 1 * time.Second apiCallLatencyThreshold time.Duration = 1 * time.Second
// We use a higher threshold for list apicalls if the cluster is big (i.e having > 500 nodes) // We use a higher threshold for list apicalls if the cluster is big (i.e having > 500 nodes)
// as list response sizes are bigger in general for big clusters. // as list response sizes are bigger in general for big clusters. We also use a higher threshold
apiListCallLatencyThreshold time.Duration = 5 * time.Second // for list calls at cluster scope (this includes non-namespaced and all-namespaced calls).
bigClusterNodeCountThreshold = 500 apiListCallLatencyThreshold time.Duration = 5 * time.Second
apiClusterScopeListCallThreshold time.Duration = 10 * time.Second
bigClusterNodeCountThreshold = 500
// Cluster Autoscaler metrics names // Cluster Autoscaler metrics names
caFunctionMetric = "cluster_autoscaler_function_duration_seconds_bucket" caFunctionMetric = "cluster_autoscaler_function_duration_seconds_bucket"
@ -371,13 +373,19 @@ func HighLatencyRequests(c clientset.Interface, nodeCount int) (int, *APIRespons
for i := range metrics.APICalls { for i := range metrics.APICalls {
latency := metrics.APICalls[i].Latency.Perc99 latency := metrics.APICalls[i].Latency.Perc99
isListCall := (metrics.APICalls[i].Verb == "LIST") isListCall := (metrics.APICalls[i].Verb == "LIST")
isClusterScopedCall := (metrics.APICalls[i].Scope == "cluster")
isBad := false isBad := false
if latency > apiCallLatencyThreshold { latencyThreshold := apiCallLatencyThreshold
if !isListCall || !isBigCluster || (latency > apiListCallLatencyThreshold) { if isListCall && isBigCluster {
isBad = true latencyThreshold = apiListCallLatencyThreshold
badMetrics++ if isClusterScopedCall {
latencyThreshold = apiClusterScopeListCallThreshold
} }
} }
if latency > latencyThreshold {
isBad = true
badMetrics++
}
if top > 0 || isBad { if top > 0 || isBad {
top-- top--
prefix := "" prefix := ""