From 75b91ff9727a2f930fc93b45dbc74fea4e9beca4 Mon Sep 17 00:00:00 2001 From: Marcin Wielgus Date: Mon, 18 Jul 2016 19:56:22 +0200 Subject: [PATCH] Extra debug information in HPA events --- .../podautoscaler/metrics/metrics_client.go | 29 +++++++++++++++++-- .../metrics/metrics_client_test.go | 6 ++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/pkg/controller/podautoscaler/metrics/metrics_client.go b/pkg/controller/podautoscaler/metrics/metrics_client.go index ebc683fde6..9319ab3086 100644 --- a/pkg/controller/podautoscaler/metrics/metrics_client.go +++ b/pkg/controller/podautoscaler/metrics/metrics_client.go @@ -27,6 +27,7 @@ import ( "k8s.io/kubernetes/pkg/api/v1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/labels" + "k8s.io/kubernetes/pkg/util/sets" heapster "k8s.io/heapster/metrics/api/v1/types" metrics_api "k8s.io/heapster/metrics/apis/metrics/v1alpha1" @@ -172,7 +173,21 @@ func (h *HeapsterMetricsClient) getCpuUtilizationForPods(namespace string, selec } if len(metrics) != len(podNames) { - return 0, time.Time{}, fmt.Errorf("metrics obtained for %d/%d of pods", len(metrics), len(podNames)) + present := sets.NewString() + for _, m := range metrics { + present.Insert(m.Name) + } + missing := make([]string, 0) + for expected := range podNames { + if !present.Has(expected) { + missing = append(missing, expected) + } + } + hint := "" + if len(missing) > 0 { + hint = fmt.Sprintf(" (sample missing pod: %s/%s)", namespace, missing[0]) + } + return 0, time.Time{}, fmt.Errorf("metrics obtained for %d/%d of pods%s", len(metrics), len(podNames), hint) } sum := int64(0) @@ -250,7 +265,17 @@ func (h *HeapsterMetricsClient) getCustomMetricForPods(metricSpec metricDefiniti sum, count, timestamp := metricSpec.aggregator(metrics) if count != len(podNames) { - return nil, time.Time{}, fmt.Errorf("metrics obtained for %d/%d of pods", count, len(podNames)) + missing := make([]string, 0) + for i, expected := range podNames { + if len(metrics.Items) > i && len(metrics.Items[i].Metrics) == 0 { + missing = append(missing, expected) + } + } + hint := "" + if len(missing) > 0 { + hint = fmt.Sprintf(" (sample missing pod: %s/%s)", namespace, missing[0]) + } + return nil, time.Time{}, fmt.Errorf("metrics obtained for %d/%d of pods%s", count, len(podNames), hint) } return &sum, timestamp, nil diff --git a/pkg/controller/podautoscaler/metrics/metrics_client_test.go b/pkg/controller/podautoscaler/metrics/metrics_client_test.go index a37056cbc5..73bce01cf3 100644 --- a/pkg/controller/podautoscaler/metrics/metrics_client_test.go +++ b/pkg/controller/podautoscaler/metrics/metrics_client_test.go @@ -184,10 +184,12 @@ func buildPod(namespace, podName string, podLabels map[string]string, phase api. } func (tc *testCase) verifyResults(t *testing.T, val *float64, timestamp time.Time, err error) { - assert.Equal(t, tc.desiredError, err) if tc.desiredError != nil { + assert.Error(t, err) + assert.Contains(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.desiredError)) return } + assert.NoError(t, err) assert.NotNil(t, val) assert.True(t, tc.desiredValue-0.001 < *val) assert.True(t, tc.desiredValue+0.001 > *val) @@ -426,7 +428,7 @@ func TestCPUEmptyMetricsForOnePod(t *testing.T) { tc := testCase{ replicas: 3, targetResource: "cpu-usage", - desiredError: fmt.Errorf("metrics obtained for 2/3 of pods"), + desiredError: fmt.Errorf("metrics obtained for 2/3 of pods (sample missing pod: test-namespace/test-pod-2)"), reportedPodMetrics: [][]int64{{100}, {300, 400}}, useMetricsApi: true, }