Merge pull request #56858 from bskiba/cm-tests

Automatic merge from submit-queue (batch tested with PRs 56858, 57040, 56979, 57051, 57052). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add e2e test for custom metrics with Prometheus and Stackdriver

**What this PR does / why we need it**:
Add e2e test for custom metrics in a setup with a container exposing metrics in Prometheus format and prometheus-to-sd sidecar container scraping the metrics and writing them to Stackdriver.
Also some test refactoring for adding new tests more easily (plan to add scale down tests and a test with scaling on two metrics).

**Release note**:
```
NONE
```

test/e2e/autoscaling/BUILD

@@ -31,6 +31,7 @@ go_library(
         "//vendor/google.golang.org/api/monitoring/v3:go_default_library",
         "//vendor/k8s.io/api/autoscaling/v2beta1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/api/policy/v1beta1:go_default_library",
         "//vendor/k8s.io/api/scheduling/v1alpha1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",

test/e2e/autoscaling/custom_metrics_autoscaling.go

@@ -20,18 +20,19 @@ import (
 	"context"
 	"time"
 
-	"golang.org/x/oauth2/google"
-	clientset "k8s.io/client-go/kubernetes"
-
-	. "github.com/onsi/ginkgo"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/kubernetes/test/e2e/framework"
-
 	gcm "google.golang.org/api/monitoring/v3"
 	as "k8s.io/api/autoscaling/v2beta1"
+	corev1 "k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/instrumentation/monitoring"
+
+	. "github.com/onsi/ginkgo"
+	"golang.org/x/oauth2/google"
 )
 
 const (
@@ -46,15 +47,32 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me
 	})
 
 	f := framework.NewDefaultFramework("horizontal-pod-autoscaling")
-	var kubeClient clientset.Interface
 
-	It("should autoscale with Custom Metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
+	It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
-		kubeClient = f.ClientSet
+		initialReplicas := 2
-		testHPA(f, kubeClient)
+		scaledReplicas := 1
+		deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
+		customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
+	})
+
+	It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
+		initialReplicas := 2
+		scaledReplicas := 1
+		deployment := monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
+		pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.ObjectMeta.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100)
+		customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name), deployment, pod, initialReplicas, scaledReplicas)
+	})
+
+	It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() {
+		initialReplicas := 2
+		scaledReplicas := 1
+		deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
+		customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
 	})
 })
 
-func testHPA(f *framework.Framework, kubeClient clientset.Interface) {
+func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hpa *as.HorizontalPodAutoscaler,
+	deployment *extensions.Deployment, pod *corev1.Pod, initialReplicas, scaledReplicas int) {
 	projectId := framework.TestContext.CloudConfig.ProjectID
 
 	ctx := context.Background()
@@ -92,51 +110,55 @@ func testHPA(f *framework.Framework, kubeClient clientset.Interface) {
 	defer monitoring.CleanupAdapter()
 
 	// Run application that exports the metric
-	err = createDeploymentsToScale(f, kubeClient)
+	err = createDeploymentToScale(f, kubeClient, deployment, pod)
 	if err != nil {
 		framework.Failf("Failed to create stackdriver-exporter pod: %v", err)
 	}
-	defer cleanupDeploymentsToScale(f, kubeClient)
+	defer cleanupDeploymentsToScale(f, kubeClient, deployment, pod)
 
-	// Autoscale the deployments
+	// Wait for the deployment to run
-	err = createPodsHPA(f, kubeClient)
+	waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, initialReplicas)
+
+	// Autoscale the deployment
+	_, err = kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(hpa)
 	if err != nil {
-		framework.Failf("Failed to create 'Pods' HPA: %v", err)
+		framework.Failf("Failed to create HPA: %v", err)
-	}
-	err = createObjectHPA(f, kubeClient)
-	if err != nil {
-		framework.Failf("Failed to create 'Objects' HPA: %v", err)
 	}
 
-	waitForReplicas(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1)
+	waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, scaledReplicas)
-	waitForReplicas(dummyDeploymentName, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1)
 }
 
-func createDeploymentsToScale(f *framework.Framework, cs clientset.Interface) error {
+func createDeploymentToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) error {
-	_, err := cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.Name, 2, 100))
+	if deployment != nil {
+		_, err := cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(deployment)
 		if err != nil {
 			return err
 		}
-	_, err = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100))
+	}
+	if pod != nil {
+		_, err := cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(pod)
 		if err != nil {
 			return err
 		}
-	_, err = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.Name, 2, 100))
+	}
-	return err
+	return nil
 }
 
-func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface) {
+func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) {
-	_ = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterDeployment, &metav1.DeleteOptions{})
+	if deployment != nil {
-	_ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterPod, &metav1.DeleteOptions{})
+		_ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(deployment.ObjectMeta.Name, &metav1.DeleteOptions{})
-	_ = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Delete(dummyDeploymentName, &metav1.DeleteOptions{})
+	}
+	if pod != nil {
+		_ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(pod.ObjectMeta.Name, &metav1.DeleteOptions{})
+	}
 }
 
-func createPodsHPA(f *framework.Framework, cs clientset.Interface) error {
+func podsHPA(namespace string) *as.HorizontalPodAutoscaler {
 	var minReplicas int32 = 1
-	_, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{
+	return &as.HorizontalPodAutoscaler{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "custom-metrics-pods-hpa",
-			Namespace: f.Namespace.ObjectMeta.Name,
+			Namespace: namespace,
 		},
 		Spec: as.HorizontalPodAutoscalerSpec{
 			Metrics: []as.MetricSpec{
@@ -156,16 +178,15 @@ func createPodsHPA(f *framework.Framework, cs clientset.Interface) error {
 				Name:       stackdriverExporterDeployment,
 			},
 		},
-	})
+	}
-	return err
 }
 
-func createObjectHPA(f *framework.Framework, cs clientset.Interface) error {
+func objectHPA(namespace string) *as.HorizontalPodAutoscaler {
 	var minReplicas int32 = 1
-	_, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{
+	return &as.HorizontalPodAutoscaler{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "custom-metrics-objects-hpa",
-			Namespace: f.Namespace.ObjectMeta.Name,
+			Namespace: namespace,
 		},
 		Spec: as.HorizontalPodAutoscalerSpec{
 			Metrics: []as.MetricSpec{
@@ -189,8 +210,7 @@ func createObjectHPA(f *framework.Framework, cs clientset.Interface) error {
 				Name:       dummyDeploymentName,
 			},
 		},
-	})
+	}
-	return err
 }
 
 func waitForReplicas(deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) {

test/e2e/instrumentation/monitoring/custom_metrics_deployments.go

@@ -18,6 +18,7 @@ package monitoring
 
 import (
 	"fmt"
+
 	gcm "google.golang.org/api/monitoring/v3"
 	corev1 "k8s.io/api/core/v1"
 	extensions "k8s.io/api/extensions/v1beta1"
@@ -27,8 +28,8 @@ import (
 )
 
 var (
-	CustomMetricName  = "foo-metric"
+	CustomMetricName  = "foo"
-	UnusedMetricName  = "unused-metric"
+	UnusedMetricName  = "unused"
 	CustomMetricValue = int64(448)
 	UnusedMetricValue = int64(446)
 	// HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for
@@ -116,6 +117,72 @@ func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.Pod
 	}
 }
 
+// PrometheusExporterDeployment is a Deployment of simple application with two containers
+// one exposing a metric in prometheus fromat and second a prometheus-to-sd container
+// that scrapes the metric and pushes it to stackdriver.
+func PrometheusExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
+	return &extensions.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: extensions.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"name": name},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"name": name,
+					},
+				},
+				Spec: prometheusExporterPodSpec(CustomMetricName, metricValue, 8080),
+			},
+			Replicas: &replicas,
+		},
+	}
+}
+
+func prometheusExporterPodSpec(metricName string, metricValue int64, port int32) corev1.PodSpec {
+	return corev1.PodSpec{
+		Containers: []corev1.Container{
+			{
+				Name:            "prometheus-exporter",
+				Image:           "gcr.io/google-containers/prometheus-dummy-exporter:v0.1.0",
+				ImagePullPolicy: corev1.PullPolicy("Always"),
+				Command: []string{"/prometheus_dummy_exporter", "--metric-name=" + metricName,
+					fmt.Sprintf("--metric-value=%v", metricValue), fmt.Sprintf("=--port=%d", port)},
+				Ports: []corev1.ContainerPort{{ContainerPort: port}},
+			},
+			{
+				Name:            "prometheus-to-sd",
+				Image:           "gcr.io/google-containers/prometheus-to-sd:v0.2.3",
+				ImagePullPolicy: corev1.PullPolicy("Always"),
+				Command: []string{"/monitor", fmt.Sprintf("--source=:http://localhost:%d", port),
+					"--stackdriver-prefix=custom.googleapis.com", "--pod-id=$(POD_ID)", "--namespace-id=$(POD_NAMESPACE)"},
+				Env: []corev1.EnvVar{
+					{
+						Name: "POD_ID",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								FieldPath: "metadata.uid",
+							},
+						},
+					},
+					{
+						Name: "POD_NAMESPACE",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								FieldPath: "metadata.namespace",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 // CreateAdapter creates Custom Metrics - Stackdriver adapter.
 func CreateAdapter() error {
 	stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")