github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #56858 from bskiba/cm-tests 

Automatic merge from submit-queue (batch tested with PRs 56858, 57040, 56979, 57051, 57052). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add e2e test for custom metrics with Prometheus and Stackdriver

**What this PR does / why we need it**:
Add e2e test for custom metrics in a setup with a container exposing metrics in Prometheus format and prometheus-to-sd sidecar container scraping the metrics and writing them to Stackdriver.
Also some test refactoring for adding new tests more easily (plan to add scale down tests and a test with scaling on two metrics).

**Release note**:
```
NONE
```
pull/6/head
Kubernetes Submit Queue 2017-12-16 16:34:39 -08:00 committed by GitHub
parent 2ae99cf146 afac46bd49
commit 9c38cef9b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 136 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test/e2e/autoscaling/BUILD
View File

@ -31,6 +31,7 @@ go_library(
"//vendor/google.golang.org/api/monitoring/v3:go_default_library",
"//vendor/k8s.io/api/autoscaling/v2beta1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/api/policy/v1beta1:go_default_library",
"//vendor/k8s.io/api/scheduling/v1alpha1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",

104
test/e2e/autoscaling/custom_metrics_autoscaling.go
View File

@ -20,18 +20,19 @@ import (
"context"
"time"
"golang.org/x/oauth2/google"
clientset "k8s.io/client-go/kubernetes"
. "github.com/onsi/ginkgo"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
gcm "google.golang.org/api/monitoring/v3"
as "k8s.io/api/autoscaling/v2beta1"
corev1 "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/instrumentation/monitoring"
. "github.com/onsi/ginkgo"
"golang.org/x/oauth2/google"
)
const (
@ -46,15 +47,32 @@ var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Me
})
f := framework.NewDefaultFramework("horizontal-pod-autoscaling")
var kubeClient clientset.Interface
It("should autoscale with Custom Metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
kubeClient = f.ClientSet
testHPA(f, kubeClient)
It("should scale down with Custom Metric of type Pod from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 2
scaledReplicas := 1
deployment := monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
})
It("should scale down with Custom Metric of type Object from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 2
scaledReplicas := 1
deployment := monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
pod := monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.ObjectMeta.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100)
customMetricTest(f, f.ClientSet, objectHPA(f.Namespace.ObjectMeta.Name), deployment, pod, initialReplicas, scaledReplicas)
})
It("should scale down with Custom Metric of type Pod from Stackdriver with Prometheus [Feature:CustomMetricsAutoscaling]", func() {
initialReplicas := 2
scaledReplicas := 1
deployment := monitoring.PrometheusExporterDeployment(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, int32(initialReplicas), 100)
customMetricTest(f, f.ClientSet, podsHPA(f.Namespace.ObjectMeta.Name), deployment, nil, initialReplicas, scaledReplicas)
})
})
func testHPA(f *framework.Framework, kubeClient clientset.Interface) {
func customMetricTest(f *framework.Framework, kubeClient clientset.Interface, hpa *as.HorizontalPodAutoscaler,
deployment *extensions.Deployment, pod *corev1.Pod, initialReplicas, scaledReplicas int) {
projectId := framework.TestContext.CloudConfig.ProjectID
ctx := context.Background()
@ -92,51 +110,55 @@ func testHPA(f *framework.Framework, kubeClient clientset.Interface) {
defer monitoring.CleanupAdapter()
// Run application that exports the metric
err = createDeploymentsToScale(f, kubeClient)
err = createDeploymentToScale(f, kubeClient, deployment, pod)
if err != nil {
framework.Failf("Failed to create stackdriver-exporter pod: %v", err)
}
defer cleanupDeploymentsToScale(f, kubeClient)
defer cleanupDeploymentsToScale(f, kubeClient, deployment, pod)
// Autoscale the deployments
err = createPodsHPA(f, kubeClient)
// Wait for the deployment to run
waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, initialReplicas)
// Autoscale the deployment
_, err = kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(hpa)
if err != nil {
framework.Failf("Failed to create 'Pods' HPA: %v", err)
}
err = createObjectHPA(f, kubeClient)
if err != nil {
framework.Failf("Failed to create 'Objects' HPA: %v", err)
framework.Failf("Failed to create HPA: %v", err)
}
waitForReplicas(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1)
waitForReplicas(dummyDeploymentName, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1)
waitForReplicas(deployment.ObjectMeta.Name, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, scaledReplicas)
}
func createDeploymentsToScale(f *framework.Framework, cs clientset.Interface) error {
_, err := cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.Name, 2, 100))
func createDeploymentToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) error {
if deployment != nil {
_, err := cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(deployment)
if err != nil {
return err
}
_, err = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100))
}
if pod != nil {
_, err := cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Create(pod)
if err != nil {
return err
}
_, err = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.Name, 2, 100))
return err
}
return nil
}
func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface) {
_ = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterDeployment, &metav1.DeleteOptions{})
_ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterPod, &metav1.DeleteOptions{})
_ = cs.ExtensionsV1beta1().Deployments(f.Namespace.ObjectMeta.Name).Delete(dummyDeploymentName, &metav1.DeleteOptions{})
func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface, deployment *extensions.Deployment, pod *corev1.Pod) {
if deployment != nil {
_ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(deployment.ObjectMeta.Name, &metav1.DeleteOptions{})
}
if pod != nil {
_ = cs.CoreV1().Pods(f.Namespace.ObjectMeta.Name).Delete(pod.ObjectMeta.Name, &metav1.DeleteOptions{})
}
}
func createPodsHPA(f *framework.Framework, cs clientset.Interface) error {
func podsHPA(namespace string) *as.HorizontalPodAutoscaler {
var minReplicas int32 = 1
_, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{
return &as.HorizontalPodAutoscaler{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-metrics-pods-hpa",
Namespace: f.Namespace.ObjectMeta.Name,
Namespace: namespace,
},
Spec: as.HorizontalPodAutoscalerSpec{
Metrics: []as.MetricSpec{
@ -156,16 +178,15 @@ func createPodsHPA(f *framework.Framework, cs clientset.Interface) error {
Name: stackdriverExporterDeployment,
},
},
})
return err
}
}
func createObjectHPA(f *framework.Framework, cs clientset.Interface) error {
func objectHPA(namespace string) *as.HorizontalPodAutoscaler {
var minReplicas int32 = 1
_, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{
return &as.HorizontalPodAutoscaler{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-metrics-objects-hpa",
Namespace: f.Namespace.ObjectMeta.Name,
Namespace: namespace,
},
Spec: as.HorizontalPodAutoscalerSpec{
Metrics: []as.MetricSpec{
@ -189,8 +210,7 @@ func createObjectHPA(f *framework.Framework, cs clientset.Interface) error {
Name: dummyDeploymentName,
},
},
})
return err
}
}
func waitForReplicas(deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) {

71
test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
View File

@ -18,6 +18,7 @@ package monitoring
import (
"fmt"
gcm "google.golang.org/api/monitoring/v3"
corev1 "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
@ -27,8 +28,8 @@ import (
)
var (
CustomMetricName = "foo-metric"
UnusedMetricName = "unused-metric"
CustomMetricName = "foo"
UnusedMetricName = "unused"
CustomMetricValue = int64(448)
UnusedMetricValue = int64(446)
// HPAPermissions is a ClusterRoleBinding that grants unauthenticated user permissions granted for
@ -116,6 +117,72 @@ func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.Pod
}
}
// PrometheusExporterDeployment is a Deployment of simple application with two containers
// one exposing a metric in prometheus fromat and second a prometheus-to-sd container
// that scrapes the metric and pushes it to stackdriver.
func PrometheusExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
return &extensions.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: extensions.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"name": name},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"name": name,
},
},
Spec: prometheusExporterPodSpec(CustomMetricName, metricValue, 8080),
},
Replicas: &replicas,
},
}
}
func prometheusExporterPodSpec(metricName string, metricValue int64, port int32) corev1.PodSpec {
return corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "prometheus-exporter",
Image: "gcr.io/google-containers/prometheus-dummy-exporter:v0.1.0",
ImagePullPolicy: corev1.PullPolicy("Always"),
Command: []string{"/prometheus_dummy_exporter", "--metric-name=" + metricName,
fmt.Sprintf("--metric-value=%v", metricValue), fmt.Sprintf("=--port=%d", port)},
Ports: []corev1.ContainerPort{{ContainerPort: port}},
},
{
Name: "prometheus-to-sd",
Image: "gcr.io/google-containers/prometheus-to-sd:v0.2.3",
ImagePullPolicy: corev1.PullPolicy("Always"),
Command: []string{"/monitor", fmt.Sprintf("--source=:http://localhost:%d", port),
"--stackdriver-prefix=custom.googleapis.com", "--pod-id=$(POD_ID)", "--namespace-id=$(POD_NAMESPACE)"},
Env: []corev1.EnvVar{
{
Name: "POD_ID",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.uid",
},
},
},
{
Name: "POD_NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
},
},
},
}
}
// CreateAdapter creates Custom Metrics - Stackdriver adapter.
func CreateAdapter() error {
stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")