Merge pull request #53205 from kawych/master

Automatic merge from submit-queue (batch tested with PRs 50223, 53205). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Create e2e tests for Custom Metrics - Stackdriver Adapter and HPA based on custom metrics from Stackdriver

**What this PR does / why we need it**:
- Add e2e test for Custom Metrics - Stackdriver Adapter
- Add 2e2 test for HPA based on custom metrics from Stackdriver
- Enable HorizontalPodAutoscalerUseRESTClients option

**Release note**:

```release-note
Horizontal pod autoscaler uses REST clients through the kube-aggregator instead of the legacy client through the API server proxy.
```

cluster/gce/gci/configure-helper.sh

@@ -1610,6 +1610,10 @@ function start-kube-controller-manager {
   if [[ -n "${CLUSTER_SIGNING_DURATION:-}" ]]; then
     params+=" --experimental-cluster-signing-duration=$CLUSTER_SIGNING_DURATION"
   fi
+  # disable using HPA metrics REST clients if metrics-server isn't enabled
+  if [[ "${ENABLE_METRICS_SERVER:-}" != "true" ]]; then
+    params+=" --horizontal-pod-autoscaler-use-rest-clients=false"
+  fi
 
   local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
   local container_env=""

cmd/kube-controller-manager/app/options/options.go

@@ -115,7 +115,7 @@ func NewCMServer() *CMServer {
 			ClusterSigningDuration:                metav1.Duration{Duration: helpers.OneYear},
 			ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 60 * time.Second},
 			EnableTaintManager:                    true,
-			HorizontalPodAutoscalerUseRESTClients: false,
+			HorizontalPodAutoscalerUseRESTClients: true,
 		},
 	}
 	s.LeaderElection.LeaderElect = true

cmd/kube-controller-manager/app/options/options_test.go

@@ -192,15 +192,16 @@ func TestAddFlags(t *testing.T) {
 				{Group: "apiregistration.k8s.io", Resource: "apiservices"},
 				{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"},
 			},
-			NodeEvictionRate:                  0.2,
-			SecondaryNodeEvictionRate:         0.05,
-			LargeClusterSizeThreshold:         100,
-			UnhealthyZoneThreshold:            0.6,
-			DisableAttachDetachReconcilerSync: true,
-			ReconcilerSyncLoopPeriod:          metav1.Duration{Duration: 30 * time.Second},
-			Controllers:                       []string{"foo", "bar"},
-			EnableTaintManager:                false,
-			UseServiceAccountCredentials:      true,
+			NodeEvictionRate:                      0.2,
+			SecondaryNodeEvictionRate:             0.05,
+			LargeClusterSizeThreshold:             100,
+			UnhealthyZoneThreshold:                0.6,
+			DisableAttachDetachReconcilerSync:     true,
+			ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 30 * time.Second},
+			Controllers:                           []string{"foo", "bar"},
+			EnableTaintManager:                    false,
+			HorizontalPodAutoscalerUseRESTClients: true,
+			UseServiceAccountCredentials:          true,
 		},
 		Kubeconfig: "/kubeconfig",
 		Master:     "192.168.4.20",

test/e2e/autoscaling/BUILD

@@ -11,6 +11,7 @@ go_library(
         "autoscaling_timer.go",
         "cluster_autoscaler_scalability.go",
         "cluster_size_autoscaling.go",
+        "custom_metrics_autoscaling.go",
         "dns_autoscaling.go",
         "framework.go",
         "horizontal_pod_autoscaling.go",
@@ -19,11 +20,15 @@ go_library(
         "//pkg/api:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
+        "//test/e2e/instrumentation/monitoring:go_default_library",
         "//test/e2e/scheduling:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/golang.org/x/oauth2/google:go_default_library",
+        "//vendor/google.golang.org/api/monitoring/v3:go_default_library",
+        "//vendor/k8s.io/api/autoscaling/v2beta1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/policy/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",

test/e2e/autoscaling/custom_metrics_autoscaling.go

@@ -0,0 +1,210 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package autoscaling
+
+import (
+	"context"
+	"time"
+
+	"golang.org/x/oauth2/google"
+	clientset "k8s.io/client-go/kubernetes"
+
+	. "github.com/onsi/ginkgo"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	gcm "google.golang.org/api/monitoring/v3"
+	as "k8s.io/api/autoscaling/v2beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/instrumentation/monitoring"
+)
+
+const (
+	stackdriverExporterDeployment = "stackdriver-exporter-deployment"
+	dummyDeploymentName           = "dummy-deployment"
+	stackdriverExporterPod        = "stackdriver-exporter-pod"
+)
+
+var _ = SIGDescribe("[HPA] Horizontal pod autoscaling (scale resource: Custom Metrics from Stackdriver)", func() {
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce")
+	})
+
+	f := framework.NewDefaultFramework("horizontal-pod-autoscaling")
+	var kubeClient clientset.Interface
+
+	It("should autoscale with Custom Metrics from Stackdriver [Feature:CustomMetricsAutoscaling]", func() {
+		kubeClient = f.ClientSet
+		testHPA(f, kubeClient)
+	})
+})
+
+func testHPA(f *framework.Framework, kubeClient clientset.Interface) {
+	projectId := framework.TestContext.CloudConfig.ProjectID
+
+	ctx := context.Background()
+	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
+
+	// Hack for running tests locally, needed to authenticate in Stackdriver
+	// If this is your use case, create application default credentials:
+	// $ gcloud auth application-default login
+	// and uncomment following lines:
+	/*
+		ts, err := google.DefaultTokenSource(oauth2.NoContext)
+		framework.Logf("Couldn't get application default credentials, %v", err)
+		if err != nil {
+			framework.Failf("Error accessing application default credentials, %v", err)
+		}
+		client := oauth2.NewClient(oauth2.NoContext, ts)
+	*/
+
+	gcmService, err := gcm.New(client)
+	if err != nil {
+		framework.Failf("Failed to create gcm service, %v", err)
+	}
+
+	// Set up a cluster: create a custom metric and set up k8s-sd adapter
+	err = monitoring.CreateDescriptors(gcmService, projectId)
+	if err != nil {
+		framework.Failf("Failed to create metric descriptor: %v", err)
+	}
+	defer monitoring.CleanupDescriptors(gcmService, projectId)
+
+	err = monitoring.CreateAdapter()
+	if err != nil {
+		framework.Failf("Failed to set up: %v", err)
+	}
+	defer monitoring.CleanupAdapter()
+
+	// Run application that exports the metric
+	err = createDeploymentsToScale(f, kubeClient)
+	if err != nil {
+		framework.Failf("Failed to create stackdriver-exporter pod: %v", err)
+	}
+	defer cleanupDeploymentsToScale(f, kubeClient)
+
+	// Autoscale the deployments
+	err = createPodsHPA(f, kubeClient)
+	if err != nil {
+		framework.Failf("Failed to create 'Pods' HPA: %v", err)
+	}
+	err = createObjectHPA(f, kubeClient)
+	if err != nil {
+		framework.Failf("Failed to create 'Objects' HPA: %v", err)
+	}
+
+	waitForReplicas(stackdriverExporterDeployment, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1)
+	waitForReplicas(dummyDeploymentName, f.Namespace.ObjectMeta.Name, kubeClient, 15*time.Minute, 1)
+}
+
+func createDeploymentsToScale(f *framework.Framework, cs clientset.Interface) error {
+	_, err := cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(stackdriverExporterDeployment, f.Namespace.Name, 2, 100))
+	if err != nil {
+		return err
+	}
+	_, err = cs.Core().Pods(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterPod(stackdriverExporterPod, f.Namespace.Name, stackdriverExporterPod, monitoring.CustomMetricName, 100))
+	if err != nil {
+		return err
+	}
+	_, err = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Create(monitoring.StackdriverExporterDeployment(dummyDeploymentName, f.Namespace.Name, 2, 100))
+	return err
+}
+
+func cleanupDeploymentsToScale(f *framework.Framework, cs clientset.Interface) {
+	_ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterDeployment, &metav1.DeleteOptions{})
+	_ = cs.Core().Pods(f.Namespace.ObjectMeta.Name).Delete(stackdriverExporterPod, &metav1.DeleteOptions{})
+	_ = cs.Extensions().Deployments(f.Namespace.ObjectMeta.Name).Delete(dummyDeploymentName, &metav1.DeleteOptions{})
+}
+
+func createPodsHPA(f *framework.Framework, cs clientset.Interface) error {
+	var minReplicas int32 = 1
+	_, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "custom-metrics-pods-hpa",
+			Namespace: f.Namespace.ObjectMeta.Name,
+		},
+		Spec: as.HorizontalPodAutoscalerSpec{
+			Metrics: []as.MetricSpec{
+				{
+					Type: as.PodsMetricSourceType,
+					Pods: &as.PodsMetricSource{
+						MetricName:         monitoring.CustomMetricName,
+						TargetAverageValue: *resource.NewQuantity(200, resource.DecimalSI),
+					},
+				},
+			},
+			MaxReplicas: 3,
+			MinReplicas: &minReplicas,
+			ScaleTargetRef: as.CrossVersionObjectReference{
+				APIVersion: "extensions/v1beta1",
+				Kind:       "Deployment",
+				Name:       stackdriverExporterDeployment,
+			},
+		},
+	})
+	return err
+}
+
+func createObjectHPA(f *framework.Framework, cs clientset.Interface) error {
+	var minReplicas int32 = 1
+	_, err := cs.AutoscalingV2beta1().HorizontalPodAutoscalers(f.Namespace.ObjectMeta.Name).Create(&as.HorizontalPodAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "custom-metrics-objects-hpa",
+			Namespace: f.Namespace.ObjectMeta.Name,
+		},
+		Spec: as.HorizontalPodAutoscalerSpec{
+			Metrics: []as.MetricSpec{
+				{
+					Type: as.ObjectMetricSourceType,
+					Object: &as.ObjectMetricSource{
+						MetricName: monitoring.CustomMetricName,
+						Target: as.CrossVersionObjectReference{
+							Kind: "Pod",
+							Name: stackdriverExporterPod,
+						},
+						TargetValue: *resource.NewQuantity(200, resource.DecimalSI),
+					},
+				},
+			},
+			MaxReplicas: 3,
+			MinReplicas: &minReplicas,
+			ScaleTargetRef: as.CrossVersionObjectReference{
+				APIVersion: "extensions/v1beta1",
+				Kind:       "Deployment",
+				Name:       dummyDeploymentName,
+			},
+		},
+	})
+	return err
+}
+
+func waitForReplicas(deploymentName, namespace string, cs clientset.Interface, timeout time.Duration, desiredReplicas int) {
+	interval := 20 * time.Second
+	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
+		deployment, err := cs.Extensions().Deployments(namespace).Get(deploymentName, metav1.GetOptions{})
+		if err != nil {
+			framework.Failf("Failed to get replication controller %s: %v", deployment, err)
+		}
+		replicas := int(deployment.Status.ReadyReplicas)
+		framework.Logf("waiting for %d replicas (current: %d)", desiredReplicas, replicas)
+		return replicas == desiredReplicas, nil // Expected number of replicas found. Exit.
+	})
+	if err != nil {
+		framework.Failf("Timeout waiting %v for %v replicas", timeout, desiredReplicas)
+	}
+}

test/e2e/instrumentation/monitoring/BUILD

@@ -9,6 +9,8 @@ go_library(
     name = "go_default_library",
     srcs = [
         "cadvisor.go",
+        "custom_metrics_deployments.go",
+        "custom_metrics_stackdriver.go",
         "influxdb.go",
         "metrics_grabber.go",
         "stackdriver.go",
@@ -23,10 +25,17 @@ go_library(
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/monitoring/v3:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library",
+        "//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
     ],
 )
 

test/e2e/instrumentation/monitoring/custom_metrics_deployments.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitoring
+
+import (
+	"fmt"
+	gcm "google.golang.org/api/monitoring/v3"
+	corev1 "k8s.io/api/core/v1"
+	extensions "k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var (
+	CustomMetricName  = "foo-metric"
+	UnusedMetricName  = "unused-metric"
+	CustomMetricValue = int64(448)
+	UnusedMetricValue = int64(446)
+)
+
+// StackdriverExporterDeployment is a Deployment of simple application that exports a metric of
+// fixed value to Stackdriver in a loop.
+func StackdriverExporterDeployment(name, namespace string, replicas int32, metricValue int64) *extensions.Deployment {
+	return &extensions.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: extensions.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"name": name},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"name": name,
+					},
+				},
+				Spec: stackdriverExporterPodSpec(CustomMetricName, metricValue),
+			},
+			Replicas: &replicas,
+		},
+	}
+}
+
+// StackdriverExporterPod is a Pod of simple application that exports a metric of fixed value to
+// Stackdriver in a loop.
+func StackdriverExporterPod(podName, namespace, podLabel, metricName string, metricValue int64) *corev1.Pod {
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: namespace,
+			Labels: map[string]string{
+				"name": podLabel,
+			},
+		},
+		Spec: stackdriverExporterPodSpec(metricName, metricValue),
+	}
+}
+
+func stackdriverExporterPodSpec(metricName string, metricValue int64) corev1.PodSpec {
+	return corev1.PodSpec{
+		Containers: []corev1.Container{
+			{
+				Name:            "stackdriver-exporter",
+				Image:           "gcr.io/google-containers/sd-dummy-exporter:v0.1.0",
+				ImagePullPolicy: corev1.PullPolicy("Always"),
+				Command:         []string{"/sd_dummy_exporter", "--pod-id=$(POD_ID)", "--metric-name=" + metricName, fmt.Sprintf("--metric-value=%v", metricValue)},
+				Env: []corev1.EnvVar{
+					{
+						Name: "POD_ID",
+						ValueFrom: &corev1.EnvVarSource{
+							FieldRef: &corev1.ObjectFieldSelector{
+								FieldPath: "metadata.uid",
+							},
+						},
+					},
+				},
+				Ports: []corev1.ContainerPort{{ContainerPort: 80}},
+			},
+		},
+	}
+}
+
+// CreateAdapter creates Custom Metrics - Stackdriver adapter.
+func CreateAdapter() error {
+	stat, err := framework.RunKubectl("create", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")
+	framework.Logf(stat)
+	return err
+}
+
+// CreateDescriptors creates descriptors for metrics: CustomMetricName and UnusedMetricName.
+func CreateDescriptors(service *gcm.Service, projectId string) error {
+	_, err := service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{
+		Name:       CustomMetricName,
+		ValueType:  "INT64",
+		Type:       "custom.googleapis.com/" + CustomMetricName,
+		MetricKind: "GAUGE",
+	}).Do()
+	if err != nil {
+		return err
+	}
+	_, err = service.Projects.MetricDescriptors.Create(fmt.Sprintf("projects/%s", projectId), &gcm.MetricDescriptor{
+		Name:       UnusedMetricName,
+		ValueType:  "INT64",
+		Type:       "custom.googleapis.com/" + UnusedMetricName,
+		MetricKind: "GAUGE",
+	}).Do()
+	return err
+}
+
+// CleanupDescriptors deletes descriptors for metrics: CustomMetricName and UnusedMetricName.
+// TODO: Cleanup time series as well
+func CleanupDescriptors(service *gcm.Service, projectId string) {
+	_, err := service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, CustomMetricName)).Do()
+	if err != nil {
+		framework.Logf("Failed to delete descriptor for metric '%s': %v", CustomMetricName, err)
+	}
+	_, err = service.Projects.MetricDescriptors.Delete(fmt.Sprintf("projects/%s/metricDescriptors/custom.googleapis.com/%s", projectId, UnusedMetricName)).Do()
+	if err != nil {
+		framework.Logf("Failed to delete descriptor for metric '%s': %v", CustomMetricName, err)
+	}
+}
+
+// CleanupAdapter deletes Custom Metrics - Stackdriver adapter deployments.
+func CleanupAdapter() error {
+	stat, err := framework.RunKubectl("delete", "-f", "https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/master/custom-metrics-stackdriver-adapter/adapter-beta.yaml")
+	framework.Logf(stat)
+	return err
+}

test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package monitoring
+
+import (
+	"context"
+	"time"
+
+	"golang.org/x/oauth2/google"
+	clientset "k8s.io/client-go/kubernetes"
+
+	. "github.com/onsi/ginkgo"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+
+	gcm "google.golang.org/api/monitoring/v3"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/client-go/discovery"
+	kubeaggrcs "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/test/e2e/framework"
+	customclient "k8s.io/metrics/pkg/client/custom_metrics"
+)
+
+const (
+	stackdriverExporterPod1  = "stackdriver-exporter-1"
+	stackdriverExporterPod2  = "stackdriver-exporter-2"
+	stackdriverExporterLabel = "stackdriver-exporter"
+)
+
+var _ = instrumentation.SIGDescribe("Stackdriver Monitoring", func() {
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce", "gke")
+	})
+
+	f := framework.NewDefaultFramework("stackdriver-monitoring")
+	var kubeClient clientset.Interface
+	var kubeAggrClient kubeaggrcs.Interface
+	var customMetricsClient customclient.CustomMetricsClient
+	var discoveryClient *discovery.DiscoveryClient
+
+	It("should run Custom Metrics - Stackdriver Adapter [Feature:StackdriverCustomMetrics]", func() {
+		kubeClient = f.ClientSet
+		kubeAggrClient = f.AggregatorClient
+		config, err := framework.LoadConfig()
+		if err != nil {
+			framework.Failf("Failed to load config: %s", err)
+		}
+		customMetricsClient = customclient.NewForConfigOrDie(config)
+		discoveryClient = discovery.NewDiscoveryClientForConfigOrDie(config)
+		testAdapter(f, kubeClient, customMetricsClient, discoveryClient)
+	})
+})
+
+func testAdapter(f *framework.Framework, kubeClient clientset.Interface, customMetricsClient customclient.CustomMetricsClient, discoveryClient *discovery.DiscoveryClient) {
+	projectId := framework.TestContext.CloudConfig.ProjectID
+
+	ctx := context.Background()
+	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
+
+	// Hack for running tests locally, needed to authenticate in Stackdriver
+	// If this is your use case, create application default credentials:
+	// $ gcloud auth application-default login
+	// and uncomment following lines (comment out the two lines above):
+	/*
+		ts, err := google.DefaultTokenSource(oauth2.NoContext)
+		framework.Logf("Couldn't get application default credentials, %v", err)
+		if err != nil {
+			framework.Failf("Error accessing application default credentials, %v", err)
+		}
+		client := oauth2.NewClient(oauth2.NoContext, ts)
+	*/
+
+	gcmService, err := gcm.New(client)
+	if err != nil {
+		framework.Failf("Failed to create gcm service, %v", err)
+	}
+
+	// Set up a cluster: create a custom metric and set up k8s-sd adapter
+	err = CreateDescriptors(gcmService, projectId)
+	if err != nil {
+		framework.Failf("Failed to create metric descriptor: %s", err)
+	}
+	defer CleanupDescriptors(gcmService, projectId)
+
+	err = CreateAdapter()
+	if err != nil {
+		framework.Failf("Failed to set up: %s", err)
+	}
+	defer CleanupAdapter()
+
+	// Run application that exports the metric
+	err = createSDExporterPods(f, kubeClient)
+	if err != nil {
+		framework.Failf("Failed to create stackdriver-exporter pod: %s", err)
+	}
+	defer cleanupSDExporterPod(f, kubeClient)
+
+	// Wait a short amount of time to create a pod and export some metrics
+	// TODO: add some events to wait for instead of fixed amount of time
+	//       i.e. pod creation, first time series exported
+	time.Sleep(60 * time.Second)
+
+	// Verify responses from Custom Metrics API
+	resources, err := discoveryClient.ServerResourcesForGroupVersion("custom.metrics.k8s.io/v1beta1")
+	if err != nil {
+		framework.Failf("Failed to retrieve a list of supported metrics: %s", err)
+	}
+	for _, resource := range resources.APIResources {
+		if resource.Name != "pods/"+CustomMetricName && resource.Name != "pods/"+UnusedMetricName {
+			framework.Failf("Unexpected metric %s. Only metric %s should be supported", resource.Name, CustomMetricName)
+		}
+	}
+	value, err := customMetricsClient.NamespacedMetrics(f.Namespace.Name).GetForObject(schema.GroupKind{Group: "", Kind: "Pod"}, stackdriverExporterPod1, CustomMetricName)
+	if err != nil {
+		framework.Failf("Failed query: %s", err)
+	}
+	if value.Value.Value() != CustomMetricValue {
+		framework.Failf("Unexpected metric value for metric %s: expected %v but received %v", CustomMetricName, CustomMetricValue, value.Value)
+	}
+	filter, err := labels.NewRequirement("name", selection.Equals, []string{stackdriverExporterLabel})
+	if err != nil {
+		framework.Failf("Couldn't create a label filter")
+	}
+	values, err := customMetricsClient.NamespacedMetrics(f.Namespace.Name).GetForObjects(schema.GroupKind{Group: "", Kind: "Pod"}, labels.NewSelector().Add(*filter), CustomMetricName)
+	if err != nil {
+		framework.Failf("Failed query: %s", err)
+	}
+	if len(values.Items) != 2 {
+		framework.Failf("Expected results for exactly 2 pods, but %v results received", len(values.Items))
+	}
+	for _, value := range values.Items {
+		if (value.DescribedObject.Name == stackdriverExporterPod1 && value.Value.Value() != CustomMetricValue) ||
+			(value.DescribedObject.Name == stackdriverExporterPod2 && value.Value.Value() != UnusedMetricValue) {
+			framework.Failf("Unexpected metric value for metric %s and pod %s: %v", CustomMetricName, value.DescribedObject.Name, value.Value.Value())
+		}
+	}
+}
+
+func cleanupSDExporterPod(f *framework.Framework, cs clientset.Interface) {
+	err := cs.Core().Pods(f.Namespace.Name).Delete(stackdriverExporterPod1, &metav1.DeleteOptions{})
+	if err != nil {
+		framework.Logf("Failed to delete %s pod: %v", stackdriverExporterPod1, err)
+	}
+	err = cs.Core().Pods(f.Namespace.Name).Delete(stackdriverExporterPod2, &metav1.DeleteOptions{})
+	if err != nil {
+		framework.Logf("Failed to delete %s pod: %v", stackdriverExporterPod2, err)
+	}
+}
+
+func createSDExporterPods(f *framework.Framework, cs clientset.Interface) error {
+	_, err := cs.Core().Pods(f.Namespace.Name).Create(StackdriverExporterPod(stackdriverExporterPod1, f.Namespace.Name, stackdriverExporterLabel, CustomMetricName, CustomMetricValue))
+	if err != nil {
+		return err
+	}
+	_, err = cs.Core().Pods(f.Namespace.Name).Create(StackdriverExporterPod(stackdriverExporterPod2, f.Namespace.Name, stackdriverExporterLabel, UnusedMetricName, UnusedMetricValue))
+	return err
+}

test/e2e/instrumentation/monitoring/stackdriver.go

@@ -76,6 +76,20 @@ func testStackdriverMonitoring(f *framework.Framework, pods, allPodsCPU int, per
 
 	ctx := context.Background()
 	client, err := google.DefaultClient(ctx, gcm.CloudPlatformScope)
+
+	// Hack for running tests locally
+	// If this is your use case, create application default credentials:
+	// $ gcloud auth application-default login
+	// and uncomment following lines (comment out the two lines above): (DON'T set the env var below)
+	/*
+		ts, err := google.DefaultTokenSource(oauth2.NoContext)
+		framework.Logf("Couldn't get application default credentials, %v", err)
+		if err != nil {
+			framework.Failf("Error accessing application default credentials, %v", err)
+		}
+		client := oauth2.NewClient(oauth2.NoContext, ts)
+	*/
+
 	gcmService, err := gcm.New(client)
 
 	// set this env var if accessing Stackdriver test endpoint (default is prod):