Merge pull request #58204 from cosmincojocar/monitoring_wrapped_azure_clients

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Instrument the Azure API calls for Prometheus monitoring

**What this PR does / why we need it**:

Instruments the Azure API calls for Prometheus monitoring.

**Special notes for your reviewer**:

This is version 2 based on the wrapped clients.

**Release note**:

```release-note
Instrument the Azure cloud provider for Prometheus monitoring.
```

cc @feiskyer @andyzhangx @jdumars @khenidak

pkg/cloudprovider/providers/azure/BUILD

@@ -19,6 +19,7 @@ go_library(
         "azure_instances.go",
         "azure_loadbalancer.go",
         "azure_managedDiskController.go",
+        "azure_metrics.go",
         "azure_routes.go",
         "azure_storage.go",
         "azure_storageaccount.go",
@@ -48,6 +49,7 @@ go_library(
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
@@ -63,6 +65,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "azure_loadbalancer_test.go",
+        "azure_metrics_test.go",
         "azure_test.go",
         "azure_util_cache_test.go",
         "azure_util_test.go",

pkg/cloudprovider/providers/azure/azure_backoff.go

@@ -89,7 +89,7 @@ func (az *Cloud) VirtualMachineClientListWithRetry() ([]compute.VirtualMachine,
 		if result.NextLink != nil {
 			err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
 				var retryErr error
-				result, retryErr = az.VirtualMachinesClient.ListNextResults(result)
+				result, retryErr = az.VirtualMachinesClient.ListNextResults(az.ResourceGroup, result)
 				if retryErr != nil {
 					glog.Errorf("VirtualMachinesClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
 						az.ResourceGroup, retryErr)
@@ -176,7 +176,7 @@ func (az *Cloud) ListLBWithRetry() ([]network.LoadBalancer, error) {
 		if result.NextLink != nil {
 			err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
 				var retryErr error
-				result, retryErr = az.LoadBalancerClient.ListNextResults(result)
+				result, retryErr = az.LoadBalancerClient.ListNextResults(az.ResourceGroup, result)
 				if retryErr != nil {
 					glog.Errorf("LoadBalancerClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
 						az.ResourceGroup,
@@ -225,7 +225,7 @@ func (az *Cloud) ListPIPWithRetry(pipResourceGroup string) ([]network.PublicIPAd
 		if result.NextLink != nil {
 			err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
 				var retryErr error
-				result, retryErr = az.PublicIPAddressesClient.ListNextResults(result)
+				result, retryErr = az.PublicIPAddressesClient.ListNextResults(az.ResourceGroup, result)
 				if retryErr != nil {
 					glog.Errorf("PublicIPAddressesClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
 						pipResourceGroup,

pkg/cloudprovider/providers/azure/azure_client.go

@@ -35,7 +35,7 @@ type VirtualMachinesClient interface {
 	CreateOrUpdate(resourceGroupName string, VMName string, parameters compute.VirtualMachine, cancel <-chan struct{}) (<-chan compute.VirtualMachine, <-chan error)
 	Get(resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (result compute.VirtualMachine, err error)
 	List(resourceGroupName string) (result compute.VirtualMachineListResult, err error)
-	ListNextResults(lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error)
+	ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error)
 }
 
 // InterfacesClient defines needed functions for azure network.InterfacesClient
@@ -51,7 +51,7 @@ type LoadBalancersClient interface {
 	Delete(resourceGroupName string, loadBalancerName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error)
 	Get(resourceGroupName string, loadBalancerName string, expand string) (result network.LoadBalancer, err error)
 	List(resourceGroupName string) (result network.LoadBalancerListResult, err error)
-	ListNextResults(lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error)
+	ListNextResults(resourceGroupName string, lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error)
 }
 
 // PublicIPAddressesClient defines needed functions for azure network.PublicIPAddressesClient
@@ -60,7 +60,7 @@ type PublicIPAddressesClient interface {
 	Delete(resourceGroupName string, publicIPAddressName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error)
 	Get(resourceGroupName string, publicIPAddressName string, expand string) (result network.PublicIPAddress, err error)
 	List(resourceGroupName string) (result network.PublicIPAddressListResult, err error)
-	ListNextResults(lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error)
+	ListNextResults(resourceGroupName string, lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error)
 }
 
 // SubnetsClient defines needed functions for azure network.SubnetsClient
@@ -84,7 +84,7 @@ type VirtualMachineScaleSetsClient interface {
 	CreateOrUpdate(resourceGroupName string, VMScaleSetName string, parameters compute.VirtualMachineScaleSet, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error)
 	Get(resourceGroupName string, VMScaleSetName string) (result compute.VirtualMachineScaleSet, err error)
 	List(resourceGroupName string) (result compute.VirtualMachineScaleSetListResult, err error)
-	ListNextResults(lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error)
+	ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error)
 	UpdateInstances(resourceGroupName string, VMScaleSetName string, VMInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error)
 }
 
@@ -93,7 +93,7 @@ type VirtualMachineScaleSetVMsClient interface {
 	Get(resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVM, err error)
 	GetInstanceView(resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVMInstanceView, err error)
 	List(resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result compute.VirtualMachineScaleSetVMListResult, err error)
-	ListNextResults(lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error)
+	ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error)
 }
 
 // RoutesClient defines needed functions for azure network.RoutesClient
@@ -158,7 +158,13 @@ func (az *azVirtualMachinesClient) CreateOrUpdate(resourceGroupName string, VMNa
 		glog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): end", resourceGroupName, VMName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, VMName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("vm", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, VMName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azVirtualMachinesClient) Get(resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (result compute.VirtualMachine, err error) {
@@ -168,7 +174,10 @@ func (az *azVirtualMachinesClient) Get(resourceGroupName string, VMName string,
 		glog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): end", resourceGroupName, VMName)
 	}()
 
-	return az.client.Get(resourceGroupName, VMName, expand)
+	mc := newMetricContext("vm", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, VMName, expand)
+	mc.Observe(err)
+	return
 }
 
 func (az *azVirtualMachinesClient) List(resourceGroupName string) (result compute.VirtualMachineListResult, err error) {
@@ -178,17 +187,23 @@ func (az *azVirtualMachinesClient) List(resourceGroupName string) (result comput
 		glog.V(10).Infof("azVirtualMachinesClient.List(%q): end", resourceGroupName)
 	}()
 
-	return az.client.List(resourceGroupName)
+	mc := newMetricContext("vm", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName)
+	mc.Observe(err)
+	return
 }
 
-func (az *azVirtualMachinesClient) ListNextResults(lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
+func (az *azVirtualMachinesClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
 	az.rateLimiter.Accept()
 	glog.V(10).Infof("azVirtualMachinesClient.ListNextResults(%q): start", lastResults)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachinesClient.ListNextResults(%q): end", lastResults)
 	}()
 
-	return az.client.ListNextResults(lastResults)
+	mc := newMetricContext("vm", "list_next_results", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListNextResults(lastResults)
+	mc.Observe(err)
+	return
 }
 
 // azInterfacesClient implements InterfacesClient.
@@ -217,7 +232,13 @@ func (az *azInterfacesClient) CreateOrUpdate(resourceGroupName string, networkIn
 		glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkInterfaceName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, networkInterfaceName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("interfaces", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, networkInterfaceName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azInterfacesClient) Get(resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) {
@@ -227,7 +248,10 @@ func (az *azInterfacesClient) Get(resourceGroupName string, networkInterfaceName
 		glog.V(10).Infof("azInterfacesClient.Get(%q,%q): end", resourceGroupName, networkInterfaceName)
 	}()
 
-	return az.client.Get(resourceGroupName, networkInterfaceName, expand)
+	mc := newMetricContext("interfaces", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, networkInterfaceName, expand)
+	mc.Observe(err)
+	return
 }
 
 func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) {
@@ -237,7 +261,10 @@ func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resource
 		glog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName)
 	}()
 
-	return az.client.GetVirtualMachineScaleSetNetworkInterface(resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, expand)
+	mc := newMetricContext("interfaces", "get_vmss_ni", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.GetVirtualMachineScaleSetNetworkInterface(resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, expand)
+	mc.Observe(err)
+	return
 }
 
 // azLoadBalancersClient implements LoadBalancersClient.
@@ -266,7 +293,13 @@ func (az *azLoadBalancersClient) CreateOrUpdate(resourceGroupName string, loadBa
 		glog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): end", resourceGroupName, loadBalancerName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, loadBalancerName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("load_balancers", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, loadBalancerName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azLoadBalancersClient) Delete(resourceGroupName string, loadBalancerName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
@@ -276,7 +309,13 @@ func (az *azLoadBalancersClient) Delete(resourceGroupName string, loadBalancerNa
 		glog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): end", resourceGroupName, loadBalancerName)
 	}()
 
-	return az.client.Delete(resourceGroupName, loadBalancerName, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("load_balancers", "delete", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Delete(resourceGroupName, loadBalancerName, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azLoadBalancersClient) Get(resourceGroupName string, loadBalancerName string, expand string) (result network.LoadBalancer, err error) {
@@ -286,7 +325,10 @@ func (az *azLoadBalancersClient) Get(resourceGroupName string, loadBalancerName
 		glog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): end", resourceGroupName, loadBalancerName)
 	}()
 
-	return az.client.Get(resourceGroupName, loadBalancerName, expand)
+	mc := newMetricContext("load_balancers", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, loadBalancerName, expand)
+	mc.Observe(err)
+	return
 }
 
 func (az *azLoadBalancersClient) List(resourceGroupName string) (result network.LoadBalancerListResult, err error) {
@@ -296,17 +338,23 @@ func (az *azLoadBalancersClient) List(resourceGroupName string) (result network.
 		glog.V(10).Infof("azLoadBalancersClient.List(%q): end", resourceGroupName)
 	}()
 
-	return az.client.List(resourceGroupName)
+	mc := newMetricContext("load_balancers", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName)
+	mc.Observe(err)
+	return
 }
 
-func (az *azLoadBalancersClient) ListNextResults(lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
+func (az *azLoadBalancersClient) ListNextResults(resourceGroupName string, lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
 	az.rateLimiter.Accept()
 	glog.V(10).Infof("azLoadBalancersClient.ListNextResults(%q): start", lastResult)
 	defer func() {
 		glog.V(10).Infof("azLoadBalancersClient.ListNextResults(%q): end", lastResult)
 	}()
 
-	return az.client.ListNextResults(lastResult)
+	mc := newMetricContext("load_balancers", "list_next_results", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListNextResults(lastResult)
+	mc.Observe(err)
+	return
 }
 
 // azPublicIPAddressesClient implements PublicIPAddressesClient.
@@ -335,7 +383,13 @@ func (az *azPublicIPAddressesClient) CreateOrUpdate(resourceGroupName string, pu
 		glog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, publicIPAddressName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, publicIPAddressName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("public_ip_addresses", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, publicIPAddressName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azPublicIPAddressesClient) Delete(resourceGroupName string, publicIPAddressName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
@@ -345,7 +399,13 @@ func (az *azPublicIPAddressesClient) Delete(resourceGroupName string, publicIPAd
 		glog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): end", resourceGroupName, publicIPAddressName)
 	}()
 
-	return az.client.Delete(resourceGroupName, publicIPAddressName, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("public_ip_addresses", "delete", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Delete(resourceGroupName, publicIPAddressName, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azPublicIPAddressesClient) Get(resourceGroupName string, publicIPAddressName string, expand string) (result network.PublicIPAddress, err error) {
@@ -355,7 +415,10 @@ func (az *azPublicIPAddressesClient) Get(resourceGroupName string, publicIPAddre
 		glog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): end", resourceGroupName, publicIPAddressName)
 	}()
 
-	return az.client.Get(resourceGroupName, publicIPAddressName, expand)
+	mc := newMetricContext("public_ip_addresses", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, publicIPAddressName, expand)
+	mc.Observe(err)
+	return
 }
 
 func (az *azPublicIPAddressesClient) List(resourceGroupName string) (result network.PublicIPAddressListResult, err error) {
@@ -365,17 +428,23 @@ func (az *azPublicIPAddressesClient) List(resourceGroupName string) (result netw
 		glog.V(10).Infof("azPublicIPAddressesClient.List(%q): end", resourceGroupName)
 	}()
 
-	return az.client.List(resourceGroupName)
+	mc := newMetricContext("public_ip_addresses", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName)
+	mc.Observe(err)
+	return
 }
 
-func (az *azPublicIPAddressesClient) ListNextResults(lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
+func (az *azPublicIPAddressesClient) ListNextResults(resourceGroupName string, lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
 	az.rateLimiter.Accept()
 	glog.V(10).Infof("azPublicIPAddressesClient.ListNextResults(%q): start", lastResults)
 	defer func() {
 		glog.V(10).Infof("azPublicIPAddressesClient.ListNextResults(%q): end", lastResults)
 	}()
 
-	return az.client.ListNextResults(lastResults)
+	mc := newMetricContext("public_ip_addresses", "list_next_results", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListNextResults(lastResults)
+	mc.Observe(err)
+	return
 }
 
 // azSubnetsClient implements SubnetsClient.
@@ -404,7 +473,13 @@ func (az *azSubnetsClient) CreateOrUpdate(resourceGroupName string, virtualNetwo
 		glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, virtualNetworkName, subnetName, subnetParameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("subnets", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, virtualNetworkName, subnetName, subnetParameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azSubnetsClient) Delete(resourceGroupName string, virtualNetworkName string, subnetName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
@@ -414,7 +489,13 @@ func (az *azSubnetsClient) Delete(resourceGroupName string, virtualNetworkName s
 		glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName)
 	}()
 
-	return az.client.Delete(resourceGroupName, virtualNetworkName, subnetName, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("subnets", "delete", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Delete(resourceGroupName, virtualNetworkName, subnetName, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azSubnetsClient) Get(resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) {
@@ -424,7 +505,10 @@ func (az *azSubnetsClient) Get(resourceGroupName string, virtualNetworkName stri
 		glog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName)
 	}()
 
-	return az.client.Get(resourceGroupName, virtualNetworkName, subnetName, expand)
+	mc := newMetricContext("subnets", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, virtualNetworkName, subnetName, expand)
+	mc.Observe(err)
+	return
 }
 
 func (az *azSubnetsClient) List(resourceGroupName string, virtualNetworkName string) (result network.SubnetListResult, err error) {
@@ -434,7 +518,10 @@ func (az *azSubnetsClient) List(resourceGroupName string, virtualNetworkName str
 		glog.V(10).Infof("azSubnetsClient.List(%q,%q): end", resourceGroupName, virtualNetworkName)
 	}()
 
-	return az.client.List(resourceGroupName, virtualNetworkName)
+	mc := newMetricContext("subnets", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName, virtualNetworkName)
+	mc.Observe(err)
+	return
 }
 
 // azSecurityGroupsClient implements SecurityGroupsClient.
@@ -463,7 +550,13 @@ func (az *azSecurityGroupsClient) CreateOrUpdate(resourceGroupName string, netwo
 		glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkSecurityGroupName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, networkSecurityGroupName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("security_groups", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, networkSecurityGroupName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azSecurityGroupsClient) Delete(resourceGroupName string, networkSecurityGroupName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
@@ -473,7 +566,13 @@ func (az *azSecurityGroupsClient) Delete(resourceGroupName string, networkSecuri
 		glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): end", resourceGroupName, networkSecurityGroupName)
 	}()
 
-	return az.client.Delete(resourceGroupName, networkSecurityGroupName, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("security_groups", "delete", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Delete(resourceGroupName, networkSecurityGroupName, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azSecurityGroupsClient) Get(resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) {
@@ -483,7 +582,10 @@ func (az *azSecurityGroupsClient) Get(resourceGroupName string, networkSecurityG
 		glog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): end", resourceGroupName, networkSecurityGroupName)
 	}()
 
-	return az.client.Get(resourceGroupName, networkSecurityGroupName, expand)
+	mc := newMetricContext("security_groups", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, networkSecurityGroupName, expand)
+	mc.Observe(err)
+	return
 }
 
 func (az *azSecurityGroupsClient) List(resourceGroupName string) (result network.SecurityGroupListResult, err error) {
@@ -493,7 +595,10 @@ func (az *azSecurityGroupsClient) List(resourceGroupName string) (result network
 		glog.V(10).Infof("azSecurityGroupsClient.List(%q): end", resourceGroupName)
 	}()
 
-	return az.client.List(resourceGroupName)
+	mc := newMetricContext("security_groups", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName)
+	mc.Observe(err)
+	return
 }
 
 // azVirtualMachineScaleSetsClient implements VirtualMachineScaleSetsClient.
@@ -522,7 +627,13 @@ func (az *azVirtualMachineScaleSetsClient) CreateOrUpdate(resourceGroupName stri
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, VMScaleSetName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, VMScaleSetName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("vmss", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, VMScaleSetName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azVirtualMachineScaleSetsClient) Get(resourceGroupName string, VMScaleSetName string) (result compute.VirtualMachineScaleSet, err error) {
@@ -532,7 +643,10 @@ func (az *azVirtualMachineScaleSetsClient) Get(resourceGroupName string, VMScale
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): end", resourceGroupName, VMScaleSetName)
 	}()
 
-	return az.client.Get(resourceGroupName, VMScaleSetName)
+	mc := newMetricContext("vmss", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, VMScaleSetName)
+	mc.Observe(err)
+	return
 }
 
 func (az *azVirtualMachineScaleSetsClient) List(resourceGroupName string) (result compute.VirtualMachineScaleSetListResult, err error) {
@@ -542,17 +656,23 @@ func (az *azVirtualMachineScaleSetsClient) List(resourceGroupName string) (resul
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q,%q): end", resourceGroupName)
 	}()
 
-	return az.client.List(resourceGroupName)
+	mc := newMetricContext("vmss", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName)
+	mc.Observe(err)
+	return
 }
 
-func (az *azVirtualMachineScaleSetsClient) ListNextResults(lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
+func (az *azVirtualMachineScaleSetsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
 	az.rateLimiter.Accept()
 	glog.V(10).Infof("azVirtualMachineScaleSetsClient.ListNextResults(%q): start", lastResults)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.ListNextResults(%q): end", lastResults)
 	}()
 
-	return az.client.ListNextResults(lastResults)
+	mc := newMetricContext("vmss", "list_next_results", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListNextResults(lastResults)
+	mc.Observe(err)
+	return
 }
 
 func (az *azVirtualMachineScaleSetsClient) UpdateInstances(resourceGroupName string, VMScaleSetName string, VMInstanceIDs compute.VirtualMachineScaleSetVMInstanceRequiredIDs, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) {
@@ -562,7 +682,13 @@ func (az *azVirtualMachineScaleSetsClient) UpdateInstances(resourceGroupName str
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%q): end", resourceGroupName, VMScaleSetName, VMInstanceIDs)
 	}()
 
-	return az.client.UpdateInstances(resourceGroupName, VMScaleSetName, VMInstanceIDs, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("vmss", "update_instances", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.UpdateInstances(resourceGroupName, VMScaleSetName, VMInstanceIDs, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 // azVirtualMachineScaleSetVMsClient implements VirtualMachineScaleSetVMsClient.
@@ -591,7 +717,10 @@ func (az *azVirtualMachineScaleSetVMsClient) Get(resourceGroupName string, VMSca
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID)
 	}()
 
-	return az.client.Get(resourceGroupName, VMScaleSetName, instanceID)
+	mc := newMetricContext("vmssvm", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, VMScaleSetName, instanceID)
+	mc.Observe(err)
+	return
 }
 
 func (az *azVirtualMachineScaleSetVMsClient) GetInstanceView(resourceGroupName string, VMScaleSetName string, instanceID string) (result compute.VirtualMachineScaleSetVMInstanceView, err error) {
@@ -601,7 +730,10 @@ func (az *azVirtualMachineScaleSetVMsClient) GetInstanceView(resourceGroupName s
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID)
 	}()
 
-	return az.client.GetInstanceView(resourceGroupName, VMScaleSetName, instanceID)
+	mc := newMetricContext("vmssvm", "get_instance_view", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.GetInstanceView(resourceGroupName, VMScaleSetName, instanceID)
+	mc.Observe(err)
+	return
 }
 
 func (az *azVirtualMachineScaleSetVMsClient) List(resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result compute.VirtualMachineScaleSetVMListResult, err error) {
@@ -611,17 +743,23 @@ func (az *azVirtualMachineScaleSetVMsClient) List(resourceGroupName string, virt
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, filter)
 	}()
 
-	return az.client.List(resourceGroupName, virtualMachineScaleSetName, filter, selectParameter, expand)
+	mc := newMetricContext("vmssvm", "list", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.List(resourceGroupName, virtualMachineScaleSetName, filter, selectParameter, expand)
+	mc.Observe(err)
+	return
 }
 
-func (az *azVirtualMachineScaleSetVMsClient) ListNextResults(lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
+func (az *azVirtualMachineScaleSetVMsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
 	az.rateLimiter.Accept()
 	glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.ListNextResults(%q,%q,%q): start", lastResults)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.ListNextResults(%q,%q,%q): end", lastResults)
 	}()
 
-	return az.client.ListNextResults(lastResults)
+	mc := newMetricContext("vmssvm", "list_next_results", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListNextResults(lastResults)
+	mc.Observe(err)
+	return
 }
 
 // azRoutesClient implements RoutesClient.
@@ -650,7 +788,13 @@ func (az *azRoutesClient) CreateOrUpdate(resourceGroupName string, routeTableNam
 		glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, routeTableName, routeName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, routeTableName, routeName, routeParameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("routes", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, routeTableName, routeName, routeParameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azRoutesClient) Delete(resourceGroupName string, routeTableName string, routeName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
@@ -660,7 +804,13 @@ func (az *azRoutesClient) Delete(resourceGroupName string, routeTableName string
 		glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): end", resourceGroupName, routeTableName, routeName)
 	}()
 
-	return az.client.Delete(resourceGroupName, routeTableName, routeName, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("routes", "delete", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Delete(resourceGroupName, routeTableName, routeName, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 // azRouteTablesClient implements RouteTablesClient.
@@ -689,7 +839,13 @@ func (az *azRouteTablesClient) CreateOrUpdate(resourceGroupName string, routeTab
 		glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, routeTableName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, routeTableName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("route_tables", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, routeTableName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azRouteTablesClient) Get(resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) {
@@ -699,7 +855,10 @@ func (az *azRouteTablesClient) Get(resourceGroupName string, routeTableName stri
 		glog.V(10).Infof("azRouteTablesClient.Get(%q,%q): end", resourceGroupName, routeTableName)
 	}()
 
-	return az.client.Get(resourceGroupName, routeTableName, expand)
+	mc := newMetricContext("route_tables", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, routeTableName, expand)
+	mc.Observe(err)
+	return
 }
 
 // azStorageAccountClient implements StorageAccountClient.
@@ -727,7 +886,13 @@ func (az *azStorageAccountClient) Create(resourceGroupName string, accountName s
 		glog.V(10).Infof("azStorageAccountClient.Create(%q,%q): end", resourceGroupName, accountName)
 	}()
 
-	return az.client.Create(resourceGroupName, accountName, parameters, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("storage_account", "create", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Create(resourceGroupName, accountName, parameters, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azStorageAccountClient) Delete(resourceGroupName string, accountName string) (result autorest.Response, err error) {
@@ -737,7 +902,10 @@ func (az *azStorageAccountClient) Delete(resourceGroupName string, accountName s
 		glog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): end", resourceGroupName, accountName)
 	}()
 
-	return az.client.Delete(resourceGroupName, accountName)
+	mc := newMetricContext("storage_account", "delete", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Delete(resourceGroupName, accountName)
+	mc.Observe(err)
+	return
 }
 
 func (az *azStorageAccountClient) ListKeys(resourceGroupName string, accountName string) (result storage.AccountListKeysResult, err error) {
@@ -747,7 +915,10 @@ func (az *azStorageAccountClient) ListKeys(resourceGroupName string, accountName
 		glog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): end", resourceGroupName, accountName)
 	}()
 
-	return az.client.ListKeys(resourceGroupName, accountName)
+	mc := newMetricContext("storage_account", "list_keys", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListKeys(resourceGroupName, accountName)
+	mc.Observe(err)
+	return
 }
 
 func (az *azStorageAccountClient) ListByResourceGroup(resourceGroupName string) (result storage.AccountListResult, err error) {
@@ -757,7 +928,10 @@ func (az *azStorageAccountClient) ListByResourceGroup(resourceGroupName string)
 		glog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): end", resourceGroupName)
 	}()
 
-	return az.client.ListByResourceGroup(resourceGroupName)
+	mc := newMetricContext("storage_account", "list_by_resource_group", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.ListByResourceGroup(resourceGroupName)
+	mc.Observe(err)
+	return
 }
 
 func (az *azStorageAccountClient) GetProperties(resourceGroupName string, accountName string) (result storage.Account, err error) {
@@ -767,7 +941,10 @@ func (az *azStorageAccountClient) GetProperties(resourceGroupName string, accoun
 		glog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): end", resourceGroupName, accountName)
 	}()
 
-	return az.client.GetProperties(resourceGroupName, accountName)
+	mc := newMetricContext("storage_account", "get_properties", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.GetProperties(resourceGroupName, accountName)
+	mc.Observe(err)
+	return
 }
 
 // azDisksClient implements DisksClient.
@@ -795,7 +972,13 @@ func (az *azDisksClient) CreateOrUpdate(resourceGroupName string, diskName strin
 		glog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): end", resourceGroupName, diskName)
 	}()
 
-	return az.client.CreateOrUpdate(resourceGroupName, diskName, diskParameter, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("disks", "create_or_update", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.CreateOrUpdate(resourceGroupName, diskName, diskParameter, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azDisksClient) Delete(resourceGroupName string, diskName string, cancel <-chan struct{}) (<-chan disk.OperationStatusResponse, <-chan error) {
@@ -805,7 +988,13 @@ func (az *azDisksClient) Delete(resourceGroupName string, diskName string, cance
 		glog.V(10).Infof("azDisksClient.Delete(%q,%q): end", resourceGroupName, diskName)
 	}()
 
-	return az.client.Delete(resourceGroupName, diskName, cancel)
+	errChan := make(chan error, 1)
+	mc := newMetricContext("disks", "delete", resourceGroupName, az.client.SubscriptionID)
+	resultChan, proxyErrChan := az.client.Delete(resourceGroupName, diskName, cancel)
+	err := <-proxyErrChan
+	mc.Observe(err)
+	errChan <- err
+	return resultChan, errChan
 }
 
 func (az *azDisksClient) Get(resourceGroupName string, diskName string) (result disk.Model, err error) {
@@ -815,5 +1004,8 @@ func (az *azDisksClient) Get(resourceGroupName string, diskName string) (result
 		glog.V(10).Infof("azDisksClient.Get(%q,%q): end", resourceGroupName, diskName)
 	}()
 
-	return az.client.Get(resourceGroupName, diskName)
+	mc := newMetricContext("disks", "get", resourceGroupName, az.client.SubscriptionID)
+	result, err = az.client.Get(resourceGroupName, diskName)
+	mc.Observe(err)
+	return
 }

pkg/cloudprovider/providers/azure/azure_fakes.go

@@ -144,7 +144,7 @@ func (fLBC fakeAzureLBClient) List(resourceGroupName string) (result network.Loa
 	return result, nil
 }
 
-func (fLBC fakeAzureLBClient) ListNextResults(lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
+func (fLBC fakeAzureLBClient) ListNextResults(resourceGroupName string, lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
 	fLBC.mutex.Lock()
 	defer fLBC.mutex.Unlock()
 	result.Response.Response = &http.Response{
@@ -264,7 +264,7 @@ func (fAPC fakeAzurePIPClient) Get(resourceGroupName string, publicIPAddressName
 	}
 }
 
-func (fAPC fakeAzurePIPClient) ListNextResults(lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
+func (fAPC fakeAzurePIPClient) ListNextResults(resourceGroupName string, lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
 	fAPC.mutex.Lock()
 	defer fAPC.mutex.Unlock()
 	return network.PublicIPAddressListResult{}, nil
@@ -411,7 +411,7 @@ func (fVMC fakeAzureVirtualMachinesClient) List(resourceGroupName string) (resul
 	result.Value = &value
 	return result, nil
 }
-func (fVMC fakeAzureVirtualMachinesClient) ListNextResults(lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
+func (fVMC fakeAzureVirtualMachinesClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
 	fVMC.mutex.Lock()
 	defer fVMC.mutex.Unlock()
 	return compute.VirtualMachineListResult{}, nil
@@ -659,7 +659,7 @@ func (fVMC fakeVirtualMachineScaleSetVMsClient) List(resourceGroupName string, v
 	return result, nil
 }
 
-func (fVMC fakeVirtualMachineScaleSetVMsClient) ListNextResults(lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
+func (fVMC fakeVirtualMachineScaleSetVMsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
 	return result, nil
 }
 
@@ -764,7 +764,7 @@ func (fVMSSC fakeVirtualMachineScaleSetsClient) List(resourceGroupName string) (
 	return result, nil
 }
 
-func (fVMSSC fakeVirtualMachineScaleSetsClient) ListNextResults(lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
+func (fVMSSC fakeVirtualMachineScaleSetsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
 	return result, nil
 }
 

pkg/cloudprovider/providers/azure/azure_metrics.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type apiCallMetrics struct {
+	latency *prometheus.HistogramVec
+	errors  *prometheus.CounterVec
+}
+
+var (
+	metricLabels = []string{
+		"request",         // API function that is being invoked
+		"resource_group",  // Resource group of the resource being monitored
+		"subscription_id", // Subscription ID of the resource being monitored
+	}
+
+	apiMetrics = registerAPIMetrics(metricLabels...)
+)
+
+type metricContext struct {
+	start      time.Time
+	attributes []string
+}
+
+func newMetricContext(prefix, request, resouceGroup, subscriptionID string) *metricContext {
+	return &metricContext{
+		start:      time.Now(),
+		attributes: []string{prefix + "_" + request, resouceGroup, subscriptionID},
+	}
+}
+
+func (mc *metricContext) Observe(err error) {
+	apiMetrics.latency.WithLabelValues(mc.attributes...).Observe(
+		time.Since(mc.start).Seconds())
+	if err != nil {
+		apiMetrics.errors.WithLabelValues(mc.attributes...).Inc()
+	}
+}
+
+func registerAPIMetrics(attributes ...string) *apiCallMetrics {
+	metrics := &apiCallMetrics{
+		latency: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name: "cloudprovider_azure_api_request_duration_seconds",
+				Help: "Latency of an Azure API call",
+			},
+			attributes,
+		),
+		errors: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "cloudprovider_azure_api_request_errors",
+				Help: "Number of errors for an Azure API call",
+			},
+			attributes,
+		),
+	}
+
+	prometheus.MustRegister(metrics.latency)
+	prometheus.MustRegister(metrics.errors)
+
+	return metrics
+}

pkg/cloudprovider/providers/azure/azure_metrics_test.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAzureMetricLabelCardinality(t *testing.T) {
+	mc := newMetricContext("test", "create", "resource_group", "subscription_id")
+	assert.Len(t, mc.attributes, len(metricLabels), "cardinalities of labels and values must match")
+}
+
+func TestAzureMetricLabelPrefix(t *testing.T) {
+	mc := newMetricContext("prefix", "request", "resource_group", "subscription_id")
+	found := false
+	for _, attribute := range mc.attributes {
+		if attribute == "prefix_request" {
+			found = true
+		}
+	}
+	assert.True(t, found, "request label must be prefixed")
+}

pkg/cloudprovider/providers/azure/azure_util_vmss.go

@@ -422,7 +422,7 @@ func (ss *scaleSet) listScaleSetsWithRetry() ([]string, error) {
 
 		if result.NextLink != nil {
 			backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-				result, err = ss.VirtualMachineScaleSetsClient.ListNextResults(result)
+				result, err = ss.VirtualMachineScaleSetsClient.ListNextResults(ss.ResourceGroup, result)
 				if err != nil {
 					glog.Errorf("VirtualMachineScaleSetsClient.ListNextResults for %v failed: %v", ss.ResourceGroup, err)
 					return false, err
@@ -468,7 +468,7 @@ func (ss *scaleSet) listScaleSetVMsWithRetry(scaleSetName string) ([]compute.Vir
 
 		if result.NextLink != nil {
 			backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-				result, err = ss.VirtualMachineScaleSetVMsClient.ListNextResults(result)
+				result, err = ss.VirtualMachineScaleSetVMsClient.ListNextResults(ss.ResourceGroup, result)
 				if err != nil {
 					glog.Errorf("VirtualMachineScaleSetVMsClient.ListNextResults for %v failed: %v", scaleSetName, err)
 					return false, err