Merge pull request #59830 from khenidak/az-ratelimit

Automatic merge from submit-queue (batch tested with PRs 59939, 59830). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Azure - ARM Read/Write rate limiting

**What this PR does / why we need it**:

Azure cloud provider currently runs with:
1. Single ARM rate limiter for both `read [put/post/delete]` and `write` operations, while ARM provide [different rates for read/write] (https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-manager-request-limits). This causes write operation to stop even if there is available write request quotas. 
2. Cloud provider uses rate limiter's `Accept()` instead of `TryAccept()` This causes control loop to wait for prolonged tike `in case of no request quota available` for **all** requests even for those does not require ARM interaction. A case for that the `Service` control loop will wait for a prolonged time trying to create `LoadBalancer` service even though it can fail and work on the next service which is `ClusterIP`. This PR moves cloud provider tp `TryAccept()`

**Which issue(s) this PR fixes**:
Fixes # https://github.com/kubernetes/kubernetes/issues/58770

**Special notes for your reviewer**:
`n/a`

**Release note**:

```release-note
- Separate current ARM rate limiter into read/write
- Improve control over how ARM rate limiter is used within Azure cloud provider
```

cc @jackfrancis (need your help carefully reviewing this one) @brendanburns @jdumars

pkg/cloudprovider/providers/azure/azure.go

@@ -97,10 +97,14 @@ type Config struct {
 	CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter" yaml:"cloudProviderBackoffJitter"`
 	// Enable rate limiting
 	CloudProviderRateLimit bool `json:"cloudProviderRateLimit" yaml:"cloudProviderRateLimit"`
-	// Rate limit QPS
+	// Rate limit QPS (Read)
 	CloudProviderRateLimitQPS float32 `json:"cloudProviderRateLimitQPS" yaml:"cloudProviderRateLimitQPS"`
 	// Rate limit Bucket Size
 	CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket" yaml:"cloudProviderRateLimitBucket"`
+	// Rate limit QPS (Write)
+	CloudProviderRateLimitQPSWrite float32 `json:"cloudProviderRateLimitQPSWrite" yaml:"cloudProviderRateLimitQPSWrite"`
+	// Rate limit Bucket Size
+	CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite" yaml:"cloudProviderRateLimitBucketWrite"`
 
 	// Use instance metadata service where possible
 	UseInstanceMetadata bool `json:"useInstanceMetadata" yaml:"useInstanceMetadata"`
@@ -168,6 +172,10 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 
 	// operationPollRateLimiter.Accept() is a no-op if rate limits are configured off.
 	operationPollRateLimiter := flowcontrol.NewFakeAlwaysRateLimiter()
+	operationPollRateLimiterWrite := flowcontrol.NewFakeAlwaysRateLimiter()
+
+	// If reader is provided (and no writer) we will
+	// use the same value for both.
 	if config.CloudProviderRateLimit {
 		// Assign rate limit defaults if no configuration was passed in
 		if config.CloudProviderRateLimitQPS == 0 {
@@ -176,19 +184,37 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 		if config.CloudProviderRateLimitBucket == 0 {
 			config.CloudProviderRateLimitBucket = rateLimitBucketDefault
 		}
+		if config.CloudProviderRateLimitQPSWrite == 0 {
+			config.CloudProviderRateLimitQPSWrite = rateLimitQPSDefault
+		}
+		if config.CloudProviderRateLimitBucketWrite == 0 {
+			config.CloudProviderRateLimitBucketWrite = rateLimitBucketDefault
+		}
+
 		operationPollRateLimiter = flowcontrol.NewTokenBucketRateLimiter(
 			config.CloudProviderRateLimitQPS,
 			config.CloudProviderRateLimitBucket)
-		glog.V(2).Infof("Azure cloudprovider using rate limit config: QPS=%g, bucket=%d",
+
+		operationPollRateLimiterWrite = flowcontrol.NewTokenBucketRateLimiter(
+			config.CloudProviderRateLimitQPSWrite,
+			config.CloudProviderRateLimitBucketWrite)
+
+		glog.V(2).Infof("Azure cloudprovider (read ops) using rate limit config: QPS=%g, bucket=%d",
 			config.CloudProviderRateLimitQPS,
 			config.CloudProviderRateLimitBucket)
+
+		glog.V(2).Infof("Azure cloudprovider (write ops) using rate limit config: QPS=%g, bucket=%d",
+			config.CloudProviderRateLimitQPSWrite,
+			config.CloudProviderRateLimitBucketWrite)
+
 	}
 
 	azClientConfig := &azClientConfig{
 		subscriptionID:          config.SubscriptionID,
 		resourceManagerEndpoint: env.ResourceManagerEndpoint,
 		servicePrincipalToken:   servicePrincipalToken,
-		rateLimiter:             operationPollRateLimiter,
+		rateLimiterReader:       operationPollRateLimiter,
+		rateLimiterWriter:       operationPollRateLimiterWrite,
 	}
 	az := Cloud{
 		Config:      *config,

pkg/cloudprovider/providers/azure/azure_client.go

@@ -18,6 +18,7 @@ package azure
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -33,6 +34,22 @@ import (
 	"k8s.io/client-go/util/flowcontrol"
 )
 
+// Helpers for rate limiting error/error channel creation
+func createARMRateLimitErr(isWrite bool, opName string) error {
+	opType := "read"
+	if isWrite {
+		opType = "write"
+	}
+	return fmt.Errorf("azure - ARM rate limited(%s) for operation:%s", opType, opName)
+}
+
+func createARMRateLimitErrChannel(isWrite bool, opName string) chan error {
+	err := createARMRateLimitErr(isWrite, opName)
+	errChan := make(chan error, 1)
+	errChan <- err
+	return errChan
+}
+
 // VirtualMachinesClient defines needed functions for azure compute.VirtualMachinesClient
 type VirtualMachinesClient interface {
 	CreateOrUpdate(resourceGroupName string, VMName string, parameters compute.VirtualMachine, cancel <-chan struct{}) (<-chan compute.VirtualMachine, <-chan error)
@@ -131,13 +148,17 @@ type azClientConfig struct {
 	subscriptionID          string
 	resourceManagerEndpoint string
 	servicePrincipalToken   *adal.ServicePrincipalToken
-	rateLimiter             flowcontrol.RateLimiter
+	// ARM Rate limiting for GET vs PUT/POST
+	//Details: https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-manager-request-limits
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 // azVirtualMachinesClient implements VirtualMachinesClient.
 type azVirtualMachinesClient struct {
-	client      compute.VirtualMachinesClient
+	client            compute.VirtualMachinesClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func getContextWithCancel() (context.Context, context.CancelFunc) {
@@ -152,13 +173,21 @@ func newAzVirtualMachinesClient(config *azClientConfig) *azVirtualMachinesClient
 	configureUserAgent(&virtualMachinesClient.Client)
 
 	return &azVirtualMachinesClient{
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
-		client:      virtualMachinesClient,
+		rateLimiterWriter: config.rateLimiterWriter,
+		client:            virtualMachinesClient,
 	}
 }
 
 func (az *azVirtualMachinesClient) CreateOrUpdate(resourceGroupName string, VMName string, parameters compute.VirtualMachine, cancel <-chan struct{}) (<-chan compute.VirtualMachine, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "NSGCreateOrUpdate")
+		resultChan := make(chan compute.VirtualMachine, 1)
+		resultChan <- compute.VirtualMachine{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): start", resourceGroupName, VMName)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachinesClient.CreateOrUpdate(%q, %q): end", resourceGroupName, VMName)
@@ -174,7 +203,11 @@ func (az *azVirtualMachinesClient) CreateOrUpdate(resourceGroupName string, VMNa
 }
 
 func (az *azVirtualMachinesClient) Get(resourceGroupName string, VMName string, expand compute.InstanceViewTypes) (result compute.VirtualMachine, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMGet")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): start", resourceGroupName, VMName)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachinesClient.Get(%q, %q): end", resourceGroupName, VMName)
@@ -187,7 +220,11 @@ func (az *azVirtualMachinesClient) Get(resourceGroupName string, VMName string,
 }
 
 func (az *azVirtualMachinesClient) List(resourceGroupName string) (result compute.VirtualMachineListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMList")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachinesClient.List(%q): start", resourceGroupName)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachinesClient.List(%q): end", resourceGroupName)
@@ -200,7 +237,11 @@ func (az *azVirtualMachinesClient) List(resourceGroupName string) (result comput
 }
 
 func (az *azVirtualMachinesClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMListNextResults")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachinesClient.ListNextResults(%q): start", lastResults)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachinesClient.ListNextResults(%q): end", lastResults)
@@ -214,8 +255,9 @@ func (az *azVirtualMachinesClient) ListNextResults(resourceGroupName string, las
 
 // azInterfacesClient implements InterfacesClient.
 type azInterfacesClient struct {
-	client      network.InterfacesClient
+	client            network.InterfacesClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzInterfacesClient(config *azClientConfig) *azInterfacesClient {
@@ -226,13 +268,21 @@ func newAzInterfacesClient(config *azClientConfig) *azInterfacesClient {
 	configureUserAgent(&interfacesClient.Client)
 
 	return &azInterfacesClient{
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
-		client:      interfacesClient,
+		rateLimiterWriter: config.rateLimiterWriter,
+		client:            interfacesClient,
 	}
 }
 
 func (az *azInterfacesClient) CreateOrUpdate(resourceGroupName string, networkInterfaceName string, parameters network.Interface, cancel <-chan struct{}) (<-chan network.Interface, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "NiCreateOrUpdate")
+		resultChan := make(chan network.Interface, 1)
+		resultChan <- network.Interface{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkInterfaceName)
 	defer func() {
 		glog.V(10).Infof("azInterfacesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkInterfaceName)
@@ -248,7 +298,11 @@ func (az *azInterfacesClient) CreateOrUpdate(resourceGroupName string, networkIn
 }
 
 func (az *azInterfacesClient) Get(resourceGroupName string, networkInterfaceName string, expand string) (result network.Interface, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "NicGet")
+		return
+	}
+
 	glog.V(10).Infof("azInterfacesClient.Get(%q,%q): start", resourceGroupName, networkInterfaceName)
 	defer func() {
 		glog.V(10).Infof("azInterfacesClient.Get(%q,%q): end", resourceGroupName, networkInterfaceName)
@@ -261,7 +315,11 @@ func (az *azInterfacesClient) Get(resourceGroupName string, networkInterfaceName
 }
 
 func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, expand string) (result network.Interface, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "NicGetVirtualMachineScaleSetNetworkInterface")
+		return
+	}
+
 	glog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName)
 	defer func() {
 		glog.V(10).Infof("azInterfacesClient.GetVirtualMachineScaleSetNetworkInterface(%q,%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName)
@@ -275,8 +333,9 @@ func (az *azInterfacesClient) GetVirtualMachineScaleSetNetworkInterface(resource
 
 // azLoadBalancersClient implements LoadBalancersClient.
 type azLoadBalancersClient struct {
-	client      network.LoadBalancersClient
+	client            network.LoadBalancersClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzLoadBalancersClient(config *azClientConfig) *azLoadBalancersClient {
@@ -287,13 +346,21 @@ func newAzLoadBalancersClient(config *azClientConfig) *azLoadBalancersClient {
 	configureUserAgent(&loadBalancerClient.Client)
 
 	return &azLoadBalancersClient{
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
-		client:      loadBalancerClient,
+		rateLimiterWriter: config.rateLimiterWriter,
+		client:            loadBalancerClient,
 	}
 }
 
 func (az *azLoadBalancersClient) CreateOrUpdate(resourceGroupName string, loadBalancerName string, parameters network.LoadBalancer, cancel <-chan struct{}) (<-chan network.LoadBalancer, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "LBCreateOrUpdate")
+		resultChan := make(chan network.LoadBalancer, 1)
+		resultChan <- network.LoadBalancer{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): start", resourceGroupName, loadBalancerName)
 	defer func() {
 		glog.V(10).Infof("azLoadBalancersClient.CreateOrUpdate(%q,%q): end", resourceGroupName, loadBalancerName)
@@ -309,7 +376,14 @@ func (az *azLoadBalancersClient) CreateOrUpdate(resourceGroupName string, loadBa
 }
 
 func (az *azLoadBalancersClient) Delete(resourceGroupName string, loadBalancerName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "LBDelete")
+		resultChan := make(chan autorest.Response, 1)
+		resultChan <- autorest.Response{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): start", resourceGroupName, loadBalancerName)
 	defer func() {
 		glog.V(10).Infof("azLoadBalancersClient.Delete(%q,%q): end", resourceGroupName, loadBalancerName)
@@ -325,7 +399,11 @@ func (az *azLoadBalancersClient) Delete(resourceGroupName string, loadBalancerNa
 }
 
 func (az *azLoadBalancersClient) Get(resourceGroupName string, loadBalancerName string, expand string) (result network.LoadBalancer, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "LBGet")
+		return
+	}
+
 	glog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): start", resourceGroupName, loadBalancerName)
 	defer func() {
 		glog.V(10).Infof("azLoadBalancersClient.Get(%q,%q): end", resourceGroupName, loadBalancerName)
@@ -338,7 +416,11 @@ func (az *azLoadBalancersClient) Get(resourceGroupName string, loadBalancerName
 }
 
 func (az *azLoadBalancersClient) List(resourceGroupName string) (result network.LoadBalancerListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "LBList")
+		return
+	}
+
 	glog.V(10).Infof("azLoadBalancersClient.List(%q): start", resourceGroupName)
 	defer func() {
 		glog.V(10).Infof("azLoadBalancersClient.List(%q): end", resourceGroupName)
@@ -351,7 +433,11 @@ func (az *azLoadBalancersClient) List(resourceGroupName string) (result network.
 }
 
 func (az *azLoadBalancersClient) ListNextResults(resourceGroupName string, lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "LBListNextResults")
+		return
+	}
+
 	glog.V(10).Infof("azLoadBalancersClient.ListNextResults(%q): start", lastResult)
 	defer func() {
 		glog.V(10).Infof("azLoadBalancersClient.ListNextResults(%q): end", lastResult)
@@ -365,8 +451,9 @@ func (az *azLoadBalancersClient) ListNextResults(resourceGroupName string, lastR
 
 // azPublicIPAddressesClient implements PublicIPAddressesClient.
 type azPublicIPAddressesClient struct {
-	client      network.PublicIPAddressesClient
+	client            network.PublicIPAddressesClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzPublicIPAddressesClient(config *azClientConfig) *azPublicIPAddressesClient {
@@ -377,13 +464,21 @@ func newAzPublicIPAddressesClient(config *azClientConfig) *azPublicIPAddressesCl
 	configureUserAgent(&publicIPAddressClient.Client)
 
 	return &azPublicIPAddressesClient{
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
-		client:      publicIPAddressClient,
+		rateLimiterWriter: config.rateLimiterWriter,
+		client:            publicIPAddressClient,
 	}
 }
 
 func (az *azPublicIPAddressesClient) CreateOrUpdate(resourceGroupName string, publicIPAddressName string, parameters network.PublicIPAddress, cancel <-chan struct{}) (<-chan network.PublicIPAddress, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "PublicIPCreateOrUpdate")
+		resultChan := make(chan network.PublicIPAddress, 1)
+		resultChan <- network.PublicIPAddress{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, publicIPAddressName)
 	defer func() {
 		glog.V(10).Infof("azPublicIPAddressesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, publicIPAddressName)
@@ -399,7 +494,14 @@ func (az *azPublicIPAddressesClient) CreateOrUpdate(resourceGroupName string, pu
 }
 
 func (az *azPublicIPAddressesClient) Delete(resourceGroupName string, publicIPAddressName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "PublicIPDelete")
+		resultChan := make(chan autorest.Response, 1)
+		resultChan <- autorest.Response{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): start", resourceGroupName, publicIPAddressName)
 	defer func() {
 		glog.V(10).Infof("azPublicIPAddressesClient.Delete(%q,%q): end", resourceGroupName, publicIPAddressName)
@@ -415,7 +517,11 @@ func (az *azPublicIPAddressesClient) Delete(resourceGroupName string, publicIPAd
 }
 
 func (az *azPublicIPAddressesClient) Get(resourceGroupName string, publicIPAddressName string, expand string) (result network.PublicIPAddress, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "PublicIPGet")
+		return
+	}
+
 	glog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): start", resourceGroupName, publicIPAddressName)
 	defer func() {
 		glog.V(10).Infof("azPublicIPAddressesClient.Get(%q,%q): end", resourceGroupName, publicIPAddressName)
@@ -428,7 +534,11 @@ func (az *azPublicIPAddressesClient) Get(resourceGroupName string, publicIPAddre
 }
 
 func (az *azPublicIPAddressesClient) List(resourceGroupName string) (result network.PublicIPAddressListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "PublicIPList")
+		return
+	}
+
 	glog.V(10).Infof("azPublicIPAddressesClient.List(%q): start", resourceGroupName)
 	defer func() {
 		glog.V(10).Infof("azPublicIPAddressesClient.List(%q): end", resourceGroupName)
@@ -441,7 +551,11 @@ func (az *azPublicIPAddressesClient) List(resourceGroupName string) (result netw
 }
 
 func (az *azPublicIPAddressesClient) ListNextResults(resourceGroupName string, lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "PublicIPListNextResults")
+		return
+	}
+
 	glog.V(10).Infof("azPublicIPAddressesClient.ListNextResults(%q): start", lastResults)
 	defer func() {
 		glog.V(10).Infof("azPublicIPAddressesClient.ListNextResults(%q): end", lastResults)
@@ -455,8 +569,9 @@ func (az *azPublicIPAddressesClient) ListNextResults(resourceGroupName string, l
 
 // azSubnetsClient implements SubnetsClient.
 type azSubnetsClient struct {
-	client      network.SubnetsClient
+	client            network.SubnetsClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzSubnetsClient(config *azClientConfig) *azSubnetsClient {
@@ -467,13 +582,21 @@ func newAzSubnetsClient(config *azClientConfig) *azSubnetsClient {
 	configureUserAgent(&subnetsClient.Client)
 
 	return &azSubnetsClient{
-		client:      subnetsClient,
+		client:            subnetsClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azSubnetsClient) CreateOrUpdate(resourceGroupName string, virtualNetworkName string, subnetName string, subnetParameters network.Subnet, cancel <-chan struct{}) (<-chan network.Subnet, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "SubnetCreateOrUpdate")
+		resultChan := make(chan network.Subnet, 1)
+		resultChan <- network.Subnet{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName)
 	defer func() {
 		glog.V(10).Infof("azSubnetsClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName)
@@ -489,7 +612,14 @@ func (az *azSubnetsClient) CreateOrUpdate(resourceGroupName string, virtualNetwo
 }
 
 func (az *azSubnetsClient) Delete(resourceGroupName string, virtualNetworkName string, subnetName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "SubnetDelete")
+		resultChan := make(chan autorest.Response, 1)
+		resultChan <- autorest.Response{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName)
 	defer func() {
 		glog.V(10).Infof("azSubnetsClient.Delete(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName)
@@ -505,7 +635,11 @@ func (az *azSubnetsClient) Delete(resourceGroupName string, virtualNetworkName s
 }
 
 func (az *azSubnetsClient) Get(resourceGroupName string, virtualNetworkName string, subnetName string, expand string) (result network.Subnet, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "SubnetGet")
+		return
+	}
+
 	glog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): start", resourceGroupName, virtualNetworkName, subnetName)
 	defer func() {
 		glog.V(10).Infof("azSubnetsClient.Get(%q,%q,%q): end", resourceGroupName, virtualNetworkName, subnetName)
@@ -518,7 +652,11 @@ func (az *azSubnetsClient) Get(resourceGroupName string, virtualNetworkName stri
 }
 
 func (az *azSubnetsClient) List(resourceGroupName string, virtualNetworkName string) (result network.SubnetListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "SubnetList")
+		return
+	}
+
 	glog.V(10).Infof("azSubnetsClient.List(%q,%q): start", resourceGroupName, virtualNetworkName)
 	defer func() {
 		glog.V(10).Infof("azSubnetsClient.List(%q,%q): end", resourceGroupName, virtualNetworkName)
@@ -532,8 +670,9 @@ func (az *azSubnetsClient) List(resourceGroupName string, virtualNetworkName str
 
 // azSecurityGroupsClient implements SecurityGroupsClient.
 type azSecurityGroupsClient struct {
-	client      network.SecurityGroupsClient
+	client            network.SecurityGroupsClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzSecurityGroupsClient(config *azClientConfig) *azSecurityGroupsClient {
@@ -544,13 +683,21 @@ func newAzSecurityGroupsClient(config *azClientConfig) *azSecurityGroupsClient {
 	configureUserAgent(&securityGroupsClient.Client)
 
 	return &azSecurityGroupsClient{
-		rateLimiter: config.rateLimiter,
+		client:            securityGroupsClient,
-		client:      securityGroupsClient,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azSecurityGroupsClient) CreateOrUpdate(resourceGroupName string, networkSecurityGroupName string, parameters network.SecurityGroup, cancel <-chan struct{}) (<-chan network.SecurityGroup, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "NSGCreateOrUpdate")
+		resultChan := make(chan network.SecurityGroup, 1)
+		resultChan <- network.SecurityGroup{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, networkSecurityGroupName)
 	defer func() {
 		glog.V(10).Infof("azSecurityGroupsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, networkSecurityGroupName)
@@ -566,7 +713,14 @@ func (az *azSecurityGroupsClient) CreateOrUpdate(resourceGroupName string, netwo
 }
 
 func (az *azSecurityGroupsClient) Delete(resourceGroupName string, networkSecurityGroupName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "NSGDelete")
+		resultChan := make(chan autorest.Response, 1)
+		resultChan <- autorest.Response{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): start", resourceGroupName, networkSecurityGroupName)
 	defer func() {
 		glog.V(10).Infof("azSecurityGroupsClient.Delete(%q,%q): end", resourceGroupName, networkSecurityGroupName)
@@ -582,7 +736,11 @@ func (az *azSecurityGroupsClient) Delete(resourceGroupName string, networkSecuri
 }
 
 func (az *azSecurityGroupsClient) Get(resourceGroupName string, networkSecurityGroupName string, expand string) (result network.SecurityGroup, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "NSGGet")
+		return
+	}
+
 	glog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): start", resourceGroupName, networkSecurityGroupName)
 	defer func() {
 		glog.V(10).Infof("azSecurityGroupsClient.Get(%q,%q): end", resourceGroupName, networkSecurityGroupName)
@@ -595,7 +753,11 @@ func (az *azSecurityGroupsClient) Get(resourceGroupName string, networkSecurityG
 }
 
 func (az *azSecurityGroupsClient) List(resourceGroupName string) (result network.SecurityGroupListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "NSGList")
+		return
+	}
+
 	glog.V(10).Infof("azSecurityGroupsClient.List(%q): start", resourceGroupName)
 	defer func() {
 		glog.V(10).Infof("azSecurityGroupsClient.List(%q): end", resourceGroupName)
@@ -609,8 +771,9 @@ func (az *azSecurityGroupsClient) List(resourceGroupName string) (result network
 
 // azVirtualMachineScaleSetsClient implements VirtualMachineScaleSetsClient.
 type azVirtualMachineScaleSetsClient struct {
-	client      computepreview.VirtualMachineScaleSetsClient
+	client            computepreview.VirtualMachineScaleSetsClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzVirtualMachineScaleSetsClient(config *azClientConfig) *azVirtualMachineScaleSetsClient {
@@ -621,13 +784,19 @@ func newAzVirtualMachineScaleSetsClient(config *azClientConfig) *azVirtualMachin
 	configureUserAgent(&virtualMachineScaleSetsClient.Client)
 
 	return &azVirtualMachineScaleSetsClient{
-		client:      virtualMachineScaleSetsClient,
+		client:            virtualMachineScaleSetsClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azVirtualMachineScaleSetsClient) CreateOrUpdate(ctx context.Context, resourceGroupName string, VMScaleSetName string, parameters computepreview.VirtualMachineScaleSet) (resp *http.Response, err error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		err = createARMRateLimitErr(true, "VMSSCreateOrUpdate")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): start", resourceGroupName, VMScaleSetName)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.CreateOrUpdate(%q,%q): end", resourceGroupName, VMScaleSetName)
@@ -646,7 +815,11 @@ func (az *azVirtualMachineScaleSetsClient) CreateOrUpdate(ctx context.Context, r
 }
 
 func (az *azVirtualMachineScaleSetsClient) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string) (result computepreview.VirtualMachineScaleSet, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMSSGet")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): start", resourceGroupName, VMScaleSetName)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.Get(%q,%q): end", resourceGroupName, VMScaleSetName)
@@ -659,7 +832,11 @@ func (az *azVirtualMachineScaleSetsClient) Get(ctx context.Context, resourceGrou
 }
 
 func (az *azVirtualMachineScaleSetsClient) List(ctx context.Context, resourceGroupName string) (result []computepreview.VirtualMachineScaleSet, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMSSList")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q,%q): start", resourceGroupName)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.List(%q,%q): end", resourceGroupName)
@@ -685,7 +862,12 @@ func (az *azVirtualMachineScaleSetsClient) List(ctx context.Context, resourceGro
 }
 
 func (az *azVirtualMachineScaleSetsClient) UpdateInstances(ctx context.Context, resourceGroupName string, VMScaleSetName string, VMInstanceIDs computepreview.VirtualMachineScaleSetVMInstanceRequiredIDs) (resp *http.Response, err error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		err = createARMRateLimitErr(true, "VMSSUpdateInstances")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%q): start", resourceGroupName, VMScaleSetName, VMInstanceIDs)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetsClient.UpdateInstances(%q,%q,%q): end", resourceGroupName, VMScaleSetName, VMInstanceIDs)
@@ -705,8 +887,9 @@ func (az *azVirtualMachineScaleSetsClient) UpdateInstances(ctx context.Context,
 
 // azVirtualMachineScaleSetVMsClient implements VirtualMachineScaleSetVMsClient.
 type azVirtualMachineScaleSetVMsClient struct {
-	client      computepreview.VirtualMachineScaleSetVMsClient
+	client            computepreview.VirtualMachineScaleSetVMsClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzVirtualMachineScaleSetVMsClient(config *azClientConfig) *azVirtualMachineScaleSetVMsClient {
@@ -717,13 +900,18 @@ func newAzVirtualMachineScaleSetVMsClient(config *azClientConfig) *azVirtualMach
 	configureUserAgent(&virtualMachineScaleSetVMsClient.Client)
 
 	return &azVirtualMachineScaleSetVMsClient{
-		client:      virtualMachineScaleSetVMsClient,
+		client:            virtualMachineScaleSetVMsClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azVirtualMachineScaleSetVMsClient) Get(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result computepreview.VirtualMachineScaleSetVM, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMSSGet")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Get(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID)
@@ -736,7 +924,11 @@ func (az *azVirtualMachineScaleSetVMsClient) Get(ctx context.Context, resourceGr
 }
 
 func (az *azVirtualMachineScaleSetVMsClient) GetInstanceView(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string) (result computepreview.VirtualMachineScaleSetVMInstanceView, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMSSGetInstanceView")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.GetInstanceView(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID)
@@ -749,7 +941,11 @@ func (az *azVirtualMachineScaleSetVMsClient) GetInstanceView(ctx context.Context
 }
 
 func (az *azVirtualMachineScaleSetVMsClient) List(ctx context.Context, resourceGroupName string, virtualMachineScaleSetName string, filter string, selectParameter string, expand string) (result []computepreview.VirtualMachineScaleSetVM, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "VMSSList")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): start", resourceGroupName, virtualMachineScaleSetName, filter)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.List(%q,%q,%q): end", resourceGroupName, virtualMachineScaleSetName, filter)
@@ -775,7 +971,11 @@ func (az *azVirtualMachineScaleSetVMsClient) List(ctx context.Context, resourceG
 }
 
 func (az *azVirtualMachineScaleSetVMsClient) Update(ctx context.Context, resourceGroupName string, VMScaleSetName string, instanceID string, parameters computepreview.VirtualMachineScaleSetVM) (resp *http.Response, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterWriter.TryAccept() {
+		err = createARMRateLimitErr(true, "VMSSUpdate")
+		return
+	}
+
 	glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Update(%q,%q,%q): start", resourceGroupName, VMScaleSetName, instanceID)
 	defer func() {
 		glog.V(10).Infof("azVirtualMachineScaleSetVMsClient.Update(%q,%q,%q): end", resourceGroupName, VMScaleSetName, instanceID)
@@ -795,8 +995,9 @@ func (az *azVirtualMachineScaleSetVMsClient) Update(ctx context.Context, resourc
 
 // azRoutesClient implements RoutesClient.
 type azRoutesClient struct {
-	client      network.RoutesClient
+	client            network.RoutesClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzRoutesClient(config *azClientConfig) *azRoutesClient {
@@ -807,13 +1008,21 @@ func newAzRoutesClient(config *azClientConfig) *azRoutesClient {
 	configureUserAgent(&routesClient.Client)
 
 	return &azRoutesClient{
-		client:      routesClient,
+		client:            routesClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azRoutesClient) CreateOrUpdate(resourceGroupName string, routeTableName string, routeName string, routeParameters network.Route, cancel <-chan struct{}) (<-chan network.Route, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "RouteCreateOrUpdate")
+		resultChan := make(chan network.Route, 1)
+		resultChan <- network.Route{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): start", resourceGroupName, routeTableName, routeName)
 	defer func() {
 		glog.V(10).Infof("azRoutesClient.CreateOrUpdate(%q,%q,%q): end", resourceGroupName, routeTableName, routeName)
@@ -829,7 +1038,14 @@ func (az *azRoutesClient) CreateOrUpdate(resourceGroupName string, routeTableNam
 }
 
 func (az *azRoutesClient) Delete(resourceGroupName string, routeTableName string, routeName string, cancel <-chan struct{}) (<-chan autorest.Response, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "RouteDelete")
+		resultChan := make(chan autorest.Response, 1)
+		resultChan <- autorest.Response{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): start", resourceGroupName, routeTableName, routeName)
 	defer func() {
 		glog.V(10).Infof("azRoutesClient.Delete(%q,%q,%q): end", resourceGroupName, routeTableName, routeName)
@@ -846,8 +1062,9 @@ func (az *azRoutesClient) Delete(resourceGroupName string, routeTableName string
 
 // azRouteTablesClient implements RouteTablesClient.
 type azRouteTablesClient struct {
-	client      network.RouteTablesClient
+	client            network.RouteTablesClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzRouteTablesClient(config *azClientConfig) *azRouteTablesClient {
@@ -858,13 +1075,21 @@ func newAzRouteTablesClient(config *azClientConfig) *azRouteTablesClient {
 	configureUserAgent(&routeTablesClient.Client)
 
 	return &azRouteTablesClient{
-		client:      routeTablesClient,
+		client:            routeTablesClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azRouteTablesClient) CreateOrUpdate(resourceGroupName string, routeTableName string, parameters network.RouteTable, cancel <-chan struct{}) (<-chan network.RouteTable, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "RouteTableCreateOrUpdate")
+		resultChan := make(chan network.RouteTable, 1)
+		resultChan <- network.RouteTable{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): start", resourceGroupName, routeTableName)
 	defer func() {
 		glog.V(10).Infof("azRouteTablesClient.CreateOrUpdate(%q,%q): end", resourceGroupName, routeTableName)
@@ -880,7 +1105,11 @@ func (az *azRouteTablesClient) CreateOrUpdate(resourceGroupName string, routeTab
 }
 
 func (az *azRouteTablesClient) Get(resourceGroupName string, routeTableName string, expand string) (result network.RouteTable, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "GetRouteTable")
+		return
+	}
+
 	glog.V(10).Infof("azRouteTablesClient.Get(%q,%q): start", resourceGroupName, routeTableName)
 	defer func() {
 		glog.V(10).Infof("azRouteTablesClient.Get(%q,%q): end", resourceGroupName, routeTableName)
@@ -894,8 +1123,9 @@ func (az *azRouteTablesClient) Get(resourceGroupName string, routeTableName stri
 
 // azStorageAccountClient implements StorageAccountClient.
 type azStorageAccountClient struct {
-	client      storage.AccountsClient
+	client            storage.AccountsClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzStorageAccountClient(config *azClientConfig) *azStorageAccountClient {
@@ -905,13 +1135,21 @@ func newAzStorageAccountClient(config *azClientConfig) *azStorageAccountClient {
 	configureUserAgent(&storageAccountClient.Client)
 
 	return &azStorageAccountClient{
-		client:      storageAccountClient,
+		client:            storageAccountClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azStorageAccountClient) Create(resourceGroupName string, accountName string, parameters storage.AccountCreateParameters, cancel <-chan struct{}) (<-chan storage.Account, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "StorageAccountCreate")
+		resultChan := make(chan storage.Account, 1)
+		resultChan <- storage.Account{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azStorageAccountClient.Create(%q,%q): start", resourceGroupName, accountName)
 	defer func() {
 		glog.V(10).Infof("azStorageAccountClient.Create(%q,%q): end", resourceGroupName, accountName)
@@ -927,7 +1165,11 @@ func (az *azStorageAccountClient) Create(resourceGroupName string, accountName s
 }
 
 func (az *azStorageAccountClient) Delete(resourceGroupName string, accountName string) (result autorest.Response, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "DeleteStorageAccount")
+		return
+	}
+
 	glog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): start", resourceGroupName, accountName)
 	defer func() {
 		glog.V(10).Infof("azStorageAccountClient.Delete(%q,%q): end", resourceGroupName, accountName)
@@ -940,7 +1182,11 @@ func (az *azStorageAccountClient) Delete(resourceGroupName string, accountName s
 }
 
 func (az *azStorageAccountClient) ListKeys(resourceGroupName string, accountName string) (result storage.AccountListKeysResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "ListStorageAccountKeys")
+		return
+	}
+
 	glog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): start", resourceGroupName, accountName)
 	defer func() {
 		glog.V(10).Infof("azStorageAccountClient.ListKeys(%q,%q): end", resourceGroupName, accountName)
@@ -953,7 +1199,11 @@ func (az *azStorageAccountClient) ListKeys(resourceGroupName string, accountName
 }
 
 func (az *azStorageAccountClient) ListByResourceGroup(resourceGroupName string) (result storage.AccountListResult, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "ListStorageAccountsByResourceGroup")
+		return
+	}
+
 	glog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): start", resourceGroupName)
 	defer func() {
 		glog.V(10).Infof("azStorageAccountClient.ListByResourceGroup(%q): end", resourceGroupName)
@@ -966,7 +1216,11 @@ func (az *azStorageAccountClient) ListByResourceGroup(resourceGroupName string)
 }
 
 func (az *azStorageAccountClient) GetProperties(resourceGroupName string, accountName string) (result storage.Account, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "GetStorageAccount/Properties")
+		return
+	}
+
 	glog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): start", resourceGroupName, accountName)
 	defer func() {
 		glog.V(10).Infof("azStorageAccountClient.GetProperties(%q,%q): end", resourceGroupName, accountName)
@@ -980,8 +1234,9 @@ func (az *azStorageAccountClient) GetProperties(resourceGroupName string, accoun
 
 // azDisksClient implements DisksClient.
 type azDisksClient struct {
-	client      disk.DisksClient
+	client            disk.DisksClient
-	rateLimiter flowcontrol.RateLimiter
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
 }
 
 func newAzDisksClient(config *azClientConfig) *azDisksClient {
@@ -991,13 +1246,21 @@ func newAzDisksClient(config *azClientConfig) *azDisksClient {
 	configureUserAgent(&disksClient.Client)
 
 	return &azDisksClient{
-		client:      disksClient,
+		client:            disksClient,
-		rateLimiter: config.rateLimiter,
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
 	}
 }
 
 func (az *azDisksClient) CreateOrUpdate(resourceGroupName string, diskName string, diskParameter disk.Model, cancel <-chan struct{}) (<-chan disk.Model, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "DiskCreateOrUpdate")
+		resultChan := make(chan disk.Model, 1)
+		resultChan <- disk.Model{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): start", resourceGroupName, diskName)
 	defer func() {
 		glog.V(10).Infof("azDisksClient.CreateOrUpdate(%q,%q): end", resourceGroupName, diskName)
@@ -1013,7 +1276,14 @@ func (az *azDisksClient) CreateOrUpdate(resourceGroupName string, diskName strin
 }
 
 func (az *azDisksClient) Delete(resourceGroupName string, diskName string, cancel <-chan struct{}) (<-chan disk.OperationStatusResponse, <-chan error) {
-	az.rateLimiter.Accept()
+	/* Write rate limiting */
+	if !az.rateLimiterWriter.TryAccept() {
+		errChan := createARMRateLimitErrChannel(true, "DiskDelete")
+		resultChan := make(chan disk.OperationStatusResponse, 1)
+		resultChan <- disk.OperationStatusResponse{}
+		return resultChan, errChan
+	}
+
 	glog.V(10).Infof("azDisksClient.Delete(%q,%q): start", resourceGroupName, diskName)
 	defer func() {
 		glog.V(10).Infof("azDisksClient.Delete(%q,%q): end", resourceGroupName, diskName)
@@ -1029,7 +1299,11 @@ func (az *azDisksClient) Delete(resourceGroupName string, diskName string, cance
 }
 
 func (az *azDisksClient) Get(resourceGroupName string, diskName string) (result disk.Model, err error) {
-	az.rateLimiter.Accept()
+	if !az.rateLimiterReader.TryAccept() {
+		err = createARMRateLimitErr(false, "GetDisk")
+		return
+	}
+
 	glog.V(10).Infof("azDisksClient.Get(%q,%q): start", resourceGroupName, diskName)
 	defer func() {
 		glog.V(10).Infof("azDisksClient.Get(%q,%q): end", resourceGroupName, diskName)