Merge pull request #67772 from andyzhangx/azuredisk-volumelimits2

Automatic merge from submit-queue (batch tested with PRs 67766, 67642, 67772). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Enable dynamic azure disk volume limits

**What this PR does / why we need it**:
Enable dynamic azure disk volume limits,
This is an azure cloud provider implementation related to feature: [Dynamic Maximum volume count](https://github.com/kubernetes/features/issues/554)

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #66269

**Special notes for your reviewer**:
This PR use `az.VirtualMachineSizesClient.List` to list all vm sizes under region, match vm size with current node size, and then got `MaxDataDiskCount`, the `GetVolumeLimits` happens in kubelet and will return `attachable-volumes-azure-disk` in node status as following example:
```
agentpool-22082114-0
    ...
    allocatable:
      attachable-volumes-azure-disk: "8"
      cpu: "2"
      ephemeral-storage: "28043041951"
      hugepages-1Gi: "0"
      hugepages-2Mi: "0"
      memory: 7034772Ki
      pods: "30"
```

**Release note**:

```
Enable dynamic azure disk volume limits
```

/sig azure
/kind feature

pkg/cloudprovider/providers/azure/azure.go

@@ -175,6 +175,9 @@ type Cloud struct {
 	VirtualMachineScaleSetsClient   VirtualMachineScaleSetsClient
 	VirtualMachineScaleSetVMsClient VirtualMachineScaleSetVMsClient
 
+	// client for vm sizes list
+	VirtualMachineSizesClient VirtualMachineSizesClient
+
 	vmCache  *timedCache
 	lbCache  *timedCache
 	nsgCache *timedCache
@@ -278,6 +281,7 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 		StorageAccountClient:            newAzStorageAccountClient(azClientConfig),
 		VirtualMachinesClient:           newAzVirtualMachinesClient(azClientConfig),
 		PublicIPAddressesClient:         newAzPublicIPAddressesClient(azClientConfig),
+		VirtualMachineSizesClient:       newAzVirtualMachineSizesClient(azClientConfig),
 		VirtualMachineScaleSetsClient:   newAzVirtualMachineScaleSetsClient(azClientConfig),
 		VirtualMachineScaleSetVMsClient: newAzVirtualMachineScaleSetVMsClient(azClientConfig),
 		FileClient:                      &azureFileClient{env: *env},

pkg/cloudprovider/providers/azure/azure_client.go

@@ -131,6 +131,11 @@ type DisksClient interface {
 	Get(ctx context.Context, resourceGroupName string, diskName string) (result compute.Disk, err error)
 }
 
+// VirtualMachineSizesClient defines needed functions for azure compute.VirtualMachineSizesClient
+type VirtualMachineSizesClient interface {
+	List(ctx context.Context, location string) (result compute.VirtualMachineSizeListResult, err error)
+}
+
 // azClientConfig contains all essential information to create an Azure client.
 type azClientConfig struct {
 	subscriptionID          string
@@ -1327,3 +1332,41 @@ func (az *azDisksClient) Get(ctx context.Context, resourceGroupName string, disk
 	mc.Observe(err)
 	return
 }
+
+// azVirtualMachineSizesClient implements VirtualMachineSizesClient.
+type azVirtualMachineSizesClient struct {
+	client            compute.VirtualMachineSizesClient
+	rateLimiterReader flowcontrol.RateLimiter
+	rateLimiterWriter flowcontrol.RateLimiter
+}
+
+func newAzVirtualMachineSizesClient(config *azClientConfig) *azVirtualMachineSizesClient {
+	VirtualMachineSizesClient := compute.NewVirtualMachineSizesClient(config.subscriptionID)
+	VirtualMachineSizesClient.BaseURI = config.resourceManagerEndpoint
+	VirtualMachineSizesClient.Authorizer = autorest.NewBearerAuthorizer(config.servicePrincipalToken)
+	VirtualMachineSizesClient.PollingDelay = 5 * time.Second
+	configureUserAgent(&VirtualMachineSizesClient.Client)
+
+	return &azVirtualMachineSizesClient{
+		rateLimiterReader: config.rateLimiterReader,
+		rateLimiterWriter: config.rateLimiterWriter,
+		client:            VirtualMachineSizesClient,
+	}
+}
+
+func (az *azVirtualMachineSizesClient) List(ctx context.Context, location string) (result compute.VirtualMachineSizeListResult, err error) {
+	if !az.rateLimiterReader.TryAccept() {
+		err = createRateLimitErr(false, "VMSizesList")
+		return
+	}
+
+	glog.V(10).Infof("azVirtualMachineSizesClient.List(%q): start", location)
+	defer func() {
+		glog.V(10).Infof("azVirtualMachineSizesClient.List(%q): end", location)
+	}()
+
+	mc := newMetricContext("vmsizes", "list", "", az.client.SubscriptionID)
+	result, err = az.client.List(ctx, location)
+	mc.Observe(err)
+	return
+}

pkg/volume/azure_dd/BUILD

@@ -75,7 +75,9 @@ go_test(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/client-go/util/testing:go_default_library",
+        "//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
     ],
 )

pkg/volume/azure_dd/azure_dd.go

@@ -17,7 +17,9 @@ limitations under the License.
 package azure_dd
 
 import (
+	"context"
 	"fmt"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute"
 	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage"
@@ -84,8 +86,12 @@ var _ volume.VolumePluginWithAttachLimits = &azureDataDiskPlugin{}
 var _ volume.ExpandableVolumePlugin = &azureDataDiskPlugin{}
 var _ volume.DeviceMountableVolumePlugin = &azureDataDiskPlugin{}
 
+// store vm size list in current region
+var vmSizeList *[]compute.VirtualMachineSize
+
 const (
 	azureDataDiskPluginName = "kubernetes.io/azure-disk"
+	defaultAzureVolumeLimit = 16
 )
 
 func ProbeVolumePlugins() []volume.VolumePlugin {
@@ -129,26 +135,66 @@ func (plugin *azureDataDiskPlugin) SupportsBulkVolumeVerification() bool {
 
 func (plugin *azureDataDiskPlugin) GetVolumeLimits() (map[string]int64, error) {
 	volumeLimits := map[string]int64{
-		util.AzureVolumeLimitKey: 16,
+		util.AzureVolumeLimitKey: defaultAzureVolumeLimit,
 	}
 
-	cloud := plugin.host.GetCloudProvider()
-
-	// if we can't fetch cloudprovider we return an error
-	// hoping external CCM or admin can set it. Returning
-	// default values from here will mean, no one can
-	// override them.
-	if cloud == nil {
-		return nil, fmt.Errorf("No cloudprovider present")
+	az, err := getCloud(plugin.host)
+	if err != nil {
+		// if we can't fetch cloudprovider we return an error
+		// hoping external CCM or admin can set it. Returning
+		// default values from here will mean, no one can
+		// override them.
+		glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName())
+		return volumeLimits, nil
 	}
 
-	if cloud.ProviderName() != azure.CloudProviderName {
-		return nil, fmt.Errorf("Expected Azure cloudprovider, got %s", cloud.ProviderName())
+	instances, ok := az.Instances()
+	if !ok {
+		glog.Warningf("Failed to get instances from cloud provider")
+		return volumeLimits, nil
+	}
+
+	instanceType, err := instances.InstanceType(context.TODO(), plugin.host.GetNodeName())
+	if err != nil {
+		glog.Errorf("Failed to get instance type from Azure cloud provider, nodeName: %s", plugin.host.GetNodeName())
+		return volumeLimits, nil
+	}
+
+	if vmSizeList == nil {
+		result, err := az.VirtualMachineSizesClient.List(context.TODO(), az.Location)
+		if err != nil || result.Value == nil {
+			glog.Errorf("failed to list vm sizes in GetVolumeLimits, plugin.host: %s, location: %s", plugin.host.GetHostName(), az.Location)
+			return volumeLimits, nil
+		}
+		vmSizeList = result.Value
+	}
+
+	volumeLimits = map[string]int64{
+		util.AzureVolumeLimitKey: getMaxDataDiskCount(instanceType, vmSizeList),
 	}
 
 	return volumeLimits, nil
 }
 
+func getMaxDataDiskCount(instanceType string, sizeList *[]compute.VirtualMachineSize) int64 {
+	if sizeList == nil {
+		return defaultAzureVolumeLimit
+	}
+
+	vmsize := strings.ToUpper(instanceType)
+	for _, size := range *sizeList {
+		if size.Name == nil || size.MaxDataDiskCount == nil {
+			glog.Errorf("failed to get vm size in getMaxDataDiskCount")
+			continue
+		}
+		if strings.ToUpper(*size.Name) == vmsize {
+			glog.V(2).Infof("got a matching size in getMaxDataDiskCount, Name: %s, MaxDataDiskCount: %s", *size.Name, *size.MaxDataDiskCount)
+			return int64(*size.MaxDataDiskCount)
+		}
+	}
+	return defaultAzureVolumeLimit
+}
+
 func (plugin *azureDataDiskPlugin) VolumeLimitKey(spec *volume.Spec) string {
 	return util.AzureVolumeLimitKey
 }

pkg/volume/azure_dd/azure_dd_test.go

@@ -20,6 +20,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-04-01/compute"
+	"github.com/Azure/go-autorest/autorest/to"
+	"github.com/stretchr/testify/assert"
+
 	"k8s.io/api/core/v1"
 	utiltesting "k8s.io/client-go/util/testing"
 	"k8s.io/kubernetes/pkg/volume"
@@ -53,3 +57,37 @@ func TestCanSupport(t *testing.T) {
 
 // fakeAzureProvider type was removed because all functions were not used
 // Testing mounting will require path calculation which depends on the cloud provider, which is faked in the above test.
+
+func TestGetMaxDataDiskCount(t *testing.T) {
+	tests := []struct {
+		instanceType string
+		sizeList     *[]compute.VirtualMachineSize
+		expectResult int64
+	}{
+		{
+			instanceType: "standard_d2_v2",
+			sizeList: &[]compute.VirtualMachineSize{
+				{Name: to.StringPtr("Standard_D2_V2"), MaxDataDiskCount: to.Int32Ptr(8)},
+				{Name: to.StringPtr("Standard_D3_V2"), MaxDataDiskCount: to.Int32Ptr(16)},
+			},
+			expectResult: 8,
+		},
+		{
+			instanceType: "NOT_EXISTING",
+			sizeList: &[]compute.VirtualMachineSize{
+				{Name: to.StringPtr("Standard_D2_V2"), MaxDataDiskCount: to.Int32Ptr(8)},
+			},
+			expectResult: defaultAzureVolumeLimit,
+		},
+		{
+			instanceType: "",
+			sizeList:     &[]compute.VirtualMachineSize{},
+			expectResult: defaultAzureVolumeLimit,
+		},
+	}
+
+	for _, test := range tests {
+		result := getMaxDataDiskCount(test.instanceType, test.sizeList)
+		assert.Equal(t, test.expectResult, result)
+	}
+}