Merge pull request #34526 from colemickens/colemickens-azure-specify-availabilityset

Automatic merge from submit-queue

azure: filter load balancer backend nodes to PrimaryAvailabilitySet (if set)

<!--  Thanks for sending a pull request!  Here are some tips for you:
1. If this is your first time, read our contributor guidelines https://github.com/kubernetes/kubernetes/blob/master/CONTRIBUTING.md and developer guide https://github.com/kubernetes/kubernetes/blob/master/docs/devel/development.md
2. If you want *faster* PR reviews, read how: https://github.com/kubernetes/kubernetes/blob/master/docs/devel/faster_reviews.md
3. Follow the instructions for writing a release note: https://github.com/kubernetes/kubernetes/blob/master/docs/devel/pull-requests.md#release-notes
-->

**What this PR does / why we need it**:
- Adds a new field (`PrimaryAvailabilitySetName`) to the Azure CloudProvider config struct
- If the field is set, only machines who are in that availabilitySet are added to the load balancer backend pool.

This is required to:
- Support more than 100 nodes in Azure (only can have 100 nodes per availability set)
- Support multiple availability sets per cluster (An Azure L4 LoadBalancer can only be pointed at nodes in a single availability set)

Without this PR, or if the field is **not** set in a cluster that contains two availabilitysets, then the following is observed:
- Azure resources are created (LB, LB rules, NSG rules, public IP)
- Azure throws errors when trying to add nodes from the "other" availability set
- The service winds up exposed to the outside world (if you manually retrieve the public ip from Azure API)
- Kubernetes controller-manager's service loop keeps retrying forever because it never finishes fully successfully
- The "external ip" property field is never updated.

**Which issue this PR fixes**: Fixes #34293

**Unknowns**:
- Naming convention: `LoadBalancedAvailabilitySet` might be more descriptive than `PrimaryAvailabilitySet`, but is also a misnomer since `kube-proxy` will still end up routing requests to all relevant nodes.
- Is it worth trying to be "smart" about it in the case the user hasn't set this field in the config? Save the first availability set name and try not to add any nodes that aren't also in that one? It may simply be better to just let this fail so the user has to choose the right setting for their use-case.

**Release note**:
<!--  Steps to write your release note:
1. Use the release-note-* labels to set the release note state (if you have access) 
2. Enter your extended release note in the below block; leaving it blank means using the PR title as the release note. If no release note is required, just write `NONE`. 
-->
```release-note
azure: add PrimaryAvailabilitySet to config, only use nodes in that set in the loadbalancer pool
```

CC: @brendandburns, @anhowe

pkg/cloudprovider/providers/azure/azure.go

@@ -33,15 +33,16 @@ const CloudProviderName = "azure"
 
 // Config holds the configuration parsed from the --cloud-config flag
 type Config struct {
-	Cloud             string `json:"cloud" yaml:"cloud"`
-	TenantID          string `json:"tenantId" yaml:"tenantId"`
-	SubscriptionID    string `json:"subscriptionId" yaml:"subscriptionId"`
-	ResourceGroup     string `json:"resourceGroup" yaml:"resourceGroup"`
-	Location          string `json:"location" yaml:"location"`
-	VnetName          string `json:"vnetName" yaml:"vnetName"`
-	SubnetName        string `json:"subnetName" yaml:"subnetName"`
-	SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"`
-	RouteTableName    string `json:"routeTableName" yaml:"routeTableName"`
+	Cloud                      string `json:"cloud" yaml:"cloud"`
+	TenantID                   string `json:"tenantId" yaml:"tenantId"`
+	SubscriptionID             string `json:"subscriptionId" yaml:"subscriptionId"`
+	ResourceGroup              string `json:"resourceGroup" yaml:"resourceGroup"`
+	Location                   string `json:"location" yaml:"location"`
+	VnetName                   string `json:"vnetName" yaml:"vnetName"`
+	SubnetName                 string `json:"subnetName" yaml:"subnetName"`
+	SecurityGroupName          string `json:"securityGroupName" yaml:"securityGroupName"`
+	RouteTableName             string `json:"routeTableName" yaml:"routeTableName"`
+	PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName" yaml:"primaryAvailabilitySetName"`
 
 	AADClientID     string `json:"aadClientId" yaml:"aadClientId"`
 	AADClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"`

pkg/cloudprovider/providers/azure/azure_loadbalancer.go

@@ -573,6 +573,17 @@ func (az *Cloud) ensureHostInPool(serviceName string, nodeName types.NodeName, b
 		return err
 	}
 
+	// Check availability set
+	if az.PrimaryAvailabilitySetName != "" {
+		expectedAvailabilitySetName := az.getAvailabilitySetID(az.PrimaryAvailabilitySetName)
+		if !strings.EqualFold(*machine.Properties.AvailabilitySet.ID, expectedAvailabilitySetName) {
+			glog.V(1).Infof(
+				"nicupdate(%s): skipping nic (%s) since it is not in the primaryAvailabilitSet(%s)",
+				serviceName, nicName, az.PrimaryAvailabilitySetName)
+			return nil
+		}
+	}
+
 	nic, err := az.InterfacesClient.Get(az.ResourceGroup, nicName, "")
 	if err != nil {
 		return err

pkg/cloudprovider/providers/azure/azure_test.go

@@ -412,7 +412,8 @@ func TestNewCloudFromJSON(t *testing.T) {
 		"subnetName": "--subnet-name--",
 		"securityGroupName": "--security-group-name--",
 		"vnetName": "--vnet-name--",
-		"routeTableName": "--route-table-name--"
+		"routeTableName": "--route-table-name--",
+		"primaryAvailabilitySetName": "--primary-availability-set-name--"
 	}`
 	validateConfig(t, config)
 }
@@ -430,6 +431,7 @@ subnetName: --subnet-name--
 securityGroupName: --security-group-name--
 vnetName: --vnet-name--
 routeTableName: --route-table-name--
+primaryAvailabilitySetName: --primary-availability-set-name--
 `
 	validateConfig(t, config)
 }
@@ -476,6 +478,9 @@ func validateConfig(t *testing.T, config string) {
 	if azureCloud.RouteTableName != "--route-table-name--" {
 		t.Errorf("got incorrect value for RouteTableName")
 	}
+	if azureCloud.PrimaryAvailabilitySetName != "--primary-availability-set-name--" {
+		t.Errorf("got incorrect value for PrimaryAvailabilitySetName")
+	}
 }
 
 func TestDecodeInstanceInfo(t *testing.T) {

pkg/cloudprovider/providers/azure/azure_util.go

@@ -32,7 +32,8 @@ const (
 	loadBalancerMinimumPriority = 500
 	loadBalancerMaximumPriority = 4096
 
-	machineResourceIDTemplate   = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s"
+	machineIDTemplate           = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s"
+	availabilitySetIDTemplate   = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s"
 	frontendIPConfigIDTemplate  = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s"
 	backendPoolIDTemplate       = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/backendAddressPools/%s"
 	loadBalancerRuleIDTemplate  = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/loadBalancingRules/%s"
@@ -43,12 +44,21 @@ const (
 // returns the full identifier of a machine
 func (az *Cloud) getMachineID(machineName string) string {
 	return fmt.Sprintf(
-		machineResourceIDTemplate,
+		machineIDTemplate,
 		az.SubscriptionID,
 		az.ResourceGroup,
 		machineName)
 }
 
+// returns the full identifier of an availabilitySet
+func (az *Cloud) getAvailabilitySetID(availabilitySetName string) string {
+	return fmt.Sprintf(
+		availabilitySetIDTemplate,
+		az.SubscriptionID,
+		az.ResourceGroup,
+		availabilitySetName)
+}
+
 // returns the full identifier of a loadbalancer frontendipconfiguration.
 func (az *Cloud) getFrontendIPConfigID(lbName, backendPoolName string) string {
 	return fmt.Sprintf(