Merge pull request #67229 from feiskyer/unzoned-disks

Automatic merge from submit-queue (batch tested with PRs 66884, 67410, 67229, 67409). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add node affinity for Azure unzoned managed disks

**What this PR does / why we need it**:

Continue of [Azure Availability Zone feature](https://github.com/kubernetes/features/issues/586).

Add node affinity for Azure unzoned managed disks, so that unzoned disks only scheduled to unzoned nodes.

This is required because Azure doesn't allow attaching unzoned disks to zoned VMs.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

Unzoned nodes would label `failure-domain.beta.kubernetes.io/zone=0` and the value is fault domain ( while availability zone is used for zoned nodes). So fault domain is used to populate unzoned disks.

Since there are at most 3 fault domains in each region, the PR adds 3 terms for them:

```yaml
kubectl describe pv pvc-bdf93a67-9c45-11e8-ba6f-000d3a07de8c
Name:              pvc-bdf93a67-9c45-11e8-ba6f-000d3a07de8c
Labels:            <none>
Annotations:       pv.kubernetes.io/bound-by-controller=yes
                   pv.kubernetes.io/provisioned-by=kubernetes.io/azure-disk
                   volumehelper.VolumeDynamicallyCreatedByKey=azure-disk-dynamic-provisioner
Finalizers:        [kubernetes.io/pv-protection]
StorageClass:      azuredisk-unzoned
Status:            Bound
Claim:             default/unzoned-pvc
Reclaim Policy:    Delete
Access Modes:      RWO
Capacity:          5Gi
Node Affinity:     
  Required Terms:  
    Term 0:        failure-domain.beta.kubernetes.io/region in [southeastasia]
                   failure-domain.beta.kubernetes.io/zone in [0]
    Term 1:        failure-domain.beta.kubernetes.io/region in [southeastasia]
                   failure-domain.beta.kubernetes.io/zone in [1]
    Term 2:        failure-domain.beta.kubernetes.io/region in [southeastasia]
                   failure-domain.beta.kubernetes.io/zone in [2]
Message:           
Source:
    Type:         AzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)
    DiskName:     k8s-5b3d7b8f-dynamic-pvc-bdf93a67-9c45-11e8-ba6f-000d3a07de8c
    DiskURI:      /subscriptions/<subscription>/resourceGroups/<rg-name>/providers/Microsoft.Compute/disks/k8s-5b3d7b8f-dynamic-pvc-bdf93a67-9c45-11e8-ba6f-000d3a07de8c
    Kind:         Managed
    FSType:       
    CachingMode:  None
    ReadOnly:     false
Events:           <none>
```

**Release note**:

```release-note
Add node affinity for Azure unzoned managed disks
```

/sig azure
/kind feature

/cc @brendandburns @khenidak @andyzhangx @msau42

pkg/cloudprovider/providers/azure/azure.go

@@ -525,3 +525,8 @@ func (az *Cloud) GetActiveZones() (sets.String, error) {
 	}
 	return zones, nil
 }
+
+// GetLocation returns the location in which k8s cluster is currently running.
+func (az *Cloud) GetLocation() string {
+	return az.Location
+}

pkg/volume/azure_dd/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//pkg/cloudprovider:go_default_library",
         "//pkg/cloudprovider/providers/azure:go_default_library",
         "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
         "//pkg/util/keymutex:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/strings:go_default_library",

pkg/volume/azure_dd/azure_dd.go

@@ -66,6 +66,9 @@ type DiskController interface {
 
 	// GetActiveZones returns all the zones in which k8s nodes are currently running.
 	GetActiveZones() (sets.String, error)
+
+	// GetLocation returns the location in which k8s cluster is currently running.
+	GetLocation() string
 }
 
 type azureDataDiskPlugin struct {

pkg/volume/azure_dd/azure_provision.go

@@ -29,6 +29,7 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
 	"k8s.io/kubernetes/pkg/features"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 )
@@ -195,8 +196,8 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
 		}
 	}
 
-	if !zoned && (zonePresent || zonesPresent) {
-		return nil, fmt.Errorf("zone or zones StorageClass parameters must be used together with zoned parameter")
+	if !zoned && (zonePresent || zonesPresent || len(allowedTopologies) > 0) {
+		return nil, fmt.Errorf("zone, zones and allowedTopologies StorageClass parameters must be used together with zoned parameter")
 	}
 
 	if cachingMode, err = normalizeCachingMode(cachingMode); err != nil {
@@ -298,18 +299,46 @@ func (p *azureDiskProvisioner) Provision(selectedNode *v1.Node, allowedTopologie
 		},
 	}
 
-	if zoned && utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
+		nodeSelectorTerms := make([]v1.NodeSelectorTerm, 0)
+
+		if zoned {
+			// Set node affinity labels based on availability zone labels.
 			requirements := make([]v1.NodeSelectorRequirement, 0)
 			for k, v := range labels {
 				requirements = append(requirements, v1.NodeSelectorRequirement{Key: k, Operator: v1.NodeSelectorOpIn, Values: []string{v}})
 			}
 
-		nodeSelectorTerm := v1.NodeSelectorTerm{
+			nodeSelectorTerms = append(nodeSelectorTerms, v1.NodeSelectorTerm{
 				MatchExpressions: requirements,
+			})
+		} else {
+			// Set node affinity labels based on fault domains.
+			// This is required because unzoned AzureDisk can't be attached to zoned nodes.
+			// There are at most 3 fault domains available in each region.
+			// Refer https://docs.microsoft.com/en-us/azure/virtual-machines/windows/manage-availability.
+			for i := 0; i < 3; i++ {
+				requirements := []v1.NodeSelectorRequirement{
+					{
+						Key:      kubeletapis.LabelZoneRegion,
+						Operator: v1.NodeSelectorOpIn,
+						Values:   []string{diskController.GetLocation()},
+					},
+					{
+						Key:      kubeletapis.LabelZoneFailureDomain,
+						Operator: v1.NodeSelectorOpIn,
+						Values:   []string{strconv.Itoa(i)},
+					},
 				}
+				nodeSelectorTerms = append(nodeSelectorTerms, v1.NodeSelectorTerm{
+					MatchExpressions: requirements,
+				})
+			}
+		}
+
 		pv.Spec.NodeAffinity = &v1.VolumeNodeAffinity{
 			Required: &v1.NodeSelector{
-				NodeSelectorTerms: []v1.NodeSelectorTerm{nodeSelectorTerm},
+				NodeSelectorTerms: nodeSelectorTerms,
 			},
 		}
 	}