diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go index 24d2f0a70c..b734ddd68e 100644 --- a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -247,6 +247,33 @@ func TestVolumeCountConflicts(t *testing.T) { }, }, } + twoVolCinderPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + Cinder: &v1.CinderVolumeSource{VolumeID: "tvp1"}, + }, + }, + { + VolumeSource: v1.VolumeSource{ + Cinder: &v1.CinderVolumeSource{VolumeID: "tvp2"}, + }, + }, + }, + }, + } + oneVolCinderPod := &v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + Cinder: &v1.CinderVolumeSource{VolumeID: "ovp"}, + }, + }, + }, + }, + } tests := []struct { newPod *v1.Pod @@ -739,6 +766,23 @@ func TestVolumeCountConflicts(t *testing.T) { fits: true, test: "two different unbound PVCs are counted towards the PV limit as two volumes", }, + // filterName:CinderVolumeFilterType + { + newPod: oneVolCinderPod, + existingPods: []*v1.Pod{twoVolCinderPod}, + filterName: CinderVolumeFilterType, + maxVols: 4, + fits: true, + test: "fits when node capacity >= new pod's Cinder volumes", + }, + { + newPod: oneVolCinderPod, + existingPods: []*v1.Pod{twoVolCinderPod}, + filterName: CinderVolumeFilterType, + maxVols: 2, + fits: false, + test: "not fit when node capacity < new pod's Cinder volumes", + }, } expectedFailureReasons := []PredicateFailureReason{ErrMaxVolumeCountExceeded} @@ -916,6 +960,8 @@ func getVolumeLimitKey(filterType string) v1.ResourceName { return v1.ResourceName(volumeutil.GCEVolumeLimitKey) case AzureDiskVolumeFilterType: return v1.ResourceName(volumeutil.AzureVolumeLimitKey) + case CinderVolumeFilterType: + return v1.ResourceName(volumeutil.CinderVolumeLimitKey) default: return v1.ResourceName(volumeutil.GetCSIAttachLimitKey(filterType)) } diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 3fcc02b0ca..8c4efb3283 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -84,6 +84,8 @@ const ( MaxGCEPDVolumeCountPred = "MaxGCEPDVolumeCount" // MaxAzureDiskVolumeCountPred defines the name of predicate MaxAzureDiskVolumeCount. MaxAzureDiskVolumeCountPred = "MaxAzureDiskVolumeCount" + // MaxCinderVolumeCountPred defines the name of predicate MaxCinderDiskVolumeCount. + MaxCinderVolumeCountPred = "MaxCinderVolumeCount" // MaxCSIVolumeCountPred defines the predicate that decides how many CSI volumes should be attached MaxCSIVolumeCountPred = "MaxCSIVolumeCountPred" // NoVolumeZoneConflictPred defines the name of predicate NoVolumeZoneConflict. @@ -112,6 +114,8 @@ const ( GCEPDVolumeFilterType = "GCE" // AzureDiskVolumeFilterType defines the filter name for AzureDiskVolumeFilter. AzureDiskVolumeFilterType = "AzureDisk" + // CinderVolumeFilterType defines the filter name for CinderVolumeFilter. + CinderVolumeFilterType = "Cinder" ) // IMPORTANT NOTE for predicate developers: @@ -133,7 +137,7 @@ var ( MatchNodeSelectorPred, PodFitsResourcesPred, NoDiskConflictPred, PodToleratesNodeTaintsPred, PodToleratesNodeNoExecuteTaintsPred, CheckNodeLabelPresencePred, CheckServiceAffinityPred, MaxEBSVolumeCountPred, MaxGCEPDVolumeCountPred, MaxCSIVolumeCountPred, - MaxAzureDiskVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred, + MaxAzureDiskVolumeCountPred, MaxCinderVolumeCountPred, CheckVolumeBindingPred, NoVolumeZoneConflictPred, CheckNodeMemoryPressurePred, CheckNodePIDPressurePred, CheckNodeDiskPressurePred, MatchInterPodAffinityPred} ) @@ -332,6 +336,9 @@ func NewMaxPDVolumeCountPredicate( case AzureDiskVolumeFilterType: filter = AzureDiskVolumeFilter volumeLimitKey = v1.ResourceName(volumeutil.AzureVolumeLimitKey) + case CinderVolumeFilterType: + filter = CinderVolumeFilter + volumeLimitKey = v1.ResourceName(volumeutil.CinderVolumeLimitKey) default: klog.Fatalf("Wrong filterName, Only Support %v %v %v ", EBSVolumeFilterType, GCEPDVolumeFilterType, AzureDiskVolumeFilterType) @@ -370,6 +377,8 @@ func getMaxVolumeFunc(filterName string) func(node *v1.Node) int { return DefaultMaxGCEPDVolumes case AzureDiskVolumeFilterType: return DefaultMaxAzureDiskVolumes + case CinderVolumeFilterType: + return volumeutil.DefaultMaxCinderVolumes default: return -1 } @@ -558,6 +567,24 @@ var AzureDiskVolumeFilter = VolumeFilter{ }, } +// CinderVolumeFilter is a VolumeFilter for filtering Cinder Volumes +// It will be deprecated once Openstack cloudprovider has been removed from in-tree. +var CinderVolumeFilter = VolumeFilter{ + FilterVolume: func(vol *v1.Volume) (string, bool) { + if vol.Cinder != nil { + return vol.Cinder.VolumeID, true + } + return "", false + }, + + FilterPersistentVolume: func(pv *v1.PersistentVolume) (string, bool) { + if pv.Spec.Cinder != nil { + return pv.Spec.Cinder.VolumeID, true + } + return "", false + }, +} + // VolumeZoneChecker contains information to check the volume zone for a predicate. type VolumeZoneChecker struct { pvInfo PersistentVolumeInfo diff --git a/pkg/scheduler/algorithmprovider/defaults/register_predicates.go b/pkg/scheduler/algorithmprovider/defaults/register_predicates.go index 4f93f83da2..3abf397c17 100644 --- a/pkg/scheduler/algorithmprovider/defaults/register_predicates.go +++ b/pkg/scheduler/algorithmprovider/defaults/register_predicates.go @@ -84,6 +84,13 @@ func init() { return predicates.NewCSIMaxVolumeLimitPredicate(args.PVInfo, args.PVCInfo) }, ) + factory.RegisterFitPredicateFactory( + predicates.MaxCinderVolumeCountPred, + func(args factory.PluginFactoryArgs) predicates.FitPredicate { + return predicates.NewMaxPDVolumeCountPredicate(predicates.CinderVolumeFilterType, args.PVInfo, args.PVCInfo) + }, + ) + // Fit is determined by inter-pod affinity. factory.RegisterFitPredicateFactory( predicates.MatchInterPodAffinityPred, diff --git a/pkg/scheduler/api/compatibility/compatibility_test.go b/pkg/scheduler/api/compatibility/compatibility_test.go index 9a04f19935..7f29c93ed3 100644 --- a/pkg/scheduler/api/compatibility/compatibility_test.go +++ b/pkg/scheduler/api/compatibility/compatibility_test.go @@ -937,6 +937,130 @@ func TestCompatibility_v1_Scheduler(t *testing.T) { }}, }, }, + "1.14": { + JSON: `{ + "kind": "Policy", + "apiVersion": "v1", + "predicates": [ + {"name": "MatchNodeSelector"}, + {"name": "PodFitsResources"}, + {"name": "PodFitsHostPorts"}, + {"name": "HostName"}, + {"name": "NoDiskConflict"}, + {"name": "NoVolumeZoneConflict"}, + {"name": "PodToleratesNodeTaints"}, + {"name": "CheckNodeMemoryPressure"}, + {"name": "CheckNodeDiskPressure"}, + {"name": "CheckNodePIDPressure"}, + {"name": "CheckNodeCondition"}, + {"name": "MaxEBSVolumeCount"}, + {"name": "MaxGCEPDVolumeCount"}, + {"name": "MaxAzureDiskVolumeCount"}, + {"name": "MaxCSIVolumeCountPred"}, + {"name": "MaxCinderVolumeCount"}, + {"name": "MatchInterPodAffinity"}, + {"name": "GeneralPredicates"}, + {"name": "CheckVolumeBinding"}, + {"name": "TestServiceAffinity", "argument": {"serviceAffinity" : {"labels" : ["region"]}}}, + {"name": "TestLabelsPresence", "argument": {"labelsPresence" : {"labels" : ["foo"], "presence":true}}} + ],"priorities": [ + {"name": "EqualPriority", "weight": 2}, + {"name": "ImageLocalityPriority", "weight": 2}, + {"name": "LeastRequestedPriority", "weight": 2}, + {"name": "BalancedResourceAllocation", "weight": 2}, + {"name": "SelectorSpreadPriority", "weight": 2}, + {"name": "NodePreferAvoidPodsPriority", "weight": 2}, + {"name": "NodeAffinityPriority", "weight": 2}, + {"name": "TaintTolerationPriority", "weight": 2}, + {"name": "InterPodAffinityPriority", "weight": 2}, + {"name": "MostRequestedPriority", "weight": 2}, + { + "name": "RequestedToCapacityRatioPriority", + "weight": 2, + "argument": { + "requestedToCapacityRatioArguments": { + "shape": [ + {"utilization": 0, "score": 0}, + {"utilization": 50, "score": 7} + ] + } + }} + ],"extenders": [{ + "urlPrefix": "/prefix", + "filterVerb": "filter", + "prioritizeVerb": "prioritize", + "weight": 1, + "bindVerb": "bind", + "enableHttps": true, + "tlsConfig": {"Insecure":true}, + "httpTimeout": 1, + "nodeCacheCapable": true, + "managedResources": [{"name":"example.com/foo","ignoredByScheduler":true}], + "ignorable":true + }] + }`, + ExpectedPolicy: schedulerapi.Policy{ + Predicates: []schedulerapi.PredicatePolicy{ + {Name: "MatchNodeSelector"}, + {Name: "PodFitsResources"}, + {Name: "PodFitsHostPorts"}, + {Name: "HostName"}, + {Name: "NoDiskConflict"}, + {Name: "NoVolumeZoneConflict"}, + {Name: "PodToleratesNodeTaints"}, + {Name: "CheckNodeMemoryPressure"}, + {Name: "CheckNodeDiskPressure"}, + {Name: "CheckNodePIDPressure"}, + {Name: "CheckNodeCondition"}, + {Name: "MaxEBSVolumeCount"}, + {Name: "MaxGCEPDVolumeCount"}, + {Name: "MaxAzureDiskVolumeCount"}, + {Name: "MaxCSIVolumeCountPred"}, + {Name: "MaxCinderVolumeCount"}, + {Name: "MatchInterPodAffinity"}, + {Name: "GeneralPredicates"}, + {Name: "CheckVolumeBinding"}, + {Name: "TestServiceAffinity", Argument: &schedulerapi.PredicateArgument{ServiceAffinity: &schedulerapi.ServiceAffinity{Labels: []string{"region"}}}}, + {Name: "TestLabelsPresence", Argument: &schedulerapi.PredicateArgument{LabelsPresence: &schedulerapi.LabelsPresence{Labels: []string{"foo"}, Presence: true}}}, + }, + Priorities: []schedulerapi.PriorityPolicy{ + {Name: "EqualPriority", Weight: 2}, + {Name: "ImageLocalityPriority", Weight: 2}, + {Name: "LeastRequestedPriority", Weight: 2}, + {Name: "BalancedResourceAllocation", Weight: 2}, + {Name: "SelectorSpreadPriority", Weight: 2}, + {Name: "NodePreferAvoidPodsPriority", Weight: 2}, + {Name: "NodeAffinityPriority", Weight: 2}, + {Name: "TaintTolerationPriority", Weight: 2}, + {Name: "InterPodAffinityPriority", Weight: 2}, + {Name: "MostRequestedPriority", Weight: 2}, + { + Name: "RequestedToCapacityRatioPriority", + Weight: 2, + Argument: &schedulerapi.PriorityArgument{ + RequestedToCapacityRatioArguments: &schedulerapi.RequestedToCapacityRatioArguments{ + UtilizationShape: []schedulerapi.UtilizationShapePoint{ + {Utilization: 0, Score: 0}, + {Utilization: 50, Score: 7}, + }}, + }, + }, + }, + ExtenderConfigs: []schedulerapi.ExtenderConfig{{ + URLPrefix: "/prefix", + FilterVerb: "filter", + PrioritizeVerb: "prioritize", + Weight: 1, + BindVerb: "bind", // 1.11 restored case-sensitivity, but allowed either "BindVerb" or "bindVerb" + EnableHTTPS: true, + TLSConfig: &schedulerapi.ExtenderTLSConfig{Insecure: true}, + HTTPTimeout: 1, + NodeCacheCapable: true, + ManagedResources: []schedulerapi.ExtenderManagedResource{{Name: v1.ResourceName("example.com/foo"), IgnoredByScheduler: true}}, + Ignorable: true, + }}, + }, + }, } registeredPredicates := sets.NewString(factory.ListRegisteredFitPredicates()...) diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go index 7b88f231a2..afdabceae9 100644 --- a/pkg/scheduler/factory/factory.go +++ b/pkg/scheduler/factory/factory.go @@ -73,7 +73,7 @@ var ( matchInterPodAffinitySet = sets.NewString(predicates.MatchInterPodAffinityPred) generalPredicatesSets = sets.NewString(predicates.GeneralPred) noDiskConflictSet = sets.NewString(predicates.NoDiskConflictPred) - maxPDVolumeCountPredicateKeys = []string{predicates.MaxGCEPDVolumeCountPred, predicates.MaxAzureDiskVolumeCountPred, predicates.MaxEBSVolumeCountPred} + maxPDVolumeCountPredicateKeys = []string{predicates.MaxGCEPDVolumeCountPred, predicates.MaxAzureDiskVolumeCountPred, predicates.MaxEBSVolumeCountPred, predicates.MaxCinderVolumeCountPred} ) // Binder knows how to write a binding. diff --git a/pkg/volume/cinder/cinder.go b/pkg/volume/cinder/cinder.go index e4e488b8d2..1cc38d4f7a 100644 --- a/pkg/volume/cinder/cinder.go +++ b/pkg/volume/cinder/cinder.go @@ -120,6 +120,32 @@ func (plugin *cinderPlugin) SupportsBulkVolumeVerification() bool { return false } +var _ volume.VolumePluginWithAttachLimits = &cinderPlugin{} + +func (plugin *cinderPlugin) GetVolumeLimits() (map[string]int64, error) { + volumeLimits := map[string]int64{ + util.CinderVolumeLimitKey: util.DefaultMaxCinderVolumes, + } + cloud := plugin.host.GetCloudProvider() + + // if we can't fetch cloudprovider we return an error + // hoping external CCM or admin can set it. Returning + // default values from here will mean, no one can + // override them. + if cloud == nil { + return nil, fmt.Errorf("No cloudprovider present") + } + + if cloud.ProviderName() != openstack.ProviderName { + return nil, fmt.Errorf("Expected Openstack cloud, found %s", cloud.ProviderName()) + } + return volumeLimits, nil +} + +func (plugin *cinderPlugin) VolumeLimitKey(spec *volume.Spec) string { + return util.CinderVolumeLimitKey +} + func (plugin *cinderPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode { return []v1.PersistentVolumeAccessMode{ v1.ReadWriteOnce, diff --git a/pkg/volume/util/attach_limit.go b/pkg/volume/util/attach_limit.go index 8325dbf755..943357b653 100644 --- a/pkg/volume/util/attach_limit.go +++ b/pkg/volume/util/attach_limit.go @@ -40,6 +40,13 @@ const ( // GCEVolumeLimitKey stores resource name that will store volume limits for GCE node GCEVolumeLimitKey = "attachable-volumes-gce-pd" + // CinderVolumeLimitKey contains Volume limit key for Cinder + CinderVolumeLimitKey = "attachable-volumes-cinder" + // DefaultMaxCinderVolumes defines the maximum number of PD Volumes for Cinder + // For Openstack we are keeping this to a high enough value so as depending on backend + // cluster admins can configure it. + DefaultMaxCinderVolumes = 256 + // CSIAttachLimitPrefix defines prefix used for CSI volumes CSIAttachLimitPrefix = "attachable-volumes-csi-"