From d85daf0f4c40c9d0f89bf75a93db7622d389443f Mon Sep 17 00:00:00 2001 From: Stephen Cuppett Date: Fri, 31 Aug 2018 09:06:16 -0400 Subject: [PATCH] Resolves #59015, extends existing regex to cover t3, r5(d) & z1d instance types From current AWS documentation: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html T3, C5, C5d, M5, M5d, R5, R5d, and z1d instances support a maximum of 28 attachments, and every instance has at least one network interface attachment. If you have no additional network interface attachments on these instances, you could attach 27 EBS volumes. --- .../max_attachable_volume_predicate_test.go | 57 ++++++++++++++++++- .../algorithm/predicates/predicates.go | 12 +--- pkg/volume/aws_ebs/aws_ebs.go | 6 +- pkg/volume/util/attach_limit.go | 8 +++ 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go index 57a3fcca9b..11434273aa 100644 --- a/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go +++ b/pkg/scheduler/algorithm/predicates/max_attachable_volume_predicate_test.go @@ -827,7 +827,7 @@ func getFakePVCInfo(filterName string) FakePersistentVolumeClaimInfo { } } -func TestMaxVolumeFunc(t *testing.T) { +func TestMaxVolumeFuncM5(t *testing.T) { node := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node-for-m5-instance", @@ -839,8 +839,59 @@ func TestMaxVolumeFunc(t *testing.T) { os.Unsetenv(KubeMaxPDVols) maxVolumeFunc := getMaxVolumeFunc(EBSVolumeFilterType) maxVolume := maxVolumeFunc(node) - if maxVolume != DefaultMaxEBSM5VolumeLimit { - t.Errorf("Expected max volume to be %d got %d", DefaultMaxEBSM5VolumeLimit, maxVolume) + if maxVolume != volumeutil.DefaultMaxEBSNitroVolumeLimit { + t.Errorf("Expected max volume to be %d got %d", volumeutil.DefaultMaxEBSNitroVolumeLimit, maxVolume) + } +} + +func TestMaxVolumeFuncT3(t *testing.T) { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-for-t3-instance", + Labels: map[string]string{ + kubeletapis.LabelInstanceType: "t3.medium", + }, + }, + } + os.Unsetenv(KubeMaxPDVols) + maxVolumeFunc := getMaxVolumeFunc(EBSVolumeFilterType) + maxVolume := maxVolumeFunc(node) + if maxVolume != volumeutil.DefaultMaxEBSNitroVolumeLimit { + t.Errorf("Expected max volume to be %d got %d", volumeutil.DefaultMaxEBSNitroVolumeLimit, maxVolume) + } +} + +func TestMaxVolumeFuncR5(t *testing.T) { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-for-r5-instance", + Labels: map[string]string{ + kubeletapis.LabelInstanceType: "r5d.xlarge", + }, + }, + } + os.Unsetenv(KubeMaxPDVols) + maxVolumeFunc := getMaxVolumeFunc(EBSVolumeFilterType) + maxVolume := maxVolumeFunc(node) + if maxVolume != volumeutil.DefaultMaxEBSNitroVolumeLimit { + t.Errorf("Expected max volume to be %d got %d", volumeutil.DefaultMaxEBSNitroVolumeLimit, maxVolume) + } +} + +func TestMaxVolumeFuncM4(t *testing.T) { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-for-m4-instance", + Labels: map[string]string{ + kubeletapis.LabelInstanceType: "m4.2xlarge", + }, + }, + } + os.Unsetenv(KubeMaxPDVols) + maxVolumeFunc := getMaxVolumeFunc(EBSVolumeFilterType) + maxVolume := maxVolumeFunc(node) + if maxVolume != volumeutil.DefaultMaxEBSVolumes { + t.Errorf("Expected max volume to be %d got %d", volumeutil.DefaultMaxEBSVolumes, maxVolume) } } diff --git a/pkg/scheduler/algorithm/predicates/predicates.go b/pkg/scheduler/algorithm/predicates/predicates.go index 61e7fb5343..4ba810bd4f 100644 --- a/pkg/scheduler/algorithm/predicates/predicates.go +++ b/pkg/scheduler/algorithm/predicates/predicates.go @@ -96,12 +96,6 @@ const ( // CheckNodePIDPressurePred defines the name of predicate CheckNodePIDPressure. CheckNodePIDPressurePred = "CheckNodePIDPressure" - // DefaultMaxEBSVolumes is the limit for volumes attached to an instance. - // Amazon recommends no more than 40; the system root volume uses at least one. - // See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits - DefaultMaxEBSVolumes = 39 - // DefaultMaxEBSM5VolumeLimit is default EBS volume limit on m5 and c5 instances - DefaultMaxEBSM5VolumeLimit = 25 // DefaultMaxGCEPDVolumes defines the maximum number of PD Volumes for GCE // GCE instances can have up to 16 PD volumes attached. DefaultMaxGCEPDVolumes = 16 @@ -380,10 +374,10 @@ func getMaxVolumeFunc(filterName string) func(node *v1.Node) int { } func getMaxEBSVolume(nodeInstanceType string) int { - if ok, _ := regexp.MatchString("^[cm]5.*", nodeInstanceType); ok { - return DefaultMaxEBSM5VolumeLimit + if ok, _ := regexp.MatchString(volumeutil.EBSNitroLimitRegex, nodeInstanceType); ok { + return volumeutil.DefaultMaxEBSNitroVolumeLimit } - return DefaultMaxEBSVolumes + return volumeutil.DefaultMaxEBSVolumes } // getMaxVolLimitFromEnv checks the max PD volumes environment variable, otherwise returning a default value diff --git a/pkg/volume/aws_ebs/aws_ebs.go b/pkg/volume/aws_ebs/aws_ebs.go index f557827ba8..1e3ef4f165 100644 --- a/pkg/volume/aws_ebs/aws_ebs.go +++ b/pkg/volume/aws_ebs/aws_ebs.go @@ -99,7 +99,7 @@ func (plugin *awsElasticBlockStorePlugin) SupportsBulkVolumeVerification() bool func (plugin *awsElasticBlockStorePlugin) GetVolumeLimits() (map[string]int64, error) { volumeLimits := map[string]int64{ - util.EBSVolumeLimitKey: 39, + util.EBSVolumeLimitKey: util.DefaultMaxEBSVolumes, } cloud := plugin.host.GetCloudProvider() @@ -127,8 +127,8 @@ func (plugin *awsElasticBlockStorePlugin) GetVolumeLimits() (map[string]int64, e return volumeLimits, nil } - if ok, _ := regexp.MatchString("^[cm]5.*", instanceType); ok { - volumeLimits[util.EBSVolumeLimitKey] = 25 + if ok, _ := regexp.MatchString(util.EBSNitroLimitRegex, instanceType); ok { + volumeLimits[util.EBSVolumeLimitKey] = util.DefaultMaxEBSNitroVolumeLimit } return volumeLimits, nil diff --git a/pkg/volume/util/attach_limit.go b/pkg/volume/util/attach_limit.go index 04bda52533..079a015006 100644 --- a/pkg/volume/util/attach_limit.go +++ b/pkg/volume/util/attach_limit.go @@ -27,6 +27,14 @@ import ( const ( // EBSVolumeLimitKey resource name that will store volume limits for EBS EBSVolumeLimitKey = "attachable-volumes-aws-ebs" + // EBSNitroLimitRegex finds nitro instance types with different limit than EBS defaults + EBSNitroLimitRegex = "^[cmr]5.*|t3|z1d" + // DefaultMaxEBSVolumes is the limit for volumes attached to an instance. + // Amazon recommends no more than 40; the system root volume uses at least one. + // See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits + DefaultMaxEBSVolumes = 39 + // DefaultMaxEBSM5VolumeLimit is default EBS volume limit on m5 and c5 instances + DefaultMaxEBSNitroVolumeLimit = 25 // AzureVolumeLimitKey stores resource name that will store volume limits for Azure AzureVolumeLimitKey = "attachable-volumes-azure-disk" // GCEVolumeLimitKey stores resource name that will store volume limits for GCE node