From f270cb1b9bc46d86ea1a2a97a2bb95d5a1a32369 Mon Sep 17 00:00:00 2001 From: Jan Safranek Date: Tue, 15 Mar 2016 17:28:59 +0100 Subject: [PATCH] Allow 39 atached EBS devices. AWS has soft support limit for 40 attached EBS devices. Assuming there is just one root device, use the rest for persistent volumes. The devices will have name /dev/xvdba - /dev/xvdcm, leaving /dev/sda - /dev/sdz to the system. Also, add better error handling and propagate error "Too many EBS volumes attached to node XYZ" to a pod. --- pkg/cloudprovider/providers/aws/aws.go | 20 +++++++++++++++---- pkg/volume/aws_ebs/aws_util.go | 12 ++++++++--- .../algorithmprovider/defaults/defaults.go | 7 ++----- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 27baccabc0..15d74e9f74 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -80,6 +80,11 @@ const MaxReadThenCreateRetries = 30 // need hardcoded defaults. const DefaultVolumeType = "gp2" +// Amazon recommends having no more that 40 volumes attached to an instance, +// and at least one of those is for the system root volume. +// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits +const DefaultMaxEBSVolumes = 39 + // Used to call aws_credentials.Init() just once var once sync.Once @@ -902,10 +907,17 @@ type mountDevice string // TODO: Also return number of mounts allowed? func (self *awsInstanceType) getEBSMountDevices() []mountDevice { // See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html + // We will generate "ba", "bb", "bc"..."bz", "ca", ..., up to DefaultMaxEBSVolumes devices := []mountDevice{} - for c := 'f'; c <= 'p'; c++ { - devices = append(devices, mountDevice(fmt.Sprintf("%c", c))) + count := 0 + for first := 'b'; count < DefaultMaxEBSVolumes; first++ { + for second := 'a'; count < DefaultMaxEBSVolumes && second <= 'z'; second++ { + device := mountDevice(fmt.Sprintf("%c%c", first, second)) + devices = append(devices, device) + count++ + } } + return devices } @@ -1014,7 +1026,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned if strings.HasPrefix(name, "/dev/xvd") { name = name[8:] } - if len(name) != 1 { + if len(name) < 1 || len(name) > 2 { glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName)) } deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId) @@ -1051,7 +1063,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned if chosen == "" { glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid) - return "", false, nil + return "", false, fmt.Errorf("Too many EBS volumes attached to node %s.", self.nodeName) } self.attaching[chosen] = volumeID diff --git a/pkg/volume/aws_ebs/aws_util.go b/pkg/volume/aws_ebs/aws_util.go index 3c86763893..51e3b2c81b 100644 --- a/pkg/volume/aws_ebs/aws_util.go +++ b/pkg/volume/aws_ebs/aws_util.go @@ -170,6 +170,8 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner) (strin // Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) { var awsCloud *aws.AWSCloud + var attachError error + for numRetries := 0; numRetries < maxRetries; numRetries++ { var err error if awsCloud == nil { @@ -186,9 +188,10 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries) } - devicePath, err := awsCloud.AttachDisk(b.volumeID, "", b.readOnly) - if err != nil { - glog.Errorf("Error attaching PD %q: %v", b.volumeID, err) + var devicePath string + devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly) + if attachError != nil { + glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError) time.Sleep(errorSleepDuration) continue } @@ -212,6 +215,9 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin } } + if attachError != nil { + return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError) + } return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID) } diff --git a/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go b/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go index 0993cbb554..6242a2ba9a 100644 --- a/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go +++ b/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go @@ -21,6 +21,7 @@ import ( "os" "strconv" + "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" @@ -31,10 +32,6 @@ import ( "github.com/golang/glog" ) -// Amazon recommends having no more that 40 volumes attached to an instance, -// and at least one of those is for the system root volume. -const DefaultMaxEBSVolumes = 39 - // GCE instances can have up to 16 PD volumes attached. const DefaultMaxGCEPDVolumes = 16 @@ -117,7 +114,7 @@ func defaultPredicates() sets.String { "MaxEBSVolumeCount", func(args factory.PluginFactoryArgs) algorithm.FitPredicate { // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly - maxVols := getMaxVols(DefaultMaxEBSVolumes) + maxVols := getMaxVols(aws.DefaultMaxEBSVolumes) return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo) }, ),