diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index 27baccabc0..15d74e9f74 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -80,6 +80,11 @@ const MaxReadThenCreateRetries = 30 // need hardcoded defaults. const DefaultVolumeType = "gp2" +// Amazon recommends having no more that 40 volumes attached to an instance, +// and at least one of those is for the system root volume. +// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits +const DefaultMaxEBSVolumes = 39 + // Used to call aws_credentials.Init() just once var once sync.Once @@ -902,10 +907,17 @@ type mountDevice string // TODO: Also return number of mounts allowed? func (self *awsInstanceType) getEBSMountDevices() []mountDevice { // See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html + // We will generate "ba", "bb", "bc"..."bz", "ca", ..., up to DefaultMaxEBSVolumes devices := []mountDevice{} - for c := 'f'; c <= 'p'; c++ { - devices = append(devices, mountDevice(fmt.Sprintf("%c", c))) + count := 0 + for first := 'b'; count < DefaultMaxEBSVolumes; first++ { + for second := 'a'; count < DefaultMaxEBSVolumes && second <= 'z'; second++ { + device := mountDevice(fmt.Sprintf("%c%c", first, second)) + devices = append(devices, device) + count++ + } } + return devices } @@ -1014,7 +1026,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned if strings.HasPrefix(name, "/dev/xvd") { name = name[8:] } - if len(name) != 1 { + if len(name) < 1 || len(name) > 2 { glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName)) } deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId) @@ -1051,7 +1063,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned if chosen == "" { glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid) - return "", false, nil + return "", false, fmt.Errorf("Too many EBS volumes attached to node %s.", self.nodeName) } self.attaching[chosen] = volumeID diff --git a/pkg/volume/aws_ebs/aws_util.go b/pkg/volume/aws_ebs/aws_util.go index 3c86763893..51e3b2c81b 100644 --- a/pkg/volume/aws_ebs/aws_util.go +++ b/pkg/volume/aws_ebs/aws_util.go @@ -170,6 +170,8 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner) (strin // Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) { var awsCloud *aws.AWSCloud + var attachError error + for numRetries := 0; numRetries < maxRetries; numRetries++ { var err error if awsCloud == nil { @@ -186,9 +188,10 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries) } - devicePath, err := awsCloud.AttachDisk(b.volumeID, "", b.readOnly) - if err != nil { - glog.Errorf("Error attaching PD %q: %v", b.volumeID, err) + var devicePath string + devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly) + if attachError != nil { + glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError) time.Sleep(errorSleepDuration) continue } @@ -212,6 +215,9 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin } } + if attachError != nil { + return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError) + } return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID) } diff --git a/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go b/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go index 0993cbb554..6242a2ba9a 100644 --- a/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go +++ b/plugin/pkg/scheduler/algorithmprovider/defaults/defaults.go @@ -21,6 +21,7 @@ import ( "os" "strconv" + "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" "k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/plugin/pkg/scheduler" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" @@ -31,10 +32,6 @@ import ( "github.com/golang/glog" ) -// Amazon recommends having no more that 40 volumes attached to an instance, -// and at least one of those is for the system root volume. -const DefaultMaxEBSVolumes = 39 - // GCE instances can have up to 16 PD volumes attached. const DefaultMaxGCEPDVolumes = 16 @@ -117,7 +114,7 @@ func defaultPredicates() sets.String { "MaxEBSVolumeCount", func(args factory.PluginFactoryArgs) algorithm.FitPredicate { // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly - maxVols := getMaxVols(DefaultMaxEBSVolumes) + maxVols := getMaxVols(aws.DefaultMaxEBSVolumes) return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo) }, ),