Merge pull request #22942 from jsafrane/devel/aws-more-devices

Allow up to 39 PVs attached to AWS node
pull/6/head
Phillip Wittrock 2016-03-15 12:41:15 -07:00
commit 8a952748a1
3 changed files with 27 additions and 12 deletions

View File

@ -80,6 +80,11 @@ const MaxReadThenCreateRetries = 30
// need hardcoded defaults. // need hardcoded defaults.
const DefaultVolumeType = "gp2" const DefaultVolumeType = "gp2"
// Amazon recommends having no more that 40 volumes attached to an instance,
// and at least one of those is for the system root volume.
// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits
const DefaultMaxEBSVolumes = 39
// Used to call aws_credentials.Init() just once // Used to call aws_credentials.Init() just once
var once sync.Once var once sync.Once
@ -902,10 +907,17 @@ type mountDevice string
// TODO: Also return number of mounts allowed? // TODO: Also return number of mounts allowed?
func (self *awsInstanceType) getEBSMountDevices() []mountDevice { func (self *awsInstanceType) getEBSMountDevices() []mountDevice {
// See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html // See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
// We will generate "ba", "bb", "bc"..."bz", "ca", ..., up to DefaultMaxEBSVolumes
devices := []mountDevice{} devices := []mountDevice{}
for c := 'f'; c <= 'p'; c++ { count := 0
devices = append(devices, mountDevice(fmt.Sprintf("%c", c))) for first := 'b'; count < DefaultMaxEBSVolumes; first++ {
for second := 'a'; count < DefaultMaxEBSVolumes && second <= 'z'; second++ {
device := mountDevice(fmt.Sprintf("%c%c", first, second))
devices = append(devices, device)
count++
}
} }
return devices return devices
} }
@ -1014,7 +1026,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned
if strings.HasPrefix(name, "/dev/xvd") { if strings.HasPrefix(name, "/dev/xvd") {
name = name[8:] name = name[8:]
} }
if len(name) != 1 { if len(name) < 1 || len(name) > 2 {
glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName)) glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName))
} }
deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId) deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId)
@ -1051,7 +1063,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned
if chosen == "" { if chosen == "" {
glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid) glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid)
return "", false, nil return "", false, fmt.Errorf("Too many EBS volumes attached to node %s.", self.nodeName)
} }
self.attaching[chosen] = volumeID self.attaching[chosen] = volumeID

View File

@ -170,6 +170,8 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner) (strin
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. // Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails.
func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) { func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) {
var awsCloud *aws.AWSCloud var awsCloud *aws.AWSCloud
var attachError error
for numRetries := 0; numRetries < maxRetries; numRetries++ { for numRetries := 0; numRetries < maxRetries; numRetries++ {
var err error var err error
if awsCloud == nil { if awsCloud == nil {
@ -186,9 +188,10 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin
glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries) glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries)
} }
devicePath, err := awsCloud.AttachDisk(b.volumeID, "", b.readOnly) var devicePath string
if err != nil { devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly)
glog.Errorf("Error attaching PD %q: %v", b.volumeID, err) if attachError != nil {
glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError)
time.Sleep(errorSleepDuration) time.Sleep(errorSleepDuration)
continue continue
} }
@ -212,6 +215,9 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin
} }
} }
if attachError != nil {
return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError)
}
return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID) return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID)
} }

View File

@ -21,6 +21,7 @@ import (
"os" "os"
"strconv" "strconv"
"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
"k8s.io/kubernetes/pkg/util/sets" "k8s.io/kubernetes/pkg/util/sets"
"k8s.io/kubernetes/plugin/pkg/scheduler" "k8s.io/kubernetes/plugin/pkg/scheduler"
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm" "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
@ -31,10 +32,6 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
) )
// Amazon recommends having no more that 40 volumes attached to an instance,
// and at least one of those is for the system root volume.
const DefaultMaxEBSVolumes = 39
// GCE instances can have up to 16 PD volumes attached. // GCE instances can have up to 16 PD volumes attached.
const DefaultMaxGCEPDVolumes = 16 const DefaultMaxGCEPDVolumes = 16
@ -117,7 +114,7 @@ func defaultPredicates() sets.String {
"MaxEBSVolumeCount", "MaxEBSVolumeCount",
func(args factory.PluginFactoryArgs) algorithm.FitPredicate { func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
// TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly // TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
maxVols := getMaxVols(DefaultMaxEBSVolumes) maxVols := getMaxVols(aws.DefaultMaxEBSVolumes)
return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo) return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
}, },
), ),