mirror of https://github.com/k3s-io/k3s
Merge pull request #22942 from jsafrane/devel/aws-more-devices
Allow up to 39 PVs attached to AWS nodepull/6/head
commit
8a952748a1
|
@ -80,6 +80,11 @@ const MaxReadThenCreateRetries = 30
|
||||||
// need hardcoded defaults.
|
// need hardcoded defaults.
|
||||||
const DefaultVolumeType = "gp2"
|
const DefaultVolumeType = "gp2"
|
||||||
|
|
||||||
|
// Amazon recommends having no more that 40 volumes attached to an instance,
|
||||||
|
// and at least one of those is for the system root volume.
|
||||||
|
// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits
|
||||||
|
const DefaultMaxEBSVolumes = 39
|
||||||
|
|
||||||
// Used to call aws_credentials.Init() just once
|
// Used to call aws_credentials.Init() just once
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
|
|
||||||
|
@ -902,10 +907,17 @@ type mountDevice string
|
||||||
// TODO: Also return number of mounts allowed?
|
// TODO: Also return number of mounts allowed?
|
||||||
func (self *awsInstanceType) getEBSMountDevices() []mountDevice {
|
func (self *awsInstanceType) getEBSMountDevices() []mountDevice {
|
||||||
// See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
|
// See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html
|
||||||
|
// We will generate "ba", "bb", "bc"..."bz", "ca", ..., up to DefaultMaxEBSVolumes
|
||||||
devices := []mountDevice{}
|
devices := []mountDevice{}
|
||||||
for c := 'f'; c <= 'p'; c++ {
|
count := 0
|
||||||
devices = append(devices, mountDevice(fmt.Sprintf("%c", c)))
|
for first := 'b'; count < DefaultMaxEBSVolumes; first++ {
|
||||||
|
for second := 'a'; count < DefaultMaxEBSVolumes && second <= 'z'; second++ {
|
||||||
|
device := mountDevice(fmt.Sprintf("%c%c", first, second))
|
||||||
|
devices = append(devices, device)
|
||||||
|
count++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return devices
|
return devices
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1014,7 +1026,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned
|
||||||
if strings.HasPrefix(name, "/dev/xvd") {
|
if strings.HasPrefix(name, "/dev/xvd") {
|
||||||
name = name[8:]
|
name = name[8:]
|
||||||
}
|
}
|
||||||
if len(name) != 1 {
|
if len(name) < 1 || len(name) > 2 {
|
||||||
glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName))
|
glog.Warningf("Unexpected EBS DeviceName: %q", aws.StringValue(blockDevice.DeviceName))
|
||||||
}
|
}
|
||||||
deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId)
|
deviceMappings[mountDevice(name)] = aws.StringValue(blockDevice.Ebs.VolumeId)
|
||||||
|
@ -1051,7 +1063,7 @@ func (self *awsInstance) getMountDevice(volumeID string, assign bool) (assigned
|
||||||
|
|
||||||
if chosen == "" {
|
if chosen == "" {
|
||||||
glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid)
|
glog.Warningf("Could not assign a mount device (all in use?). mappings=%v, valid=%v", deviceMappings, valid)
|
||||||
return "", false, nil
|
return "", false, fmt.Errorf("Too many EBS volumes attached to node %s.", self.nodeName)
|
||||||
}
|
}
|
||||||
|
|
||||||
self.attaching[chosen] = volumeID
|
self.attaching[chosen] = volumeID
|
||||||
|
|
|
@ -170,6 +170,8 @@ func (util *AWSDiskUtil) CreateVolume(c *awsElasticBlockStoreProvisioner) (strin
|
||||||
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails.
|
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails.
|
||||||
func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) {
|
func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) {
|
||||||
var awsCloud *aws.AWSCloud
|
var awsCloud *aws.AWSCloud
|
||||||
|
var attachError error
|
||||||
|
|
||||||
for numRetries := 0; numRetries < maxRetries; numRetries++ {
|
for numRetries := 0; numRetries < maxRetries; numRetries++ {
|
||||||
var err error
|
var err error
|
||||||
if awsCloud == nil {
|
if awsCloud == nil {
|
||||||
|
@ -186,9 +188,10 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin
|
||||||
glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries)
|
glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries)
|
||||||
}
|
}
|
||||||
|
|
||||||
devicePath, err := awsCloud.AttachDisk(b.volumeID, "", b.readOnly)
|
var devicePath string
|
||||||
if err != nil {
|
devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly)
|
||||||
glog.Errorf("Error attaching PD %q: %v", b.volumeID, err)
|
if attachError != nil {
|
||||||
|
glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError)
|
||||||
time.Sleep(errorSleepDuration)
|
time.Sleep(errorSleepDuration)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
@ -212,6 +215,9 @@ func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.Strin
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if attachError != nil {
|
||||||
|
return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError)
|
||||||
|
}
|
||||||
return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID)
|
return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,7 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
|
"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
|
||||||
"k8s.io/kubernetes/pkg/util/sets"
|
"k8s.io/kubernetes/pkg/util/sets"
|
||||||
"k8s.io/kubernetes/plugin/pkg/scheduler"
|
"k8s.io/kubernetes/plugin/pkg/scheduler"
|
||||||
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
|
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
|
||||||
|
@ -31,10 +32,6 @@ import (
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Amazon recommends having no more that 40 volumes attached to an instance,
|
|
||||||
// and at least one of those is for the system root volume.
|
|
||||||
const DefaultMaxEBSVolumes = 39
|
|
||||||
|
|
||||||
// GCE instances can have up to 16 PD volumes attached.
|
// GCE instances can have up to 16 PD volumes attached.
|
||||||
const DefaultMaxGCEPDVolumes = 16
|
const DefaultMaxGCEPDVolumes = 16
|
||||||
|
|
||||||
|
@ -117,7 +114,7 @@ func defaultPredicates() sets.String {
|
||||||
"MaxEBSVolumeCount",
|
"MaxEBSVolumeCount",
|
||||||
func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
|
func(args factory.PluginFactoryArgs) algorithm.FitPredicate {
|
||||||
// TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
|
// TODO: allow for generically parameterized scheduler predicates, because this is a bit ugly
|
||||||
maxVols := getMaxVols(DefaultMaxEBSVolumes)
|
maxVols := getMaxVols(aws.DefaultMaxEBSVolumes)
|
||||||
return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
|
return predicates.NewMaxPDVolumeCountPredicate(predicates.EBSVolumeFilter, maxVols, args.PVInfo, args.PVCInfo)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
|
Loading…
Reference in New Issue