github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

attacher/detacher refactor

pull/8/head
NickrenREN 2018-08-01 13:42:04 +08:00
parent 05acb32aca
commit c7e4466873
5 changed files with 118 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
pkg/kubelet/volumemanager/cache/desired_state_of_world.go vendored
View File

@ -150,6 +150,10 @@ type volumeToMount struct {
// the volume.Attacher interface
pluginIsAttachable bool
// pluginIsDeviceMountable indicates that the plugin for this volume implements
// the volume.DeviceMounter interface
pluginIsDeviceMountable bool
// volumeGidValue contains the value of the GID annotation, if present.
volumeGidValue string
@ -220,13 +224,16 @@ func (dsw *desiredStateOfWorld) AddPodToVolume(
volumeName = util.GetUniqueVolumeNameForNonAttachableVolume(podName, volumePlugin, volumeSpec)
}
deviceMountable := dsw.isDeviceMountableVolume(volumeSpec)
if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists {
dsw.volumesToMount[volumeName] = volumeToMount{
volumeName: volumeName,
podsToMount: make(map[types.UniquePodName]podToMount),
pluginIsAttachable: attachable,
volumeGidValue: volumeGidValue,
reportedInUse: false,
volumeName: volumeName,
podsToMount: make(map[types.UniquePodName]podToMount),
pluginIsAttachable: attachable,
pluginIsDeviceMountable: deviceMountable,
volumeGidValue: volumeGidValue,
reportedInUse: false,
}
}
@ -346,14 +353,15 @@ func (dsw *desiredStateOfWorld) GetVolumesToMount() []VolumeToMount {
volumesToMount,
VolumeToMount{
VolumeToMount: operationexecutor.VolumeToMount{
VolumeName: volumeName,
PodName: podName,
Pod: podObj.pod,
VolumeSpec: podObj.volumeSpec,
PluginIsAttachable: volumeObj.pluginIsAttachable,
OuterVolumeSpecName: podObj.outerVolumeSpecName,
VolumeGidValue: volumeObj.volumeGidValue,
ReportedInUse: volumeObj.reportedInUse}})
VolumeName: volumeName,
PodName: podName,
Pod: podObj.pod,
VolumeSpec: podObj.volumeSpec,
PluginIsAttachable: volumeObj.pluginIsAttachable,
PluginIsDeviceMountable: volumeObj.pluginIsDeviceMountable,
OuterVolumeSpecName: podObj.outerVolumeSpecName,
VolumeGidValue: volumeObj.volumeGidValue,
ReportedInUse: volumeObj.reportedInUse}})
}
}
return volumesToMount
@ -371,3 +379,15 @@ func (dsw *desiredStateOfWorld) isAttachableVolume(volumeSpec *volume.Spec) bool
return false
}
func (dsw *desiredStateOfWorld) isDeviceMountableVolume(volumeSpec *volume.Spec) bool {
deviceMountableVolumePlugin, _ := dsw.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec)
if deviceMountableVolumePlugin != nil {
volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter()
if err == nil && volumeDeviceMounter != nil {
return true
}
}
return false
}

34
pkg/volume/plugins.go
View File

@ -202,9 +202,17 @@ type ProvisionableVolumePlugin interface {
// AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment
// to a node before mounting.
type AttachableVolumePlugin interface {
VolumePlugin
DeviceMountableVolumePlugin
NewAttacher() (Attacher, error)
NewDetacher() (Detacher, error)
}
// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used
// for volumes that requires mount device to a node before binding to volume to pod.
type DeviceMountableVolumePlugin interface {
VolumePlugin
NewDeviceMounter() (DeviceMounter, error)
NewDeviceUnmounter() (DeviceUnmounter, error)
GetDeviceMountRefs(deviceMountPath string) ([]string, error)
}
@ -757,6 +765,30 @@ func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVo
return nil, nil
}
// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec.
func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginBySpec(spec)
if err != nil {
return nil, err
}
if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
return deviceMountableVolumePlugin, nil
}
return nil, nil
}
// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name.
func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginByName(name)
if err != nil {
return nil, err
}
if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
return deviceMountableVolumePlugin, nil
}
return nil, nil
}
// FindExpandablePluginBySpec fetches a persistent volume plugin by spec.
func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) {
volumePlugin, err := pm.FindPluginBySpec(spec)

8
pkg/volume/util/operationexecutor/operation_executor.go
View File

@ -329,6 +329,10 @@ type VolumeToMount struct {
// the volume.Attacher interface
PluginIsAttachable bool
// PluginIsDeviceMountable indicates that the plugin for this volume implements
// the volume.DeviceMounter interface
PluginIsDeviceMountable bool
// VolumeGidValue contains the value of the GID annotation, if present.
VolumeGidValue string
@ -738,8 +742,8 @@ func (oe *operationExecutor) MountVolume(
podName := nestedpendingoperations.EmptyUniquePodName
// TODO: remove this -- not necessary
if !volumeToMount.PluginIsAttachable {
// Non-attachable volume plugins can execute mount for multiple pods
if !volumeToMount.PluginIsAttachable && !volumeToMount.PluginIsDeviceMountable {
// volume plugins which are Non-attachable and Non-deviceMountable can execute mount for multiple pods
// referencing the same volume in parallel
podName = util.GetUniquePodName(volumeToMount.Pod)
}

52
pkg/volume/util/operationexecutor/operation_generator.go
View File

@ -478,6 +478,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
volumeAttacher, _ = attachableVolumePlugin.NewAttacher()
}
// get deviceMounter, if possible
deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec)
var volumeDeviceMounter volume.DeviceMounter
if deviceMountableVolumePlugin != nil {
volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter()
}
var fsGroup *int64
if volumeToMount.Pod.Spec.SecurityContext != nil &&
volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil {
@ -485,28 +492,31 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
}
mountVolumeFunc := func() (error, error) {
devicePath := volumeToMount.DevicePath
if volumeAttacher != nil {
// Wait for attachable volumes to finish attaching
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath)))
devicePath, err := volumeAttacher.WaitForAttach(
volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout)
devicePath, err = volumeAttacher.WaitForAttach(
volumeToMount.VolumeSpec, devicePath, volumeToMount.Pod, waitForAttachTimeout)
if err != nil {
// On failure, return error. Caller will log and retry.
return volumeToMount.GenerateError("MountVolume.WaitForAttach failed", err)
}
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
}
if volumeDeviceMounter != nil {
deviceMountPath, err :=
volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec)
volumeDeviceMounter.GetDeviceMountPath(volumeToMount.VolumeSpec)
if err != nil {
// On failure, return error. Caller will log and retry.
return volumeToMount.GenerateError("MountVolume.GetDeviceMountPath failed", err)
}
// Mount device to global mount path
err = volumeAttacher.MountDevice(
err = volumeDeviceMounter.MountDevice(
volumeToMount.VolumeSpec,
devicePath,
deviceMountPath)
@ -532,7 +542,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
if resizeSimpleError != nil || resizeDetailedError != nil {
return resizeSimpleError, resizeDetailedError
}
}
if og.checkNodeCapabilitiesBeforeMount {
@ -718,20 +727,31 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
deviceToDetach AttachedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater,
mounter mount.Interface) (volumetypes.GeneratedOperations, error) {
// Get attacher plugin
attachableVolumePlugin, err :=
og.volumePluginMgr.FindAttachablePluginByName(deviceToDetach.PluginName)
if err != nil || attachableVolumePlugin == nil {
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindAttachablePluginBySpec failed", err)
// Get DeviceMounter plugin
deviceMountableVolumePlugin, err :=
og.volumePluginMgr.FindDeviceMountablePluginByName(deviceToDetach.PluginName)
if err != nil || deviceMountableVolumePlugin == nil {
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindDeviceMountablePluginByName failed", err)
}
volumeDeviceUmounter, err := deviceMountableVolumePlugin.NewDeviceUnmounter()
if err != nil {
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceUmounter failed", err)
}
volumeDetacher, err := attachableVolumePlugin.NewDetacher()
volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter()
if err != nil {
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDetacher failed", err)
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceMounter failed", err)
}
unmountDeviceFunc := func() (error, error) {
deviceMountPath := deviceToDetach.DeviceMountPath
refs, err := attachableVolumePlugin.GetDeviceMountRefs(deviceMountPath)
//deviceMountPath := deviceToDetach.DeviceMountPath
deviceMountPath, err :=
volumeDeviceMounter.GetDeviceMountPath(deviceToDetach.VolumeSpec)
if err != nil {
// On failure, return error. Caller will log and retry.
return deviceToDetach.GenerateError("GetDeviceMountPath failed", err)
}
refs, err := deviceMountableVolumePlugin.GetDeviceMountRefs(deviceMountPath)
if err != nil || mount.HasMountRefs(deviceMountPath, refs) {
if err == nil {
@ -740,7 +760,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
return deviceToDetach.GenerateError("GetDeviceMountRefs check failed", err)
}
// Execute unmount
unmountDeviceErr := volumeDetacher.UnmountDevice(deviceMountPath)
unmountDeviceErr := volumeDeviceUmounter.UnmountDevice(deviceMountPath)
if unmountDeviceErr != nil {
// On failure, return error. Caller will log and retry.
return deviceToDetach.GenerateError("UnmountDevice failed", unmountDeviceErr)
@ -775,7 +795,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
return volumetypes.GeneratedOperations{
OperationFunc: unmountDeviceFunc,
CompleteFunc: util.OperationCompleteHook(attachableVolumePlugin.GetPluginName(), "unmount_device"),
CompleteFunc: util.OperationCompleteHook(deviceMountableVolumePlugin.GetPluginName(), "unmount_device"),
EventRecorderFunc: nil, // nil because we do not want to generate event on error
}, nil
}

10
pkg/volume/volume.go
View File

@ -204,6 +204,8 @@ type Deleter interface {
// Attacher can attach a volume to a node.
type Attacher interface {
DeviceMounter
// Attaches the volume specified by the given spec to the node with the given Name.
// On success, returns the device path where the device was attached on the
// node.
@ -219,7 +221,10 @@ type Attacher interface {
// is returned. Otherwise, if the device does not attach after
// the given timeout period, an error will be returned.
WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error)
}
// DeviceMounter can mount a block volume to a global path.
type DeviceMounter interface {
// GetDeviceMountPath returns a path where the device should
// be mounted after it is attached. This is a global mount
// point which should be bind mounted for individual volumes.
@ -227,6 +232,7 @@ type Attacher interface {
// MountDevice mounts the disk to a global path which
// individual pods can then bind mount
// Note that devicePath can be empty if the volume plugin does not implement any of Attach and WaitForAttach methods.
MountDevice(spec *Spec, devicePath string, deviceMountPath string) error
}
@ -240,11 +246,15 @@ type BulkVolumeVerifier interface {
// Detacher can detach a volume from a node.
type Detacher interface {
DeviceUnmounter
// Detach the given volume from the node with the given Name.
// volumeName is name of the volume as returned from plugin's
// GetVolumeName().
Detach(volumeName string, nodeName types.NodeName) error
}
// DeviceUnmounter can unmount a block volume from the global path.
type DeviceUnmounter interface {
// UnmountDevice unmounts the global mount of the disk. This
// should only be called once all bind mounts have been
// unmounted.