diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index fd52aaaa83..d4745ee0b1 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -150,6 +150,10 @@ type volumeToMount struct { // the volume.Attacher interface pluginIsAttachable bool + // pluginIsDeviceMountable indicates that the plugin for this volume implements + // the volume.DeviceMounter interface + pluginIsDeviceMountable bool + // volumeGidValue contains the value of the GID annotation, if present. volumeGidValue string @@ -220,13 +224,16 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( volumeName = util.GetUniqueVolumeNameForNonAttachableVolume(podName, volumePlugin, volumeSpec) } + deviceMountable := dsw.isDeviceMountableVolume(volumeSpec) + if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists { dsw.volumesToMount[volumeName] = volumeToMount{ - volumeName: volumeName, - podsToMount: make(map[types.UniquePodName]podToMount), - pluginIsAttachable: attachable, - volumeGidValue: volumeGidValue, - reportedInUse: false, + volumeName: volumeName, + podsToMount: make(map[types.UniquePodName]podToMount), + pluginIsAttachable: attachable, + pluginIsDeviceMountable: deviceMountable, + volumeGidValue: volumeGidValue, + reportedInUse: false, } } @@ -346,14 +353,15 @@ func (dsw *desiredStateOfWorld) GetVolumesToMount() []VolumeToMount { volumesToMount, VolumeToMount{ VolumeToMount: operationexecutor.VolumeToMount{ - VolumeName: volumeName, - PodName: podName, - Pod: podObj.pod, - VolumeSpec: podObj.volumeSpec, - PluginIsAttachable: volumeObj.pluginIsAttachable, - OuterVolumeSpecName: podObj.outerVolumeSpecName, - VolumeGidValue: volumeObj.volumeGidValue, - ReportedInUse: volumeObj.reportedInUse}}) + VolumeName: volumeName, + PodName: podName, + Pod: podObj.pod, + VolumeSpec: podObj.volumeSpec, + PluginIsAttachable: volumeObj.pluginIsAttachable, + PluginIsDeviceMountable: volumeObj.pluginIsDeviceMountable, + OuterVolumeSpecName: podObj.outerVolumeSpecName, + VolumeGidValue: volumeObj.volumeGidValue, + ReportedInUse: volumeObj.reportedInUse}}) } } return volumesToMount @@ -371,3 +379,15 @@ func (dsw *desiredStateOfWorld) isAttachableVolume(volumeSpec *volume.Spec) bool return false } + +func (dsw *desiredStateOfWorld) isDeviceMountableVolume(volumeSpec *volume.Spec) bool { + deviceMountableVolumePlugin, _ := dsw.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec) + if deviceMountableVolumePlugin != nil { + volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter() + if err == nil && volumeDeviceMounter != nil { + return true + } + } + + return false +} diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index e2c7e05bb1..8ea4584f99 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -202,9 +202,17 @@ type ProvisionableVolumePlugin interface { // AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment // to a node before mounting. type AttachableVolumePlugin interface { - VolumePlugin + DeviceMountableVolumePlugin NewAttacher() (Attacher, error) NewDetacher() (Detacher, error) +} + +// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used +// for volumes that requires mount device to a node before binding to volume to pod. +type DeviceMountableVolumePlugin interface { + VolumePlugin + NewDeviceMounter() (DeviceMounter, error) + NewDeviceUnmounter() (DeviceUnmounter, error) GetDeviceMountRefs(deviceMountPath string) ([]string, error) } @@ -757,6 +765,30 @@ func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVo return nil, nil } +// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec. +func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { + return deviceMountableVolumePlugin, nil + } + return nil, nil +} + +// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name. +func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { + return deviceMountableVolumePlugin, nil + } + return nil, nil +} + // FindExpandablePluginBySpec fetches a persistent volume plugin by spec. func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) { volumePlugin, err := pm.FindPluginBySpec(spec) diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 1a44c62d4a..8983ae48d3 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -329,6 +329,10 @@ type VolumeToMount struct { // the volume.Attacher interface PluginIsAttachable bool + // PluginIsDeviceMountable indicates that the plugin for this volume implements + // the volume.DeviceMounter interface + PluginIsDeviceMountable bool + // VolumeGidValue contains the value of the GID annotation, if present. VolumeGidValue string @@ -738,8 +742,8 @@ func (oe *operationExecutor) MountVolume( podName := nestedpendingoperations.EmptyUniquePodName // TODO: remove this -- not necessary - if !volumeToMount.PluginIsAttachable { - // Non-attachable volume plugins can execute mount for multiple pods + if !volumeToMount.PluginIsAttachable && !volumeToMount.PluginIsDeviceMountable { + // volume plugins which are Non-attachable and Non-deviceMountable can execute mount for multiple pods // referencing the same volume in parallel podName = util.GetUniquePodName(volumeToMount.Pod) } diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 99c52e4b2e..57c7adfa7c 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -478,6 +478,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc( volumeAttacher, _ = attachableVolumePlugin.NewAttacher() } + // get deviceMounter, if possible + deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec) + var volumeDeviceMounter volume.DeviceMounter + if deviceMountableVolumePlugin != nil { + volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter() + } + var fsGroup *int64 if volumeToMount.Pod.Spec.SecurityContext != nil && volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil { @@ -485,28 +492,31 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } mountVolumeFunc := func() (error, error) { + devicePath := volumeToMount.DevicePath if volumeAttacher != nil { // Wait for attachable volumes to finish attaching glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) - devicePath, err := volumeAttacher.WaitForAttach( - volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout) + devicePath, err = volumeAttacher.WaitForAttach( + volumeToMount.VolumeSpec, devicePath, volumeToMount.Pod, waitForAttachTimeout) if err != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.WaitForAttach failed", err) } glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + } + if volumeDeviceMounter != nil { deviceMountPath, err := - volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) + volumeDeviceMounter.GetDeviceMountPath(volumeToMount.VolumeSpec) if err != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.GetDeviceMountPath failed", err) } // Mount device to global mount path - err = volumeAttacher.MountDevice( + err = volumeDeviceMounter.MountDevice( volumeToMount.VolumeSpec, devicePath, deviceMountPath) @@ -532,7 +542,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc( if resizeSimpleError != nil || resizeDetailedError != nil { return resizeSimpleError, resizeDetailedError } - } if og.checkNodeCapabilitiesBeforeMount { @@ -718,20 +727,31 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (volumetypes.GeneratedOperations, error) { - // Get attacher plugin - attachableVolumePlugin, err := - og.volumePluginMgr.FindAttachablePluginByName(deviceToDetach.PluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindAttachablePluginBySpec failed", err) + // Get DeviceMounter plugin + deviceMountableVolumePlugin, err := + og.volumePluginMgr.FindDeviceMountablePluginByName(deviceToDetach.PluginName) + if err != nil || deviceMountableVolumePlugin == nil { + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindDeviceMountablePluginByName failed", err) + } + volumeDeviceUmounter, err := deviceMountableVolumePlugin.NewDeviceUnmounter() + if err != nil { + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceUmounter failed", err) } - volumeDetacher, err := attachableVolumePlugin.NewDetacher() + volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter() if err != nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDetacher failed", err) + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceMounter failed", err) } + unmountDeviceFunc := func() (error, error) { - deviceMountPath := deviceToDetach.DeviceMountPath - refs, err := attachableVolumePlugin.GetDeviceMountRefs(deviceMountPath) + //deviceMountPath := deviceToDetach.DeviceMountPath + deviceMountPath, err := + volumeDeviceMounter.GetDeviceMountPath(deviceToDetach.VolumeSpec) + if err != nil { + // On failure, return error. Caller will log and retry. + return deviceToDetach.GenerateError("GetDeviceMountPath failed", err) + } + refs, err := deviceMountableVolumePlugin.GetDeviceMountRefs(deviceMountPath) if err != nil || mount.HasMountRefs(deviceMountPath, refs) { if err == nil { @@ -740,7 +760,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( return deviceToDetach.GenerateError("GetDeviceMountRefs check failed", err) } // Execute unmount - unmountDeviceErr := volumeDetacher.UnmountDevice(deviceMountPath) + unmountDeviceErr := volumeDeviceUmounter.UnmountDevice(deviceMountPath) if unmountDeviceErr != nil { // On failure, return error. Caller will log and retry. return deviceToDetach.GenerateError("UnmountDevice failed", unmountDeviceErr) @@ -775,7 +795,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( return volumetypes.GeneratedOperations{ OperationFunc: unmountDeviceFunc, - CompleteFunc: util.OperationCompleteHook(attachableVolumePlugin.GetPluginName(), "unmount_device"), + CompleteFunc: util.OperationCompleteHook(deviceMountableVolumePlugin.GetPluginName(), "unmount_device"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 5f633214fc..013e0dc419 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -204,6 +204,8 @@ type Deleter interface { // Attacher can attach a volume to a node. type Attacher interface { + DeviceMounter + // Attaches the volume specified by the given spec to the node with the given Name. // On success, returns the device path where the device was attached on the // node. @@ -219,7 +221,10 @@ type Attacher interface { // is returned. Otherwise, if the device does not attach after // the given timeout period, an error will be returned. WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) +} +// DeviceMounter can mount a block volume to a global path. +type DeviceMounter interface { // GetDeviceMountPath returns a path where the device should // be mounted after it is attached. This is a global mount // point which should be bind mounted for individual volumes. @@ -227,6 +232,7 @@ type Attacher interface { // MountDevice mounts the disk to a global path which // individual pods can then bind mount + // Note that devicePath can be empty if the volume plugin does not implement any of Attach and WaitForAttach methods. MountDevice(spec *Spec, devicePath string, deviceMountPath string) error } @@ -240,11 +246,15 @@ type BulkVolumeVerifier interface { // Detacher can detach a volume from a node. type Detacher interface { + DeviceUnmounter // Detach the given volume from the node with the given Name. // volumeName is name of the volume as returned from plugin's // GetVolumeName(). Detach(volumeName string, nodeName types.NodeName) error +} +// DeviceUnmounter can unmount a block volume from the global path. +type DeviceUnmounter interface { // UnmountDevice unmounts the global mount of the disk. This // should only be called once all bind mounts have been // unmounted.