diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index d72fbe4e6c..7dceaba273 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -294,6 +294,10 @@ func (plugin *TestPlugin) NewAttacher() (volume.Attacher, error) { return &attacher, nil } +func (plugin *TestPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *TestPlugin) NewDetacher() (volume.Detacher, error) { detacher := testPluginDetacher{ detachedVolumeMap: plugin.detachedVolumeMap, @@ -302,6 +306,10 @@ func (plugin *TestPlugin) NewDetacher() (volume.Detacher, error) { return &detacher, nil } +func (plugin *TestPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (plugin *TestPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { return []string{}, nil } diff --git a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go index fd52aaaa83..d4745ee0b1 100644 --- a/pkg/kubelet/volumemanager/cache/desired_state_of_world.go +++ b/pkg/kubelet/volumemanager/cache/desired_state_of_world.go @@ -150,6 +150,10 @@ type volumeToMount struct { // the volume.Attacher interface pluginIsAttachable bool + // pluginIsDeviceMountable indicates that the plugin for this volume implements + // the volume.DeviceMounter interface + pluginIsDeviceMountable bool + // volumeGidValue contains the value of the GID annotation, if present. volumeGidValue string @@ -220,13 +224,16 @@ func (dsw *desiredStateOfWorld) AddPodToVolume( volumeName = util.GetUniqueVolumeNameForNonAttachableVolume(podName, volumePlugin, volumeSpec) } + deviceMountable := dsw.isDeviceMountableVolume(volumeSpec) + if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists { dsw.volumesToMount[volumeName] = volumeToMount{ - volumeName: volumeName, - podsToMount: make(map[types.UniquePodName]podToMount), - pluginIsAttachable: attachable, - volumeGidValue: volumeGidValue, - reportedInUse: false, + volumeName: volumeName, + podsToMount: make(map[types.UniquePodName]podToMount), + pluginIsAttachable: attachable, + pluginIsDeviceMountable: deviceMountable, + volumeGidValue: volumeGidValue, + reportedInUse: false, } } @@ -346,14 +353,15 @@ func (dsw *desiredStateOfWorld) GetVolumesToMount() []VolumeToMount { volumesToMount, VolumeToMount{ VolumeToMount: operationexecutor.VolumeToMount{ - VolumeName: volumeName, - PodName: podName, - Pod: podObj.pod, - VolumeSpec: podObj.volumeSpec, - PluginIsAttachable: volumeObj.pluginIsAttachable, - OuterVolumeSpecName: podObj.outerVolumeSpecName, - VolumeGidValue: volumeObj.volumeGidValue, - ReportedInUse: volumeObj.reportedInUse}}) + VolumeName: volumeName, + PodName: podName, + Pod: podObj.pod, + VolumeSpec: podObj.volumeSpec, + PluginIsAttachable: volumeObj.pluginIsAttachable, + PluginIsDeviceMountable: volumeObj.pluginIsDeviceMountable, + OuterVolumeSpecName: podObj.outerVolumeSpecName, + VolumeGidValue: volumeObj.volumeGidValue, + ReportedInUse: volumeObj.reportedInUse}}) } } return volumesToMount @@ -371,3 +379,15 @@ func (dsw *desiredStateOfWorld) isAttachableVolume(volumeSpec *volume.Spec) bool return false } + +func (dsw *desiredStateOfWorld) isDeviceMountableVolume(volumeSpec *volume.Spec) bool { + deviceMountableVolumePlugin, _ := dsw.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec) + if deviceMountableVolumePlugin != nil { + volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter() + if err == nil && volumeDeviceMounter != nil { + return true + } + } + + return false +} diff --git a/pkg/volume/aws_ebs/attacher.go b/pkg/volume/aws_ebs/attacher.go index e4178068dc..da25691d41 100644 --- a/pkg/volume/aws_ebs/attacher.go +++ b/pkg/volume/aws_ebs/attacher.go @@ -39,8 +39,12 @@ type awsElasticBlockStoreAttacher struct { var _ volume.Attacher = &awsElasticBlockStoreAttacher{} +var _ volume.DeviceMounter = &awsElasticBlockStoreAttacher{} + var _ volume.AttachableVolumePlugin = &awsElasticBlockStorePlugin{} +var _ volume.DeviceMountableVolumePlugin = &awsElasticBlockStorePlugin{} + func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) { awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -53,6 +57,10 @@ func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) }, nil } +func (plugin *awsElasticBlockStorePlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *awsElasticBlockStorePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mounter.GetMountRefs(deviceMountPath) @@ -236,6 +244,8 @@ type awsElasticBlockStoreDetacher struct { var _ volume.Detacher = &awsElasticBlockStoreDetacher{} +var _ volume.DeviceUnmounter = &awsElasticBlockStoreDetacher{} + func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) { awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -248,6 +258,10 @@ func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) }, nil } +func (plugin *awsElasticBlockStorePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (detacher *awsElasticBlockStoreDetacher) Detach(volumeName string, nodeName types.NodeName) error { volumeID := aws.KubernetesVolumeID(path.Base(volumeName)) diff --git a/pkg/volume/azure_dd/attacher.go b/pkg/volume/azure_dd/attacher.go index 2a76ef09c2..556496b24a 100644 --- a/pkg/volume/azure_dd/attacher.go +++ b/pkg/volume/azure_dd/attacher.go @@ -52,6 +52,9 @@ type azureDiskAttacher struct { var _ volume.Attacher = &azureDiskAttacher{} var _ volume.Detacher = &azureDiskDetacher{} +var _ volume.DeviceMounter = &azureDiskAttacher{} +var _ volume.DeviceUnmounter = &azureDiskDetacher{} + // acquire lock to get an lun number var getLunMutex = keymutex.NewKeyMutex() diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 278276fd9b..b191f59c86 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -79,6 +79,7 @@ var _ volume.ProvisionableVolumePlugin = &azureDataDiskPlugin{} var _ volume.AttachableVolumePlugin = &azureDataDiskPlugin{} var _ volume.VolumePluginWithAttachLimits = &azureDataDiskPlugin{} var _ volume.ExpandableVolumePlugin = &azureDataDiskPlugin{} +var _ volume.DeviceMountableVolumePlugin = &azureDataDiskPlugin{} const ( azureDataDiskPluginName = "kubernetes.io/azure-disk" @@ -276,3 +277,11 @@ func (plugin *azureDataDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([ m := plugin.host.GetMounter(plugin.GetPluginName()) return m.GetMountRefs(deviceMountPath) } + +func (plugin *azureDataDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + +func (plugin *azureDataDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} diff --git a/pkg/volume/cinder/attacher.go b/pkg/volume/cinder/attacher.go index e650954328..2cdc6da7a5 100644 --- a/pkg/volume/cinder/attacher.go +++ b/pkg/volume/cinder/attacher.go @@ -40,8 +40,12 @@ type cinderDiskAttacher struct { var _ volume.Attacher = &cinderDiskAttacher{} +var _ volume.DeviceMounter = &cinderDiskAttacher{} + var _ volume.AttachableVolumePlugin = &cinderPlugin{} +var _ volume.DeviceMountableVolumePlugin = &cinderPlugin{} + const ( probeVolumeInitDelay = 1 * time.Second probeVolumeFactor = 2.0 @@ -67,6 +71,10 @@ func (plugin *cinderPlugin) NewAttacher() (volume.Attacher, error) { }, nil } +func (plugin *cinderPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mounter.GetMountRefs(deviceMountPath) @@ -299,6 +307,8 @@ type cinderDiskDetacher struct { var _ volume.Detacher = &cinderDiskDetacher{} +var _ volume.DeviceUnmounter = &cinderDiskDetacher{} + func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) { cinder, err := plugin.getCloudProvider() if err != nil { @@ -310,6 +320,10 @@ func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) { }, nil } +func (plugin *cinderPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error { backoff := wait.Backoff{ Duration: operationFinishInitDelay, diff --git a/pkg/volume/csi/csi_attacher.go b/pkg/volume/csi/csi_attacher.go index f5d1601925..1b33926e9f 100644 --- a/pkg/volume/csi/csi_attacher.go +++ b/pkg/volume/csi/csi_attacher.go @@ -56,6 +56,8 @@ type csiAttacher struct { // volume.Attacher methods var _ volume.Attacher = &csiAttacher{} +var _ volume.DeviceMounter = &csiAttacher{} + func (c *csiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) { if spec == nil { glog.Error(log("attacher.Attach missing volume.Spec")) @@ -373,6 +375,8 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo var _ volume.Detacher = &csiAttacher{} +var _ volume.DeviceUnmounter = &csiAttacher{} + func (c *csiAttacher) Detach(volumeName string, nodeName types.NodeName) error { // volumeName in format driverNamevolumeHandle generated by plugin.GetVolumeName() if volumeName == "" { diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index c485d8815c..cc9e3c9790 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -298,6 +298,8 @@ func (p *csiPlugin) SupportsBulkVolumeVerification() bool { // volume.AttachableVolumePlugin methods var _ volume.AttachableVolumePlugin = &csiPlugin{} +var _ volume.DeviceMountableVolumePlugin = &csiPlugin{} + func (p *csiPlugin) NewAttacher() (volume.Attacher, error) { k8s := p.host.GetKubeClient() if k8s == nil { @@ -312,6 +314,10 @@ func (p *csiPlugin) NewAttacher() (volume.Attacher, error) { }, nil } +func (p *csiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return p.NewAttacher() +} + func (p *csiPlugin) NewDetacher() (volume.Detacher, error) { k8s := p.host.GetKubeClient() if k8s == nil { @@ -326,6 +332,10 @@ func (p *csiPlugin) NewDetacher() (volume.Detacher, error) { }, nil } +func (p *csiPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return p.NewDetacher() +} + func (p *csiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { m := p.host.GetMounter(p.GetPluginName()) return m.GetMountRefs(deviceMountPath) diff --git a/pkg/volume/fc/attacher.go b/pkg/volume/fc/attacher.go index 69d14fbff1..7030a94999 100644 --- a/pkg/volume/fc/attacher.go +++ b/pkg/volume/fc/attacher.go @@ -40,8 +40,12 @@ type fcAttacher struct { var _ volume.Attacher = &fcAttacher{} +var _ volume.DeviceMounter = &fcAttacher{} + var _ volume.AttachableVolumePlugin = &fcPlugin{} +var _ volume.DeviceMountableVolumePlugin = &fcPlugin{} + func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) { return &fcAttacher{ host: plugin.host, @@ -49,6 +53,10 @@ func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) { }, nil } +func (plugin *fcPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *fcPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mounter.GetMountRefs(deviceMountPath) @@ -129,6 +137,8 @@ type fcDetacher struct { var _ volume.Detacher = &fcDetacher{} +var _ volume.DeviceUnmounter = &fcDetacher{} + func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) { return &fcDetacher{ mounter: plugin.host.GetMounter(plugin.GetPluginName()), @@ -136,6 +146,10 @@ func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) { }, nil } +func (plugin *fcPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (detacher *fcDetacher) Detach(volumeName string, nodeName types.NodeName) error { return nil } diff --git a/pkg/volume/flexvolume/attacher.go b/pkg/volume/flexvolume/attacher.go index 2464b3f239..f7767d3894 100644 --- a/pkg/volume/flexvolume/attacher.go +++ b/pkg/volume/flexvolume/attacher.go @@ -31,6 +31,8 @@ type flexVolumeAttacher struct { var _ volume.Attacher = &flexVolumeAttacher{} +var _ volume.DeviceMounter = &flexVolumeAttacher{} + // Attach is part of the volume.Attacher interface func (a *flexVolumeAttacher) Attach(spec *volume.Spec, hostName types.NodeName) (string, error) { diff --git a/pkg/volume/flexvolume/detacher.go b/pkg/volume/flexvolume/detacher.go index 9111f52e6c..6b57602584 100644 --- a/pkg/volume/flexvolume/detacher.go +++ b/pkg/volume/flexvolume/detacher.go @@ -32,6 +32,8 @@ type flexVolumeDetacher struct { var _ volume.Detacher = &flexVolumeDetacher{} +var _ volume.DeviceUnmounter = &flexVolumeDetacher{} + // Detach is part of the volume.Detacher interface. func (d *flexVolumeDetacher) Detach(volumeName string, hostName types.NodeName) error { diff --git a/pkg/volume/flexvolume/plugin.go b/pkg/volume/flexvolume/plugin.go index e67318bc85..70d4b009f6 100644 --- a/pkg/volume/flexvolume/plugin.go +++ b/pkg/volume/flexvolume/plugin.go @@ -58,6 +58,8 @@ type flexVolumeAttachablePlugin struct { var _ volume.AttachableVolumePlugin = &flexVolumeAttachablePlugin{} var _ volume.PersistentVolumePlugin = &flexVolumePlugin{} +var _ volume.DeviceMountableVolumePlugin = &flexVolumeAttachablePlugin{} + type PluginFactory interface { NewFlexVolumePlugin(pluginDir, driverName string, runner exec.Interface) (volume.VolumePlugin, error) } @@ -218,11 +220,19 @@ func (plugin *flexVolumeAttachablePlugin) NewAttacher() (volume.Attacher, error) return &flexVolumeAttacher{plugin}, nil } +func (plugin *flexVolumeAttachablePlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + // NewDetacher is part of the volume.AttachableVolumePlugin interface. func (plugin *flexVolumeAttachablePlugin) NewDetacher() (volume.Detacher, error) { return &flexVolumeDetacher{plugin}, nil } +func (plugin *flexVolumeAttachablePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + // ConstructVolumeSpec is part of the volume.AttachableVolumePlugin interface. func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { flexVolume := &api.Volume{ diff --git a/pkg/volume/gce_pd/attacher.go b/pkg/volume/gce_pd/attacher.go index 6b73edbc66..1a84fc4cba 100644 --- a/pkg/volume/gce_pd/attacher.go +++ b/pkg/volume/gce_pd/attacher.go @@ -40,8 +40,12 @@ type gcePersistentDiskAttacher struct { var _ volume.Attacher = &gcePersistentDiskAttacher{} +var _ volume.DeviceMounter = &gcePersistentDiskAttacher{} + var _ volume.AttachableVolumePlugin = &gcePersistentDiskPlugin{} +var _ volume.DeviceMountableVolumePlugin = &gcePersistentDiskPlugin{} + func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) { gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -54,6 +58,10 @@ func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) { }, nil } +func (plugin *gcePersistentDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *gcePersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { mounter := plugin.host.GetMounter(plugin.GetPluginName()) return mounter.GetMountRefs(deviceMountPath) @@ -226,6 +234,8 @@ type gcePersistentDiskDetacher struct { var _ volume.Detacher = &gcePersistentDiskDetacher{} +var _ volume.DeviceUnmounter = &gcePersistentDiskDetacher{} + func (plugin *gcePersistentDiskPlugin) NewDetacher() (volume.Detacher, error) { gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -238,6 +248,10 @@ func (plugin *gcePersistentDiskPlugin) NewDetacher() (volume.Detacher, error) { }, nil } +func (plugin *gcePersistentDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + // Detach checks with the GCE cloud provider if the specified volume is already // attached to the specified node. If the volume is not attached, it succeeds // (returns nil). If it is attached, Detach issues a call to the GCE cloud diff --git a/pkg/volume/iscsi/attacher.go b/pkg/volume/iscsi/attacher.go index d7fa9eb9ec..82eccfd3d2 100644 --- a/pkg/volume/iscsi/attacher.go +++ b/pkg/volume/iscsi/attacher.go @@ -40,8 +40,12 @@ type iscsiAttacher struct { var _ volume.Attacher = &iscsiAttacher{} +var _ volume.DeviceMounter = &iscsiAttacher{} + var _ volume.AttachableVolumePlugin = &iscsiPlugin{} +var _ volume.DeviceMountableVolumePlugin = &iscsiPlugin{} + func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) { return &iscsiAttacher{ host: plugin.host, @@ -50,6 +54,10 @@ func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) { }, nil } +func (plugin *iscsiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *iscsiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) { mounter := plugin.host.GetMounter(iscsiPluginName) return mounter.GetMountRefs(deviceMountPath) @@ -133,6 +141,8 @@ type iscsiDetacher struct { var _ volume.Detacher = &iscsiDetacher{} +var _ volume.DeviceUnmounter = &iscsiDetacher{} + func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) { return &iscsiDetacher{ host: plugin.host, @@ -141,6 +151,10 @@ func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) { }, nil } +func (plugin *iscsiPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (detacher *iscsiDetacher) Detach(volumeName string, nodeName types.NodeName) error { return nil } diff --git a/pkg/volume/photon_pd/attacher.go b/pkg/volume/photon_pd/attacher.go index 368b9bea94..016fd65b83 100644 --- a/pkg/volume/photon_pd/attacher.go +++ b/pkg/volume/photon_pd/attacher.go @@ -40,8 +40,13 @@ type photonPersistentDiskAttacher struct { } var _ volume.Attacher = &photonPersistentDiskAttacher{} + +var _ volume.DeviceMounter = &photonPersistentDiskAttacher{} + var _ volume.AttachableVolumePlugin = &photonPersistentDiskPlugin{} +var _ volume.DeviceMountableVolumePlugin = &photonPersistentDiskPlugin{} + func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error) { photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -55,6 +60,10 @@ func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error) }, nil } +func (plugin *photonPersistentDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + // Attaches the volume specified by the given spec to the given host. // On success, returns the device path where the device was attached on the // node. @@ -229,6 +238,8 @@ type photonPersistentDiskDetacher struct { var _ volume.Detacher = &photonPersistentDiskDetacher{} +var _ volume.DeviceUnmounter = &photonPersistentDiskDetacher{} + func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) { photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -242,6 +253,10 @@ func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) }, nil } +func (plugin *photonPersistentDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + // Detach the given device from the given host. func (detacher *photonPersistentDiskDetacher) Detach(volumeName string, nodeName types.NodeName) error { diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index e2c7e05bb1..8ea4584f99 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -202,9 +202,17 @@ type ProvisionableVolumePlugin interface { // AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment // to a node before mounting. type AttachableVolumePlugin interface { - VolumePlugin + DeviceMountableVolumePlugin NewAttacher() (Attacher, error) NewDetacher() (Detacher, error) +} + +// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used +// for volumes that requires mount device to a node before binding to volume to pod. +type DeviceMountableVolumePlugin interface { + VolumePlugin + NewDeviceMounter() (DeviceMounter, error) + NewDeviceUnmounter() (DeviceUnmounter, error) GetDeviceMountRefs(deviceMountPath string) ([]string, error) } @@ -757,6 +765,30 @@ func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVo return nil, nil } +// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec. +func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginBySpec(spec) + if err != nil { + return nil, err + } + if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { + return deviceMountableVolumePlugin, nil + } + return nil, nil +} + +// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name. +func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) { + volumePlugin, err := pm.FindPluginByName(name) + if err != nil { + return nil, err + } + if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok { + return deviceMountableVolumePlugin, nil + } + return nil, nil +} + // FindExpandablePluginBySpec fetches a persistent volume plugin by spec. func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) { volumePlugin, err := pm.FindPluginBySpec(spec) diff --git a/pkg/volume/rbd/attacher.go b/pkg/volume/rbd/attacher.go index 628f985ecc..70fc15ea61 100644 --- a/pkg/volume/rbd/attacher.go +++ b/pkg/volume/rbd/attacher.go @@ -34,6 +34,11 @@ func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) { return plugin.newAttacherInternal(&RBDUtil{}) } +// NewDeviceMounter implements DeviceMountableVolumePlugin.NewDeviceMounter +func (plugin *rbdPlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *rbdPlugin) newAttacherInternal(manager diskManager) (volume.Attacher, error) { return &rbdAttacher{ plugin: plugin, @@ -47,6 +52,11 @@ func (plugin *rbdPlugin) NewDetacher() (volume.Detacher, error) { return plugin.newDetacherInternal(&RBDUtil{}) } +// NewDeviceUnmounter implements DeviceMountableVolumePlugin.NewDeviceUnmounter +func (plugin *rbdPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (plugin *rbdPlugin) newDetacherInternal(manager diskManager) (volume.Detacher, error) { return &rbdDetacher{ plugin: plugin, @@ -70,6 +80,8 @@ type rbdAttacher struct { var _ volume.Attacher = &rbdAttacher{} +var _ volume.DeviceMounter = &rbdAttacher{} + // Attach implements Attacher.Attach. // We do not lock image here, because it requires kube-controller-manager to // access external `rbd` utility. And there is no need since AttachDetach @@ -172,6 +184,8 @@ type rbdDetacher struct { var _ volume.Detacher = &rbdDetacher{} +var _ volume.DeviceUnmounter = &rbdDetacher{} + // UnmountDevice implements Detacher.UnmountDevice. It unmounts the global // mount of the RBD image. This is called once all bind mounts have been // unmounted. diff --git a/pkg/volume/rbd/rbd.go b/pkg/volume/rbd/rbd.go index 20776de47f..b94e307ba9 100644 --- a/pkg/volume/rbd/rbd.go +++ b/pkg/volume/rbd/rbd.go @@ -61,6 +61,7 @@ var _ volume.ProvisionableVolumePlugin = &rbdPlugin{} var _ volume.AttachableVolumePlugin = &rbdPlugin{} var _ volume.ExpandableVolumePlugin = &rbdPlugin{} var _ volume.BlockVolumePlugin = &rbdPlugin{} +var _ volume.DeviceMountableVolumePlugin = &rbdPlugin{} const ( rbdPluginName = "kubernetes.io/rbd" diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 17aa410635..cacbcb3ca3 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -242,6 +242,7 @@ var _ DeletableVolumePlugin = &FakeVolumePlugin{} var _ ProvisionableVolumePlugin = &FakeVolumePlugin{} var _ AttachableVolumePlugin = &FakeVolumePlugin{} var _ VolumePluginWithAttachLimits = &FakeVolumePlugin{} +var _ DeviceMountableVolumePlugin = &FakeVolumePlugin{} func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume { volume := &FakeVolume{} @@ -372,6 +373,10 @@ func (plugin *FakeVolumePlugin) NewAttacher() (Attacher, error) { return plugin.getFakeVolume(&plugin.Attachers), nil } +func (plugin *FakeVolumePlugin) NewDeviceMounter() (DeviceMounter, error) { + return plugin.NewAttacher() +} + func (plugin *FakeVolumePlugin) GetAttachers() (Attachers []*FakeVolume) { plugin.RLock() defer plugin.RUnlock() @@ -391,6 +396,10 @@ func (plugin *FakeVolumePlugin) NewDetacher() (Detacher, error) { return plugin.getFakeVolume(&plugin.Detachers), nil } +func (plugin *FakeVolumePlugin) NewDeviceUnmounter() (DeviceUnmounter, error) { + return plugin.NewDetacher() +} + func (plugin *FakeVolumePlugin) GetDetachers() (Detachers []*FakeVolume) { plugin.RLock() defer plugin.RUnlock() diff --git a/pkg/volume/util/operationexecutor/operation_executor.go b/pkg/volume/util/operationexecutor/operation_executor.go index 1a44c62d4a..8983ae48d3 100644 --- a/pkg/volume/util/operationexecutor/operation_executor.go +++ b/pkg/volume/util/operationexecutor/operation_executor.go @@ -329,6 +329,10 @@ type VolumeToMount struct { // the volume.Attacher interface PluginIsAttachable bool + // PluginIsDeviceMountable indicates that the plugin for this volume implements + // the volume.DeviceMounter interface + PluginIsDeviceMountable bool + // VolumeGidValue contains the value of the GID annotation, if present. VolumeGidValue string @@ -738,8 +742,8 @@ func (oe *operationExecutor) MountVolume( podName := nestedpendingoperations.EmptyUniquePodName // TODO: remove this -- not necessary - if !volumeToMount.PluginIsAttachable { - // Non-attachable volume plugins can execute mount for multiple pods + if !volumeToMount.PluginIsAttachable && !volumeToMount.PluginIsDeviceMountable { + // volume plugins which are Non-attachable and Non-deviceMountable can execute mount for multiple pods // referencing the same volume in parallel podName = util.GetUniquePodName(volumeToMount.Pod) } diff --git a/pkg/volume/util/operationexecutor/operation_executor_test.go b/pkg/volume/util/operationexecutor/operation_executor_test.go index 7d427e48f0..c7d0c9c809 100644 --- a/pkg/volume/util/operationexecutor/operation_executor_test.go +++ b/pkg/volume/util/operationexecutor/operation_executor_test.go @@ -48,7 +48,7 @@ const ( var _ OperationGenerator = &fakeOperationGenerator{} -func TestOperationExecutor_MountVolume_ConcurrentMountForNonAttachablePlugins(t *testing.T) { +func TestOperationExecutor_MountVolume_ConcurrentMountForNonAttachableAndNonDevicemountablePlugins(t *testing.T) { // Arrange ch, quit, oe := setup() volumesToMount := make([]VolumeToMount, numVolumesToMount) @@ -60,10 +60,11 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForNonAttachablePlugins(t podName := "pod-" + strconv.Itoa((i + 1)) pod := getTestPodWithSecret(podName, secretName) volumesToMount[i] = VolumeToMount{ - Pod: pod, - VolumeName: volumeName, - PluginIsAttachable: false, // this field determines whether the plugin is attachable - ReportedInUse: true, + Pod: pod, + VolumeName: volumeName, + PluginIsAttachable: false, // this field determines whether the plugin is attachable + PluginIsDeviceMountable: false, // this field determines whether the plugin is devicemountable + ReportedInUse: true, } oe.MountVolume(0 /* waitForAttachTimeOut */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */, false /* isRemount */) } @@ -99,6 +100,31 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForAttachablePlugins(t *te } } +func TestOperationExecutor_MountVolume_ConcurrentMountForDeviceMountablePlugins(t *testing.T) { + // Arrange + ch, quit, oe := setup() + volumesToMount := make([]VolumeToMount, numVolumesToAttach) + pdName := "pd-volume" + volumeName := v1.UniqueVolumeName(pdName) + // Act + for i := range volumesToMount { + podName := "pod-" + strconv.Itoa((i + 1)) + pod := getTestPodWithGCEPD(podName, pdName) + volumesToMount[i] = VolumeToMount{ + Pod: pod, + VolumeName: volumeName, + PluginIsDeviceMountable: true, // this field determines whether the plugin is devicemountable + ReportedInUse: true, + } + oe.MountVolume(0 /* waitForAttachTimeout */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */, false /* isRemount */) + } + + // Assert + if !isOperationRunSerially(ch, quit) { + t.Fatalf("Mount operations should not start concurrently for devicemountable volumes") + } +} + func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins(t *testing.T) { // Arrange ch, quit, oe := setup() diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 99c52e4b2e..57c7adfa7c 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -478,6 +478,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc( volumeAttacher, _ = attachableVolumePlugin.NewAttacher() } + // get deviceMounter, if possible + deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec) + var volumeDeviceMounter volume.DeviceMounter + if deviceMountableVolumePlugin != nil { + volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter() + } + var fsGroup *int64 if volumeToMount.Pod.Spec.SecurityContext != nil && volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil { @@ -485,28 +492,31 @@ func (og *operationGenerator) GenerateMountVolumeFunc( } mountVolumeFunc := func() (error, error) { + devicePath := volumeToMount.DevicePath if volumeAttacher != nil { // Wait for attachable volumes to finish attaching glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) - devicePath, err := volumeAttacher.WaitForAttach( - volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout) + devicePath, err = volumeAttacher.WaitForAttach( + volumeToMount.VolumeSpec, devicePath, volumeToMount.Pod, waitForAttachTimeout) if err != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.WaitForAttach failed", err) } glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) + } + if volumeDeviceMounter != nil { deviceMountPath, err := - volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) + volumeDeviceMounter.GetDeviceMountPath(volumeToMount.VolumeSpec) if err != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.GetDeviceMountPath failed", err) } // Mount device to global mount path - err = volumeAttacher.MountDevice( + err = volumeDeviceMounter.MountDevice( volumeToMount.VolumeSpec, devicePath, deviceMountPath) @@ -532,7 +542,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc( if resizeSimpleError != nil || resizeDetailedError != nil { return resizeSimpleError, resizeDetailedError } - } if og.checkNodeCapabilitiesBeforeMount { @@ -718,20 +727,31 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (volumetypes.GeneratedOperations, error) { - // Get attacher plugin - attachableVolumePlugin, err := - og.volumePluginMgr.FindAttachablePluginByName(deviceToDetach.PluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindAttachablePluginBySpec failed", err) + // Get DeviceMounter plugin + deviceMountableVolumePlugin, err := + og.volumePluginMgr.FindDeviceMountablePluginByName(deviceToDetach.PluginName) + if err != nil || deviceMountableVolumePlugin == nil { + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindDeviceMountablePluginByName failed", err) + } + volumeDeviceUmounter, err := deviceMountableVolumePlugin.NewDeviceUnmounter() + if err != nil { + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceUmounter failed", err) } - volumeDetacher, err := attachableVolumePlugin.NewDetacher() + volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter() if err != nil { - return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDetacher failed", err) + return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceMounter failed", err) } + unmountDeviceFunc := func() (error, error) { - deviceMountPath := deviceToDetach.DeviceMountPath - refs, err := attachableVolumePlugin.GetDeviceMountRefs(deviceMountPath) + //deviceMountPath := deviceToDetach.DeviceMountPath + deviceMountPath, err := + volumeDeviceMounter.GetDeviceMountPath(deviceToDetach.VolumeSpec) + if err != nil { + // On failure, return error. Caller will log and retry. + return deviceToDetach.GenerateError("GetDeviceMountPath failed", err) + } + refs, err := deviceMountableVolumePlugin.GetDeviceMountRefs(deviceMountPath) if err != nil || mount.HasMountRefs(deviceMountPath, refs) { if err == nil { @@ -740,7 +760,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( return deviceToDetach.GenerateError("GetDeviceMountRefs check failed", err) } // Execute unmount - unmountDeviceErr := volumeDetacher.UnmountDevice(deviceMountPath) + unmountDeviceErr := volumeDeviceUmounter.UnmountDevice(deviceMountPath) if unmountDeviceErr != nil { // On failure, return error. Caller will log and retry. return deviceToDetach.GenerateError("UnmountDevice failed", unmountDeviceErr) @@ -775,7 +795,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc( return volumetypes.GeneratedOperations{ OperationFunc: unmountDeviceFunc, - CompleteFunc: util.OperationCompleteHook(attachableVolumePlugin.GetPluginName(), "unmount_device"), + CompleteFunc: util.OperationCompleteHook(deviceMountableVolumePlugin.GetPluginName(), "unmount_device"), EventRecorderFunc: nil, // nil because we do not want to generate event on error }, nil } diff --git a/pkg/volume/volume.go b/pkg/volume/volume.go index 5f633214fc..013e0dc419 100644 --- a/pkg/volume/volume.go +++ b/pkg/volume/volume.go @@ -204,6 +204,8 @@ type Deleter interface { // Attacher can attach a volume to a node. type Attacher interface { + DeviceMounter + // Attaches the volume specified by the given spec to the node with the given Name. // On success, returns the device path where the device was attached on the // node. @@ -219,7 +221,10 @@ type Attacher interface { // is returned. Otherwise, if the device does not attach after // the given timeout period, an error will be returned. WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) +} +// DeviceMounter can mount a block volume to a global path. +type DeviceMounter interface { // GetDeviceMountPath returns a path where the device should // be mounted after it is attached. This is a global mount // point which should be bind mounted for individual volumes. @@ -227,6 +232,7 @@ type Attacher interface { // MountDevice mounts the disk to a global path which // individual pods can then bind mount + // Note that devicePath can be empty if the volume plugin does not implement any of Attach and WaitForAttach methods. MountDevice(spec *Spec, devicePath string, deviceMountPath string) error } @@ -240,11 +246,15 @@ type BulkVolumeVerifier interface { // Detacher can detach a volume from a node. type Detacher interface { + DeviceUnmounter // Detach the given volume from the node with the given Name. // volumeName is name of the volume as returned from plugin's // GetVolumeName(). Detach(volumeName string, nodeName types.NodeName) error +} +// DeviceUnmounter can unmount a block volume from the global path. +type DeviceUnmounter interface { // UnmountDevice unmounts the global mount of the disk. This // should only be called once all bind mounts have been // unmounted. diff --git a/pkg/volume/vsphere_volume/attacher.go b/pkg/volume/vsphere_volume/attacher.go index ad6698de84..c49be010c3 100644 --- a/pkg/volume/vsphere_volume/attacher.go +++ b/pkg/volume/vsphere_volume/attacher.go @@ -38,8 +38,13 @@ type vsphereVMDKAttacher struct { } var _ volume.Attacher = &vsphereVMDKAttacher{} + +var _ volume.DeviceMounter = &vsphereVMDKAttacher{} + var _ volume.AttachableVolumePlugin = &vsphereVolumePlugin{} +var _ volume.DeviceMountableVolumePlugin = &vsphereVolumePlugin{} + // Singleton key mutex for keeping attach operations for the same host atomic var attachdetachMutex = keymutex.NewKeyMutex() @@ -55,6 +60,10 @@ func (plugin *vsphereVolumePlugin) NewAttacher() (volume.Attacher, error) { }, nil } +func (plugin *vsphereVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) { + return plugin.NewAttacher() +} + // Attaches the volume specified by the given spec to the given host. // On success, returns the device path where the device was attached on the // node. @@ -237,6 +246,8 @@ type vsphereVMDKDetacher struct { var _ volume.Detacher = &vsphereVMDKDetacher{} +var _ volume.DeviceUnmounter = &vsphereVMDKDetacher{} + func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) { vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider()) if err != nil { @@ -249,6 +260,10 @@ func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) { }, nil } +func (plugin *vsphereVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) { + return plugin.NewDetacher() +} + // Detach the given device from the given node. func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.NodeName) error {