mirror of https://github.com/k3s-io/k3s
Merge pull request #66884 from NickrenREN/attacher-detacher-refactor
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Attacher/Detacher refactor for local storage Proposal link: https://github.com/kubernetes/community/pull/2438 **What this PR does / why we need it**: Attacher/Detacher refactor for the plugins which just need to mount device, but do not need to attach, such as local storage plugin. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: ```release-note Attacher/Detacher refactor for local storage ``` /sig storage /kind featurepull/8/head
commit
c5e74d128d
|
@ -294,6 +294,10 @@ func (plugin *TestPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
return &attacher, nil
|
return &attacher, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *TestPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *TestPlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *TestPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
detacher := testPluginDetacher{
|
detacher := testPluginDetacher{
|
||||||
detachedVolumeMap: plugin.detachedVolumeMap,
|
detachedVolumeMap: plugin.detachedVolumeMap,
|
||||||
|
@ -302,6 +306,10 @@ func (plugin *TestPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
return &detacher, nil
|
return &detacher, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *TestPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *TestPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (plugin *TestPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
return []string{}, nil
|
return []string{}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -150,6 +150,10 @@ type volumeToMount struct {
|
||||||
// the volume.Attacher interface
|
// the volume.Attacher interface
|
||||||
pluginIsAttachable bool
|
pluginIsAttachable bool
|
||||||
|
|
||||||
|
// pluginIsDeviceMountable indicates that the plugin for this volume implements
|
||||||
|
// the volume.DeviceMounter interface
|
||||||
|
pluginIsDeviceMountable bool
|
||||||
|
|
||||||
// volumeGidValue contains the value of the GID annotation, if present.
|
// volumeGidValue contains the value of the GID annotation, if present.
|
||||||
volumeGidValue string
|
volumeGidValue string
|
||||||
|
|
||||||
|
@ -220,13 +224,16 @@ func (dsw *desiredStateOfWorld) AddPodToVolume(
|
||||||
volumeName = util.GetUniqueVolumeNameForNonAttachableVolume(podName, volumePlugin, volumeSpec)
|
volumeName = util.GetUniqueVolumeNameForNonAttachableVolume(podName, volumePlugin, volumeSpec)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
deviceMountable := dsw.isDeviceMountableVolume(volumeSpec)
|
||||||
|
|
||||||
if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists {
|
if _, volumeExists := dsw.volumesToMount[volumeName]; !volumeExists {
|
||||||
dsw.volumesToMount[volumeName] = volumeToMount{
|
dsw.volumesToMount[volumeName] = volumeToMount{
|
||||||
volumeName: volumeName,
|
volumeName: volumeName,
|
||||||
podsToMount: make(map[types.UniquePodName]podToMount),
|
podsToMount: make(map[types.UniquePodName]podToMount),
|
||||||
pluginIsAttachable: attachable,
|
pluginIsAttachable: attachable,
|
||||||
volumeGidValue: volumeGidValue,
|
pluginIsDeviceMountable: deviceMountable,
|
||||||
reportedInUse: false,
|
volumeGidValue: volumeGidValue,
|
||||||
|
reportedInUse: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -346,14 +353,15 @@ func (dsw *desiredStateOfWorld) GetVolumesToMount() []VolumeToMount {
|
||||||
volumesToMount,
|
volumesToMount,
|
||||||
VolumeToMount{
|
VolumeToMount{
|
||||||
VolumeToMount: operationexecutor.VolumeToMount{
|
VolumeToMount: operationexecutor.VolumeToMount{
|
||||||
VolumeName: volumeName,
|
VolumeName: volumeName,
|
||||||
PodName: podName,
|
PodName: podName,
|
||||||
Pod: podObj.pod,
|
Pod: podObj.pod,
|
||||||
VolumeSpec: podObj.volumeSpec,
|
VolumeSpec: podObj.volumeSpec,
|
||||||
PluginIsAttachable: volumeObj.pluginIsAttachable,
|
PluginIsAttachable: volumeObj.pluginIsAttachable,
|
||||||
OuterVolumeSpecName: podObj.outerVolumeSpecName,
|
PluginIsDeviceMountable: volumeObj.pluginIsDeviceMountable,
|
||||||
VolumeGidValue: volumeObj.volumeGidValue,
|
OuterVolumeSpecName: podObj.outerVolumeSpecName,
|
||||||
ReportedInUse: volumeObj.reportedInUse}})
|
VolumeGidValue: volumeObj.volumeGidValue,
|
||||||
|
ReportedInUse: volumeObj.reportedInUse}})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return volumesToMount
|
return volumesToMount
|
||||||
|
@ -371,3 +379,15 @@ func (dsw *desiredStateOfWorld) isAttachableVolume(volumeSpec *volume.Spec) bool
|
||||||
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (dsw *desiredStateOfWorld) isDeviceMountableVolume(volumeSpec *volume.Spec) bool {
|
||||||
|
deviceMountableVolumePlugin, _ := dsw.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeSpec)
|
||||||
|
if deviceMountableVolumePlugin != nil {
|
||||||
|
volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter()
|
||||||
|
if err == nil && volumeDeviceMounter != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
|
@ -39,8 +39,12 @@ type awsElasticBlockStoreAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &awsElasticBlockStoreAttacher{}
|
var _ volume.Attacher = &awsElasticBlockStoreAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &awsElasticBlockStoreAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &awsElasticBlockStorePlugin{}
|
var _ volume.AttachableVolumePlugin = &awsElasticBlockStorePlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &awsElasticBlockStorePlugin{}
|
||||||
|
|
||||||
func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) {
|
func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -53,6 +57,10 @@ func (plugin *awsElasticBlockStorePlugin) NewAttacher() (volume.Attacher, error)
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *awsElasticBlockStorePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *awsElasticBlockStorePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (plugin *awsElasticBlockStorePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
||||||
return mounter.GetMountRefs(deviceMountPath)
|
return mounter.GetMountRefs(deviceMountPath)
|
||||||
|
@ -236,6 +244,8 @@ type awsElasticBlockStoreDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &awsElasticBlockStoreDetacher{}
|
var _ volume.Detacher = &awsElasticBlockStoreDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &awsElasticBlockStoreDetacher{}
|
||||||
|
|
||||||
func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
awsCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -248,6 +258,10 @@ func (plugin *awsElasticBlockStorePlugin) NewDetacher() (volume.Detacher, error)
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *awsElasticBlockStorePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (detacher *awsElasticBlockStoreDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
func (detacher *awsElasticBlockStoreDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
||||||
volumeID := aws.KubernetesVolumeID(path.Base(volumeName))
|
volumeID := aws.KubernetesVolumeID(path.Base(volumeName))
|
||||||
|
|
||||||
|
|
|
@ -52,6 +52,9 @@ type azureDiskAttacher struct {
|
||||||
var _ volume.Attacher = &azureDiskAttacher{}
|
var _ volume.Attacher = &azureDiskAttacher{}
|
||||||
var _ volume.Detacher = &azureDiskDetacher{}
|
var _ volume.Detacher = &azureDiskDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &azureDiskAttacher{}
|
||||||
|
var _ volume.DeviceUnmounter = &azureDiskDetacher{}
|
||||||
|
|
||||||
// acquire lock to get an lun number
|
// acquire lock to get an lun number
|
||||||
var getLunMutex = keymutex.NewKeyMutex()
|
var getLunMutex = keymutex.NewKeyMutex()
|
||||||
|
|
||||||
|
|
|
@ -79,6 +79,7 @@ var _ volume.ProvisionableVolumePlugin = &azureDataDiskPlugin{}
|
||||||
var _ volume.AttachableVolumePlugin = &azureDataDiskPlugin{}
|
var _ volume.AttachableVolumePlugin = &azureDataDiskPlugin{}
|
||||||
var _ volume.VolumePluginWithAttachLimits = &azureDataDiskPlugin{}
|
var _ volume.VolumePluginWithAttachLimits = &azureDataDiskPlugin{}
|
||||||
var _ volume.ExpandableVolumePlugin = &azureDataDiskPlugin{}
|
var _ volume.ExpandableVolumePlugin = &azureDataDiskPlugin{}
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &azureDataDiskPlugin{}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
azureDataDiskPluginName = "kubernetes.io/azure-disk"
|
azureDataDiskPluginName = "kubernetes.io/azure-disk"
|
||||||
|
@ -276,3 +277,11 @@ func (plugin *azureDataDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([
|
||||||
m := plugin.host.GetMounter(plugin.GetPluginName())
|
m := plugin.host.GetMounter(plugin.GetPluginName())
|
||||||
return m.GetMountRefs(deviceMountPath)
|
return m.GetMountRefs(deviceMountPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *azureDataDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (plugin *azureDataDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
|
@ -40,8 +40,12 @@ type cinderDiskAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &cinderDiskAttacher{}
|
var _ volume.Attacher = &cinderDiskAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &cinderDiskAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &cinderPlugin{}
|
var _ volume.AttachableVolumePlugin = &cinderPlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &cinderPlugin{}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
probeVolumeInitDelay = 1 * time.Second
|
probeVolumeInitDelay = 1 * time.Second
|
||||||
probeVolumeFactor = 2.0
|
probeVolumeFactor = 2.0
|
||||||
|
@ -67,6 +71,10 @@ func (plugin *cinderPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *cinderPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
||||||
return mounter.GetMountRefs(deviceMountPath)
|
return mounter.GetMountRefs(deviceMountPath)
|
||||||
|
@ -299,6 +307,8 @@ type cinderDiskDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &cinderDiskDetacher{}
|
var _ volume.Detacher = &cinderDiskDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &cinderDiskDetacher{}
|
||||||
|
|
||||||
func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
cinder, err := plugin.getCloudProvider()
|
cinder, err := plugin.getCloudProvider()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -310,6 +320,10 @@ func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *cinderPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error {
|
func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error {
|
||||||
backoff := wait.Backoff{
|
backoff := wait.Backoff{
|
||||||
Duration: operationFinishInitDelay,
|
Duration: operationFinishInitDelay,
|
||||||
|
|
|
@ -56,6 +56,8 @@ type csiAttacher struct {
|
||||||
// volume.Attacher methods
|
// volume.Attacher methods
|
||||||
var _ volume.Attacher = &csiAttacher{}
|
var _ volume.Attacher = &csiAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &csiAttacher{}
|
||||||
|
|
||||||
func (c *csiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
|
func (c *csiAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
|
||||||
if spec == nil {
|
if spec == nil {
|
||||||
glog.Error(log("attacher.Attach missing volume.Spec"))
|
glog.Error(log("attacher.Attach missing volume.Spec"))
|
||||||
|
@ -373,6 +375,8 @@ func (c *csiAttacher) MountDevice(spec *volume.Spec, devicePath string, deviceMo
|
||||||
|
|
||||||
var _ volume.Detacher = &csiAttacher{}
|
var _ volume.Detacher = &csiAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &csiAttacher{}
|
||||||
|
|
||||||
func (c *csiAttacher) Detach(volumeName string, nodeName types.NodeName) error {
|
func (c *csiAttacher) Detach(volumeName string, nodeName types.NodeName) error {
|
||||||
// volumeName in format driverName<SEP>volumeHandle generated by plugin.GetVolumeName()
|
// volumeName in format driverName<SEP>volumeHandle generated by plugin.GetVolumeName()
|
||||||
if volumeName == "" {
|
if volumeName == "" {
|
||||||
|
|
|
@ -298,6 +298,8 @@ func (p *csiPlugin) SupportsBulkVolumeVerification() bool {
|
||||||
// volume.AttachableVolumePlugin methods
|
// volume.AttachableVolumePlugin methods
|
||||||
var _ volume.AttachableVolumePlugin = &csiPlugin{}
|
var _ volume.AttachableVolumePlugin = &csiPlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &csiPlugin{}
|
||||||
|
|
||||||
func (p *csiPlugin) NewAttacher() (volume.Attacher, error) {
|
func (p *csiPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
k8s := p.host.GetKubeClient()
|
k8s := p.host.GetKubeClient()
|
||||||
if k8s == nil {
|
if k8s == nil {
|
||||||
|
@ -312,6 +314,10 @@ func (p *csiPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *csiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return p.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (p *csiPlugin) NewDetacher() (volume.Detacher, error) {
|
func (p *csiPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
k8s := p.host.GetKubeClient()
|
k8s := p.host.GetKubeClient()
|
||||||
if k8s == nil {
|
if k8s == nil {
|
||||||
|
@ -326,6 +332,10 @@ func (p *csiPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *csiPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return p.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (p *csiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (p *csiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
m := p.host.GetMounter(p.GetPluginName())
|
m := p.host.GetMounter(p.GetPluginName())
|
||||||
return m.GetMountRefs(deviceMountPath)
|
return m.GetMountRefs(deviceMountPath)
|
||||||
|
|
|
@ -40,8 +40,12 @@ type fcAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &fcAttacher{}
|
var _ volume.Attacher = &fcAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &fcAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &fcPlugin{}
|
var _ volume.AttachableVolumePlugin = &fcPlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &fcPlugin{}
|
||||||
|
|
||||||
func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) {
|
func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
return &fcAttacher{
|
return &fcAttacher{
|
||||||
host: plugin.host,
|
host: plugin.host,
|
||||||
|
@ -49,6 +53,10 @@ func (plugin *fcPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *fcPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *fcPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (plugin *fcPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
||||||
return mounter.GetMountRefs(deviceMountPath)
|
return mounter.GetMountRefs(deviceMountPath)
|
||||||
|
@ -129,6 +137,8 @@ type fcDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &fcDetacher{}
|
var _ volume.Detacher = &fcDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &fcDetacher{}
|
||||||
|
|
||||||
func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
return &fcDetacher{
|
return &fcDetacher{
|
||||||
mounter: plugin.host.GetMounter(plugin.GetPluginName()),
|
mounter: plugin.host.GetMounter(plugin.GetPluginName()),
|
||||||
|
@ -136,6 +146,10 @@ func (plugin *fcPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *fcPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (detacher *fcDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
func (detacher *fcDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,6 +31,8 @@ type flexVolumeAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &flexVolumeAttacher{}
|
var _ volume.Attacher = &flexVolumeAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &flexVolumeAttacher{}
|
||||||
|
|
||||||
// Attach is part of the volume.Attacher interface
|
// Attach is part of the volume.Attacher interface
|
||||||
func (a *flexVolumeAttacher) Attach(spec *volume.Spec, hostName types.NodeName) (string, error) {
|
func (a *flexVolumeAttacher) Attach(spec *volume.Spec, hostName types.NodeName) (string, error) {
|
||||||
|
|
||||||
|
|
|
@ -32,6 +32,8 @@ type flexVolumeDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &flexVolumeDetacher{}
|
var _ volume.Detacher = &flexVolumeDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &flexVolumeDetacher{}
|
||||||
|
|
||||||
// Detach is part of the volume.Detacher interface.
|
// Detach is part of the volume.Detacher interface.
|
||||||
func (d *flexVolumeDetacher) Detach(volumeName string, hostName types.NodeName) error {
|
func (d *flexVolumeDetacher) Detach(volumeName string, hostName types.NodeName) error {
|
||||||
|
|
||||||
|
|
|
@ -58,6 +58,8 @@ type flexVolumeAttachablePlugin struct {
|
||||||
var _ volume.AttachableVolumePlugin = &flexVolumeAttachablePlugin{}
|
var _ volume.AttachableVolumePlugin = &flexVolumeAttachablePlugin{}
|
||||||
var _ volume.PersistentVolumePlugin = &flexVolumePlugin{}
|
var _ volume.PersistentVolumePlugin = &flexVolumePlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &flexVolumeAttachablePlugin{}
|
||||||
|
|
||||||
type PluginFactory interface {
|
type PluginFactory interface {
|
||||||
NewFlexVolumePlugin(pluginDir, driverName string, runner exec.Interface) (volume.VolumePlugin, error)
|
NewFlexVolumePlugin(pluginDir, driverName string, runner exec.Interface) (volume.VolumePlugin, error)
|
||||||
}
|
}
|
||||||
|
@ -218,11 +220,19 @@ func (plugin *flexVolumeAttachablePlugin) NewAttacher() (volume.Attacher, error)
|
||||||
return &flexVolumeAttacher{plugin}, nil
|
return &flexVolumeAttacher{plugin}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *flexVolumeAttachablePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
// NewDetacher is part of the volume.AttachableVolumePlugin interface.
|
// NewDetacher is part of the volume.AttachableVolumePlugin interface.
|
||||||
func (plugin *flexVolumeAttachablePlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *flexVolumeAttachablePlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
return &flexVolumeDetacher{plugin}, nil
|
return &flexVolumeDetacher{plugin}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *flexVolumeAttachablePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
// ConstructVolumeSpec is part of the volume.AttachableVolumePlugin interface.
|
// ConstructVolumeSpec is part of the volume.AttachableVolumePlugin interface.
|
||||||
func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
|
func (plugin *flexVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
|
||||||
flexVolume := &api.Volume{
|
flexVolume := &api.Volume{
|
||||||
|
|
|
@ -40,8 +40,12 @@ type gcePersistentDiskAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &gcePersistentDiskAttacher{}
|
var _ volume.Attacher = &gcePersistentDiskAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &gcePersistentDiskAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &gcePersistentDiskPlugin{}
|
var _ volume.AttachableVolumePlugin = &gcePersistentDiskPlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &gcePersistentDiskPlugin{}
|
||||||
|
|
||||||
func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) {
|
func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -54,6 +58,10 @@ func (plugin *gcePersistentDiskPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *gcePersistentDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *gcePersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (plugin *gcePersistentDiskPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
mounter := plugin.host.GetMounter(plugin.GetPluginName())
|
||||||
return mounter.GetMountRefs(deviceMountPath)
|
return mounter.GetMountRefs(deviceMountPath)
|
||||||
|
@ -226,6 +234,8 @@ type gcePersistentDiskDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &gcePersistentDiskDetacher{}
|
var _ volume.Detacher = &gcePersistentDiskDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &gcePersistentDiskDetacher{}
|
||||||
|
|
||||||
func (plugin *gcePersistentDiskPlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *gcePersistentDiskPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
gceCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -238,6 +248,10 @@ func (plugin *gcePersistentDiskPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *gcePersistentDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
// Detach checks with the GCE cloud provider if the specified volume is already
|
// Detach checks with the GCE cloud provider if the specified volume is already
|
||||||
// attached to the specified node. If the volume is not attached, it succeeds
|
// attached to the specified node. If the volume is not attached, it succeeds
|
||||||
// (returns nil). If it is attached, Detach issues a call to the GCE cloud
|
// (returns nil). If it is attached, Detach issues a call to the GCE cloud
|
||||||
|
|
|
@ -40,8 +40,12 @@ type iscsiAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &iscsiAttacher{}
|
var _ volume.Attacher = &iscsiAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &iscsiAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &iscsiPlugin{}
|
var _ volume.AttachableVolumePlugin = &iscsiPlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &iscsiPlugin{}
|
||||||
|
|
||||||
func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) {
|
func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
return &iscsiAttacher{
|
return &iscsiAttacher{
|
||||||
host: plugin.host,
|
host: plugin.host,
|
||||||
|
@ -50,6 +54,10 @@ func (plugin *iscsiPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *iscsiPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *iscsiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
func (plugin *iscsiPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
|
||||||
mounter := plugin.host.GetMounter(iscsiPluginName)
|
mounter := plugin.host.GetMounter(iscsiPluginName)
|
||||||
return mounter.GetMountRefs(deviceMountPath)
|
return mounter.GetMountRefs(deviceMountPath)
|
||||||
|
@ -133,6 +141,8 @@ type iscsiDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &iscsiDetacher{}
|
var _ volume.Detacher = &iscsiDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &iscsiDetacher{}
|
||||||
|
|
||||||
func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
return &iscsiDetacher{
|
return &iscsiDetacher{
|
||||||
host: plugin.host,
|
host: plugin.host,
|
||||||
|
@ -141,6 +151,10 @@ func (plugin *iscsiPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *iscsiPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (detacher *iscsiDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
func (detacher *iscsiDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,8 +40,13 @@ type photonPersistentDiskAttacher struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ volume.Attacher = &photonPersistentDiskAttacher{}
|
var _ volume.Attacher = &photonPersistentDiskAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &photonPersistentDiskAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &photonPersistentDiskPlugin{}
|
var _ volume.AttachableVolumePlugin = &photonPersistentDiskPlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &photonPersistentDiskPlugin{}
|
||||||
|
|
||||||
func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error) {
|
func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -55,6 +60,10 @@ func (plugin *photonPersistentDiskPlugin) NewAttacher() (volume.Attacher, error)
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *photonPersistentDiskPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
// Attaches the volume specified by the given spec to the given host.
|
// Attaches the volume specified by the given spec to the given host.
|
||||||
// On success, returns the device path where the device was attached on the
|
// On success, returns the device path where the device was attached on the
|
||||||
// node.
|
// node.
|
||||||
|
@ -229,6 +238,8 @@ type photonPersistentDiskDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &photonPersistentDiskDetacher{}
|
var _ volume.Detacher = &photonPersistentDiskDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &photonPersistentDiskDetacher{}
|
||||||
|
|
||||||
func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
photonCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -242,6 +253,10 @@ func (plugin *photonPersistentDiskPlugin) NewDetacher() (volume.Detacher, error)
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *photonPersistentDiskPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
// Detach the given device from the given host.
|
// Detach the given device from the given host.
|
||||||
func (detacher *photonPersistentDiskDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
func (detacher *photonPersistentDiskDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
||||||
|
|
||||||
|
|
|
@ -202,9 +202,17 @@ type ProvisionableVolumePlugin interface {
|
||||||
// AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment
|
// AttachableVolumePlugin is an extended interface of VolumePlugin and is used for volumes that require attachment
|
||||||
// to a node before mounting.
|
// to a node before mounting.
|
||||||
type AttachableVolumePlugin interface {
|
type AttachableVolumePlugin interface {
|
||||||
VolumePlugin
|
DeviceMountableVolumePlugin
|
||||||
NewAttacher() (Attacher, error)
|
NewAttacher() (Attacher, error)
|
||||||
NewDetacher() (Detacher, error)
|
NewDetacher() (Detacher, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeviceMountableVolumePlugin is an extended interface of VolumePlugin and is used
|
||||||
|
// for volumes that requires mount device to a node before binding to volume to pod.
|
||||||
|
type DeviceMountableVolumePlugin interface {
|
||||||
|
VolumePlugin
|
||||||
|
NewDeviceMounter() (DeviceMounter, error)
|
||||||
|
NewDeviceUnmounter() (DeviceUnmounter, error)
|
||||||
GetDeviceMountRefs(deviceMountPath string) ([]string, error)
|
GetDeviceMountRefs(deviceMountPath string) ([]string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -757,6 +765,30 @@ func (pm *VolumePluginMgr) FindAttachablePluginByName(name string) (AttachableVo
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FindDeviceMountablePluginBySpec fetches a persistent volume plugin by spec.
|
||||||
|
func (pm *VolumePluginMgr) FindDeviceMountablePluginBySpec(spec *Spec) (DeviceMountableVolumePlugin, error) {
|
||||||
|
volumePlugin, err := pm.FindPluginBySpec(spec)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
|
||||||
|
return deviceMountableVolumePlugin, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindDeviceMountablePluginByName fetches a devicemountable volume plugin by name.
|
||||||
|
func (pm *VolumePluginMgr) FindDeviceMountablePluginByName(name string) (DeviceMountableVolumePlugin, error) {
|
||||||
|
volumePlugin, err := pm.FindPluginByName(name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if deviceMountableVolumePlugin, ok := volumePlugin.(DeviceMountableVolumePlugin); ok {
|
||||||
|
return deviceMountableVolumePlugin, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
// FindExpandablePluginBySpec fetches a persistent volume plugin by spec.
|
// FindExpandablePluginBySpec fetches a persistent volume plugin by spec.
|
||||||
func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) {
|
func (pm *VolumePluginMgr) FindExpandablePluginBySpec(spec *Spec) (ExpandableVolumePlugin, error) {
|
||||||
volumePlugin, err := pm.FindPluginBySpec(spec)
|
volumePlugin, err := pm.FindPluginBySpec(spec)
|
||||||
|
|
|
@ -34,6 +34,11 @@ func (plugin *rbdPlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
return plugin.newAttacherInternal(&RBDUtil{})
|
return plugin.newAttacherInternal(&RBDUtil{})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewDeviceMounter implements DeviceMountableVolumePlugin.NewDeviceMounter
|
||||||
|
func (plugin *rbdPlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *rbdPlugin) newAttacherInternal(manager diskManager) (volume.Attacher, error) {
|
func (plugin *rbdPlugin) newAttacherInternal(manager diskManager) (volume.Attacher, error) {
|
||||||
return &rbdAttacher{
|
return &rbdAttacher{
|
||||||
plugin: plugin,
|
plugin: plugin,
|
||||||
|
@ -47,6 +52,11 @@ func (plugin *rbdPlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
return plugin.newDetacherInternal(&RBDUtil{})
|
return plugin.newDetacherInternal(&RBDUtil{})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewDeviceUnmounter implements DeviceMountableVolumePlugin.NewDeviceUnmounter
|
||||||
|
func (plugin *rbdPlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *rbdPlugin) newDetacherInternal(manager diskManager) (volume.Detacher, error) {
|
func (plugin *rbdPlugin) newDetacherInternal(manager diskManager) (volume.Detacher, error) {
|
||||||
return &rbdDetacher{
|
return &rbdDetacher{
|
||||||
plugin: plugin,
|
plugin: plugin,
|
||||||
|
@ -70,6 +80,8 @@ type rbdAttacher struct {
|
||||||
|
|
||||||
var _ volume.Attacher = &rbdAttacher{}
|
var _ volume.Attacher = &rbdAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &rbdAttacher{}
|
||||||
|
|
||||||
// Attach implements Attacher.Attach.
|
// Attach implements Attacher.Attach.
|
||||||
// We do not lock image here, because it requires kube-controller-manager to
|
// We do not lock image here, because it requires kube-controller-manager to
|
||||||
// access external `rbd` utility. And there is no need since AttachDetach
|
// access external `rbd` utility. And there is no need since AttachDetach
|
||||||
|
@ -172,6 +184,8 @@ type rbdDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &rbdDetacher{}
|
var _ volume.Detacher = &rbdDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &rbdDetacher{}
|
||||||
|
|
||||||
// UnmountDevice implements Detacher.UnmountDevice. It unmounts the global
|
// UnmountDevice implements Detacher.UnmountDevice. It unmounts the global
|
||||||
// mount of the RBD image. This is called once all bind mounts have been
|
// mount of the RBD image. This is called once all bind mounts have been
|
||||||
// unmounted.
|
// unmounted.
|
||||||
|
|
|
@ -61,6 +61,7 @@ var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
|
||||||
var _ volume.AttachableVolumePlugin = &rbdPlugin{}
|
var _ volume.AttachableVolumePlugin = &rbdPlugin{}
|
||||||
var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
|
var _ volume.ExpandableVolumePlugin = &rbdPlugin{}
|
||||||
var _ volume.BlockVolumePlugin = &rbdPlugin{}
|
var _ volume.BlockVolumePlugin = &rbdPlugin{}
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &rbdPlugin{}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
rbdPluginName = "kubernetes.io/rbd"
|
rbdPluginName = "kubernetes.io/rbd"
|
||||||
|
|
|
@ -242,6 +242,7 @@ var _ DeletableVolumePlugin = &FakeVolumePlugin{}
|
||||||
var _ ProvisionableVolumePlugin = &FakeVolumePlugin{}
|
var _ ProvisionableVolumePlugin = &FakeVolumePlugin{}
|
||||||
var _ AttachableVolumePlugin = &FakeVolumePlugin{}
|
var _ AttachableVolumePlugin = &FakeVolumePlugin{}
|
||||||
var _ VolumePluginWithAttachLimits = &FakeVolumePlugin{}
|
var _ VolumePluginWithAttachLimits = &FakeVolumePlugin{}
|
||||||
|
var _ DeviceMountableVolumePlugin = &FakeVolumePlugin{}
|
||||||
|
|
||||||
func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume {
|
func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume {
|
||||||
volume := &FakeVolume{}
|
volume := &FakeVolume{}
|
||||||
|
@ -372,6 +373,10 @@ func (plugin *FakeVolumePlugin) NewAttacher() (Attacher, error) {
|
||||||
return plugin.getFakeVolume(&plugin.Attachers), nil
|
return plugin.getFakeVolume(&plugin.Attachers), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *FakeVolumePlugin) NewDeviceMounter() (DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *FakeVolumePlugin) GetAttachers() (Attachers []*FakeVolume) {
|
func (plugin *FakeVolumePlugin) GetAttachers() (Attachers []*FakeVolume) {
|
||||||
plugin.RLock()
|
plugin.RLock()
|
||||||
defer plugin.RUnlock()
|
defer plugin.RUnlock()
|
||||||
|
@ -391,6 +396,10 @@ func (plugin *FakeVolumePlugin) NewDetacher() (Detacher, error) {
|
||||||
return plugin.getFakeVolume(&plugin.Detachers), nil
|
return plugin.getFakeVolume(&plugin.Detachers), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *FakeVolumePlugin) NewDeviceUnmounter() (DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
func (plugin *FakeVolumePlugin) GetDetachers() (Detachers []*FakeVolume) {
|
func (plugin *FakeVolumePlugin) GetDetachers() (Detachers []*FakeVolume) {
|
||||||
plugin.RLock()
|
plugin.RLock()
|
||||||
defer plugin.RUnlock()
|
defer plugin.RUnlock()
|
||||||
|
|
|
@ -329,6 +329,10 @@ type VolumeToMount struct {
|
||||||
// the volume.Attacher interface
|
// the volume.Attacher interface
|
||||||
PluginIsAttachable bool
|
PluginIsAttachable bool
|
||||||
|
|
||||||
|
// PluginIsDeviceMountable indicates that the plugin for this volume implements
|
||||||
|
// the volume.DeviceMounter interface
|
||||||
|
PluginIsDeviceMountable bool
|
||||||
|
|
||||||
// VolumeGidValue contains the value of the GID annotation, if present.
|
// VolumeGidValue contains the value of the GID annotation, if present.
|
||||||
VolumeGidValue string
|
VolumeGidValue string
|
||||||
|
|
||||||
|
@ -738,8 +742,8 @@ func (oe *operationExecutor) MountVolume(
|
||||||
podName := nestedpendingoperations.EmptyUniquePodName
|
podName := nestedpendingoperations.EmptyUniquePodName
|
||||||
|
|
||||||
// TODO: remove this -- not necessary
|
// TODO: remove this -- not necessary
|
||||||
if !volumeToMount.PluginIsAttachable {
|
if !volumeToMount.PluginIsAttachable && !volumeToMount.PluginIsDeviceMountable {
|
||||||
// Non-attachable volume plugins can execute mount for multiple pods
|
// volume plugins which are Non-attachable and Non-deviceMountable can execute mount for multiple pods
|
||||||
// referencing the same volume in parallel
|
// referencing the same volume in parallel
|
||||||
podName = util.GetUniquePodName(volumeToMount.Pod)
|
podName = util.GetUniquePodName(volumeToMount.Pod)
|
||||||
}
|
}
|
||||||
|
|
|
@ -48,7 +48,7 @@ const (
|
||||||
|
|
||||||
var _ OperationGenerator = &fakeOperationGenerator{}
|
var _ OperationGenerator = &fakeOperationGenerator{}
|
||||||
|
|
||||||
func TestOperationExecutor_MountVolume_ConcurrentMountForNonAttachablePlugins(t *testing.T) {
|
func TestOperationExecutor_MountVolume_ConcurrentMountForNonAttachableAndNonDevicemountablePlugins(t *testing.T) {
|
||||||
// Arrange
|
// Arrange
|
||||||
ch, quit, oe := setup()
|
ch, quit, oe := setup()
|
||||||
volumesToMount := make([]VolumeToMount, numVolumesToMount)
|
volumesToMount := make([]VolumeToMount, numVolumesToMount)
|
||||||
|
@ -60,10 +60,11 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForNonAttachablePlugins(t
|
||||||
podName := "pod-" + strconv.Itoa((i + 1))
|
podName := "pod-" + strconv.Itoa((i + 1))
|
||||||
pod := getTestPodWithSecret(podName, secretName)
|
pod := getTestPodWithSecret(podName, secretName)
|
||||||
volumesToMount[i] = VolumeToMount{
|
volumesToMount[i] = VolumeToMount{
|
||||||
Pod: pod,
|
Pod: pod,
|
||||||
VolumeName: volumeName,
|
VolumeName: volumeName,
|
||||||
PluginIsAttachable: false, // this field determines whether the plugin is attachable
|
PluginIsAttachable: false, // this field determines whether the plugin is attachable
|
||||||
ReportedInUse: true,
|
PluginIsDeviceMountable: false, // this field determines whether the plugin is devicemountable
|
||||||
|
ReportedInUse: true,
|
||||||
}
|
}
|
||||||
oe.MountVolume(0 /* waitForAttachTimeOut */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */, false /* isRemount */)
|
oe.MountVolume(0 /* waitForAttachTimeOut */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */, false /* isRemount */)
|
||||||
}
|
}
|
||||||
|
@ -99,6 +100,31 @@ func TestOperationExecutor_MountVolume_ConcurrentMountForAttachablePlugins(t *te
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOperationExecutor_MountVolume_ConcurrentMountForDeviceMountablePlugins(t *testing.T) {
|
||||||
|
// Arrange
|
||||||
|
ch, quit, oe := setup()
|
||||||
|
volumesToMount := make([]VolumeToMount, numVolumesToAttach)
|
||||||
|
pdName := "pd-volume"
|
||||||
|
volumeName := v1.UniqueVolumeName(pdName)
|
||||||
|
// Act
|
||||||
|
for i := range volumesToMount {
|
||||||
|
podName := "pod-" + strconv.Itoa((i + 1))
|
||||||
|
pod := getTestPodWithGCEPD(podName, pdName)
|
||||||
|
volumesToMount[i] = VolumeToMount{
|
||||||
|
Pod: pod,
|
||||||
|
VolumeName: volumeName,
|
||||||
|
PluginIsDeviceMountable: true, // this field determines whether the plugin is devicemountable
|
||||||
|
ReportedInUse: true,
|
||||||
|
}
|
||||||
|
oe.MountVolume(0 /* waitForAttachTimeout */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */, false /* isRemount */)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assert
|
||||||
|
if !isOperationRunSerially(ch, quit) {
|
||||||
|
t.Fatalf("Mount operations should not start concurrently for devicemountable volumes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins(t *testing.T) {
|
func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins(t *testing.T) {
|
||||||
// Arrange
|
// Arrange
|
||||||
ch, quit, oe := setup()
|
ch, quit, oe := setup()
|
||||||
|
|
|
@ -478,6 +478,13 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
|
||||||
volumeAttacher, _ = attachableVolumePlugin.NewAttacher()
|
volumeAttacher, _ = attachableVolumePlugin.NewAttacher()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// get deviceMounter, if possible
|
||||||
|
deviceMountableVolumePlugin, _ := og.volumePluginMgr.FindDeviceMountablePluginBySpec(volumeToMount.VolumeSpec)
|
||||||
|
var volumeDeviceMounter volume.DeviceMounter
|
||||||
|
if deviceMountableVolumePlugin != nil {
|
||||||
|
volumeDeviceMounter, _ = deviceMountableVolumePlugin.NewDeviceMounter()
|
||||||
|
}
|
||||||
|
|
||||||
var fsGroup *int64
|
var fsGroup *int64
|
||||||
if volumeToMount.Pod.Spec.SecurityContext != nil &&
|
if volumeToMount.Pod.Spec.SecurityContext != nil &&
|
||||||
volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil {
|
volumeToMount.Pod.Spec.SecurityContext.FSGroup != nil {
|
||||||
|
@ -485,28 +492,31 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
|
||||||
}
|
}
|
||||||
|
|
||||||
mountVolumeFunc := func() (error, error) {
|
mountVolumeFunc := func() (error, error) {
|
||||||
|
devicePath := volumeToMount.DevicePath
|
||||||
if volumeAttacher != nil {
|
if volumeAttacher != nil {
|
||||||
// Wait for attachable volumes to finish attaching
|
// Wait for attachable volumes to finish attaching
|
||||||
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath)))
|
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath)))
|
||||||
|
|
||||||
devicePath, err := volumeAttacher.WaitForAttach(
|
devicePath, err = volumeAttacher.WaitForAttach(
|
||||||
volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout)
|
volumeToMount.VolumeSpec, devicePath, volumeToMount.Pod, waitForAttachTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// On failure, return error. Caller will log and retry.
|
// On failure, return error. Caller will log and retry.
|
||||||
return volumeToMount.GenerateError("MountVolume.WaitForAttach failed", err)
|
return volumeToMount.GenerateError("MountVolume.WaitForAttach failed", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
|
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
|
||||||
|
}
|
||||||
|
|
||||||
|
if volumeDeviceMounter != nil {
|
||||||
deviceMountPath, err :=
|
deviceMountPath, err :=
|
||||||
volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec)
|
volumeDeviceMounter.GetDeviceMountPath(volumeToMount.VolumeSpec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// On failure, return error. Caller will log and retry.
|
// On failure, return error. Caller will log and retry.
|
||||||
return volumeToMount.GenerateError("MountVolume.GetDeviceMountPath failed", err)
|
return volumeToMount.GenerateError("MountVolume.GetDeviceMountPath failed", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mount device to global mount path
|
// Mount device to global mount path
|
||||||
err = volumeAttacher.MountDevice(
|
err = volumeDeviceMounter.MountDevice(
|
||||||
volumeToMount.VolumeSpec,
|
volumeToMount.VolumeSpec,
|
||||||
devicePath,
|
devicePath,
|
||||||
deviceMountPath)
|
deviceMountPath)
|
||||||
|
@ -532,7 +542,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
|
||||||
if resizeSimpleError != nil || resizeDetailedError != nil {
|
if resizeSimpleError != nil || resizeDetailedError != nil {
|
||||||
return resizeSimpleError, resizeDetailedError
|
return resizeSimpleError, resizeDetailedError
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if og.checkNodeCapabilitiesBeforeMount {
|
if og.checkNodeCapabilitiesBeforeMount {
|
||||||
|
@ -718,20 +727,31 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
|
||||||
deviceToDetach AttachedVolume,
|
deviceToDetach AttachedVolume,
|
||||||
actualStateOfWorld ActualStateOfWorldMounterUpdater,
|
actualStateOfWorld ActualStateOfWorldMounterUpdater,
|
||||||
mounter mount.Interface) (volumetypes.GeneratedOperations, error) {
|
mounter mount.Interface) (volumetypes.GeneratedOperations, error) {
|
||||||
// Get attacher plugin
|
// Get DeviceMounter plugin
|
||||||
attachableVolumePlugin, err :=
|
deviceMountableVolumePlugin, err :=
|
||||||
og.volumePluginMgr.FindAttachablePluginByName(deviceToDetach.PluginName)
|
og.volumePluginMgr.FindDeviceMountablePluginByName(deviceToDetach.PluginName)
|
||||||
if err != nil || attachableVolumePlugin == nil {
|
if err != nil || deviceMountableVolumePlugin == nil {
|
||||||
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindAttachablePluginBySpec failed", err)
|
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.FindDeviceMountablePluginByName failed", err)
|
||||||
|
}
|
||||||
|
volumeDeviceUmounter, err := deviceMountableVolumePlugin.NewDeviceUnmounter()
|
||||||
|
if err != nil {
|
||||||
|
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceUmounter failed", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
volumeDetacher, err := attachableVolumePlugin.NewDetacher()
|
volumeDeviceMounter, err := deviceMountableVolumePlugin.NewDeviceMounter()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDetacher failed", err)
|
return volumetypes.GeneratedOperations{}, deviceToDetach.GenerateErrorDetailed("UnmountDevice.NewDeviceMounter failed", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
unmountDeviceFunc := func() (error, error) {
|
unmountDeviceFunc := func() (error, error) {
|
||||||
deviceMountPath := deviceToDetach.DeviceMountPath
|
//deviceMountPath := deviceToDetach.DeviceMountPath
|
||||||
refs, err := attachableVolumePlugin.GetDeviceMountRefs(deviceMountPath)
|
deviceMountPath, err :=
|
||||||
|
volumeDeviceMounter.GetDeviceMountPath(deviceToDetach.VolumeSpec)
|
||||||
|
if err != nil {
|
||||||
|
// On failure, return error. Caller will log and retry.
|
||||||
|
return deviceToDetach.GenerateError("GetDeviceMountPath failed", err)
|
||||||
|
}
|
||||||
|
refs, err := deviceMountableVolumePlugin.GetDeviceMountRefs(deviceMountPath)
|
||||||
|
|
||||||
if err != nil || mount.HasMountRefs(deviceMountPath, refs) {
|
if err != nil || mount.HasMountRefs(deviceMountPath, refs) {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
@ -740,7 +760,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
|
||||||
return deviceToDetach.GenerateError("GetDeviceMountRefs check failed", err)
|
return deviceToDetach.GenerateError("GetDeviceMountRefs check failed", err)
|
||||||
}
|
}
|
||||||
// Execute unmount
|
// Execute unmount
|
||||||
unmountDeviceErr := volumeDetacher.UnmountDevice(deviceMountPath)
|
unmountDeviceErr := volumeDeviceUmounter.UnmountDevice(deviceMountPath)
|
||||||
if unmountDeviceErr != nil {
|
if unmountDeviceErr != nil {
|
||||||
// On failure, return error. Caller will log and retry.
|
// On failure, return error. Caller will log and retry.
|
||||||
return deviceToDetach.GenerateError("UnmountDevice failed", unmountDeviceErr)
|
return deviceToDetach.GenerateError("UnmountDevice failed", unmountDeviceErr)
|
||||||
|
@ -775,7 +795,7 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
|
||||||
|
|
||||||
return volumetypes.GeneratedOperations{
|
return volumetypes.GeneratedOperations{
|
||||||
OperationFunc: unmountDeviceFunc,
|
OperationFunc: unmountDeviceFunc,
|
||||||
CompleteFunc: util.OperationCompleteHook(attachableVolumePlugin.GetPluginName(), "unmount_device"),
|
CompleteFunc: util.OperationCompleteHook(deviceMountableVolumePlugin.GetPluginName(), "unmount_device"),
|
||||||
EventRecorderFunc: nil, // nil because we do not want to generate event on error
|
EventRecorderFunc: nil, // nil because we do not want to generate event on error
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -204,6 +204,8 @@ type Deleter interface {
|
||||||
|
|
||||||
// Attacher can attach a volume to a node.
|
// Attacher can attach a volume to a node.
|
||||||
type Attacher interface {
|
type Attacher interface {
|
||||||
|
DeviceMounter
|
||||||
|
|
||||||
// Attaches the volume specified by the given spec to the node with the given Name.
|
// Attaches the volume specified by the given spec to the node with the given Name.
|
||||||
// On success, returns the device path where the device was attached on the
|
// On success, returns the device path where the device was attached on the
|
||||||
// node.
|
// node.
|
||||||
|
@ -219,7 +221,10 @@ type Attacher interface {
|
||||||
// is returned. Otherwise, if the device does not attach after
|
// is returned. Otherwise, if the device does not attach after
|
||||||
// the given timeout period, an error will be returned.
|
// the given timeout period, an error will be returned.
|
||||||
WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error)
|
WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeviceMounter can mount a block volume to a global path.
|
||||||
|
type DeviceMounter interface {
|
||||||
// GetDeviceMountPath returns a path where the device should
|
// GetDeviceMountPath returns a path where the device should
|
||||||
// be mounted after it is attached. This is a global mount
|
// be mounted after it is attached. This is a global mount
|
||||||
// point which should be bind mounted for individual volumes.
|
// point which should be bind mounted for individual volumes.
|
||||||
|
@ -227,6 +232,7 @@ type Attacher interface {
|
||||||
|
|
||||||
// MountDevice mounts the disk to a global path which
|
// MountDevice mounts the disk to a global path which
|
||||||
// individual pods can then bind mount
|
// individual pods can then bind mount
|
||||||
|
// Note that devicePath can be empty if the volume plugin does not implement any of Attach and WaitForAttach methods.
|
||||||
MountDevice(spec *Spec, devicePath string, deviceMountPath string) error
|
MountDevice(spec *Spec, devicePath string, deviceMountPath string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,11 +246,15 @@ type BulkVolumeVerifier interface {
|
||||||
|
|
||||||
// Detacher can detach a volume from a node.
|
// Detacher can detach a volume from a node.
|
||||||
type Detacher interface {
|
type Detacher interface {
|
||||||
|
DeviceUnmounter
|
||||||
// Detach the given volume from the node with the given Name.
|
// Detach the given volume from the node with the given Name.
|
||||||
// volumeName is name of the volume as returned from plugin's
|
// volumeName is name of the volume as returned from plugin's
|
||||||
// GetVolumeName().
|
// GetVolumeName().
|
||||||
Detach(volumeName string, nodeName types.NodeName) error
|
Detach(volumeName string, nodeName types.NodeName) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeviceUnmounter can unmount a block volume from the global path.
|
||||||
|
type DeviceUnmounter interface {
|
||||||
// UnmountDevice unmounts the global mount of the disk. This
|
// UnmountDevice unmounts the global mount of the disk. This
|
||||||
// should only be called once all bind mounts have been
|
// should only be called once all bind mounts have been
|
||||||
// unmounted.
|
// unmounted.
|
||||||
|
|
|
@ -38,8 +38,13 @@ type vsphereVMDKAttacher struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ volume.Attacher = &vsphereVMDKAttacher{}
|
var _ volume.Attacher = &vsphereVMDKAttacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMounter = &vsphereVMDKAttacher{}
|
||||||
|
|
||||||
var _ volume.AttachableVolumePlugin = &vsphereVolumePlugin{}
|
var _ volume.AttachableVolumePlugin = &vsphereVolumePlugin{}
|
||||||
|
|
||||||
|
var _ volume.DeviceMountableVolumePlugin = &vsphereVolumePlugin{}
|
||||||
|
|
||||||
// Singleton key mutex for keeping attach operations for the same host atomic
|
// Singleton key mutex for keeping attach operations for the same host atomic
|
||||||
var attachdetachMutex = keymutex.NewKeyMutex()
|
var attachdetachMutex = keymutex.NewKeyMutex()
|
||||||
|
|
||||||
|
@ -55,6 +60,10 @@ func (plugin *vsphereVolumePlugin) NewAttacher() (volume.Attacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *vsphereVolumePlugin) NewDeviceMounter() (volume.DeviceMounter, error) {
|
||||||
|
return plugin.NewAttacher()
|
||||||
|
}
|
||||||
|
|
||||||
// Attaches the volume specified by the given spec to the given host.
|
// Attaches the volume specified by the given spec to the given host.
|
||||||
// On success, returns the device path where the device was attached on the
|
// On success, returns the device path where the device was attached on the
|
||||||
// node.
|
// node.
|
||||||
|
@ -237,6 +246,8 @@ type vsphereVMDKDetacher struct {
|
||||||
|
|
||||||
var _ volume.Detacher = &vsphereVMDKDetacher{}
|
var _ volume.Detacher = &vsphereVMDKDetacher{}
|
||||||
|
|
||||||
|
var _ volume.DeviceUnmounter = &vsphereVMDKDetacher{}
|
||||||
|
|
||||||
func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) {
|
func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
vsphereCloud, err := getCloudProvider(plugin.host.GetCloudProvider())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -249,6 +260,10 @@ func (plugin *vsphereVolumePlugin) NewDetacher() (volume.Detacher, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (plugin *vsphereVolumePlugin) NewDeviceUnmounter() (volume.DeviceUnmounter, error) {
|
||||||
|
return plugin.NewDetacher()
|
||||||
|
}
|
||||||
|
|
||||||
// Detach the given device from the given node.
|
// Detach the given device from the given node.
|
||||||
func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
func (detacher *vsphereVMDKDetacher) Detach(volumeName string, nodeName types.NodeName) error {
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue