github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

BlockVolumesSupport: CRI, VolumeManager and OperationExecutor changes 

This patch contains following changes.
- container runtime changes for adding block devices
- volumemanager changes
- operationexecutor changes
pull/6/head
mtanino 2017-09-07 14:16:23 -04:00
parent d0301aa6e8
commit 8903e8cd85
29 changed files with 3090 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/controller/volume/attachdetach/attach_detach_controller.go
View File

@ -45,6 +45,7 @@ import (
"k8s.io/kubernetes/pkg/util/io"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)
@ -136,6 +137,7 @@ func NewAttachDetachController(
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "attachdetach-controller"})
blkutil := volumeutil.NewBlockVolumePathHandler()
adc.desiredStateOfWorld = cache.NewDesiredStateOfWorld(&adc.volumePluginMgr)
adc.actualStateOfWorld = cache.NewActualStateOfWorld(&adc.volumePluginMgr)
@ -144,7 +146,8 @@ func NewAttachDetachController(
kubeClient,
&adc.volumePluginMgr,
recorder,
false)) // flag for experimental binary check for volume mount
false, // flag for experimental binary check for volume mount
blkutil))
adc.nodeStatusUpdater = statusupdater.NewNodeStatusUpdater(
kubeClient, nodeInformer.Lister(), adc.actualStateOfWorld)
@ -515,6 +518,10 @@ func (adc *attachDetachController) GetPluginDir(podUID string) string {
return ""
}
func (adc *attachDetachController) GetVolumeDevicePluginDir(podUID string) string {
return ""
}
func (adc *attachDetachController) GetPodVolumeDir(podUID types.UID, pluginName, volumeName string) string {
return ""
}
@ -523,6 +530,10 @@ func (adc *attachDetachController) GetPodPluginDir(podUID types.UID, pluginName
return ""
}
func (adc *attachDetachController) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string {
return ""
}
func (adc *attachDetachController) GetKubeClient() clientset.Interface {
return adc.kubeClient
}

56
pkg/controller/volume/attachdetach/reconciler/reconciler_test.go
View File

@ -50,7 +50,13 @@ func Test_Run_Positive_DoNothing(t *testing.T) {
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, controller.NoResyncPeriodFunc())
nsu := statusupdater.NewNodeStatusUpdater(
fakeKubeClient, informerFactory.Core().V1().Nodes().Lister(), asw)
@ -80,7 +86,13 @@ func Test_Run_Positive_OneDesiredVolumeAttach(t *testing.T) {
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */)
reconciler := NewReconciler(
reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder)
@ -126,7 +138,13 @@ func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithUnmountedVolume(t *te
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */)
reconciler := NewReconciler(
reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder)
@ -193,7 +211,13 @@ func Test_Run_Positive_OneDesiredVolumeAttachThenDetachWithMountedVolume(t *test
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */)
reconciler := NewReconciler(
reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder)
@ -260,7 +284,13 @@ func Test_Run_Negative_OneDesiredVolumeAttachThenDetachWithUnmountedVolumeUpdate
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
nsu := statusupdater.NewFakeNodeStatusUpdater(true /* returnError */)
reconciler := NewReconciler(
reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder)
@ -330,7 +360,13 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteMany(t *testing.
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */)
reconciler := NewReconciler(
reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder)
@ -416,7 +452,13 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing.
asw := cache.NewActualStateOfWorld(volumePluginMgr)
fakeKubeClient := controllervolumetesting.CreateTestClient()
fakeRecorder := &record.FakeRecorder{}
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(fakeKubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
fakeKubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */)
reconciler := NewReconciler(
reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, fakeRecorder)

13
pkg/controller/volume/expand/expand_controller.go
View File

@ -42,6 +42,7 @@ import (
"k8s.io/kubernetes/pkg/util/io"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
)
@ -117,12 +118,14 @@ func NewExpandController(
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
recorder := eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "volume_expand"})
blkutil := util.NewBlockVolumePathHandler()
expc.opExecutor = operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
&expc.volumePluginMgr,
recorder,
false))
false,
blkutil))
expc.resizeMap = cache.NewVolumeResizeMap(expc.kubeClient)
@ -203,10 +206,18 @@ func (expc *expandController) GetPluginDir(pluginName string) string {
return ""
}
func (expc *expandController) GetVolumeDevicePluginDir(pluginName string) string {
return ""
}
func (expc *expandController) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string {
return ""
}
func (expc *expandController) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string {
return ""
}
func (expc *expandController) GetPodPluginDir(podUID types.UID, pluginName string) string {
return ""
}

8
pkg/controller/volume/persistentvolume/volume_host.go
View File

@ -37,6 +37,10 @@ func (ctrl *PersistentVolumeController) GetPluginDir(pluginName string) string {
return ""
}
func (ctrl *PersistentVolumeController) GetVolumeDevicePluginDir(pluginName string) string {
return ""
}
func (ctrl *PersistentVolumeController) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string {
return ""
}
@ -45,6 +49,10 @@ func (ctrl *PersistentVolumeController) GetPodPluginDir(podUID types.UID, plugin
return ""
}
func (ctrl *PersistentVolumeController) GetPodVolumeDeviceDir(ppodUID types.UID, pluginName string) string {
return ""
}
func (ctrl *PersistentVolumeController) GetKubeClient() clientset.Interface {
return ctrl.kubeClient
}

1
pkg/kubelet/config/defaults.go
View File

@ -19,6 +19,7 @@ package config
const (
DefaultKubeletPodsDirName = "pods"
DefaultKubeletVolumesDirName = "volumes"
DefaultKubeletVolumeDevicesDirName = "volumeDevices"
DefaultKubeletPluginsDirName = "plugins"
DefaultKubeletContainersDirName = "containers"
DefaultKubeletPluginContainersDirName = "plugin-containers"

5
pkg/kubelet/container/runtime.go
View File

@ -446,9 +446,14 @@ type RunContainerOptions struct {
type VolumeInfo struct {
// Mounter is the volume's mounter
Mounter volume.Mounter
// BlockVolumeMapper is the Block volume's mapper
BlockVolumeMapper volume.BlockVolumeMapper
// SELinuxLabeled indicates whether this volume has had the
// pod's SELinux label applied to it or not
SELinuxLabeled bool
// Whether the volume permission is set to read-only or not
// This value is passed from volume.spec
ReadOnly bool
}
type VolumeMap map[string]VolumeInfo

2
pkg/kubelet/events/event.go
View File

@ -53,6 +53,8 @@ const (
FailedMountVolume = "FailedMount"
VolumeResizeFailed = "VolumeResizeFailed"
FailedUnMountVolume = "FailedUnMount"
FailedMapVolume = "FailedMapVolume"
FailedUnmapDevice = "FailedUnmapDevice"
WarnAlreadyMountedVolume = "AlreadyMountedVolume"
SuccessfulDetachVolume = "SuccessfulDetachVolume"
SuccessfulMountVolume = "SuccessfulMountVolume"

28
pkg/kubelet/kubelet_getters.go
View File

@ -64,6 +64,21 @@ func (kl *Kubelet) getPluginDir(pluginName string) string {
return filepath.Join(kl.getPluginsDir(), pluginName)
}
// getVolumeDevicePluginsDir returns the full path to the directory under which plugin
// directories are created. Plugins can use these directories for data that
// they need to persist. Plugins should create subdirectories under this named
// after their own names.
func (kl *Kubelet) getVolumeDevicePluginsDir() string {
return filepath.Join(kl.getRootDir(), config.DefaultKubeletPluginsDirName)
}
// getVolumeDevicePluginDir returns a data directory name for a given plugin name.
// Plugins can use these directories to store data that they need to persist.
// For per-pod plugin data, see getVolumeDevicePluginsDir.
func (kl *Kubelet) getVolumeDevicePluginDir(pluginName string) string {
return filepath.Join(kl.getVolumeDevicePluginsDir(), pluginName, config.DefaultKubeletVolumeDevicesDirName)
}
// GetPodDir returns the full path to the per-pod data directory for the
// specified pod. This directory may not exist if the pod does not exist.
func (kl *Kubelet) GetPodDir(podUID types.UID) string {
@ -90,6 +105,19 @@ func (kl *Kubelet) getPodVolumeDir(podUID types.UID, pluginName string, volumeNa
return filepath.Join(kl.getPodVolumesDir(podUID), pluginName, volumeName)
}
// getPodVolumeDevicesDir returns the full path to the per-pod data directory under
// which volumes are created for the specified pod. This directory may not
// exist if the pod does not exist.
func (kl *Kubelet) getPodVolumeDevicesDir(podUID types.UID) string {
return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletVolumeDevicesDirName)
}
// getPodVolumeDeviceDir returns the full path to the directory which represents the
// named plugin for specified pod. This directory may not exist if the pod does not exist.
func (kl *Kubelet) getPodVolumeDeviceDir(podUID types.UID, pluginName string) string {
return filepath.Join(kl.getPodVolumeDevicesDir(podUID), pluginName)
}
// getPodPluginsDir returns the full path to the per-pod data directory under
// which plugins may store data for the specified pod. This directory may not
// exist if the pod does not exist.

50
pkg/kubelet/kubelet_pods.go
View File

@ -64,6 +64,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/util/format"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
"k8s.io/kubernetes/third_party/forked/golang/expansion"
@ -91,9 +92,9 @@ func (kl *Kubelet) GetActivePods() []*v1.Pod {
return activePods
}
// makeDevices determines the devices for the given container.
// makeGPUDevices determines the devices for the given container.
// Experimental.
func (kl *Kubelet) makeDevices(pod *v1.Pod, container *v1.Container) ([]kubecontainer.DeviceInfo, error) {
func (kl *Kubelet) makeGPUDevices(pod *v1.Pod, container *v1.Container) ([]kubecontainer.DeviceInfo, error) {
if container.Resources.Limits.NvidiaGPU().IsZero() {
return nil, nil
}
@ -128,6 +129,39 @@ func makeAbsolutePath(goos, path string) string {
return "c:\\" + path
}
// makeBlockVolumes maps the raw block devices specified in the path of the container
// Experimental
func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVolumes kubecontainer.VolumeMap, blkutil volumeutil.BlockVolumePathHandler) ([]kubecontainer.DeviceInfo, error) {
var devices []kubecontainer.DeviceInfo
for _, device := range container.VolumeDevices {
// check path is absolute
if !filepath.IsAbs(device.DevicePath) {
return nil, fmt.Errorf("error DevicePath `%s` must be an absolute path", device.DevicePath)
}
vol, ok := podVolumes[device.Name]
if !ok || vol.BlockVolumeMapper == nil {
glog.Errorf("Block volume cannot be satisfied for container %q, because the volume is missing or the volume mapper is nil: %+v", container.Name, device)
return nil, fmt.Errorf("cannot find volume %q to pass into container %q", device.Name, container.Name)
}
// Get a symbolic link associated to a block device under pod device path
dirPath, volName := vol.BlockVolumeMapper.GetPodDeviceMapPath()
symlinkPath := path.Join(dirPath, volName)
if islinkExist, checkErr := blkutil.IsSymlinkExist(symlinkPath); checkErr != nil {
return nil, checkErr
} else if islinkExist {
// Check readOnly in PVCVolumeSource and set read only permission if it's true.
permission := "mrw"
if vol.ReadOnly {
permission = "r"
}
glog.V(4).Infof("Device will be attached to container %q. Path on host: %v", container.Name, symlinkPath)
devices = append(devices, kubecontainer.DeviceInfo{PathOnHost: symlinkPath, PathInContainer: device.DevicePath, Permissions: permission})
}
}
return devices, nil
}
// makeMounts determines the mount points for the given container.
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) {
// Kubernetes only mounts on /etc/hosts if:
@ -416,12 +450,22 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
opts.PortMappings = kubecontainer.MakePortMappings(container)
// TODO(random-liu): Move following convert functions into pkg/kubelet/container
devices, err := kl.makeDevices(pod, container)
devices, err := kl.makeGPUDevices(pod, container)
if err != nil {
return nil, err
}
opts.Devices = append(opts.Devices, devices...)
// TODO: remove feature gate check after no longer needed
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
blkutil := volumeutil.NewBlockVolumePathHandler()
blkVolumes, err := kl.makeBlockVolumes(pod, container, volumes, blkutil)
if err != nil {
return nil, err
}
opts.Devices = append(opts.Devices, blkVolumes...)
}
mounts, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes)
if err != nil {
return nil, err

134
pkg/kubelet/kubelet_pods_test.go
View File

@ -47,6 +47,7 @@ import (
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/server/portforward"
"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
)
func TestMakeAbsolutePath(t *testing.T) {
@ -352,6 +353,139 @@ func TestMakeMounts(t *testing.T) {
}
}
func TestMakeBlockVolumes(t *testing.T) {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
defer testKubelet.Cleanup()
kubelet := testKubelet.kubelet
testCases := map[string]struct {
container v1.Container
podVolumes kubecontainer.VolumeMap
expectErr bool
expectedErrMsg string
expectedDevices []kubecontainer.DeviceInfo
}{
"valid volumeDevices in container": {
podVolumes: kubecontainer.VolumeMap{
"disk1": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/", volName: "sda"}},
"disk2": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/disk/by-path/", volName: "diskPath"}, ReadOnly: true},
"disk3": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/disk/by-id/", volName: "diskUuid"}},
"disk4": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/var/lib/", volName: "rawdisk"}, ReadOnly: true},
},
container: v1.Container{
Name: "container1",
VolumeDevices: []v1.VolumeDevice{
{
DevicePath: "/dev/sda",
Name: "disk1",
},
{
DevicePath: "/dev/xvda",
Name: "disk2",
},
{
DevicePath: "/dev/xvdb",
Name: "disk3",
},
{
DevicePath: "/mnt/rawdisk",
Name: "disk4",
},
},
},
expectedDevices: []kubecontainer.DeviceInfo{
{
PathInContainer: "/dev/sda",
PathOnHost: "/dev/sda",
Permissions: "mrw",
},
{
PathInContainer: "/dev/xvda",
PathOnHost: "/dev/disk/by-path/diskPath",
Permissions: "r",
},
{
PathInContainer: "/dev/xvdb",
PathOnHost: "/dev/disk/by-id/diskUuid",
Permissions: "mrw",
},
{
PathInContainer: "/mnt/rawdisk",
PathOnHost: "/var/lib/rawdisk",
Permissions: "r",
},
},
expectErr: false,
},
"invalid absolute Path": {
podVolumes: kubecontainer.VolumeMap{
"disk": kubecontainer.VolumeInfo{BlockVolumeMapper: &stubBlockVolume{dirPath: "/dev/", volName: "sda"}},
},
container: v1.Container{
VolumeDevices: []v1.VolumeDevice{
{
DevicePath: "must/be/absolute",
Name: "disk",
},
},
},
expectErr: true,
expectedErrMsg: "error DevicePath `must/be/absolute` must be an absolute path",
},
"volume doesn't exist": {
podVolumes: kubecontainer.VolumeMap{},
container: v1.Container{
VolumeDevices: []v1.VolumeDevice{
{
DevicePath: "/dev/sdaa",
Name: "disk",
},
},
},
expectErr: true,
expectedErrMsg: "cannot find volume \"disk\" to pass into container \"\"",
},
"volume BlockVolumeMapper is nil": {
podVolumes: kubecontainer.VolumeMap{
"disk": kubecontainer.VolumeInfo{},
},
container: v1.Container{
VolumeDevices: []v1.VolumeDevice{
{
DevicePath: "/dev/sdzz",
Name: "disk",
},
},
},
expectErr: true,
expectedErrMsg: "cannot find volume \"disk\" to pass into container \"\"",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
pod := v1.Pod{
Spec: v1.PodSpec{
HostNetwork: true,
},
}
blkutil := volumetest.NewBlockVolumePathHandler()
blkVolumes, err := kubelet.makeBlockVolumes(&pod, &tc.container, tc.podVolumes, blkutil)
// validate only the error if we expect an error
if tc.expectErr {
if err == nil || err.Error() != tc.expectedErrMsg {
t.Fatalf("expected error message `%s` but got `%v`", tc.expectedErrMsg, err)
}
return
}
// otherwise validate the devices
if err != nil {
t.Fatal(err)
}
assert.Equal(t, tc.expectedDevices, blkVolumes, "devices of container %+v", tc.container)
})
}
}
func TestNodeHostsFileContent(t *testing.T) {
testCases := []struct {
hostsFileName string

20
pkg/kubelet/kubelet_volumes_test.go
View File

@ -448,3 +448,23 @@ func (f *stubVolume) SetUp(fsGroup *int64) error {
func (f *stubVolume) SetUpAt(dir string, fsGroup *int64) error {
return nil
}
type stubBlockVolume struct {
dirPath string
volName string
}
func (f *stubBlockVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
return "", nil
}
func (f *stubBlockVolume) GetPodDeviceMapPath() (string, string) {
return f.dirPath, f.volName
}
func (f *stubBlockVolume) SetUpDevice() (string, error) {
return "", nil
}
func (f *stubBlockVolume) TearDownDevice(mapPath string, devicePath string) error {
return nil
}

8
pkg/kubelet/volume_host.go
View File

@ -86,10 +86,18 @@ type kubeletVolumeHost struct {
mountPodManager mountpod.Manager
}
func (kvh *kubeletVolumeHost) GetVolumeDevicePluginDir(pluginName string) string {
return kvh.kubelet.getVolumeDevicePluginDir(pluginName)
}
func (kvh *kubeletVolumeHost) GetPodVolumeDir(podUID types.UID, pluginName string, volumeName string) string {
return kvh.kubelet.getPodVolumeDir(podUID, pluginName, volumeName)
}
func (kvh *kubeletVolumeHost) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string {
return kvh.kubelet.getPodVolumeDeviceDir(podUID, pluginName)
}
func (kvh *kubeletVolumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string {
return kvh.kubelet.getPodPluginDir(podUID, pluginName)
}

13
pkg/kubelet/volumemanager/cache/actual_state_of_world.go vendored
View File

@ -58,7 +58,7 @@ type ActualStateOfWorld interface {
// volume, reset the pod's remountRequired value.
// If a volume with the name volumeName does not exist in the list of
// attached volumes, an error is returned.
AddPodToVolume(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, outerVolumeSpecName string, volumeGidValue string) error
AddPodToVolume(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, blockVolumeMapper volume.BlockVolumeMapper, outerVolumeSpecName string, volumeGidValue string) error
// MarkRemountRequired marks each volume that is successfully attached and
// mounted for the specified pod as requiring remount (if the plugin for the
@ -254,6 +254,9 @@ type mountedPod struct {
// mounter used to mount
mounter volume.Mounter
// mapper used to block volumes support
blockVolumeMapper volume.BlockVolumeMapper
// outerVolumeSpecName is the volume.Spec.Name() of the volume as referenced
// directly in the pod. If the volume was referenced through a persistent
// volume claim, this contains the volume.Spec.Name() of the persistent
@ -287,6 +290,7 @@ func (asw *actualStateOfWorld) MarkVolumeAsMounted(
podUID types.UID,
volumeName v1.UniqueVolumeName,
mounter volume.Mounter,
blockVolumeMapper volume.BlockVolumeMapper,
outerVolumeSpecName string,
volumeGidValue string) error {
return asw.AddPodToVolume(
@ -294,6 +298,7 @@ func (asw *actualStateOfWorld) MarkVolumeAsMounted(
podUID,
volumeName,
mounter,
blockVolumeMapper,
outerVolumeSpecName,
volumeGidValue)
}
@ -385,6 +390,7 @@ func (asw *actualStateOfWorld) AddPodToVolume(
podUID types.UID,
volumeName v1.UniqueVolumeName,
mounter volume.Mounter,
blockVolumeMapper volume.BlockVolumeMapper,
outerVolumeSpecName string,
volumeGidValue string) error {
asw.Lock()
@ -403,6 +409,7 @@ func (asw *actualStateOfWorld) AddPodToVolume(
podName: podName,
podUID: podUID,
mounter: mounter,
blockVolumeMapper: blockVolumeMapper,
outerVolumeSpecName: outerVolumeSpecName,
volumeGidValue: volumeGidValue,
}
@ -682,5 +689,7 @@ func getMountedVolume(
PluginName: attachedVolume.pluginName,
PodUID: mountedPod.podUID,
Mounter: mountedPod.mounter,
VolumeGidValue: mountedPod.volumeGidValue}}
BlockVolumeMapper: mountedPod.blockVolumeMapper,
VolumeGidValue: mountedPod.volumeGidValue,
VolumeSpec: attachedVolume.spec}}
}

32
pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go vendored
View File

@ -204,9 +204,14 @@ func Test_AddPodToVolume_Positive_ExistingVolumeNewNode(t *testing.T) {
t.Fatalf("NewMounter failed. Expected: <no error> Actual: <%v>", err)
}
mapper, err := plugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{})
if err != nil {
t.Fatalf("NewBlockVolumeMapper failed. Expected: <no error> Actual: <%v>", err)
}
// Act
err = asw.AddPodToVolume(
podName, pod.UID, generatedVolumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */)
podName, pod.UID, generatedVolumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */)
// Assert
if err != nil {
@ -263,15 +268,20 @@ func Test_AddPodToVolume_Positive_ExistingVolumeExistingNode(t *testing.T) {
t.Fatalf("NewMounter failed. Expected: <no error> Actual: <%v>", err)
}
mapper, err := plugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{})
if err != nil {
t.Fatalf("NewBlockVolumeMapper failed. Expected: <no error> Actual: <%v>", err)
}
err = asw.AddPodToVolume(
podName, pod.UID, generatedVolumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */)
podName, pod.UID, generatedVolumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */)
if err != nil {
t.Fatalf("AddPodToVolume failed. Expected: <no error> Actual: <%v>", err)
}
// Act
err = asw.AddPodToVolume(
podName, pod.UID, generatedVolumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */)
podName, pod.UID, generatedVolumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */)
// Assert
if err != nil {
@ -318,6 +328,15 @@ func Test_AddPodToVolume_Negative_VolumeDoesntExist(t *testing.T) {
volumeSpec,
err)
}
blockplugin, err := volumePluginMgr.FindMapperPluginBySpec(volumeSpec)
if err != nil {
t.Fatalf(
"volumePluginMgr.FindMapperPluginBySpec failed to find volume plugin for %#v with: %v",
volumeSpec,
err)
}
volumeName, err := volumehelper.GetUniqueVolumeNameFromSpec(
plugin, volumeSpec)
@ -328,9 +347,14 @@ func Test_AddPodToVolume_Negative_VolumeDoesntExist(t *testing.T) {
t.Fatalf("NewMounter failed. Expected: <no error> Actual: <%v>", err)
}
mapper, err := blockplugin.NewBlockVolumeMapper(volumeSpec, pod, volume.VolumeOptions{})
if err != nil {
t.Fatalf("NewBlockVolumeMapper failed. Expected: <no error> Actual: <%v>", err)
}
// Act
err = asw.AddPodToVolume(
podName, pod.UID, volumeName, mounter, volumeSpec.Name(), "" /* volumeGidValue */)
podName, pod.UID, volumeName, mounter, mapper, volumeSpec.Name(), "" /* volumeGidValue */)
// Assert
if err == nil {

54
pkg/kubelet/volumemanager/populator/desired_state_of_world_populator.go
View File

@ -31,7 +31,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/pod"
@ -260,11 +262,12 @@ func (dswp *desiredStateOfWorldPopulator) processPodVolumes(pod *v1.Pod) {
}
allVolumesAdded := true
mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers)
// Process volume spec for each volume defined in pod
for _, podVolume := range pod.Spec.Volumes {
volumeSpec, volumeGidValue, err :=
dswp.createVolumeSpec(podVolume, pod.Namespace)
dswp.createVolumeSpec(podVolume, pod.Name, pod.Namespace, mountsMap, devicesMap)
if err != nil {
glog.Errorf(
"Error processing volume %q for pod %q: %v",
@ -336,7 +339,7 @@ func (dswp *desiredStateOfWorldPopulator) deleteProcessedPod(
// specified volume. It dereference any PVC to get PV objects, if needed.
// Returns an error if unable to obtain the volume at this time.
func (dswp *desiredStateOfWorldPopulator) createVolumeSpec(
podVolume v1.Volume, podNamespace string) (*volume.Spec, string, error) {
podVolume v1.Volume, podName string, podNamespace string, mountsMap map[string]bool, devicesMap map[string]bool) (*volume.Spec, string, error) {
if pvcSource :=
podVolume.VolumeSource.PersistentVolumeClaim; pvcSource != nil {
glog.V(10).Infof(
@ -381,6 +384,31 @@ func (dswp *desiredStateOfWorldPopulator) createVolumeSpec(
pvcSource.ClaimName,
pvcUID)
// TODO: remove feature gate check after no longer needed
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
volumeMode, err := volumehelper.GetVolumeMode(volumeSpec)
if err != nil {
return nil, "", err
}
// Error if a container has volumeMounts but the volumeMode of PVC isn't Filesystem
if mountsMap[podVolume.Name] && volumeMode != v1.PersistentVolumeFilesystem {
return nil, "", fmt.Errorf(
"Volume %q has volumeMode %q, but is specified in volumeMounts for pod %q/%q",
podVolume.Name,
volumeMode,
podNamespace,
podName)
}
// Error if a container has volumeDevices but the volumeMode of PVC isn't Block
if devicesMap[podVolume.Name] && volumeMode != v1.PersistentVolumeBlock {
return nil, "", fmt.Errorf(
"Volume %q has volumeMode %q, but is specified in volumeDevices for pod %q/%q",
podVolume.Name,
volumeMode,
podNamespace,
podName)
}
}
return volumeSpec, volumeGidValue, nil
}
@ -449,6 +477,28 @@ func (dswp *desiredStateOfWorldPopulator) getPVSpec(
return volume.NewSpecFromPersistentVolume(pv, pvcReadOnly), volumeGidValue, nil
}
func (dswp *desiredStateOfWorldPopulator) makeVolumeMap(containers []v1.Container) (map[string]bool, map[string]bool) {
volumeDevicesMap := make(map[string]bool)
volumeMountsMap := make(map[string]bool)
for _, container := range containers {
if container.VolumeMounts != nil {
for _, mount := range container.VolumeMounts {
volumeMountsMap[mount.Name] = true
}
}
// TODO: remove feature gate check after no longer needed
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) &&
container.VolumeDevices != nil {
for _, device := range container.VolumeDevices {
volumeDevicesMap[device.Name] = true
}
}
}
return volumeMountsMap, volumeDevicesMap
}
func getPVVolumeGidAnnotationValue(pv *v1.PersistentVolume) string {
if volumeGid, ok := pv.Annotations[volumehelper.VolumeGidAnnotationKey]; ok {
return volumeGid

446
pkg/kubelet/volumemanager/populator/desired_state_of_world_populator_test.go
View File

@ -22,7 +22,10 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
"k8s.io/kubernetes/pkg/kubelet/configmap"
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
@ -37,54 +40,37 @@ import (
)
func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) {
fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t)
fakeClient := &fake.Clientset{}
fakeSecretManager := secret.NewFakeManager()
fakeConfigMapManager := configmap.NewFakeManager()
fakePodManager := kubepod.NewBasicPodManager(
podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager)
fakesDSW := cache.NewDesiredStateOfWorld(fakeVolumePluginMgr)
fakeRuntime := &containertest.FakeRuntime{}
fakeStatusManager := status.NewManager(fakeClient, fakePodManager, &statustest.FakePodDeletionSafetyProvider{})
dswp := &desiredStateOfWorldPopulator{
kubeClient: fakeClient,
loopSleepDuration: 100 * time.Millisecond,
getPodStatusRetryDuration: 2 * time.Second,
podManager: fakePodManager,
podStatusProvider: fakeStatusManager,
desiredStateOfWorld: fakesDSW,
pods: processedPods{
processedPods: make(map[types.UniquePodName]bool)},
kubeContainerRuntime: fakeRuntime,
keepTerminatedPodVolumes: false,
}
pod := &v1.Pod{
// create dswp
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "dswp-test-pod",
UID: "dswp-test-pod-uid",
Namespace: "dswp-test",
Name: "dswp-test-volume-name",
},
Spec: v1.PodSpec{
Volumes: []v1.Volume{
Spec: v1.PersistentVolumeSpec{
ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"},
},
}
pvc := &v1.PersistentVolumeClaim{
Spec: v1.PersistentVolumeClaimSpec{
VolumeName: "dswp-test-volume-name",
},
Status: v1.PersistentVolumeClaimStatus{
Phase: v1.ClaimBound,
},
}
dswp, fakePodManager, fakesDSW := createDswpWithVolume(t, pv, pvc)
// create pod
containers := []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "dswp-test-volume-name",
VolumeSource: v1.VolumeSource{
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
PDName: "dswp-test-fake-device",
},
},
Name: "dswp-test-volume-name",
MountPath: "/mnt",
},
},
},
Status: v1.PodStatus{
Phase: v1.PodPhase("Running"),
},
}
pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers)
fakePodManager.AddPod(pod)
@ -158,6 +144,320 @@ func TestFindAndAddNewPods_FindAndRemoveDeletedPods(t *testing.T) {
}
func TestFindAndAddNewPods_FindAndRemoveDeletedPods_Valid_Block_VolumeDevices(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// create dswp
mode := v1.PersistentVolumeBlock
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "dswp-test-volume-name",
},
Spec: v1.PersistentVolumeSpec{
ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "block-bound"},
VolumeMode: &mode,
},
}
pvc := &v1.PersistentVolumeClaim{
Spec: v1.PersistentVolumeClaimSpec{
VolumeName: "dswp-test-volume-name",
},
Status: v1.PersistentVolumeClaimStatus{
Phase: v1.ClaimBound,
},
}
dswp, fakePodManager, fakesDSW := createDswpWithVolume(t, pv, pvc)
// create pod
containers := []v1.Container{
{
VolumeDevices: []v1.VolumeDevice{
{
Name: "dswp-test-volume-name",
DevicePath: "/dev/sdb",
},
},
},
}
pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers)
fakePodManager.AddPod(pod)
podName := volumehelper.GetUniquePodName(pod)
generatedVolumeName := "fake-plugin/" + pod.Spec.Volumes[0].Name
dswp.findAndAddNewPods()
if !dswp.pods.processedPods[podName] {
t.Fatalf("Failed to record that the volumes for the specified pod: %s have been processed by the populator", podName)
}
expectedVolumeName := v1.UniqueVolumeName(generatedVolumeName)
volumeExists := fakesDSW.VolumeExists(expectedVolumeName)
if !volumeExists {
t.Fatalf(
"VolumeExists(%q) failed. Expected: <true> Actual: <%v>",
expectedVolumeName,
volumeExists)
}
if podExistsInVolume := fakesDSW.PodExistsInVolume(
podName, expectedVolumeName); !podExistsInVolume {
t.Fatalf(
"DSW PodExistsInVolume returned incorrect value. Expected: <true> Actual: <%v>",
podExistsInVolume)
}
verifyVolumeExistsInVolumesToMount(
t, v1.UniqueVolumeName(generatedVolumeName), false /* expectReportedInUse */, fakesDSW)
//let the pod be terminated
podGet, exist := fakePodManager.GetPodByName(pod.Namespace, pod.Name)
if !exist {
t.Fatalf("Failed to get pod by pod name: %s and namespace: %s", pod.Name, pod.Namespace)
}
podGet.Status.Phase = v1.PodFailed
//pod is added to fakePodManager but fakeRuntime can not get the pod,so here findAndRemoveDeletedPods() will remove the pod and volumes it is mounted
dswp.findAndRemoveDeletedPods()
if dswp.pods.processedPods[podName] {
t.Fatalf("Failed to remove pods from desired state of world since they no longer exist")
}
volumeExists = fakesDSW.VolumeExists(expectedVolumeName)
if volumeExists {
t.Fatalf(
"VolumeExists(%q) failed. Expected: <false> Actual: <%v>",
expectedVolumeName,
volumeExists)
}
if podExistsInVolume := fakesDSW.PodExistsInVolume(
podName, expectedVolumeName); podExistsInVolume {
t.Fatalf(
"DSW PodExistsInVolume returned incorrect value. Expected: <false> Actual: <%v>",
podExistsInVolume)
}
volumesToMount := fakesDSW.GetVolumesToMount()
for _, volume := range volumesToMount {
if volume.VolumeName == expectedVolumeName {
t.Fatalf(
"Found volume %v in the list of desired state of world volumes to mount. Expected not",
expectedVolumeName)
}
}
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
func TestCreateVolumeSpec_Valid_File_VolumeMounts(t *testing.T) {
// create dswp
mode := v1.PersistentVolumeFilesystem
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "dswp-test-volume-name",
},
Spec: v1.PersistentVolumeSpec{
ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"},
VolumeMode: &mode,
},
}
pvc := &v1.PersistentVolumeClaim{
Spec: v1.PersistentVolumeClaimSpec{
VolumeName: "dswp-test-volume-name",
},
Status: v1.PersistentVolumeClaimStatus{
Phase: v1.ClaimBound,
},
}
dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc)
// create pod
containers := []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "dswp-test-volume-name",
MountPath: "/mnt",
},
},
},
}
pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers)
fakePodManager.AddPod(pod)
mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers)
volumeSpec, _, err :=
dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap)
// Assert
if volumeSpec == nil || err != nil {
t.Fatalf("Failed to create volumeSpec with combination of filesystem mode and volumeMounts. err: %v", err)
}
}
func TestCreateVolumeSpec_Valid_Block_VolumeDevices(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// create dswp
mode := v1.PersistentVolumeBlock
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "dswp-test-volume-name",
},
Spec: v1.PersistentVolumeSpec{
ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "block-bound"},
VolumeMode: &mode,
},
}
pvc := &v1.PersistentVolumeClaim{
Spec: v1.PersistentVolumeClaimSpec{
VolumeName: "dswp-test-volume-name",
},
Status: v1.PersistentVolumeClaimStatus{
Phase: v1.ClaimBound,
},
}
dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc)
// create pod
containers := []v1.Container{
{
VolumeDevices: []v1.VolumeDevice{
{
Name: "dswp-test-volume-name",
DevicePath: "/dev/sdb",
},
},
},
}
pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers)
fakePodManager.AddPod(pod)
mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers)
volumeSpec, _, err :=
dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap)
// Assert
if volumeSpec == nil || err != nil {
t.Fatalf("Failed to create volumeSpec with combination of block mode and volumeDevices. err: %v", err)
}
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
func TestCreateVolumeSpec_Invalid_File_VolumeDevices(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// create dswp
mode := v1.PersistentVolumeFilesystem
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "dswp-test-volume-name",
},
Spec: v1.PersistentVolumeSpec{
ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "file-bound"},
VolumeMode: &mode,
},
}
pvc := &v1.PersistentVolumeClaim{
Spec: v1.PersistentVolumeClaimSpec{
VolumeName: "dswp-test-volume-name",
},
Status: v1.PersistentVolumeClaimStatus{
Phase: v1.ClaimBound,
},
}
dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc)
// create pod
containers := []v1.Container{
{
VolumeDevices: []v1.VolumeDevice{
{
Name: "dswp-test-volume-name",
DevicePath: "/dev/sdb",
},
},
},
}
pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "file-bound", containers)
fakePodManager.AddPod(pod)
mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers)
volumeSpec, _, err :=
dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap)
// Assert
if volumeSpec != nil || err == nil {
t.Fatalf("Unexpected volumeMode and volumeMounts/volumeDevices combination is accepted")
}
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
func TestCreateVolumeSpec_Invalid_Block_VolumeMounts(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// create dswp
mode := v1.PersistentVolumeBlock
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: "dswp-test-volume-name",
},
Spec: v1.PersistentVolumeSpec{
ClaimRef: &v1.ObjectReference{Namespace: "ns", Name: "block-bound"},
VolumeMode: &mode,
},
}
pvc := &v1.PersistentVolumeClaim{
Spec: v1.PersistentVolumeClaimSpec{
VolumeName: "dswp-test-volume-name",
},
Status: v1.PersistentVolumeClaimStatus{
Phase: v1.ClaimBound,
},
}
dswp, fakePodManager, _ := createDswpWithVolume(t, pv, pvc)
// create pod
containers := []v1.Container{
{
VolumeMounts: []v1.VolumeMount{
{
Name: "dswp-test-volume-name",
MountPath: "/mnt",
},
},
},
}
pod := createPodWithVolume("dswp-test-pod", "dswp-test-volume-name", "block-bound", containers)
fakePodManager.AddPod(pod)
mountsMap, devicesMap := dswp.makeVolumeMap(pod.Spec.Containers)
volumeSpec, _, err :=
dswp.createVolumeSpec(pod.Spec.Volumes[0], pod.Name, pod.Namespace, mountsMap, devicesMap)
// Assert
if volumeSpec != nil || err == nil {
t.Fatalf("Unexpected volumeMode and volumeMounts/volumeDevices combination is accepted")
}
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
func verifyVolumeExistsInVolumesToMount(t *testing.T, expectedVolumeName v1.UniqueVolumeName, expectReportedInUse bool, dsw cache.DesiredStateOfWorld) {
volumesToMount := dsw.GetVolumesToMount()
for _, volume := range volumesToMount {
@ -179,3 +479,67 @@ func verifyVolumeExistsInVolumesToMount(t *testing.T, expectedVolumeName v1.Uniq
expectedVolumeName,
volumesToMount)
}
func createPodWithVolume(pod, pv, pvc string, containers []v1.Container) *v1.Pod {
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: pod,
UID: "dswp-test-pod-uid",
Namespace: "dswp-test",
},
Spec: v1.PodSpec{
Volumes: []v1.Volume{
{
Name: pv,
VolumeSource: v1.VolumeSource{
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
PDName: "dswp-test-fake-device",
},
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: pvc,
},
},
},
},
Containers: containers,
},
Status: v1.PodStatus{
Phase: v1.PodPhase("Running"),
},
}
}
func createDswpWithVolume(t *testing.T, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) (*desiredStateOfWorldPopulator, kubepod.Manager, cache.DesiredStateOfWorld) {
fakeVolumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t)
fakeClient := &fake.Clientset{}
fakeClient.AddReactor("get", "persistentvolumeclaims", func(action core.Action) (bool, runtime.Object, error) {
return true, pvc, nil
})
fakeClient.AddReactor("get", "persistentvolumes", func(action core.Action) (bool, runtime.Object, error) {
return true, pv, nil
})
fakeSecretManager := secret.NewFakeManager()
fakeConfigMapManager := configmap.NewFakeManager()
fakePodManager := kubepod.NewBasicPodManager(
podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager)
fakesDSW := cache.NewDesiredStateOfWorld(fakeVolumePluginMgr)
fakeRuntime := &containertest.FakeRuntime{}
fakeStatusManager := status.NewManager(fakeClient, fakePodManager, &statustest.FakePodDeletionSafetyProvider{})
dswp := &desiredStateOfWorldPopulator{
kubeClient: fakeClient,
loopSleepDuration: 100 * time.Millisecond,
getPodStatusRetryDuration: 2 * time.Second,
podManager: fakePodManager,
podStatusProvider: fakeStatusManager,
desiredStateOfWorld: fakesDSW,
pods: processedPods{
processedPods: make(map[types.UniquePodName]bool)},
kubeContainerRuntime: fakeRuntime,
keepTerminatedPodVolumes: false,
}
return dswp, fakePodManager, fakesDSW
}

170
pkg/kubelet/volumemanager/reconciler/reconciler.go
View File

@ -22,6 +22,7 @@ package reconciler
import (
"fmt"
"io/ioutil"
"os"
"path"
"time"
@ -30,13 +31,15 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet/config"
"k8s.io/kubernetes/pkg/kubelet/volumemanager/cache"
utilfile "k8s.io/kubernetes/pkg/util/file"
"k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/util/strings"
utilstrings "k8s.io/kubernetes/pkg/util/strings"
volumepkg "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util/nestedpendingoperations"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
@ -171,10 +174,12 @@ func (rc *reconciler) reconcile() {
// Ensure volumes that should be unmounted are unmounted.
for _, mountedVolume := range rc.actualStateOfWorld.GetMountedVolumes() {
if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName) {
// Volume is mounted, unmount it
glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", ""))
err := rc.operationExecutor.UnmountVolume(
mountedVolume.MountedVolume, rc.actualStateOfWorld)
volumeHandler, err := operationexecutor.NewVolumeHandler(mountedVolume.VolumeSpec, rc.operationExecutor)
if err != nil {
glog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.NewVolumeHandler for UnmountVolume failed"), err).Error())
continue
}
err = volumeHandler.UnmountVolumeHandler(mountedVolume.MountedVolume, rc.actualStateOfWorld)
if err != nil &&
!nestedpendingoperations.IsAlreadyExists(err) &&
!exponentialbackoff.IsExponentialBackoff(err) {
@ -239,12 +244,12 @@ func (rc *reconciler) reconcile() {
if isRemount {
remountingLogStr = "Volume is already mounted to pod, but remount was requested."
}
glog.V(12).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr))
err := rc.operationExecutor.MountVolume(
rc.waitForAttachTimeout,
volumeToMount.VolumeToMount,
rc.actualStateOfWorld,
isRemount)
volumeHandler, err := operationexecutor.NewVolumeHandler(volumeToMount.VolumeSpec, rc.operationExecutor)
if err != nil {
glog.Errorf(volumeToMount.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.NewVolumeHandler for MountVolume failed"), err).Error())
continue
}
err = volumeHandler.MountVolumeHandler(rc.waitForAttachTimeout, volumeToMount.VolumeToMount, rc.actualStateOfWorld, isRemount, remountingLogStr)
if err != nil &&
!nestedpendingoperations.IsAlreadyExists(err) &&
!exponentialbackoff.IsExponentialBackoff(err) {
@ -268,10 +273,12 @@ func (rc *reconciler) reconcile() {
if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName) &&
!rc.operationExecutor.IsOperationPending(attachedVolume.VolumeName, nestedpendingoperations.EmptyUniquePodName) {
if attachedVolume.GloballyMounted {
// Volume is globally mounted to device, unmount it
glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", ""))
err := rc.operationExecutor.UnmountDevice(
attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.mounter)
volumeHandler, err := operationexecutor.NewVolumeHandler(attachedVolume.VolumeSpec, rc.operationExecutor)
if err != nil {
glog.Errorf(attachedVolume.GenerateErrorDetailed(fmt.Sprintf("operationExecutor.NewVolumeHandler for UnmountDevice failed"), err).Error())
continue
}
err = volumeHandler.UnmountDeviceHandler(attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.mounter)
if err != nil &&
!nestedpendingoperations.IsAlreadyExists(err) &&
!exponentialbackoff.IsExponentialBackoff(err) {
@ -332,6 +339,7 @@ type podVolume struct {
volumeSpecName string
mountPath string
pluginName string
volumeMode v1.PersistentVolumeMode
}
type reconstructedVolume struct {
@ -345,6 +353,7 @@ type reconstructedVolume struct {
devicePath string
reportedInUse bool
mounter volumepkg.Mounter
blockVolumeMapper volumepkg.BlockVolumeMapper
}
// reconstructFromDisk scans the volume directories under the given pod directory. If the volume is not
@ -419,20 +428,44 @@ func (rc *reconciler) syncStates(podsDir string) {
// Reconstruct Volume object and reconstructedVolume data structure by reading the pod's volume directories
func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume, error) {
// plugin initializations
plugin, err := rc.volumePluginMgr.FindPluginByName(volume.pluginName)
if err != nil {
return nil, err
}
volumeSpec, err := plugin.ConstructVolumeSpec(volume.volumeSpecName, volume.mountPath)
attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginByName(volume.pluginName)
if err != nil {
return nil, err
}
// Create volumeSpec
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
UID: types.UID(volume.podName),
},
}
attachablePlugin, err := rc.volumePluginMgr.FindAttachablePluginByName(volume.pluginName)
// TODO: remove feature gate check after no longer needed
var mapperPlugin volumepkg.BlockVolumePlugin
tmpSpec := &volumepkg.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{}}}
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
mapperPlugin, err = rc.volumePluginMgr.FindMapperPluginByName(volume.pluginName)
if err != nil {
return nil, err
}
tmpSpec = &volumepkg.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{VolumeMode: &volume.volumeMode}}}
}
volumeHandler, err := operationexecutor.NewVolumeHandler(tmpSpec, rc.operationExecutor)
if err != nil {
return nil, err
}
volumeSpec, err := volumeHandler.ReconstructVolumeHandler(
plugin,
mapperPlugin,
pod.UID,
volume.podName,
volume.volumeSpecName,
volume.mountPath,
volume.pluginName)
if err != nil {
return nil, err
}
@ -448,17 +481,14 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume,
uniqueVolumeName = volumehelper.GetUniqueVolumeNameForNonAttachableVolume(volume.podName, plugin, volumeSpec)
}
if attachablePlugin != nil {
if isNotMount, mountCheckErr := rc.mounter.IsLikelyNotMountPoint(volume.mountPath); mountCheckErr != nil {
return nil, fmt.Errorf("Could not check whether the volume %q (spec.Name: %q) pod %q (UID: %q) is mounted with: %v",
uniqueVolumeName,
volumeSpec.Name(),
volume.podName,
pod.UID,
mountCheckErr)
} else if isNotMount {
return nil, fmt.Errorf("Volume: %q is not mounted", uniqueVolumeName)
}
// Check existence of mount point for filesystem volume or symbolic link for block volume
isExist, checkErr := volumeHandler.CheckVolumeExistence(volume.mountPath, volumeSpec.Name(), rc.mounter, uniqueVolumeName, volume.podName, pod.UID, attachablePlugin)
if checkErr != nil {
return nil, err
}
// If mount or symlink doesn't exist, volume reconstruction should be failed
if !isExist {
return nil, fmt.Errorf("Volume: %q is not mounted", uniqueVolumeName)
}
volumeMounter, newMounterErr := plugin.NewMounter(
@ -467,7 +497,7 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume,
volumepkg.VolumeOptions{})
if newMounterErr != nil {
return nil, fmt.Errorf(
"MountVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
"reconstructVolume.NewMounter failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
uniqueVolumeName,
volumeSpec.Name(),
volume.podName,
@ -475,6 +505,27 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume,
newMounterErr)
}
// TODO: remove feature gate check after no longer needed
var volumeMapper volumepkg.BlockVolumeMapper
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
var newMapperErr error
if mapperPlugin != nil {
volumeMapper, newMapperErr = mapperPlugin.NewBlockVolumeMapper(
volumeSpec,
pod,
volumepkg.VolumeOptions{})
if newMapperErr != nil {
return nil, fmt.Errorf(
"reconstructVolume.NewBlockVolumeMapper failed for volume %q (spec.Name: %q) pod %q (UID: %q) with: %v",
uniqueVolumeName,
volumeSpec.Name(),
volume.podName,
pod.UID,
newMapperErr)
}
}
}
reconstructedVolume := &reconstructedVolume{
volumeName: uniqueVolumeName,
podName: volume.podName,
@ -489,6 +540,7 @@ func (rc *reconciler) reconstructVolume(volume podVolume) (*reconstructedVolume,
volumeGidValue: "",
devicePath: "",
mounter: volumeMounter,
blockVolumeMapper: volumeMapper,
}
return reconstructedVolume, nil
}
@ -518,7 +570,6 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*re
volumeToMount.VolumeName, volume.outerVolumeSpecName)
}
}
for _, volume := range volumesNeedUpdate {
err := rc.actualStateOfWorld.MarkVolumeAsAttached(
volume.volumeName, volume.volumeSpec, "" /* nodeName */, volume.devicePath)
@ -532,6 +583,7 @@ func (rc *reconciler) updateStates(volumesNeedUpdate map[v1.UniqueVolumeName]*re
types.UID(volume.podName),
volume.volumeName,
volume.mounter,
volume.blockVolumeMapper,
volume.outerVolumeSpecName,
volume.volumeGidValue)
if err != nil {
@ -574,33 +626,45 @@ func getVolumesFromPodDir(podDir string) ([]podVolume, error) {
}
podName := podsDirInfo[i].Name()
podDir := path.Join(podDir, podName)
volumesDir := path.Join(podDir, config.DefaultKubeletVolumesDirName)
volumesDirInfo, err := ioutil.ReadDir(volumesDir)
if err != nil {
glog.Errorf("Could not read volume directory %q: %v", volumesDir, err)
continue
}
for _, volumeDir := range volumesDirInfo {
pluginName := volumeDir.Name()
volumePluginPath := path.Join(volumesDir, pluginName)
volumePluginDirs, err := utilfile.ReadDirNoStat(volumePluginPath)
if err != nil {
glog.Errorf("Could not read volume plugin directory %q: %v", volumePluginPath, err)
// Find filesystem volume information
// ex. filesystem volume: /pods/{podUid}/volume/{escapeQualifiedPluginName}/{volumeName}
volumesDirs := map[v1.PersistentVolumeMode]string{
v1.PersistentVolumeFilesystem: path.Join(podDir, config.DefaultKubeletVolumesDirName),
}
// TODO: remove feature gate check after no longer needed
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
// Find block volume information
// ex. block volume: /pods/{podUid}/volumeDevices/{escapeQualifiedPluginName}/{volumeName}
volumesDirs[v1.PersistentVolumeBlock] = path.Join(podDir, config.DefaultKubeletVolumeDevicesDirName)
}
for volumeMode, volumesDir := range volumesDirs {
var volumesDirInfo []os.FileInfo
if volumesDirInfo, err = ioutil.ReadDir(volumesDir); err != nil {
// Just skip the loop becuase given volumesDir doesn't exist depending on volumeMode
continue
}
unescapePluginName := strings.UnescapeQualifiedNameForDisk(pluginName)
for _, volumeName := range volumePluginDirs {
mountPath := path.Join(volumePluginPath, volumeName)
volumes = append(volumes, podVolume{
podName: volumetypes.UniquePodName(podName),
volumeSpecName: volumeName,
mountPath: mountPath,
pluginName: unescapePluginName,
})
for _, volumeDir := range volumesDirInfo {
pluginName := volumeDir.Name()
volumePluginPath := path.Join(volumesDir, pluginName)
volumePluginDirs, err := utilfile.ReadDirNoStat(volumePluginPath)
if err != nil {
glog.Errorf("Could not read volume plugin directory %q: %v", volumePluginPath, err)
continue
}
unescapePluginName := utilstrings.UnescapeQualifiedNameForDisk(pluginName)
for _, volumeName := range volumePluginDirs {
mountPath := path.Join(volumePluginPath, volumeName)
glog.V(5).Infof("podName: %v, mount path from volume plugin directory: %v, ", podName, mountPath)
volumes = append(volumes, podVolume{
podName: volumetypes.UniquePodName(podName),
volumeSpecName: volumeName,
mountPath: mountPath,
pluginName: unescapePluginName,
volumeMode: volumeMode,
})
}
}
}
}
glog.V(10).Infof("Get volumes from pod directory %q %+v", podDir, volumes)

554
pkg/kubelet/volumemanager/reconciler/reconciler_test.go
View File

@ -23,10 +23,12 @@ import (
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
k8stypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
"k8s.io/client-go/tools/record"
@ -61,7 +63,14 @@ func Test_Run_Positive_DoNothing(t *testing.T) {
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler,
))
reconciler := NewReconciler(
kubeClient,
false, /* controllerAttachDetachEnabled */
@ -99,7 +108,13 @@ func Test_Run_Positive_VolumeAttachAndMount(t *testing.T) {
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
false, /* controllerAttachDetachEnabled */
@ -171,7 +186,13 @@ func Test_Run_Positive_VolumeMountControllerAttachEnabled(t *testing.T) {
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
true, /* controllerAttachDetachEnabled */
@ -244,7 +265,13 @@ func Test_Run_Positive_VolumeAttachMountUnmountDetach(t *testing.T) {
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
false, /* controllerAttachDetachEnabled */
@ -328,7 +355,13 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) {
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(kubeClient, volumePluginMgr, fakeRecorder, false /* checkNodeCapabilitiesBeforeMount */))
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
true, /* controllerAttachDetachEnabled */
@ -399,6 +432,517 @@ func Test_Run_Positive_VolumeUnmountControllerAttachEnabled(t *testing.T) {
assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin))
}
// Populates desiredStateOfWorld cache with one volume/pod.
// Calls Run()
// Verifies there are attach/get map paths/setupDevice calls and
// no detach/teardownDevice calls.
func Test_Run_Positive_VolumeAttachAndMap(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// Arrange
volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t)
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
false, /* controllerAttachDetachEnabled */
reconcilerLoopSleepDuration,
reconcilerSyncStatesSleepPeriod,
waitForAttachTimeout,
nodeName,
dsw,
asw,
hasAddedPods,
oex,
&mount.FakeMounter{},
volumePluginMgr,
kubeletPodsDir)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
UID: "pod1uid",
},
Spec: v1.PodSpec{},
}
mode := v1.PersistentVolumeBlock
gcepv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"},
Spec: v1.PersistentVolumeSpec{
Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")},
PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}},
AccessModes: []v1.PersistentVolumeAccessMode{
v1.ReadWriteOnce,
v1.ReadOnlyMany,
},
VolumeMode: &mode,
},
}
volumeSpec := &volume.Spec{
PersistentVolume: gcepv,
}
podName := volumehelper.GetUniquePodName(pod)
generatedVolumeName, err := dsw.AddPodToVolume(
podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */)
// Assert
if err != nil {
t.Fatalf("AddPodToVolume failed. Expected: <no error> Actual: <%v>", err)
}
// Act
runReconciler(reconciler)
waitForMount(t, fakePlugin, generatedVolumeName, asw)
// Assert
assert.NoError(t, volumetesting.VerifyAttachCallCount(
1 /* expectedAttachCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount(
1 /* expectedWaitForAttachCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount(
1 /* expectedGetGlobalMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount(
1 /* expectedPodDeviceMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount(
1 /* expectedSetUpDeviceCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin))
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
// Populates desiredStateOfWorld cache with one volume/pod.
// Enables controllerAttachDetachEnabled.
// Calls Run()
// Verifies there are two get map path calls, a setupDevice call
// and no teardownDevice call.
// Verifies there are no attach/detach calls.
func Test_Run_Positive_BlockVolumeMapControllerAttachEnabled(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// Arrange
volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t)
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
true, /* controllerAttachDetachEnabled */
reconcilerLoopSleepDuration,
reconcilerSyncStatesSleepPeriod,
waitForAttachTimeout,
nodeName,
dsw,
asw,
hasAddedPods,
oex,
&mount.FakeMounter{},
volumePluginMgr,
kubeletPodsDir)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
UID: "pod1uid",
},
Spec: v1.PodSpec{},
}
mode := v1.PersistentVolumeBlock
gcepv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"},
Spec: v1.PersistentVolumeSpec{
Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")},
PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}},
AccessModes: []v1.PersistentVolumeAccessMode{
v1.ReadWriteOnce,
v1.ReadOnlyMany,
},
VolumeMode: &mode,
},
}
volumeSpec := &volume.Spec{
PersistentVolume: gcepv,
}
podName := volumehelper.GetUniquePodName(pod)
generatedVolumeName, err := dsw.AddPodToVolume(
podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */)
dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName})
// Assert
if err != nil {
t.Fatalf("AddPodToVolume failed. Expected: <no error> Actual: <%v>", err)
}
// Act
runReconciler(reconciler)
waitForMount(t, fakePlugin, generatedVolumeName, asw)
// Assert
assert.NoError(t, volumetesting.VerifyZeroAttachCalls(fakePlugin))
assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount(
1 /* expectedWaitForAttachCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount(
1 /* expectedGetGlobalMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount(
1 /* expectedPodDeviceMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount(
1 /* expectedSetUpCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin))
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
// Populates desiredStateOfWorld cache with one volume/pod.
// Calls Run()
// Verifies there is one attach call, two get map path calls,
// setupDevice call and no detach calls.
// Deletes volume/pod from desired state of world.
// Verifies one detach/teardownDevice calls are issued.
func Test_Run_Positive_BlockVolumeAttachMapUnmapDetach(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// Arrange
volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t)
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
false, /* controllerAttachDetachEnabled */
reconcilerLoopSleepDuration,
reconcilerSyncStatesSleepPeriod,
waitForAttachTimeout,
nodeName,
dsw,
asw,
hasAddedPods,
oex,
&mount.FakeMounter{},
volumePluginMgr,
kubeletPodsDir)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
UID: "pod1uid",
},
Spec: v1.PodSpec{},
}
mode := v1.PersistentVolumeBlock
gcepv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"},
Spec: v1.PersistentVolumeSpec{
Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")},
PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}},
AccessModes: []v1.PersistentVolumeAccessMode{
v1.ReadWriteOnce,
v1.ReadOnlyMany,
},
VolumeMode: &mode,
},
}
volumeSpec := &volume.Spec{
PersistentVolume: gcepv,
}
podName := volumehelper.GetUniquePodName(pod)
generatedVolumeName, err := dsw.AddPodToVolume(
podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */)
// Assert
if err != nil {
t.Fatalf("AddPodToVolume failed. Expected: <no error> Actual: <%v>", err)
}
// Act
runReconciler(reconciler)
waitForMount(t, fakePlugin, generatedVolumeName, asw)
// Assert
assert.NoError(t, volumetesting.VerifyAttachCallCount(
1 /* expectedAttachCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount(
1 /* expectedWaitForAttachCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount(
1 /* expectedGetGlobalMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount(
1 /* expectedPodDeviceMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount(
1 /* expectedSetUpCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin))
// Act
dsw.DeletePodFromVolume(podName, generatedVolumeName)
waitForDetach(t, fakePlugin, generatedVolumeName, asw)
// Assert
assert.NoError(t, volumetesting.VerifyTearDownDeviceCallCount(
1 /* expectedTearDownDeviceCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyDetachCallCount(
1 /* expectedDetachCallCount */, fakePlugin))
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
// Populates desiredStateOfWorld cache with one volume/pod.
// Enables controllerAttachDetachEnabled.
// Calls Run()
// Verifies two map path calls are made and no teardownDevice/detach calls.
// Deletes volume/pod from desired state of world.
// Verifies one teardownDevice call is made.
// Verifies there are no attach/detach calls made.
func Test_Run_Positive_VolumeUnmapControllerAttachEnabled(t *testing.T) {
// Enable BlockVolume feature gate
utilfeature.DefaultFeatureGate.Set("BlockVolume=true")
// Arrange
volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t)
dsw := cache.NewDesiredStateOfWorld(volumePluginMgr)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
kubeClient := createTestClient()
fakeRecorder := &record.FakeRecorder{}
fakeHandler := volumetesting.NewBlockVolumePathHandler()
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
kubeClient,
volumePluginMgr,
fakeRecorder,
false, /* checkNodeCapabilitiesBeforeMount */
fakeHandler))
reconciler := NewReconciler(
kubeClient,
true, /* controllerAttachDetachEnabled */
reconcilerLoopSleepDuration,
reconcilerSyncStatesSleepPeriod,
waitForAttachTimeout,
nodeName,
dsw,
asw,
hasAddedPods,
oex,
&mount.FakeMounter{},
volumePluginMgr,
kubeletPodsDir)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
UID: "pod1uid",
},
Spec: v1.PodSpec{},
}
mode := v1.PersistentVolumeBlock
gcepv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{UID: "001", Name: "volume-name"},
Spec: v1.PersistentVolumeSpec{
Capacity: v1.ResourceList{v1.ResourceName(v1.ResourceStorage): resource.MustParse("10G")},
PersistentVolumeSource: v1.PersistentVolumeSource{GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{}},
AccessModes: []v1.PersistentVolumeAccessMode{
v1.ReadWriteOnce,
v1.ReadOnlyMany,
},
VolumeMode: &mode,
},
}
volumeSpec := &volume.Spec{
PersistentVolume: gcepv,
}
podName := volumehelper.GetUniquePodName(pod)
generatedVolumeName, err := dsw.AddPodToVolume(
podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */)
// Assert
if err != nil {
t.Fatalf("AddPodToVolume failed. Expected: <no error> Actual: <%v>", err)
}
// Act
runReconciler(reconciler)
dsw.MarkVolumesReportedInUse([]v1.UniqueVolumeName{generatedVolumeName})
waitForMount(t, fakePlugin, generatedVolumeName, asw)
// Assert
assert.NoError(t, volumetesting.VerifyZeroAttachCalls(fakePlugin))
assert.NoError(t, volumetesting.VerifyWaitForAttachCallCount(
1 /* expectedWaitForAttachCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetGlobalMapPathCallCount(
1 /* expectedGetGlobalMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyGetPodDeviceMapPathCallCount(
1 /* expectedPodDeviceMapPathCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifySetUpDeviceCallCount(
1 /* expectedSetUpCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroTearDownDeviceCallCount(fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin))
// Act
dsw.DeletePodFromVolume(podName, generatedVolumeName)
waitForDetach(t, fakePlugin, generatedVolumeName, asw)
// Assert
assert.NoError(t, volumetesting.VerifyTearDownDeviceCallCount(
1 /* expectedTearDownDeviceCallCount */, fakePlugin))
assert.NoError(t, volumetesting.VerifyZeroDetachCallCount(fakePlugin))
// Rollback feature gate to false.
utilfeature.DefaultFeatureGate.Set("BlockVolume=false")
}
func Test_GenerateMapVolumeFunc_Plugin_Not_Found(t *testing.T) {
testCases := map[string]struct {
volumePlugins []volume.VolumePlugin
expectErr bool
expectedErrMsg string
}{
"volumePlugin is nil": {
volumePlugins: []volume.VolumePlugin{},
expectErr: true,
expectedErrMsg: "MapVolume.FindMapperPluginBySpec failed",
},
"blockVolumePlugin is nil": {
volumePlugins: volumetesting.NewFakeFileVolumePlugin(),
expectErr: true,
expectedErrMsg: "MapVolume.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
volumePluginMgr := &volume.VolumePluginMgr{}
volumePluginMgr.InitPlugins(tc.volumePlugins, nil, nil)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
nil, /* kubeClient */
volumePluginMgr,
nil, /* fakeRecorder */
false, /* checkNodeCapabilitiesBeforeMount */
nil))
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
UID: "pod1uid",
},
Spec: v1.PodSpec{},
}
volumeToMount := operationexecutor.VolumeToMount{Pod: pod, VolumeSpec: &volume.Spec{}}
err := oex.MapVolume(waitForAttachTimeout, volumeToMount, asw)
// Assert
if assert.Error(t, err) {
assert.Contains(t, err.Error(), tc.expectedErrMsg)
}
})
}
}
func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) {
testCases := map[string]struct {
volumePlugins []volume.VolumePlugin
expectErr bool
expectedErrMsg string
}{
"volumePlugin is nil": {
volumePlugins: []volume.VolumePlugin{},
expectErr: true,
expectedErrMsg: "UnmapVolume.FindMapperPluginByName failed",
},
"blockVolumePlugin is nil": {
volumePlugins: volumetesting.NewFakeFileVolumePlugin(),
expectErr: true,
expectedErrMsg: "UnmapVolume.FindMapperPluginByName failed to find BlockVolumeMapper plugin. Volume plugin is nil.",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
volumePluginMgr := &volume.VolumePluginMgr{}
volumePluginMgr.InitPlugins(tc.volumePlugins, nil, nil)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
nil, /* kubeClient */
volumePluginMgr,
nil, /* fakeRecorder */
false, /* checkNodeCapabilitiesBeforeMount */
nil))
volumeToUnmount := operationexecutor.MountedVolume{PluginName: "fake-file-plugin"}
err := oex.UnmapVolume(volumeToUnmount, asw)
// Assert
if assert.Error(t, err) {
assert.Contains(t, err.Error(), tc.expectedErrMsg)
}
})
}
}
func Test_GenerateUnmapDeviceFunc_Plugin_Not_Found(t *testing.T) {
testCases := map[string]struct {
volumePlugins []volume.VolumePlugin
expectErr bool
expectedErrMsg string
}{
"volumePlugin is nil": {
volumePlugins: []volume.VolumePlugin{},
expectErr: true,
expectedErrMsg: "UnmapDevice.FindMapperPluginBySpec failed",
},
"blockVolumePlugin is nil": {
volumePlugins: volumetesting.NewFakeFileVolumePlugin(),
expectErr: true,
expectedErrMsg: "UnmapDevice.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
volumePluginMgr := &volume.VolumePluginMgr{}
volumePluginMgr.InitPlugins(tc.volumePlugins, nil, nil)
asw := cache.NewActualStateOfWorld(nodeName, volumePluginMgr)
oex := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator(
nil, /* kubeClient */
volumePluginMgr,
nil, /* fakeRecorder */
false, /* checkNodeCapabilitiesBeforeMount */
nil))
var mounter mount.Interface
deviceToDetach := operationexecutor.AttachedVolume{VolumeSpec: &volume.Spec{}}
err := oex.UnmapDevice(deviceToDetach, asw, mounter)
// Assert
if assert.Error(t, err) {
assert.Contains(t, err.Error(), tc.expectedErrMsg)
}
})
}
}
func waitForMount(
t *testing.T,
fakePlugin *volumetesting.FakeVolumePlugin,

11
pkg/kubelet/volumemanager/volume_manager.go
View File

@ -41,6 +41,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
"k8s.io/kubernetes/pkg/volume/util/types"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
@ -168,8 +169,8 @@ func NewVolumeManager(
kubeClient,
volumePluginMgr,
recorder,
checkNodeCapabilitiesBeforeMount),
),
checkNodeCapabilitiesBeforeMount,
util.NewBlockVolumePathHandler())),
}
vm.desiredStateOfWorldPopulator = populator.NewDesiredStateOfWorldPopulator(
@ -253,7 +254,11 @@ func (vm *volumeManager) Run(sourcesReady config.SourcesReady, stopCh <-chan str
func (vm *volumeManager) GetMountedVolumesForPod(podName types.UniquePodName) container.VolumeMap {
podVolumes := make(container.VolumeMap)
for _, mountedVolume := range vm.actualStateOfWorld.GetMountedVolumesForPod(podName) {
podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{Mounter: mountedVolume.Mounter}
podVolumes[mountedVolume.OuterVolumeSpecName] = container.VolumeInfo{
Mounter: mountedVolume.Mounter,
BlockVolumeMapper: mountedVolume.BlockVolumeMapper,
ReadOnly: mountedVolume.VolumeSpec.ReadOnly,
}
}
return podVolumes
}

58
pkg/volume/plugins.go
View File

@ -208,6 +208,26 @@ type ExpandableVolumePlugin interface {
RequiresFSResize() bool
}
// BlockVolumePlugin is an extend interface of VolumePlugin and is used for block volumes support.
type BlockVolumePlugin interface {
VolumePlugin
// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
// Ownership of the spec pointer in *not* transferred.
// - spec: The v1.Volume spec
// - pod: The enclosing pod
NewBlockVolumeMapper(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (BlockVolumeMapper, error)
// NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state.
// - name: The volume name, as per the v1.Volume spec.
// - podUID: The UID of the enclosing pod
NewBlockVolumeUnmapper(name string, podUID types.UID) (BlockVolumeUnmapper, error)
// ConstructBlockVolumeSpec constructs a volume spec based on the given
// podUID, volume name and a pod device map path.
// The spec may have incomplete information due to limited information
// from input. This function is used by volume manager to reconstruct
// volume spec by reading the volume directories from disk.
ConstructBlockVolumeSpec(podUID types.UID, volumeName, mountPath string) (*Spec, error)
}
// VolumeHost is an interface that plugins can use to access the kubelet.
type VolumeHost interface {
// GetPluginDir returns the absolute path to a directory under which
@ -216,6 +236,11 @@ type VolumeHost interface {
// GetPodPluginDir().
GetPluginDir(pluginName string) string
// GetVolumeDevicePluginDir returns the absolute path to a directory
// under which a given plugin may store data.
// ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/
GetVolumeDevicePluginDir(pluginName string) string
// GetPodVolumeDir returns the absolute path a directory which
// represents the named volume under the named plugin for the given
// pod. If the specified pod does not exist, the result of this call
@ -228,6 +253,13 @@ type VolumeHost interface {
// directory might not actually exist on disk yet.
GetPodPluginDir(podUID types.UID, pluginName string) string
// GetPodVolumeDeviceDir returns the absolute path a directory which
// represents the named plugin for the given pod.
// If the specified pod does not exist, the result of this call
// might not exist.
// ex. pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/
GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string
// GetKubeClient returns a client interface
GetKubeClient() clientset.Interface
@ -675,6 +707,32 @@ func (pm *VolumePluginMgr) FindExpandablePluginByName(name string) (ExpandableVo
return nil, nil
}
// FindMapperPluginBySpec fetches a block volume plugin by spec.
func (pm *VolumePluginMgr) FindMapperPluginBySpec(spec *Spec) (BlockVolumePlugin, error) {
volumePlugin, err := pm.FindPluginBySpec(spec)
if err != nil {
return nil, err
}
if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok {
return blockVolumePlugin, nil
}
return nil, nil
}
// FindMapperPluginByName fetches a block volume plugin by name.
func (pm *VolumePluginMgr) FindMapperPluginByName(name string) (BlockVolumePlugin, error) {
volumePlugin, err := pm.FindPluginByName(name)
if err != nil {
return nil, err
}
if blockVolumePlugin, ok := volumePlugin.(BlockVolumePlugin); ok {
return blockVolumePlugin, nil
}
return nil, nil
}
// NewPersistentVolumeRecyclerPodTemplate creates a template for a recycler
// pod. By default, a recycler pod simply runs "rm -rf" on a volume and tests
// for emptiness. Most attributes of the template will be correct for most

333
pkg/volume/testing/testing.go
View File

@ -39,6 +39,7 @@ import (
"k8s.io/kubernetes/pkg/util/mount"
utilstrings "k8s.io/kubernetes/pkg/util/strings"
. "k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)
@ -81,10 +82,18 @@ func (f *fakeVolumeHost) GetPluginDir(podUID string) string {
return path.Join(f.rootDir, "plugins", podUID)
}
func (f *fakeVolumeHost) GetVolumeDevicePluginDir(podUID string) string {
return path.Join(f.rootDir, "plugins", podUID)
}
func (f *fakeVolumeHost) GetPodVolumeDir(podUID types.UID, pluginName, volumeName string) string {
return path.Join(f.rootDir, "pods", string(podUID), "volumes", pluginName, volumeName)
}
func (f *fakeVolumeHost) GetPodVolumeDeviceDir(podUID types.UID, pluginName string) string {
return path.Join(f.rootDir, "pods", string(podUID), "volumeDevices", pluginName)
}
func (f *fakeVolumeHost) GetPodPluginDir(podUID types.UID, pluginName string) string {
return path.Join(f.rootDir, "pods", string(podUID), "plugins", pluginName)
}
@ -194,13 +203,16 @@ type FakeVolumePlugin struct {
NewAttacherCallCount int
NewDetacherCallCount int
Mounters []*FakeVolume
Unmounters []*FakeVolume
Attachers []*FakeVolume
Detachers []*FakeVolume
Mounters []*FakeVolume
Unmounters []*FakeVolume
Attachers []*FakeVolume
Detachers []*FakeVolume
BlockVolumeMappers []*FakeVolume
BlockVolumeUnmappers []*FakeVolume
}
var _ VolumePlugin = &FakeVolumePlugin{}
var _ BlockVolumePlugin = &FakeVolumePlugin{}
var _ RecyclableVolumePlugin = &FakeVolumePlugin{}
var _ DeletableVolumePlugin = &FakeVolumePlugin{}
var _ ProvisionableVolumePlugin = &FakeVolumePlugin{}
@ -280,6 +292,44 @@ func (plugin *FakeVolumePlugin) GetUnmounters() (Unmounters []*FakeVolume) {
return plugin.Unmounters
}
// Block volume support
func (plugin *FakeVolumePlugin) NewBlockVolumeMapper(spec *Spec, pod *v1.Pod, opts VolumeOptions) (BlockVolumeMapper, error) {
plugin.Lock()
defer plugin.Unlock()
volume := plugin.getFakeVolume(&plugin.BlockVolumeMappers)
if pod != nil {
volume.PodUID = pod.UID
}
volume.VolName = spec.Name()
volume.Plugin = plugin
return volume, nil
}
// Block volume support
func (plugin *FakeVolumePlugin) GetBlockVolumeMapper() (BlockVolumeMappers []*FakeVolume) {
plugin.RLock()
defer plugin.RUnlock()
return plugin.BlockVolumeMappers
}
// Block volume support
func (plugin *FakeVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (BlockVolumeUnmapper, error) {
plugin.Lock()
defer plugin.Unlock()
volume := plugin.getFakeVolume(&plugin.BlockVolumeUnmappers)
volume.PodUID = podUID
volume.VolName = volName
volume.Plugin = plugin
return volume, nil
}
// Block volume support
func (plugin *FakeVolumePlugin) GetBlockVolumeUnmapper() (BlockVolumeUnmappers []*FakeVolume) {
plugin.RLock()
defer plugin.RUnlock()
return plugin.BlockVolumeUnmappers
}
func (plugin *FakeVolumePlugin) NewAttacher() (Attacher, error) {
plugin.Lock()
defer plugin.Unlock()
@ -345,10 +395,66 @@ func (plugin *FakeVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string
}, nil
}
// Block volume support
func (plugin *FakeVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mountPath string) (*Spec, error) {
return &Spec{
Volume: &v1.Volume{
Name: volumeName,
},
}, nil
}
func (plugin *FakeVolumePlugin) GetDeviceMountRefs(deviceMountPath string) ([]string, error) {
return []string{}, nil
}
type FakeFileVolumePlugin struct {
}
func (plugin *FakeFileVolumePlugin) Init(host VolumeHost) error {
return nil
}
func (plugin *FakeFileVolumePlugin) GetPluginName() string {
return "fake-file-plugin"
}
func (plugin *FakeFileVolumePlugin) GetVolumeName(spec *Spec) (string, error) {
return "", nil
}
func (plugin *FakeFileVolumePlugin) CanSupport(spec *Spec) bool {
return true
}
func (plugin *FakeFileVolumePlugin) RequiresRemount() bool {
return false
}
func (plugin *FakeFileVolumePlugin) SupportsMountOption() bool {
return false
}
func (plugin *FakeFileVolumePlugin) SupportsBulkVolumeVerification() bool {
return false
}
func (plugin *FakeFileVolumePlugin) NewMounter(spec *Spec, podRef *v1.Pod, opts VolumeOptions) (Mounter, error) {
return nil, nil
}
func (plugin *FakeFileVolumePlugin) NewUnmounter(name string, podUID types.UID) (Unmounter, error) {
return nil, nil
}
func (plugin *FakeFileVolumePlugin) ConstructVolumeSpec(volumeName, mountPath string) (*Spec, error) {
return nil, nil
}
func NewFakeFileVolumePlugin() []VolumePlugin {
return []VolumePlugin{&FakeFileVolumePlugin{}}
}
type FakeVolume struct {
sync.RWMutex
PodUID types.UID
@ -364,6 +470,10 @@ type FakeVolume struct {
MountDeviceCallCount int
UnmountDeviceCallCount int
GetDeviceMountPathCallCount int
SetUpDeviceCallCount int
TearDownDeviceCallCount int
GlobalMapPathCallCount int
PodDeviceMapPathCallCount int
}
func (_ *FakeVolume) GetAttributes() Attributes {
@ -422,6 +532,76 @@ func (fv *FakeVolume) TearDownAt(dir string) error {
return os.RemoveAll(dir)
}
// Block volume support
func (fv *FakeVolume) SetUpDevice() (string, error) {
fv.Lock()
defer fv.Unlock()
fv.SetUpDeviceCallCount++
return "", nil
}
// Block volume support
func (fv *FakeVolume) GetSetUpDeviceCallCount() int {
fv.RLock()
defer fv.RUnlock()
return fv.SetUpDeviceCallCount
}
// Block volume support
func (fv *FakeVolume) GetGlobalMapPath(spec *Spec) (string, error) {
fv.RLock()
defer fv.RUnlock()
fv.GlobalMapPathCallCount++
return fv.getGlobalMapPath()
}
// Block volume support
func (fv *FakeVolume) getGlobalMapPath() (string, error) {
return path.Join(fv.Plugin.Host.GetVolumeDevicePluginDir(utilstrings.EscapeQualifiedNameForDisk(fv.Plugin.PluginName)), "pluginDependentPath"), nil
}
// Block volume support
func (fv *FakeVolume) GetGlobalMapPathCallCount() int {
fv.RLock()
defer fv.RUnlock()
return fv.GlobalMapPathCallCount
}
// Block volume support
func (fv *FakeVolume) GetPodDeviceMapPath() (string, string) {
fv.RLock()
defer fv.RUnlock()
fv.PodDeviceMapPathCallCount++
return fv.getPodDeviceMapPath()
}
// Block volume support
func (fv *FakeVolume) getPodDeviceMapPath() (string, string) {
return path.Join(fv.Plugin.Host.GetPodVolumeDeviceDir(fv.PodUID, utilstrings.EscapeQualifiedNameForDisk(fv.Plugin.PluginName))), fv.VolName
}
// Block volume support
func (fv *FakeVolume) GetPodDeviceMapPathCallCount() int {
fv.RLock()
defer fv.RUnlock()
return fv.PodDeviceMapPathCallCount
}
// Block volume support
func (fv *FakeVolume) TearDownDevice(mapPath string, devicePath string) error {
fv.Lock()
defer fv.Unlock()
fv.TearDownDeviceCallCount++
return nil
}
// Block volume support
func (fv *FakeVolume) GetTearDownDeviceCallCount() int {
fv.RLock()
defer fv.RUnlock()
return fv.TearDownDeviceCallCount
}
func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error) {
fv.Lock()
defer fv.Unlock()
@ -439,7 +619,7 @@ func (fv *FakeVolume) WaitForAttach(spec *Spec, devicePath string, pod *v1.Pod,
fv.Lock()
defer fv.Unlock()
fv.WaitForAttachCallCount++
return "", nil
return "/dev/sdb", nil
}
func (fv *FakeVolume) GetWaitForAttachCallCount() int {
@ -540,6 +720,62 @@ func (fc *FakeProvisioner) Provision() (*v1.PersistentVolume, error) {
return pv, nil
}
var _ util.BlockVolumePathHandler = &FakeVolumePathHandler{}
//NewDeviceHandler Create a new IoHandler implementation
func NewBlockVolumePathHandler() util.BlockVolumePathHandler {
return &FakeVolumePathHandler{}
}
type FakeVolumePathHandler struct {
sync.RWMutex
}
func (fv *FakeVolumePathHandler) MapDevice(devicePath string, mapDir string, linkName string) error {
// nil is success, else error
return nil
}
func (fv *FakeVolumePathHandler) UnmapDevice(mapDir string, linkName string) error {
// nil is success, else error
return nil
}
func (fv *FakeVolumePathHandler) RemoveMapPath(mapPath string) error {
// nil is success, else error
return nil
}
func (fv *FakeVolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) {
// nil is success, else error
return true, nil
}
func (fv *FakeVolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) {
// nil is success, else error
return []string{}, nil
}
func (fv *FakeVolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
// nil is success, else error
return "", nil
}
func (fv *FakeVolumePathHandler) AttachFileDevice(path string) (string, error) {
// nil is success, else error
return "", nil
}
func (fv *FakeVolumePathHandler) GetLoopDevice(path string) (string, error) {
// nil is success, else error
return "/dev/loop1", nil
}
func (fv *FakeVolumePathHandler) RemoveLoopDevice(device string) error {
// nil is success, else error
return nil
}
// FindEmptyDirectoryUsageOnTmpfs finds the expected usage of an empty directory existing on
// a tmpfs filesystem on this system.
func FindEmptyDirectoryUsageOnTmpfs() (*resource.Quantity, error) {
@ -758,6 +994,93 @@ func VerifyZeroDetachCallCount(fakeVolumePlugin *FakeVolumePlugin) error {
return nil
}
// VerifySetUpDeviceCallCount ensures that at least one of the Mappers for this
// plugin has the expectedSetUpDeviceCallCount number of calls. Otherwise it
// returns an error.
func VerifySetUpDeviceCallCount(
expectedSetUpDeviceCallCount int,
fakeVolumePlugin *FakeVolumePlugin) error {
for _, mapper := range fakeVolumePlugin.GetBlockVolumeMapper() {
actualCallCount := mapper.GetSetUpDeviceCallCount()
if actualCallCount >= expectedSetUpDeviceCallCount {
return nil
}
}
return fmt.Errorf(
"No Mapper have expected SetUpDeviceCallCount. Expected: <%v>.",
expectedSetUpDeviceCallCount)
}
// VerifyTearDownDeviceCallCount ensures that at least one of the Unmappers for this
// plugin has the expectedTearDownDeviceCallCount number of calls. Otherwise it
// returns an error.
func VerifyTearDownDeviceCallCount(
expectedTearDownDeviceCallCount int,
fakeVolumePlugin *FakeVolumePlugin) error {
for _, unmapper := range fakeVolumePlugin.GetBlockVolumeUnmapper() {
actualCallCount := unmapper.GetTearDownDeviceCallCount()
if actualCallCount >= expectedTearDownDeviceCallCount {
return nil
}
}
return fmt.Errorf(
"No Unmapper have expected TearDownDeviceCallCount. Expected: <%v>.",
expectedTearDownDeviceCallCount)
}
// VerifyZeroTearDownDeviceCallCount ensures that all Mappers for this plugin have a
// zero TearDownDeviceCallCount. Otherwise it returns an error.
func VerifyZeroTearDownDeviceCallCount(fakeVolumePlugin *FakeVolumePlugin) error {
for _, unmapper := range fakeVolumePlugin.GetBlockVolumeUnmapper() {
actualCallCount := unmapper.GetTearDownDeviceCallCount()
if actualCallCount != 0 {
return fmt.Errorf(
"At least one unmapper has non-zero TearDownDeviceCallCount: <%v>.",
actualCallCount)
}
}
return nil
}
// VerifyGetGlobalMapPathCallCount ensures that at least one of the Mappers for this
// plugin has the expectedGlobalMapPathCallCount number of calls. Otherwise it returns
// an error.
func VerifyGetGlobalMapPathCallCount(
expectedGlobalMapPathCallCount int,
fakeVolumePlugin *FakeVolumePlugin) error {
for _, mapper := range fakeVolumePlugin.GetBlockVolumeMapper() {
actualCallCount := mapper.GetGlobalMapPathCallCount()
if actualCallCount == expectedGlobalMapPathCallCount {
return nil
}
}
return fmt.Errorf(
"No Mappers have expected GetGlobalMapPathCallCount. Expected: <%v>.",
expectedGlobalMapPathCallCount)
}
// VerifyGetPodDeviceMapPathCallCount ensures that at least one of the Mappers for this
// plugin has the expectedPodDeviceMapPathCallCount number of calls. Otherwise it returns
// an error.
func VerifyGetPodDeviceMapPathCallCount(
expectedPodDeviceMapPathCallCount int,
fakeVolumePlugin *FakeVolumePlugin) error {
for _, mapper := range fakeVolumePlugin.GetBlockVolumeMapper() {
actualCallCount := mapper.GetPodDeviceMapPathCallCount()
if actualCallCount == expectedPodDeviceMapPathCallCount {
return nil
}
}
return fmt.Errorf(
"No Mappers have expected GetPodDeviceMapPathCallCount. Expected: <%v>.",
expectedPodDeviceMapPathCallCount)
}
// GetTestVolumePluginMgr creates, initializes, and returns a test volume plugin
// manager and fake volume plugin using a fake volume host.
func GetTestVolumePluginMgr(

290
pkg/volume/util/operationexecutor/operation_executor.go
View File

@ -29,7 +29,9 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
expandcache "k8s.io/kubernetes/pkg/controller/volume/expand/cache"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
@ -105,6 +107,28 @@ type OperationExecutor interface {
// actual state of the world to reflect that.
UnmountDevice(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error
// MapVolume is used when the volumeMode is 'Block'.
// This method creates a symbolic link to the volume from both the pod
// specified in volumeToMount and global map path.
// Specifically it will:
// * Wait for the device to finish attaching (for attachable volumes only).
// * Update actual state of world to reflect volume is globally mounted/mapped.
// * Map volume to global map path using symbolic link.
// * Map the volume to the pod device map path using symbolic link.
// * Update actual state of world to reflect volume is mounted/mapped to the pod path.
MapVolume(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater) error
// UnmapVolume unmaps symbolic link to the volume from both the pod device
// map path in volumeToUnmount and global map path.
// And then, updates the actual state of the world to reflect that.
UnmapVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error
// UnmapDevice checks number of symbolic links under global map path.
// If number of reference is zero, remove global map path directory and
// free a volume for detach.
// It then updates the actual state of the world to reflect that.
UnmapDevice(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error
// VerifyControllerAttachedVolume checks if the specified volume is present
// in the specified nodes AttachedVolumes Status field. It uses kubeClient
// to fetch the node object.
@ -139,7 +163,7 @@ func NewOperationExecutor(
// state of the world cache after successful mount/unmount.
type ActualStateOfWorldMounterUpdater interface {
// Marks the specified volume as mounted to the specified pod
MarkVolumeAsMounted(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, outerVolumeSpecName string, volumeGidValue string) error
MarkVolumeAsMounted(podName volumetypes.UniquePodName, podUID types.UID, volumeName v1.UniqueVolumeName, mounter volume.Mounter, blockVolumeMapper volume.BlockVolumeMapper, outerVolumeSpecName string, volumeGidValue string) error
// Marks the specified volume as unmounted from the specified pod
MarkVolumeAsUnmounted(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName) error
@ -495,8 +519,16 @@ type MountedVolume struct {
// by kubelet to create container.VolumeMap.
Mounter volume.Mounter
// BlockVolumeMapper is the volume mapper used to map this volume. It is required
// by kubelet to create container.VolumeMap.
BlockVolumeMapper volume.BlockVolumeMapper
// VolumeGidValue contains the value of the GID annotation, if present.
VolumeGidValue string
// VolumeSpec is a volume spec containing the specification for the volume
// that should be mounted.
VolumeSpec *volume.Spec
}
// GenerateMsgDetailed returns detailed msgs for mounted volumes
@ -733,6 +765,68 @@ func (oe *operationExecutor) ExpandVolume(pvcWithResizeRequest *expandcache.PVCW
return oe.pendingOperations.Run(uniqueVolumeKey, "", expandFunc, opCompleteFunc)
}
func (oe *operationExecutor) MapVolume(
waitForAttachTimeout time.Duration,
volumeToMount VolumeToMount,
actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
mapFunc, plugin, err := oe.operationGenerator.GenerateMapVolumeFunc(
waitForAttachTimeout, volumeToMount, actualStateOfWorld)
if err != nil {
return err
}
// Avoid executing map from multiple pods referencing the
// same volume in parallel
podName := nestedpendingoperations.EmptyUniquePodName
// TODO: remove this -- not necessary
if !volumeToMount.PluginIsAttachable {
// Non-attachable volume plugins can execute mount for multiple pods
// referencing the same volume in parallel
podName = volumehelper.GetUniquePodName(volumeToMount.Pod)
}
opCompleteFunc := util.OperationCompleteHook(plugin, "map_volume")
return oe.pendingOperations.Run(
volumeToMount.VolumeName, podName, mapFunc, opCompleteFunc)
}
func (oe *operationExecutor) UnmapVolume(
volumeToUnmount MountedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
unmapFunc, plugin, err :=
oe.operationGenerator.GenerateUnmapVolumeFunc(volumeToUnmount, actualStateOfWorld)
if err != nil {
return err
}
// All volume plugins can execute unmap for multiple pods referencing the
// same volume in parallel
podName := volumetypes.UniquePodName(volumeToUnmount.PodUID)
opCompleteFunc := util.OperationCompleteHook(plugin, "unmap_volume")
return oe.pendingOperations.Run(
volumeToUnmount.VolumeName, podName, unmapFunc, opCompleteFunc)
}
func (oe *operationExecutor) UnmapDevice(
deviceToDetach AttachedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater,
mounter mount.Interface) error {
unmapDeviceFunc, plugin, err :=
oe.operationGenerator.GenerateUnmapDeviceFunc(deviceToDetach, actualStateOfWorld, mounter)
if err != nil {
return err
}
// Avoid executing unmap device from multiple pods referencing
// the same volume in parallel
podName := nestedpendingoperations.EmptyUniquePodName
opCompleteFunc := util.OperationCompleteHook(plugin, "unmap_device")
return oe.pendingOperations.Run(
deviceToDetach.VolumeName, podName, unmapDeviceFunc, opCompleteFunc)
}
func (oe *operationExecutor) VerifyControllerAttachedVolume(
volumeToMount VolumeToMount,
nodeName types.NodeName,
@ -748,6 +842,200 @@ func (oe *operationExecutor) VerifyControllerAttachedVolume(
volumeToMount.VolumeName, "" /* podName */, verifyControllerAttachedVolumeFunc, opCompleteFunc)
}
// VolumeStateHandler defines a set of operations for handling mount/unmount/detach/reconstruct volume-related operations
type VolumeStateHandler interface {
// Volume is attached, mount/map it
MountVolumeHandler(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, isRemount bool, remountingLogStr string) error
// Volume is mounted/mapped, unmount/unmap it
UnmountVolumeHandler(mountedVolume MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error
// Volume is not referenced from pod, unmount/unmap and detach it
UnmountDeviceHandler(attachedVolume AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error
// Reconstruct volume from mount path
ReconstructVolumeHandler(plugin volume.VolumePlugin, mapperPlugin volume.BlockVolumePlugin, uid types.UID, podName volumetypes.UniquePodName, volumeSpecName string, mountPath string, pluginName string) (*volume.Spec, error)
// check mount path if volume still exists
CheckVolumeExistence(mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, attachable volume.AttachableVolumePlugin) (bool, error)
}
// NewVolumeHandler return a new instance of volumeHandler depens on a volumeMode
func NewVolumeHandler(volumeSpec *volume.Spec, oe OperationExecutor) (VolumeStateHandler, error) {
// TODO: remove feature gate check after no longer needed
var volumeHandler VolumeStateHandler
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
volumeMode, err := volumehelper.GetVolumeMode(volumeSpec)
if err != nil {
return nil, err
}
if volumeMode == v1.PersistentVolumeFilesystem {
volumeHandler = NewFilesystemVolumeHandler(oe)
} else {
volumeHandler = NewBlockVolumeHandler(oe)
}
} else {
volumeHandler = NewFilesystemVolumeHandler(oe)
}
return volumeHandler, nil
}
// NewFilesystemVolumeHandler returns a new instance of FilesystemVolumeHandler.
func NewFilesystemVolumeHandler(operationExecutor OperationExecutor) FilesystemVolumeHandler {
return FilesystemVolumeHandler{
oe: operationExecutor}
}
// NewBlockVolumeHandler returns a new instance of BlockVolumeHandler.
func NewBlockVolumeHandler(operationExecutor OperationExecutor) BlockVolumeHandler {
return BlockVolumeHandler{
oe: operationExecutor}
}
// FilesystemVolumeHandler is VolumeHandler for Filesystem volume
type FilesystemVolumeHandler struct {
oe OperationExecutor
}
// BlockVolumeHandler is VolumeHandler for Block volume
type BlockVolumeHandler struct {
oe OperationExecutor
}
// MountVolumeHandler mount/remount a volume when a volume is attached
// This method is handler for filesystem volume
func (f FilesystemVolumeHandler) MountVolumeHandler(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, isRemount bool, remountingLogStr string) error {
glog.V(12).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MountVolume", remountingLogStr))
err := f.oe.MountVolume(
waitForAttachTimeout,
volumeToMount,
actualStateOfWorld,
isRemount)
return err
}
// UnmountVolumeHandler unmount a volume if a volume is mounted
// This method is handler for filesystem volume
func (f FilesystemVolumeHandler) UnmountVolumeHandler(mountedVolume MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", ""))
err := f.oe.UnmountVolume(
mountedVolume,
actualStateOfWorld)
return err
}
// UnmountDeviceHandler unmount and detach a device if a volume isn't referenced
// This method is handler for filesystem volume
func (f FilesystemVolumeHandler) UnmountDeviceHandler(attachedVolume AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error {
glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountDevice", ""))
err := f.oe.UnmountDevice(
attachedVolume,
actualStateOfWorld,
mounter)
return err
}
// ReconstructVolumeHandler create volumeSpec from mount path
// This method is handler for filesystem volume
func (f FilesystemVolumeHandler) ReconstructVolumeHandler(plugin volume.VolumePlugin, _ volume.BlockVolumePlugin, _ types.UID, _ volumetypes.UniquePodName, volumeSpecName string, mountPath string, _ string) (*volume.Spec, error) {
glog.V(12).Infof("Starting operationExecutor.ReconstructVolumepodName")
volumeSpec, err := plugin.ConstructVolumeSpec(volumeSpecName, mountPath)
if err != nil {
return nil, err
}
return volumeSpec, nil
}
// CheckVolumeExistence checks mount path directory if volume still exists, return true if volume is there
// Also return true for non-attachable volume case without mount point check
// This method is handler for filesystem volume
func (f FilesystemVolumeHandler) CheckVolumeExistence(mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, attachable volume.AttachableVolumePlugin) (bool, error) {
if attachable != nil {
var isNotMount bool
var mountCheckErr error
if isNotMount, mountCheckErr = mounter.IsLikelyNotMountPoint(mountPath); mountCheckErr != nil {
return false, fmt.Errorf("Could not check whether the volume %q (spec.Name: %q) pod %q (UID: %q) is mounted with: %v",
uniqueVolumeName,
volumeName,
podName,
podUID,
mountCheckErr)
}
return !isNotMount, nil
}
return true, nil
}
// MountVolumeHandler creates a map to device if a volume is attached
// This method is handler for block volume
func (b BlockVolumeHandler) MountVolumeHandler(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorld ActualStateOfWorldMounterUpdater, _ bool, _ string) error {
glog.V(12).Infof(volumeToMount.GenerateMsgDetailed("Starting operationExecutor.MapVolume", ""))
err := b.oe.MapVolume(
waitForAttachTimeout,
volumeToMount,
actualStateOfWorld)
return err
}
// UnmountVolumeHandler unmap a volume if a volume is mapped
// This method is handler for block volume
func (b BlockVolumeHandler) UnmountVolumeHandler(mountedVolume MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmapVolume", ""))
err := b.oe.UnmapVolume(
mountedVolume,
actualStateOfWorld)
return err
}
// UnmountDeviceHandler detach a device and remove loopback if a volume isn't referenced
// This method is handler for block volume
func (b BlockVolumeHandler) UnmountDeviceHandler(attachedVolume AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) error {
glog.V(12).Infof(attachedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmapDevice", ""))
err := b.oe.UnmapDevice(
attachedVolume,
actualStateOfWorld,
mounter)
return err
}
// ReconstructVolumeHandler create volumeSpec from mount path
// This method is handler for block volume
func (b BlockVolumeHandler) ReconstructVolumeHandler(_ volume.VolumePlugin, mapperPlugin volume.BlockVolumePlugin, uid types.UID, podName volumetypes.UniquePodName, volumeSpecName string, mountPath string, pluginName string) (*volume.Spec, error) {
glog.V(12).Infof("Starting operationExecutor.ReconstructVolume")
if mapperPlugin == nil {
return nil, fmt.Errorf("Could not find block volume plugin %q (spec.Name: %q) pod %q (UID: %q)",
pluginName,
volumeSpecName,
podName,
uid)
}
// mountPath contains volumeName on the path. In the case of block volume, {volumeName} is symbolic link
// corresponding to raw block device.
// ex. mountPath: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
volumeSpec, err := mapperPlugin.ConstructBlockVolumeSpec(uid, volumeSpecName, mountPath)
if err != nil {
return nil, err
}
return volumeSpec, nil
}
// CheckVolumeExistence checks mount path directory if volume still exists, then return
// true if volume is there. Either plugin is attachable or non-attachable, the plugin
// should have symbolic link associated to raw block device under pod device map
// if volume exists.
// This method is handler for block volume
func (b BlockVolumeHandler) CheckVolumeExistence(mountPath, volumeName string, mounter mount.Interface, uniqueVolumeName v1.UniqueVolumeName, podName volumetypes.UniquePodName, podUID types.UID, _ volume.AttachableVolumePlugin) (bool, error) {
blkutil := util.NewBlockVolumePathHandler()
var islinkExist bool
var checkErr error
if islinkExist, checkErr = blkutil.IsSymlinkExist(mountPath); checkErr != nil {
return false, fmt.Errorf("Could not check whether the block volume %q (spec.Name: %q) pod %q (UID: %q) is mapped to: %v",
uniqueVolumeName,
volumeName,
podName,
podUID,
checkErr)
}
return islinkExist, nil
}
// TODO: this is a workaround for the unmount device issue caused by gci mounter.
// In GCI cluster, if gci mounter is used for mounting, the container started by mounter
// script will cause additional mounts created in the container. Since these mounts are

132
pkg/volume/util/operationexecutor/operation_executor_test.go
View File

@ -40,6 +40,10 @@ const (
numVolumesToDetach = 2
numVolumesToVerifyAttached = 2
numVolumesToVerifyControllerAttached = 2
numVolumesToMap = 2
numAttachableVolumesToUnmap = 2
numNonAttachableVolumesToUnmap = 2
numDevicesToUnmap = 2
)
var _ OperationGenerator = &fakeOperationGenerator{}
@ -228,6 +232,113 @@ func TestOperationExecutor_VerifyControllerAttachedVolumeConcurrently(t *testing
}
}
func TestOperationExecutor_MapVolume_ConcurrentMapForNonAttachablePlugins(t *testing.T) {
// Arrange
ch, quit, oe := setup()
volumesToMount := make([]VolumeToMount, numVolumesToMap)
secretName := "secret-volume"
volumeName := v1.UniqueVolumeName(secretName)
// Act
for i := range volumesToMount {
podName := "pod-" + strconv.Itoa((i + 1))
pod := getTestPodWithSecret(podName, secretName)
volumesToMount[i] = VolumeToMount{
Pod: pod,
VolumeName: volumeName,
PluginIsAttachable: false, // this field determines whether the plugin is attachable
ReportedInUse: true,
}
oe.MapVolume(0 /* waitForAttachTimeOut */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */)
}
// Assert
if !isOperationRunConcurrently(ch, quit, numVolumesToMap) {
t.Fatalf("Unable to start map operations in Concurrent for non-attachable volumes")
}
}
func TestOperationExecutor_MapVolume_ConcurrentMapForAttachablePlugins(t *testing.T) {
// Arrange
ch, quit, oe := setup()
volumesToMount := make([]VolumeToMount, numVolumesToAttach)
pdName := "pd-volume"
volumeName := v1.UniqueVolumeName(pdName)
// Act
for i := range volumesToMount {
podName := "pod-" + strconv.Itoa((i + 1))
pod := getTestPodWithGCEPD(podName, pdName)
volumesToMount[i] = VolumeToMount{
Pod: pod,
VolumeName: volumeName,
PluginIsAttachable: true, // this field determines whether the plugin is attachable
ReportedInUse: true,
}
oe.MapVolume(0 /* waitForAttachTimeout */, volumesToMount[i], nil /* actualStateOfWorldMounterUpdater */)
}
// Assert
if !isOperationRunSerially(ch, quit) {
t.Fatalf("Map operations should not start concurrently for attachable volumes")
}
}
func TestOperationExecutor_UnmapVolume_ConcurrentUnmapForAllPlugins(t *testing.T) {
// Arrange
ch, quit, oe := setup()
volumesToUnmount := make([]MountedVolume, numAttachableVolumesToUnmap+numNonAttachableVolumesToUnmap)
pdName := "pd-volume"
secretName := "secret-volume"
// Act
for i := 0; i < numNonAttachableVolumesToUnmap+numAttachableVolumesToUnmap; i++ {
podName := "pod-" + strconv.Itoa(i+1)
if i < numNonAttachableVolumesToUnmap {
pod := getTestPodWithSecret(podName, secretName)
volumesToUnmount[i] = MountedVolume{
PodName: volumetypes.UniquePodName(podName),
VolumeName: v1.UniqueVolumeName(secretName),
PodUID: pod.UID,
}
} else {
pod := getTestPodWithGCEPD(podName, pdName)
volumesToUnmount[i] = MountedVolume{
PodName: volumetypes.UniquePodName(podName),
VolumeName: v1.UniqueVolumeName(pdName),
PodUID: pod.UID,
}
}
oe.UnmapVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */)
}
// Assert
if !isOperationRunConcurrently(ch, quit, numNonAttachableVolumesToUnmap+numAttachableVolumesToUnmap) {
t.Fatalf("Unable to start unmap operations concurrently for volume plugins")
}
}
func TestOperationExecutor_UnmapDeviceConcurrently(t *testing.T) {
// Arrange
ch, quit, oe := setup()
attachedVolumes := make([]AttachedVolume, numDevicesToUnmap)
pdName := "pd-volume"
// Act
for i := range attachedVolumes {
attachedVolumes[i] = AttachedVolume{
VolumeName: v1.UniqueVolumeName(pdName),
NodeName: "node-name",
}
oe.UnmapDevice(attachedVolumes[i], nil /* actualStateOfWorldMounterUpdater */, nil /* mount.Interface */)
}
// Assert
if !isOperationRunSerially(ch, quit) {
t.Fatalf("Unmap device operations should not start concurrently")
}
}
type fakeOperationGenerator struct {
ch chan interface{}
quit chan interface{}
@ -302,6 +413,27 @@ func (fopg *fakeOperationGenerator) GenerateBulkVolumeVerifyFunc(
}, nil
}
func (fopg *fakeOperationGenerator) GenerateMapVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateUnmapVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GenerateUnmapDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error) {
return func() error {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil
}, "", nil
}
func (fopg *fakeOperationGenerator) GetVolumePluginMgr() *volume.VolumePluginMgr {
return nil
}

387
pkg/volume/util/operationexecutor/operation_generator.go
View File

@ -18,6 +18,7 @@ package operationexecutor
import (
"fmt"
"strings"
"time"
"github.com/golang/glog"
@ -55,19 +56,24 @@ type operationGenerator struct {
// which verifies that the components (binaries, etc.) required to mount
// the volume are available on the underlying node before attempting mount.
checkNodeCapabilitiesBeforeMount bool
// blkUtil provides volume path related operations for block volume
blkUtil util.BlockVolumePathHandler
}
// NewOperationGenerator is returns instance of operationGenerator
func NewOperationGenerator(kubeClient clientset.Interface,
volumePluginMgr *volume.VolumePluginMgr,
recorder record.EventRecorder,
checkNodeCapabilitiesBeforeMount bool) OperationGenerator {
checkNodeCapabilitiesBeforeMount bool,
blkUtil util.BlockVolumePathHandler) OperationGenerator {
return &operationGenerator{
kubeClient: kubeClient,
volumePluginMgr: volumePluginMgr,
recorder: recorder,
checkNodeCapabilitiesBeforeMount: checkNodeCapabilitiesBeforeMount,
blkUtil: blkUtil,
}
}
@ -94,6 +100,15 @@ type OperationGenerator interface {
// Generates the function needed to check if the attach_detach controller has attached the volume plugin
GenerateVerifyControllerAttachedVolumeFunc(volumeToMount VolumeToMount, nodeName types.NodeName, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error)
// Generates the MapVolume function needed to perform the map of a volume plugin
GenerateMapVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater) (func() error, string, error)
// Generates the UnmapVolume function needed to perform the unmap of a volume plugin
GenerateUnmapVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error)
// Generates the UnmapDevice function needed to perform the unmap of a device
GenerateUnmapDeviceFunc(deviceToDetach AttachedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, mounter mount.Interface) (func() error, string, error)
// GetVolumePluginMgr returns volume plugin manager
GetVolumePluginMgr() *volume.VolumePluginMgr
@ -434,7 +449,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
return volumeToMount.GenerateErrorDetailed("MountVolume.WaitForAttach failed", err)
}
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", ""))
glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
deviceMountPath, err :=
volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec)
@ -501,6 +516,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
volumeToMount.Pod.UID,
volumeToMount.VolumeName,
volumeMounter,
nil,
volumeToMount.OuterVolumeSpecName,
volumeToMount.VolumeGidValue)
if markVolMountedErr != nil {
@ -604,19 +620,9 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
// use mounter.PathIsDevice to check if the path is a device,
// if so use mounter.DeviceOpened to check if the device is in use anywhere
// else on the system. Retry if it returns true.
isDevicePath, devicePathErr := mounter.PathIsDevice(deviceToDetach.DevicePath)
var deviceOpened bool
var deviceOpenedErr error
if !isDevicePath && devicePathErr == nil {
// not a device path or path doesn't exist
//TODO: refer to #36092
glog.V(3).Infof("Not checking device path %s", deviceToDetach.DevicePath)
deviceOpened = false
} else {
deviceOpened, deviceOpenedErr = mounter.DeviceOpened(deviceToDetach.DevicePath)
if deviceOpenedErr != nil {
return deviceToDetach.GenerateErrorDetailed("UnmountDevice.DeviceOpened failed", deviceOpenedErr)
}
deviceOpened, deviceOpenedErr := isDeviceOpened(deviceToDetach, mounter)
if deviceOpenedErr != nil {
return deviceOpenedErr
}
// The device is still in use elsewhere. Caller will log and retry.
if deviceOpened {
@ -639,6 +645,337 @@ func (og *operationGenerator) GenerateUnmountDeviceFunc(
}, attachableVolumePlugin.GetPluginName(), nil
}
// GenerateMapVolumeFunc marks volume as mounted based on following steps.
// If plugin is attachable, call WaitForAttach() and then mark the device
// as mounted. On next step, SetUpDevice is called without dependent of
// plugin type, but this method mainly is targeted for none attachable plugin.
// After setup is done, create symbolic links on both global map path and pod
// device map path. Once symbolic links are created, take fd lock by
// loopback for the device to avoid silent volume replacement. This lock
// will be realased once no one uses the device.
// If all steps are completed, the volume is marked as unmounted.
func (og *operationGenerator) GenerateMapVolumeFunc(
waitForAttachTimeout time.Duration,
volumeToMount VolumeToMount,
actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) {
// Get block volume mapper plugin
var blockVolumeMapper volume.BlockVolumeMapper
blockVolumePlugin, err :=
og.volumePluginMgr.FindMapperPluginBySpec(volumeToMount.VolumeSpec)
if err != nil {
return nil, "", volumeToMount.GenerateErrorDetailed("MapVolume.FindMapperPluginBySpec failed", err)
}
if blockVolumePlugin == nil {
return nil, "", volumeToMount.GenerateErrorDetailed("MapVolume.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil)
}
affinityErr := checkNodeAffinity(og, volumeToMount, blockVolumePlugin)
if affinityErr != nil {
return nil, blockVolumePlugin.GetPluginName(), affinityErr
}
blockVolumeMapper, newMapperErr := blockVolumePlugin.NewBlockVolumeMapper(
volumeToMount.VolumeSpec,
volumeToMount.Pod,
volume.VolumeOptions{})
if newMapperErr != nil {
eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.NewBlockVolumeMapper initialization failed", newMapperErr)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error())
return nil, blockVolumePlugin.GetPluginName(), detailedErr
}
// Get attacher, if possible
attachableVolumePlugin, _ :=
og.volumePluginMgr.FindAttachablePluginBySpec(volumeToMount.VolumeSpec)
var volumeAttacher volume.Attacher
if attachableVolumePlugin != nil {
volumeAttacher, _ = attachableVolumePlugin.NewAttacher()
}
return func() error {
var devicePath string
if volumeAttacher != nil {
// Wait for attachable volumes to finish attaching
glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath)))
devicePath, err = volumeAttacher.WaitForAttach(
volumeToMount.VolumeSpec, volumeToMount.DevicePath, volumeToMount.Pod, waitForAttachTimeout)
if err != nil {
// On failure, return error. Caller will log and retry.
return volumeToMount.GenerateErrorDetailed("MapVolume.WaitForAttach failed", err)
}
glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath)))
// Update actual state of world to reflect volume is globally mounted
markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted(
volumeToMount.VolumeName)
if markDeviceMappedErr != nil {
// On failure, return error. Caller will log and retry.
return volumeToMount.GenerateErrorDetailed("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr)
}
}
// A plugin doesn't have attacher also needs to map device to global map path with SetUpDevice()
pluginDevicePath, mapErr := blockVolumeMapper.SetUpDevice()
if mapErr != nil {
// On failure, return error. Caller will log and retry.
eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.SetUp failed", mapErr)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error())
return detailedErr
}
// Update devicePath for none attachable plugin case
if len(devicePath) == 0 {
if len(pluginDevicePath) != 0 {
devicePath = pluginDevicePath
} else {
return volumeToMount.GenerateErrorDetailed("MapVolume failed", fmt.Errorf("Device path of the volume is empty"))
}
}
// Set up global map path under the given plugin directory using symbolic link
globalMapPath, err :=
blockVolumeMapper.GetGlobalMapPath(volumeToMount.VolumeSpec)
if err != nil {
// On failure, return error. Caller will log and retry.
return volumeToMount.GenerateErrorDetailed("MapVolume.GetDeviceMountPath failed", err)
}
mapErr = og.blkUtil.MapDevice(devicePath, globalMapPath, string(volumeToMount.Pod.UID))
if mapErr != nil {
// On failure, return error. Caller will log and retry.
eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.MapDevice failed", mapErr)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error())
return detailedErr
}
// Device mapping for global map path succeeded
simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MapVolume.MapDevice succeeded", fmt.Sprintf("globalMapPath %q", globalMapPath))
verbosity := glog.Level(4)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.SuccessfulMountVolume, simpleMsg)
glog.V(verbosity).Infof(detailedMsg)
// Map device to pod device map path under the given pod directory using symbolic link
volumeMapPath, volName := blockVolumeMapper.GetPodDeviceMapPath()
mapErr = og.blkUtil.MapDevice(devicePath, volumeMapPath, volName)
if mapErr != nil {
// On failure, return error. Caller will log and retry.
eventErr, detailedErr := volumeToMount.GenerateError("MapVolume.MapDevice failed", mapErr)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FailedMapVolume, eventErr.Error())
return detailedErr
}
// Take filedescriptor lock to keep a block device opened. Otherwise, there is a case
// that the block device is silently removed and attached another device with same name.
// Container runtime can't handler this problem. To avoid unexpected condition fd lock
// for the block device is required.
_, err = og.blkUtil.AttachFileDevice(devicePath)
if err != nil {
return volumeToMount.GenerateErrorDetailed("MapVolume.AttachFileDevice failed", err)
}
// Device mapping for pod device map path succeeded
simpleMsg, detailedMsg = volumeToMount.GenerateMsg("MapVolume.MapDevice succeeded", fmt.Sprintf("volumeMapPath %q", volumeMapPath))
verbosity = glog.Level(1)
og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeNormal, kevents.SuccessfulMountVolume, simpleMsg)
glog.V(verbosity).Infof(detailedMsg)
// Update actual state of world
markVolMountedErr := actualStateOfWorld.MarkVolumeAsMounted(
volumeToMount.PodName,
volumeToMount.Pod.UID,
volumeToMount.VolumeName,
nil,
blockVolumeMapper,
volumeToMount.OuterVolumeSpecName,
volumeToMount.VolumeGidValue)
if markVolMountedErr != nil {
// On failure, return error. Caller will log and retry.
return volumeToMount.GenerateErrorDetailed("MapVolume.MarkVolumeAsMounted failed", markVolMountedErr)
}
return nil
}, blockVolumePlugin.GetPluginName(), nil
}
// GenerateUnmapVolumeFunc marks volume as unmonuted based on following steps.
// Remove symbolic links from pod device map path dir and global map path dir.
// Once those cleanups are done, remove pod device map path dir.
// If all steps are completed, the volume is marked as unmounted.
func (og *operationGenerator) GenerateUnmapVolumeFunc(
volumeToUnmount MountedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) {
// Get block volume unmapper plugin
var blockVolumeUnmapper volume.BlockVolumeUnmapper
blockVolumePlugin, err :=
og.volumePluginMgr.FindMapperPluginByName(volumeToUnmount.PluginName)
if err != nil {
return nil, "", volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed", err)
}
if blockVolumePlugin == nil {
return nil, "", volumeToUnmount.GenerateErrorDetailed("UnmapVolume.FindMapperPluginByName failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil)
}
blockVolumeUnmapper, newUnmapperErr := blockVolumePlugin.NewBlockVolumeUnmapper(
volumeToUnmount.InnerVolumeSpecName, volumeToUnmount.PodUID)
if newUnmapperErr != nil {
return nil, blockVolumePlugin.GetPluginName(), volumeToUnmount.GenerateErrorDetailed("UnmapVolume.NewUnmapper failed", newUnmapperErr)
}
return func() error {
// Try to unmap volumeName symlink under pod device map path dir
// pods/{podUid}/volumeDevices/{escapeQualifiedPluginName}/{volumeName}
podDeviceUnmapPath, volName := blockVolumeUnmapper.GetPodDeviceMapPath()
unmapDeviceErr := og.blkUtil.UnmapDevice(podDeviceUnmapPath, volName)
if unmapDeviceErr != nil {
// On failure, return error. Caller will log and retry.
return volumeToUnmount.GenerateErrorDetailed("UnmapVolume.UnmapDevice on pod device map path failed", unmapDeviceErr)
}
// Try to unmap podUID symlink under global map path dir
// plugins/kubernetes.io/{PluginName}/volumeDevices/{volumePluginDependentPath}/{podUID}
globalUnmapPath, err :=
blockVolumeUnmapper.GetGlobalMapPath(volumeToUnmount.VolumeSpec)
if err != nil {
// On failure, return error. Caller will log and retry.
return volumeToUnmount.GenerateErrorDetailed("UnmapVolume.GetGlobalUnmapPath failed", err)
}
unmapDeviceErr = og.blkUtil.UnmapDevice(globalUnmapPath, string(volumeToUnmount.PodUID))
if unmapDeviceErr != nil {
// On failure, return error. Caller will log and retry.
return volumeToUnmount.GenerateErrorDetailed("UnmapVolume.UnmapDevice on global map path failed", unmapDeviceErr)
}
glog.Infof(
"UnmapVolume succeeded for volume %q (OuterVolumeSpecName: %q) pod %q (UID: %q). InnerVolumeSpecName %q. PluginName %q, VolumeGidValue %q",
volumeToUnmount.VolumeName,
volumeToUnmount.OuterVolumeSpecName,
volumeToUnmount.PodName,
volumeToUnmount.PodUID,
volumeToUnmount.InnerVolumeSpecName,
volumeToUnmount.PluginName,
volumeToUnmount.VolumeGidValue)
// Update actual state of world
markVolUnmountedErr := actualStateOfWorld.MarkVolumeAsUnmounted(
volumeToUnmount.PodName, volumeToUnmount.VolumeName)
if markVolUnmountedErr != nil {
// On failure, just log and exit
glog.Errorf(volumeToUnmount.GenerateErrorDetailed("UnmapVolume.MarkVolumeAsUnmounted failed", markVolUnmountedErr).Error())
}
return nil
}, blockVolumePlugin.GetPluginName(), nil
}
// GenerateUnmapDeviceFunc marks device as unmounted based on following steps.
// Check under globalMapPath dir if there isn't pod's symbolic links in it.
// If symbolick link isn't there, the device isn't referenced from Pods.
// Call plugin TearDownDevice to clean-up device connection, stored data under
// globalMapPath, these operations depend on plugin implementation.
// Once TearDownDevice is completed, remove globalMapPath dir.
// After globalMapPath is removed, fd lock by loopback for the device can
// be released safely because no one can consume the device at this point.
// At last, device open status will be checked just in case.
// If all steps are completed, the device is marked as unmounted.
func (og *operationGenerator) GenerateUnmapDeviceFunc(
deviceToDetach AttachedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater,
mounter mount.Interface) (func() error, string, error) {
// Get block volume mapper plugin
var blockVolumeMapper volume.BlockVolumeMapper
blockVolumePlugin, err :=
og.volumePluginMgr.FindMapperPluginBySpec(deviceToDetach.VolumeSpec)
if err != nil {
return nil, "", deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginBySpec failed", err)
}
if blockVolumePlugin == nil {
return nil, "", deviceToDetach.GenerateErrorDetailed("UnmapDevice.FindMapperPluginBySpec failed to find BlockVolumeMapper plugin. Volume plugin is nil.", nil)
}
blockVolumeMapper, newMapperErr := blockVolumePlugin.NewBlockVolumeMapper(
deviceToDetach.VolumeSpec,
nil, /* Pod */
volume.VolumeOptions{})
if newMapperErr != nil {
return nil, "", deviceToDetach.GenerateErrorDetailed("UnmapDevice.NewBlockVolumeMapper initialization failed", newMapperErr)
}
blockVolumeUnmapper, newUnmapperErr := blockVolumePlugin.NewBlockVolumeUnmapper(
string(deviceToDetach.VolumeName),
"" /* podUID */)
if newUnmapperErr != nil {
return nil, blockVolumePlugin.GetPluginName(), deviceToDetach.GenerateErrorDetailed("UnmapDevice.NewUnmapper failed", newUnmapperErr)
}
return func() error {
// Search under globalMapPath dir if all symbolic links from pods have been removed already.
// If symbolick links are there, pods may still refer the volume.
globalMapPath, err :=
blockVolumeMapper.GetGlobalMapPath(deviceToDetach.VolumeSpec)
if err != nil {
// On failure, return error. Caller will log and retry.
return deviceToDetach.GenerateErrorDetailed("UnmapDevice.GetGlobalMapPath failed", err)
}
refs, err := og.blkUtil.GetDeviceSymlinkRefs(deviceToDetach.DevicePath, globalMapPath)
if err != nil {
return deviceToDetach.GenerateErrorDetailed("UnmapDevice.GetDeviceSymlinkRefs check failed", err)
}
if len(refs) > 0 {
err = fmt.Errorf("The device %q is still referenced from other Pods %v", globalMapPath, refs)
return deviceToDetach.GenerateErrorDetailed("UnmapDevice failed", err)
}
// Execute tear down device
unmapErr := blockVolumeUnmapper.TearDownDevice(globalMapPath, deviceToDetach.DevicePath)
if unmapErr != nil {
// On failure, return error. Caller will log and retry.
return deviceToDetach.GenerateErrorDetailed("UnmapDevice.TearDownDevice failed", unmapErr)
}
// Plugin finished TearDownDevice(). Now globalMapPath dir and plugin's stored data
// on the dir are unnecessary, clean up it.
removeMapPathErr := og.blkUtil.RemoveMapPath(globalMapPath)
if removeMapPathErr != nil {
// On failure, return error. Caller will log and retry.
return deviceToDetach.GenerateErrorDetailed("UnmapDevice failed", removeMapPathErr)
}
// The block volume is not referenced from Pods. Release file descriptor lock.
glog.V(5).Infof("UnmapDevice: deviceToDetach.DevicePath: %v", deviceToDetach.DevicePath)
loopPath, err := og.blkUtil.GetLoopDevice(deviceToDetach.DevicePath)
if err != nil {
glog.Warningf(deviceToDetach.GenerateMsgDetailed("UnmapDevice: Couldn't find loopback device which takes file descriptor lock", fmt.Sprintf("device path: %q", deviceToDetach.DevicePath)))
} else {
err = og.blkUtil.RemoveLoopDevice(loopPath)
if err != nil {
return deviceToDetach.GenerateErrorDetailed("UnmapDevice.AttachFileDevice failed", err)
}
}
// Before logging that UnmapDevice succeeded and moving on,
// use mounter.PathIsDevice to check if the path is a device,
// if so use mounter.DeviceOpened to check if the device is in use anywhere
// else on the system. Retry if it returns true.
deviceOpened, deviceOpenedErr := isDeviceOpened(deviceToDetach, mounter)
if deviceOpenedErr != nil {
return deviceOpenedErr
}
// The device is still in use elsewhere. Caller will log and retry.
if deviceOpened {
return deviceToDetach.GenerateErrorDetailed(
"UnmapDevice failed",
fmt.Errorf("the device is in use when it was no longer expected to be in use"))
}
glog.Infof(deviceToDetach.GenerateMsgDetailed("UnmapDevice succeeded", ""))
// Update actual state of world
markDeviceUnmountedErr := actualStateOfWorld.MarkDeviceAsUnmounted(
deviceToDetach.VolumeName)
if markDeviceUnmountedErr != nil {
// On failure, return error. Caller will log and retry.
return deviceToDetach.GenerateErrorDetailed("MarkDeviceAsUnmounted failed", markDeviceUnmountedErr)
}
return nil
}, blockVolumePlugin.GetPluginName(), nil
}
func (og *operationGenerator) GenerateVerifyControllerAttachedVolumeFunc(
volumeToMount VolumeToMount,
nodeName types.NodeName,
@ -833,3 +1170,23 @@ func checkNodeAffinity(og *operationGenerator, volumeToMount VolumeToMount, plug
}
return nil
}
// isDeviceOpened checks the device status if the device is in use anywhere else on the system
func isDeviceOpened(deviceToDetach AttachedVolume, mounter mount.Interface) (bool, error) {
isDevicePath, devicePathErr := mounter.PathIsDevice(deviceToDetach.DevicePath)
var deviceOpened bool
var deviceOpenedErr error
if !isDevicePath && devicePathErr == nil ||
(devicePathErr != nil && strings.Contains(devicePathErr.Error(), "does not exist")) {
// not a device path or path doesn't exist
//TODO: refer to #36092
glog.V(3).Infof("The path isn't device path or doesn't exist. Skip checking device path: %s", deviceToDetach.DevicePath)
deviceOpened = false
} else {
deviceOpened, deviceOpenedErr = mounter.DeviceOpened(deviceToDetach.DevicePath)
if deviceOpenedErr != nil {
return false, deviceToDetach.GenerateErrorDetailed("DeviceOpened failed", deviceOpenedErr)
}
}
return deviceOpened, nil
}

210
pkg/volume/util/util.go
View File

@ -21,7 +21,7 @@ import (
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/golang/glog"
@ -30,6 +30,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api/legacyscheme"
@ -38,7 +39,14 @@ import (
"k8s.io/kubernetes/pkg/util/mount"
)
const readyFileName = "ready"
const (
readyFileName = "ready"
losetupPath = "losetup"
ErrDeviceNotFound = "device not found"
ErrDeviceNotSupported = "device not supported"
ErrNotAvailable = "not available"
)
// IsReady checks for the existence of a regular file
// called 'ready' in the given directory and returns
@ -270,3 +278,201 @@ func stringToSet(str, delimiter string) (sets.String, error) {
}
return zonesSet, nil
}
// BlockVolumePathHandler defines a set of operations for handling block volume-related operations
type BlockVolumePathHandler interface {
// MapDevice creates a symbolic link to block device under specified map path
MapDevice(devicePath string, mapPath string, linkName string) error
// UnmapDevice removes a symbolic link to block device under specified map path
UnmapDevice(mapPath string, linkName string) error
// RemovePath removes a file or directory on specified map path
RemoveMapPath(mapPath string) error
// IsSymlinkExist retruns true if specified symbolic link exists
IsSymlinkExist(mapPath string) (bool, error)
// GetDeviceSymlinkRefs searches symbolic links under global map path
GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error)
// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
// corresponding to map path symlink, and then return global map path with pod uuid.
FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error)
// AttachFileDevice takes a path to a regular file and makes it available as an
// attached block device.
AttachFileDevice(path string) (string, error)
// GetLoopDevice returns the full path to the loop device associated with the given path.
GetLoopDevice(path string) (string, error)
// RemoveLoopDevice removes specified loopback device
RemoveLoopDevice(device string) error
}
// NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler.
func NewBlockVolumePathHandler() BlockVolumePathHandler {
var volumePathHandler VolumePathHandler
return volumePathHandler
}
// VolumePathHandler is path related operation handlers for block volume
type VolumePathHandler struct {
}
// MapDevice creates a symbolic link to block device under specified map path
func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string) error {
// Example of global map path:
// globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid}
// linkName: {podUid}
//
// Example of pod device map path:
// podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
// linkName: {volumeName}
if len(devicePath) == 0 {
return fmt.Errorf("Failed to map device to map path. devicePath is empty")
}
if len(mapPath) == 0 {
return fmt.Errorf("Failed to map device to map path. mapPath is empty")
}
if !filepath.IsAbs(mapPath) {
return fmt.Errorf("The map path should be absolute: map path: %s", mapPath)
}
glog.V(5).Infof("MapDevice: devicePath %s", devicePath)
glog.V(5).Infof("MapDevice: mapPath %s", mapPath)
glog.V(5).Infof("MapDevice: linkName %s", linkName)
// Check and create mapPath
_, err := os.Stat(mapPath)
if err != nil && !os.IsNotExist(err) {
glog.Errorf("cannot validate map path: %s", mapPath)
return err
}
if err = os.MkdirAll(mapPath, 0750); err != nil {
return fmt.Errorf("Failed to mkdir %s, error %v", mapPath, err)
}
// Remove old symbolic link(or file) then create new one.
// This should be done because current symbolic link is
// stale accross node reboot.
linkPath := path.Join(mapPath, string(linkName))
if err = os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
return err
}
err = os.Symlink(devicePath, linkPath)
return err
}
// UnmapDevice removes a symbolic link associated to block device under specified map path
func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string) error {
if len(mapPath) == 0 {
return fmt.Errorf("Failed to unmap device from map path. mapPath is empty")
}
glog.V(5).Infof("UnmapDevice: mapPath %s", mapPath)
glog.V(5).Infof("UnmapDevice: linkName %s", linkName)
// Check symbolic link exists
linkPath := path.Join(mapPath, string(linkName))
if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil {
return checkErr
} else if !islinkExist {
glog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath)
return nil
}
err := os.Remove(linkPath)
return err
}
// RemoveMapPath removes a file or directory on specified map path
func (v VolumePathHandler) RemoveMapPath(mapPath string) error {
if len(mapPath) == 0 {
return fmt.Errorf("Failed to remove map path. mapPath is empty")
}
glog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath)
err := os.RemoveAll(mapPath)
if err != nil && !os.IsNotExist(err) {
return err
}
return nil
}
// IsSymlinkExist returns true if specified file exists and the type is symbolik link.
// If file doesn't exist, or file exists but not symbolick link, return false with no error.
// On other cases, return false with error from Lstat().
func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) {
fi, err := os.Lstat(mapPath)
if err == nil {
// If file exits and it's symbolick link, return true and no error
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
return true, nil
}
// If file exits but it's not symbolick link, return fale and no error
return false, nil
}
// If file doesn't exist, return false and no error
if os.IsNotExist(err) {
return false, nil
}
// Return error from Lstat()
return false, err
}
// GetDeviceSymlinkRefs searches symbolic links under global map path
func (v VolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) {
var refs []string
files, err := ioutil.ReadDir(mapPath)
if err != nil {
return nil, fmt.Errorf("Directory cannot read %v", err)
}
for _, file := range files {
if file.Mode()&os.ModeSymlink != os.ModeSymlink {
continue
}
filename := file.Name()
filepath, err := os.Readlink(path.Join(mapPath, filename))
if err != nil {
return nil, fmt.Errorf("Symbolic link cannot be retrieved %v", err)
}
glog.V(5).Infof("GetDeviceSymlinkRefs: filepath: %v, devPath: %v", filepath, devPath)
if filepath == devPath {
refs = append(refs, path.Join(mapPath, filename))
}
}
glog.V(5).Infof("GetDeviceSymlinkRefs: refs %v", refs)
return refs, nil
}
// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
// corresponding to map path symlink, and then return global map path with pod uuid.
// ex. mapPath symlink: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} -> /dev/sdX
// globalMapPath/{pod uuid}: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} -> /dev/sdX
func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
var globalMapPathUUID string
// Find symbolic link named pod uuid under plugin dir
err := filepath.Walk(pluginDir, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if (fi.Mode()&os.ModeSymlink == os.ModeSymlink) && (fi.Name() == string(podUID)) {
glog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath)
if res, err := compareSymlinks(path, mapPath); err == nil && res {
globalMapPathUUID = path
}
}
return nil
})
if err != nil {
return "", err
}
glog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID)
// Return path contains global map path + {pod uuid}
return globalMapPathUUID, nil
}
func compareSymlinks(global, pod string) (bool, error) {
devGlobal, err := os.Readlink(global)
if err != nil {
return false, err
}
devPod, err := os.Readlink(pod)
if err != nil {
return false, err
}
glog.V(5).Infof("CompareSymlinks: devGloBal %s, devPod %s", devGlobal, devPod)
if devGlobal == devPod {
return true, nil
}
return false, nil
}

106
pkg/volume/util/util_linux.go Normal file
View File

@ -0,0 +1,106 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"errors"
"fmt"
"os"
"os/exec"
"strings"
"github.com/golang/glog"
)
// AttachFileDevice takes a path to a regular file and makes it available as an
// attached block device.
func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
blockDevicePath, err := v.GetLoopDevice(path)
if err != nil && err.Error() != ErrDeviceNotFound {
return "", err
}
// If no existing loop device for the path, create one
if blockDevicePath == "" {
glog.V(4).Infof("Creating device for path: %s", path)
blockDevicePath, err = makeLoopDevice(path)
if err != nil {
return "", err
}
}
return blockDevicePath, nil
}
// GetLoopDevice returns the full path to the loop device associated with the given path.
func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
_, err := os.Stat(path)
if os.IsNotExist(err) {
return "", errors.New(ErrNotAvailable)
}
if err != nil {
return "", fmt.Errorf("not attachable: %v", err)
}
args := []string{"-j", path}
cmd := exec.Command(losetupPath, args...)
out, err := cmd.CombinedOutput()
if err != nil {
glog.V(2).Infof("Failed device discover command for path %s: %v", path, err)
return "", err
}
return parseLosetupOutputForDevice(out)
}
func makeLoopDevice(path string) (string, error) {
args := []string{"-f", "--show", path}
cmd := exec.Command(losetupPath, args...)
out, err := cmd.CombinedOutput()
if err != nil {
glog.V(2).Infof("Failed device create command for path %s: %v", path, err)
return "", err
}
return parseLosetupOutputForDevice(out)
}
// RemoveLoopDevice removes specified loopback device
func (v VolumePathHandler) RemoveLoopDevice(device string) error {
args := []string{"-d", device}
cmd := exec.Command(losetupPath, args...)
out, err := cmd.CombinedOutput()
if err != nil {
if !strings.Contains(string(out), "No such device or address") {
return err
}
}
return nil
}
func parseLosetupOutputForDevice(output []byte) (string, error) {
if len(output) == 0 {
return "", errors.New(ErrDeviceNotFound)
}
// losetup returns device in the format:
// /dev/loop1: [0073]:148662 (/dev/sda)
device := strings.TrimSpace(strings.SplitN(string(output), ":", 2)[0])
if len(device) == 0 {
return "", errors.New(ErrDeviceNotFound)
}
return device, nil
}

39
pkg/volume/util/util_unsupported.go Normal file
View File

@ -0,0 +1,39 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
)
// AttachFileDevice takes a path to a regular file and makes it available as an
// attached block device.
func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
return "", fmt.Errorf("AttachFileDevice not supported for this build.")
}
// GetLoopDevice returns the full path to the loop device associated with the given path.
func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
return "", fmt.Errorf("GetLoopDevice not supported for this build.")
}
// RemoveLoopDevice removes specified loopback device
func (v VolumePathHandler) RemoveLoopDevice(device string) error {
return fmt.Errorf("RemoveLoopDevice not supported for this build.")
}

21
pkg/volume/util/volumehelper/volumehelper.go
View File

@ -136,3 +136,24 @@ func NewSafeFormatAndMountFromHost(pluginName string, host volume.VolumeHost) *m
exec := host.GetExec(pluginName)
return &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}
}
// GetVolumeMode retrieves VolumeMode from pv.
// If the volume doesn't have PersistentVolume, it's an inline volume,
// should return volumeMode as filesystem to keep existing behavior.
func GetVolumeMode(volumeSpec *volume.Spec) (v1.PersistentVolumeMode, error) {
if volumeSpec == nil || volumeSpec.PersistentVolume == nil {
return v1.PersistentVolumeFilesystem, nil
}
if volumeSpec.PersistentVolume.Spec.VolumeMode != nil {
return *volumeSpec.PersistentVolume.Spec.VolumeMode, nil
}
return "", fmt.Errorf("cannot get volumeMode for volume: %v", volumeSpec.Name())
}
// GetPersistentVolumeClaimVolumeMode retrieves VolumeMode from pvc.
func GetPersistentVolumeClaimVolumeMode(claim *v1.PersistentVolumeClaim) (v1.PersistentVolumeMode, error) {
if claim.Spec.VolumeMode != nil {
return *claim.Spec.VolumeMode, nil
}
return "", fmt.Errorf("cannot get volumeMode from pvc: %v", claim.Name)
}

41
pkg/volume/volume.go
View File

@ -37,6 +37,19 @@ type Volume interface {
MetricsProvider
}
// BlockVolume interface provides methods to generate global map path
// and pod device map path.
type BlockVolume interface {
// GetGlobalMapPath returns a global map path which contains
// symbolic links associated to a block device.
// ex. plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid}
GetGlobalMapPath(spec *Spec) (string, error)
// GetPodDeviceMapPath returns a pod device map path
// and name of a symbolic link associated to a block device.
// ex. pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
GetPodDeviceMapPath() (string, string)
}
// MetricsProvider exposes metrics (e.g. used,available space) related to a
// Volume.
type MetricsProvider interface {
@ -132,6 +145,34 @@ type Unmounter interface {
TearDownAt(dir string) error
}
// BlockVolumeMapper interface provides methods to set up/map the volume.
type BlockVolumeMapper interface {
BlockVolume
// SetUpDevice prepares the volume to a self-determined directory path,
// which may or may not exist yet and returns combination of physical
// device path of a block volume and error.
// If the plugin is non-attachable, it should prepare the device
// in /dev/ (or where appropriate) and return unique device path.
// Unique device path across kubelet node reboot is required to avoid
// unexpected block volume destruction.
// If the plugin is attachable, it should not do anything here,
// just return empty string for device path.
// Instead, attachable plugin have to return unique device path
// at attacher.Attach() and attacher.WaitForAttach().
// This may be called more than once, so implementations must be idempotent.
SetUpDevice() (string, error)
}
// BlockVolumeUnmapper interface provides methods to cleanup/unmap the volumes.
type BlockVolumeUnmapper interface {
BlockVolume
// TearDownDevice removes traces of the SetUpDevice procedure under
// a self-determined directory.
// If the plugin is non-attachable, this method detaches the volume
// from a node.
TearDownDevice(mapPath string, devicePath string) error
}
// Provisioner is an interface that creates templates for PersistentVolumes
// and can create the volume as a new resource in the infrastructure provider.
type Provisioner interface {