Feature: Implements Raw Block Volume Support for in-tree vSphere plugin

- add raw block volume support for vsphere volume provisioner
- set VolumeMode and any FSType in vsphere volume dynamic provision if feature
BlockVolume enabled
- implement BlockVolumePlugin, BlockVolumeMapper, BlockVolumeUnmapper interfaces.
- add tests for vsphere block volume

pkg/volume/vsphere_volume/BUILD

@@ -11,21 +11,25 @@ go_library(
     srcs = [
         "attacher.go",
         "vsphere_volume.go",
+        "vsphere_volume_block.go",
         "vsphere_volume_util.go",
     ],
     importpath = "k8s.io/kubernetes/pkg/volume/vsphere_volume",
     deps = [
         "//pkg/cloudprovider/providers/vsphere:go_default_library",
         "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/util/keymutex:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//pkg/util/strings:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/util:go_default_library",
+        "//pkg/volume/util/volumepathhandler:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
     ],
@@ -35,6 +39,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "attacher_test.go",
+        "vsphere_volume_block_test.go",
         "vsphere_volume_test.go",
     ],
     embed = [":go_default_library"],
@@ -46,6 +51,7 @@ go_test(
         "//pkg/volume:go_default_library",
         "//pkg/volume/testing:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/client-go/util/testing:go_default_library",
         "//staging/src/k8s.io/cloud-provider:go_default_library",

pkg/volume/vsphere_volume/vsphere_volume.go

@@ -27,6 +27,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/mount"
 	utilstrings "k8s.io/kubernetes/pkg/util/strings"
 	"k8s.io/kubernetes/pkg/volume"
@@ -356,9 +358,6 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol
 	if !util.AccessModesContainedInAll(v.plugin.GetAccessModes(), v.options.PVC.Spec.AccessModes) {
 		return nil, fmt.Errorf("invalid AccessModes %v: only AccessModes %v are supported", v.options.PVC.Spec.AccessModes, v.plugin.GetAccessModes())
 	}
-	if util.CheckPersistentVolumeClaimModeBlock(v.options.PVC) {
-		return nil, fmt.Errorf("%s does not support block volume provisioning", v.plugin.GetPluginName())
-	}
 
 	volSpec, err := v.manager.CreateVolume(v)
 	if err != nil {
@@ -369,6 +368,15 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol
 		volSpec.Fstype = "ext4"
 	}
 
+	var volumeMode *v1.PersistentVolumeMode
+	if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
+		volumeMode = v.options.PVC.Spec.VolumeMode
+		if volumeMode != nil && *volumeMode == v1.PersistentVolumeBlock {
+			glog.V(5).Infof("vSphere block volume should not have any FSType")
+			volSpec.Fstype = ""
+		}
+	}
+
 	pv := &v1.PersistentVolume{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   v.options.PVName,
@@ -383,6 +391,7 @@ func (v *vsphereVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopol
 			Capacity: v1.ResourceList{
 				v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dKi", volSpec.Size)),
 			},
+			VolumeMode: volumeMode,
 			PersistentVolumeSource: v1.PersistentVolumeSource{
 				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
 					VolumePath:        volSpec.Path,

pkg/volume/vsphere_volume/vsphere_volume_block.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere_volume
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/util/mount"
+	kstrings "k8s.io/kubernetes/pkg/util/strings"
+	"k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
+)
+
+var _ volume.BlockVolumePlugin = &vsphereVolumePlugin{}
+
+func (plugin *vsphereVolumePlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
+
+	pluginDir := plugin.host.GetPluginDir(plugin.GetPluginName())
+	blkUtil := volumepathhandler.NewBlockVolumePathHandler()
+	globalMapPathUUID, err := blkUtil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
+	if err != nil {
+		glog.Errorf("Failed to find GlobalMapPathUUID from Pod: %s with error: %+v", podUID, err)
+		return nil, err
+	}
+	glog.V(5).Infof("globalMapPathUUID: %v", globalMapPathUUID)
+	globalMapPath := filepath.Dir(globalMapPathUUID)
+	if len(globalMapPath) <= 1 {
+		return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
+	}
+	return getVolumeSpecFromGlobalMapPath(globalMapPath)
+}
+
+func getVolumeSpecFromGlobalMapPath(globalMapPath string) (*volume.Spec, error) {
+	// Construct volume spec from globalMapPath
+	// globalMapPath example:
+	//   plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
+	//   plugins/kubernetes.io/vsphere-volume/volumeDevices/[datastore1]\\040volumes/myDisk
+	volPath := filepath.Base(globalMapPath)
+	volPath = strings.Replace(volPath, "\\040", "", -1)
+	if len(volPath) <= 1 {
+		return nil, fmt.Errorf("failed to get volume path from global path=%s", globalMapPath)
+	}
+	block := v1.PersistentVolumeBlock
+	vsphereVolume := &v1.PersistentVolume{
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+					VolumePath: volPath,
+				},
+			},
+			VolumeMode: &block,
+		},
+	}
+	return volume.NewSpecFromPersistentVolume(vsphereVolume, true), nil
+}
+
+func (plugin *vsphereVolumePlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
+	// If this called via GenerateUnmapDeviceFunc(), pod is nil.
+	// Pass empty string as dummy uid since uid isn't used in the case.
+	var uid types.UID
+	if pod != nil {
+		uid = pod.UID
+	}
+	return plugin.newBlockVolumeMapperInternal(spec, uid, &VsphereDiskUtil{}, plugin.host.GetMounter(plugin.GetPluginName()))
+}
+
+func (plugin *vsphereVolumePlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager vdManager, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
+	volumeSource, _, err := getVolumeSource(spec)
+	if err != nil {
+		glog.Errorf("Failed to get Volume source from volume Spec: %+v with error: %+v", *spec, err)
+		return nil, err
+	}
+	volPath := volumeSource.VolumePath
+	return &vsphereBlockVolumeMapper{
+		vsphereVolume: &vsphereVolume{
+			volName:         spec.Name(),
+			podUID:          podUID,
+			volPath:         volPath,
+			manager:         manager,
+			mounter:         mounter,
+			plugin:          plugin,
+			MetricsProvider: volume.NewMetricsStatFS(getPath(podUID, spec.Name(), plugin.host)),
+		},
+	}, nil
+
+}
+
+func (plugin *vsphereVolumePlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
+	return plugin.newUnmapperInternal(volName, podUID, &VsphereDiskUtil{})
+}
+
+func (plugin *vsphereVolumePlugin) newUnmapperInternal(volName string, podUID types.UID, manager vdManager) (volume.BlockVolumeUnmapper, error) {
+	return &vsphereBlockVolumeUnmapper{
+		vsphereVolume: &vsphereVolume{
+			volName: volName,
+			podUID:  podUID,
+			volPath: volName,
+			manager: manager,
+			plugin:  plugin,
+		},
+	}, nil
+}
+
+var _ volume.BlockVolumeMapper = &vsphereBlockVolumeMapper{}
+
+type vsphereBlockVolumeMapper struct {
+	*vsphereVolume
+}
+
+func (v vsphereBlockVolumeMapper) SetUpDevice() (string, error) {
+	return "", nil
+}
+
+func (v vsphereBlockVolumeMapper) MapDevice(devicePath, globalMapPath, volumeMapPath, volumeMapName string, podUID types.UID) error {
+	return util.MapBlockVolume(devicePath, globalMapPath, volumeMapPath, volumeMapName, podUID)
+}
+
+var _ volume.BlockVolumeUnmapper = &vsphereBlockVolumeUnmapper{}
+
+type vsphereBlockVolumeUnmapper struct {
+	*vsphereVolume
+}
+
+func (v *vsphereBlockVolumeUnmapper) TearDownDevice(mapPath, devicePath string) error {
+	return nil
+}
+
+// GetGlobalMapPath returns global map path and error
+// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumePath
+func (v *vsphereVolume) GetGlobalMapPath(spec *volume.Spec) (string, error) {
+	volumeSource, _, err := getVolumeSource(spec)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(v.plugin.host.GetVolumeDevicePluginDir(vsphereVolumePluginName), string(volumeSource.VolumePath)), nil
+}
+
+func (v *vsphereVolume) GetPodDeviceMapPath() (string, string) {
+	return v.plugin.host.GetPodVolumeDeviceDir(v.podUID, kstrings.EscapeQualifiedNameForDisk(vsphereVolumePluginName)), v.volName
+}

pkg/volume/vsphere_volume/vsphere_volume_block_test.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vsphere_volume
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	utiltesting "k8s.io/client-go/util/testing"
+	"k8s.io/kubernetes/pkg/volume"
+	volumetest "k8s.io/kubernetes/pkg/volume/testing"
+)
+
+var (
+	testVolumePath = "volPath1"
+	testGlobalPath = "plugins/kubernetes.io/vsphere-volume/volumeDevices/volPath1"
+	testPodPath    = "pods/poduid/volumeDevices/kubernetes.io~vsphere-volume"
+)
+
+func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) {
+	// make our test path for fake GlobalMapPath
+	// /tmp symbolized our pluginDir
+	// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/vsphere-volume/volumeDevices/
+	tmpVDir, err := utiltesting.MkTmpdir("vsphereBlockVolume")
+	if err != nil {
+		t.Fatalf("cant' make a temp dir: %s", err)
+	}
+	// deferred clean up
+	defer os.RemoveAll(tmpVDir)
+
+	expectedGlobalPath := path.Join(tmpVDir, testGlobalPath)
+
+	// Bad Path
+	badspec, err := getVolumeSpecFromGlobalMapPath("")
+	if badspec != nil || err == nil {
+		t.Errorf("Expected not to get spec from GlobalMapPath but did")
+	}
+
+	// Good Path
+	spec, err := getVolumeSpecFromGlobalMapPath(expectedGlobalPath)
+	if spec == nil || err != nil {
+		t.Fatalf("Failed to get spec from GlobalMapPath: %s", err)
+	}
+	if spec.PersistentVolume.Spec.VsphereVolume.VolumePath != testVolumePath {
+		t.Fatalf("Invalid volumePath from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.VsphereVolume.VolumePath)
+	}
+	block := v1.PersistentVolumeBlock
+	specMode := spec.PersistentVolume.Spec.VolumeMode
+	if &specMode == nil {
+		t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", &specMode, block)
+	}
+	if *specMode != block {
+		t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block)
+	}
+}
+
+func TestGetPodAndPluginMapPaths(t *testing.T) {
+	tmpVDir, err := utiltesting.MkTmpdir("vsphereBlockVolume")
+	if err != nil {
+		t.Fatalf("cant' make a temp dir: %s", err)
+	}
+	// deferred clean up
+	defer os.RemoveAll(tmpVDir)
+
+	expectedGlobalPath := path.Join(tmpVDir, testGlobalPath)
+	expectedPodPath := path.Join(tmpVDir, testPodPath)
+
+	spec := getTestVolume(true) // block volume
+	pluginMgr := volume.VolumePluginMgr{}
+	pluginMgr.InitPlugins(ProbeVolumePlugins(), nil, volumetest.NewFakeVolumeHost(tmpVDir, nil, nil))
+	plugin, err := pluginMgr.FindMapperPluginByName(vsphereVolumePluginName)
+	if err != nil {
+		os.RemoveAll(tmpVDir)
+		t.Fatalf("Can't find the plugin by name: %q", vsphereVolumePluginName)
+	}
+	if plugin.GetPluginName() != vsphereVolumePluginName {
+		t.Fatalf("Wrong name: %s", plugin.GetPluginName())
+	}
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID: types.UID("poduid"),
+		},
+	}
+	mapper, err := plugin.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a new Mounter: %v", err)
+	}
+	if mapper == nil {
+		t.Fatalf("Got a nil Mounter")
+	}
+
+	// GetGlobalMapPath
+	globalMapPath, err := mapper.GetGlobalMapPath(spec)
+	if err != nil || len(globalMapPath) == 0 {
+		t.Fatalf("Invalid GlobalMapPath from spec: %s", spec.PersistentVolume.Spec.VsphereVolume.VolumePath)
+	}
+	if globalMapPath != expectedGlobalPath {
+		t.Errorf("Failed to get GlobalMapPath: %s %s", globalMapPath, expectedGlobalPath)
+	}
+
+	// GetPodDeviceMapPath
+	devicePath, volumeName := mapper.GetPodDeviceMapPath()
+	if devicePath != expectedPodPath {
+		t.Errorf("Got unexpected pod path: %s, expected %s", devicePath, expectedPodPath)
+	}
+	if volumeName != testVolumePath {
+		t.Errorf("Got unexpected volNamne: %s, expected %s", volumeName, testVolumePath)
+	}
+}
+
+func getTestVolume(isBlock bool) *volume.Spec {
+	pv := &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: testVolumePath,
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+					VolumePath: testVolumePath,
+				},
+			},
+		},
+	}
+	if isBlock {
+		blockMode := v1.PersistentVolumeBlock
+		pv.Spec.VolumeMode = &blockMode
+	}
+	return volume.NewSpecFromPersistentVolume(pv, true)
+}