Merge pull request #63841 from andyzhangx/azuredisk-block-device

Automatic merge from submit-queue (batch tested with PRs 64102, 63303, 64150, 63841). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

add block device support for azure disk

**What this PR does / why we need it**:
add block device support for azure disk

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #61821

**Special notes for your reviewer**:
detailed steps about how it work:
https://github.com/andyzhangx/demo/tree/master/linux/block/azuredisk

**Release note**:

```
add block device support for azure disk
```

@karataliu @feiskyer @khenidak
pull/8/head
Kubernetes Submit Queue 2018-05-23 04:53:25 -07:00 committed by GitHub
commit eacf6f05b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 327 additions and 19 deletions

View File

@ -12,6 +12,7 @@ go_library(
"attacher.go",
"azure_common.go",
"azure_dd.go",
"azure_dd_block.go",
"azure_mounter.go",
"azure_provision.go",
] + select({
@ -60,6 +61,7 @@ go_library(
"//pkg/util/strings:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/util:go_default_library",
"//pkg/volume/util/volumepathhandler:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2017-12-01/compute:go_default_library",
"//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2017-10-01/storage:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
@ -89,6 +91,7 @@ go_test(
name = "go_default_test",
srcs = [
"azure_common_test.go",
"azure_dd_block_test.go",
"azure_dd_test.go",
],
embed = [":go_default_library"],
@ -97,6 +100,8 @@ go_test(
"//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/util/testing:go_default_library",
],
)

View File

@ -57,7 +57,7 @@ var getLunMutex = keymutex.NewKeyMutex()
// Attach attaches a volume.Spec to an Azure VM referenced by NodeName, returning the disk's LUN
func (a *azureDiskAttacher) Attach(spec *volume.Spec, nodeName types.NodeName) (string, error) {
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
glog.Warningf("failed to get azure disk spec (%v)", err)
return "", err
@ -114,7 +114,7 @@ func (a *azureDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty
volumeSpecMap := make(map[string]*volume.Spec)
volumeIDList := []string{}
for _, spec := range specs {
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
glog.Errorf("azureDisk - Error getting volume (%q) source : %v", spec.Name(), err)
continue
@ -151,7 +151,7 @@ func (a *azureDiskAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName ty
func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, _ *v1.Pod, timeout time.Duration) (string, error) {
var err error
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return "", err
}
@ -203,7 +203,7 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string,
// this is generalized for both managed and blob disks
// we also prefix the hash with m/b based on disk kind
func (a *azureDiskAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) {
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return "", err
}
@ -250,7 +250,7 @@ func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath str
}
}
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return err
}

View File

@ -46,6 +46,7 @@ type dataDisk struct {
volumeName string
diskName string
podUID types.UID
plugin *azureDataDiskPlugin
}
var (
@ -82,7 +83,7 @@ func makeGlobalPDPath(host volume.VolumeHost, diskUri string, isManaged bool) (s
return pdPath, nil
}
func makeDataDisk(volumeName string, podUID types.UID, diskName string, host volume.VolumeHost) *dataDisk {
func makeDataDisk(volumeName string, podUID types.UID, diskName string, host volume.VolumeHost, plugin *azureDataDiskPlugin) *dataDisk {
var metricProvider volume.MetricsProvider
if podUID != "" {
metricProvider = volume.NewMetricsStatFS(getPath(podUID, volumeName, host))
@ -93,19 +94,20 @@ func makeDataDisk(volumeName string, podUID types.UID, diskName string, host vol
volumeName: volumeName,
diskName: diskName,
podUID: podUID,
plugin: plugin,
}
}
func getVolumeSource(spec *volume.Spec) (*v1.AzureDiskVolumeSource, error) {
func getVolumeSource(spec *volume.Spec) (volumeSource *v1.AzureDiskVolumeSource, readOnly bool, err error) {
if spec.Volume != nil && spec.Volume.AzureDisk != nil {
return spec.Volume.AzureDisk, nil
return spec.Volume.AzureDisk, spec.Volume.AzureDisk.ReadOnly != nil && *spec.Volume.AzureDisk.ReadOnly, nil
}
if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.AzureDisk != nil {
return spec.PersistentVolume.Spec.AzureDisk, nil
return spec.PersistentVolume.Spec.AzureDisk, spec.ReadOnly, nil
}
return nil, fmt.Errorf("azureDisk - Spec does not reference an Azure disk volume type")
return nil, false, fmt.Errorf("azureDisk - Spec does not reference an Azure disk volume type")
}
func normalizeKind(kind string) (v1.AzureDataDiskKind, error) {

View File

@ -81,7 +81,7 @@ func (plugin *azureDataDiskPlugin) GetPluginName() string {
}
func (plugin *azureDataDiskPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return "", err
}
@ -140,12 +140,12 @@ func (plugin *azureDataDiskPlugin) NewDetacher() (volume.Detacher, error) {
}
func (plugin *azureDataDiskPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return nil, err
}
disk := makeDataDisk(spec.Name(), "", volumeSource.DiskName, plugin.host)
disk := makeDataDisk(spec.Name(), "", volumeSource.DiskName, plugin.host, plugin)
return &azureDiskDeleter{
spec: spec,
@ -166,11 +166,11 @@ func (plugin *azureDataDiskPlugin) NewProvisioner(options volume.VolumeOptions)
}
func (plugin *azureDataDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, options volume.VolumeOptions) (volume.Mounter, error) {
volumeSource, err := getVolumeSource(spec)
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return nil, err
}
disk := makeDataDisk(spec.Name(), pod.UID, volumeSource.DiskName, plugin.host)
disk := makeDataDisk(spec.Name(), pod.UID, volumeSource.DiskName, plugin.host, plugin)
return &azureDiskMounter{
plugin: plugin,
@ -181,7 +181,7 @@ func (plugin *azureDataDiskPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, op
}
func (plugin *azureDataDiskPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
disk := makeDataDisk(volName, podUID, "", plugin.host)
disk := makeDataDisk(volName, podUID, "", plugin.host, plugin)
return &azureDiskUnmounter{
plugin: plugin,

View File

@ -0,0 +1,156 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure_dd
import (
"fmt"
"path/filepath"
"github.com/golang/glog"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/util/mount"
kstrings "k8s.io/kubernetes/pkg/util/strings"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
)
var _ volume.VolumePlugin = &azureDataDiskPlugin{}
var _ volume.PersistentVolumePlugin = &azureDataDiskPlugin{}
var _ volume.BlockVolumePlugin = &azureDataDiskPlugin{}
var _ volume.DeletableVolumePlugin = &azureDataDiskPlugin{}
var _ volume.ProvisionableVolumePlugin = &azureDataDiskPlugin{}
func (plugin *azureDataDiskPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
pluginDir := plugin.host.GetVolumeDevicePluginDir(azureDataDiskPluginName)
blkutil := volumepathhandler.NewBlockVolumePathHandler()
globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
if err != nil {
return nil, err
}
glog.V(5).Infof("constructing block volume spec from globalMapPathUUID: %s", globalMapPathUUID)
globalMapPath := filepath.Dir(globalMapPathUUID)
if len(globalMapPath) <= 1 {
return nil, fmt.Errorf("failed to get volume plugin information from globalMapPathUUID: %v", globalMapPathUUID)
}
return getVolumeSpecFromGlobalMapPath(globalMapPath, volumeName)
}
func getVolumeSpecFromGlobalMapPath(globalMapPath, volumeName string) (*volume.Spec, error) {
// Get volume spec information from globalMapPath
// globalMapPath example:
// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumeID}
// plugins/kubernetes.io/azure-disk/volumeDevices/vol-XXXXXX
diskName := filepath.Base(globalMapPath)
if len(diskName) <= 1 {
return nil, fmt.Errorf("failed to get diskName from global path=%s", globalMapPath)
}
glog.V(5).Infof("got diskName(%s) from globalMapPath: %s", globalMapPath, diskName)
block := v1.PersistentVolumeBlock
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: volumeName,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
AzureDisk: &v1.AzureDiskVolumeSource{
DiskName: diskName,
},
},
VolumeMode: &block,
},
}
return volume.NewSpecFromPersistentVolume(pv, true), nil
}
// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
func (plugin *azureDataDiskPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
// If this is called via GenerateUnmapDeviceFunc(), pod is nil.
// Pass empty string as dummy uid since uid isn't used in the case.
var uid types.UID
if pod != nil {
uid = pod.UID
}
return plugin.newBlockVolumeMapperInternal(spec, uid, plugin.host.GetMounter(plugin.GetPluginName()))
}
func (plugin *azureDataDiskPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, mounter mount.Interface) (volume.BlockVolumeMapper, error) {
volumeSource, readOnly, err := getVolumeSource(spec)
if err != nil {
return nil, err
}
disk := makeDataDisk(spec.Name(), podUID, volumeSource.DiskName, plugin.host, plugin)
return &azureDataDiskMapper{
dataDisk: disk,
readOnly: readOnly,
}, nil
}
func (plugin *azureDataDiskPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
return plugin.newUnmapperInternal(volName, podUID, plugin.host.GetMounter(plugin.GetPluginName()))
}
func (plugin *azureDataDiskPlugin) newUnmapperInternal(volName string, podUID types.UID, mounter mount.Interface) (volume.BlockVolumeUnmapper, error) {
disk := makeDataDisk(volName, podUID, "", plugin.host, plugin)
return &azureDataDiskUnmapper{dataDisk: disk}, nil
}
func (c *azureDataDiskUnmapper) TearDownDevice(mapPath, devicePath string) error {
return nil
}
type azureDataDiskUnmapper struct {
*dataDisk
}
var _ volume.BlockVolumeUnmapper = &azureDataDiskUnmapper{}
type azureDataDiskMapper struct {
*dataDisk
readOnly bool
}
var _ volume.BlockVolumeMapper = &azureDataDiskMapper{}
func (b *azureDataDiskMapper) SetUpDevice() (string, error) {
return "", nil
}
// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/volumeID
// plugins/kubernetes.io/azure-disk/volumeDevices/vol-XXXXXX
func (disk *dataDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) {
volumeSource, _, err := getVolumeSource(spec)
if err != nil {
return "", err
}
return filepath.Join(disk.plugin.host.GetVolumeDevicePluginDir(azureDataDiskPluginName), string(volumeSource.DiskName)), nil
}
// GetPodDeviceMapPath returns pod device map path and volume name
// path: pods/{podUid}/volumeDevices/kubernetes.io~azure
func (disk *dataDisk) GetPodDeviceMapPath() (string, string) {
name := azureDataDiskPluginName
return disk.plugin.host.GetPodVolumeDeviceDir(disk.podUID, kstrings.EscapeQualifiedNameForDisk(name)), disk.volumeName
}

View File

@ -0,0 +1,145 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure_dd
import (
"os"
"path/filepath"
"testing"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utiltesting "k8s.io/client-go/util/testing"
"k8s.io/kubernetes/pkg/volume"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
)
const (
testDiskName = "disk1"
testPVName = "pv1"
testGlobalPath = "plugins/kubernetes.io/azure-disk/volumeDevices/disk1"
testPodPath = "pods/poduid/volumeDevices/kubernetes.io~azure-disk"
)
func TestGetVolumeSpecFromGlobalMapPath(t *testing.T) {
// make our test path for fake GlobalMapPath
// /tmp symbolized our pluginDir
// /tmp/testGlobalPathXXXXX/plugins/kubernetes.io/azure-disk/volumeDevices/disk1
tmpVDir, err := utiltesting.MkTmpdir("azureDiskBlockTest")
if err != nil {
t.Fatalf("can't make a temp dir: %v", err)
}
//deferred clean up
defer os.RemoveAll(tmpVDir)
expectedGlobalPath := filepath.Join(tmpVDir, testGlobalPath)
//Bad Path
badspec, err := getVolumeSpecFromGlobalMapPath("", "")
if badspec != nil || err == nil {
t.Errorf("Expected not to get spec from GlobalMapPath but did")
}
// Good Path
spec, err := getVolumeSpecFromGlobalMapPath(expectedGlobalPath, "")
if spec == nil || err != nil {
t.Fatalf("Failed to get spec from GlobalMapPath: %v", err)
}
if spec.PersistentVolume.Spec.AzureDisk.DiskName != testDiskName {
t.Errorf("Invalid pdName from GlobalMapPath spec: %s", spec.PersistentVolume.Spec.AzureDisk.DiskName)
}
block := v1.PersistentVolumeBlock
specMode := spec.PersistentVolume.Spec.VolumeMode
if &specMode == nil {
t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", &specMode, block)
}
if *specMode != block {
t.Errorf("Invalid volumeMode from GlobalMapPath spec: %v expected: %v", *specMode, block)
}
}
func getTestVolume(readOnly bool, path string, isBlock bool) *volume.Spec {
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: testPVName,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
AzureDisk: &v1.AzureDiskVolumeSource{
DiskName: testDiskName,
},
},
},
}
if isBlock {
blockMode := v1.PersistentVolumeBlock
pv.Spec.VolumeMode = &blockMode
}
return volume.NewSpecFromPersistentVolume(pv, readOnly)
}
func TestGetPodAndPluginMapPaths(t *testing.T) {
tmpVDir, err := utiltesting.MkTmpdir("azureDiskBlockTest")
if err != nil {
t.Fatalf("can't make a temp dir: %v", err)
}
//deferred clean up
defer os.RemoveAll(tmpVDir)
expectedGlobalPath := filepath.Join(tmpVDir, testGlobalPath)
expectedPodPath := filepath.Join(tmpVDir, testPodPath)
spec := getTestVolume(false, tmpVDir, true /*isBlock*/)
plugMgr := volume.VolumePluginMgr{}
plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, volumetest.NewFakeVolumeHost(tmpVDir, nil, nil))
plug, err := plugMgr.FindMapperPluginByName(azureDataDiskPluginName)
if err != nil {
os.RemoveAll(tmpVDir)
t.Fatalf("Can't find the plugin by name: %q", azureDataDiskPluginName)
}
if plug.GetPluginName() != azureDataDiskPluginName {
t.Fatalf("Wrong name: %s", plug.GetPluginName())
}
pod := &v1.Pod{ObjectMeta: metav1.ObjectMeta{UID: types.UID("poduid")}}
mapper, err := plug.NewBlockVolumeMapper(spec, pod, volume.VolumeOptions{})
if err != nil {
t.Fatalf("Failed to make a new Mounter: %v", err)
}
if mapper == nil {
t.Fatalf("Got a nil Mounter")
}
//GetGlobalMapPath
gMapPath, err := mapper.GetGlobalMapPath(spec)
if err != nil || len(gMapPath) == 0 {
t.Fatalf("Invalid GlobalMapPath from spec: %s, error: %v", spec.PersistentVolume.Spec.AzureDisk.DiskName, err)
}
if gMapPath != expectedGlobalPath {
t.Errorf("Failed to get GlobalMapPath: %s, expected %s", gMapPath, expectedGlobalPath)
}
//GetPodDeviceMapPath
gDevicePath, gVolName := mapper.GetPodDeviceMapPath()
if gDevicePath != expectedPodPath {
t.Errorf("Got unexpected pod path: %s, expected %s", gDevicePath, expectedPodPath)
}
if gVolName != testPVName {
t.Errorf("Got unexpected volNamne: %s, expected %s", gVolName, testPVName)
}
}

View File

@ -44,7 +44,7 @@ var _ volume.Mounter = &azureDiskMounter{}
func (m *azureDiskMounter) GetAttributes() volume.Attributes {
readOnly := false
volumeSource, err := getVolumeSource(m.spec)
volumeSource, _, err := getVolumeSource(m.spec)
if err != nil {
glog.Infof("azureDisk - mounter failed to get volume source for spec %s %v", m.spec.Name(), err)
} else if volumeSource.ReadOnly != nil {
@ -71,7 +71,7 @@ func (m *azureDiskMounter) GetPath() string {
func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
mounter := m.plugin.host.GetMounter(m.plugin.GetPluginName())
volumeSource, err := getVolumeSource(m.spec)
volumeSource, _, err := getVolumeSource(m.spec)
if err != nil {
glog.Infof("azureDisk - mounter failed to get volume source for spec %s", m.spec.Name())

View File

@ -46,7 +46,7 @@ func (d *azureDiskDeleter) GetPath() string {
}
func (d *azureDiskDeleter) Delete() error {
volumeSource, err := getVolumeSource(d.spec)
volumeSource, _, err := getVolumeSource(d.spec)
if err != nil {
return err
}