github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #51252 from andyzhangx/azuredisk-windows 

Automatic merge from submit-queue

Azuredisk mount on windows node

**What this PR does / why we need it**:
This PR will enable azure disk on windows node, customer could create a pod mounted with azure disk on windows node. 
There are a few pending items still left:
1) Current fstype would be forced as NTFS, will change if there is such requirement
2) GetDeviceNameFromMount function is not implemented(empty) because in Linux, we could use "cat /proc/mounts" to read all mounting points in OS easily, but in Windows, there is no such place, I am still figuring out. The empty function would cause a few warning logging, but it will not affect the main logic now.

**Special notes for your reviewer**:
1. This PR depends on https://github.com/kubernetes/kubernetes/pull/51240, which allow windows mount path in config validation
2. There is a bug in docker on windows(https://github.com/moby/moby/issues/34729), the ContainerPath could only be a drive letter now(e.g. D:), dir path would fail in the end.

The example pod with mount path is like below:

```
kind: Pod
apiVersion: v1
metadata:
  name: pod-uses-shared-hdd-5g
  labels:
    name: storage
spec:
  containers:
  - image: microsoft/iis
    name: az-c-01
    volumeMounts:
    - name: blobdisk01
      mountPath: 'F:'
  nodeSelector:
    beta.kubernetes.io/os: windows
  volumes:
  - name: blobdisk01
    persistentVolumeClaim:
      claimName: pv-dd-shared-hdd-5
```

**Release note**:

```release-note
pull/6/head
Kubernetes Submit Queue 2017-09-12 17:43:13 -07:00 committed by GitHub
parent 108ee22096 82c909cc99
commit 39659ac1dd
16 changed files with 691 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/util/mount/BUILD
View File

@ -20,6 +20,9 @@ go_library(
"mount_linux.go",
"nsenter_mount.go",
],
"@io_bazel_rules_go//go/platform:windows_amd64": [
"mount_windows.go",
],
"//conditions:default": [],
}),
deps = [
@ -44,6 +47,9 @@ go_test(
"mount_linux_test.go",
"nsenter_mount_test.go",
],
"@io_bazel_rules_go//go/platform:windows_amd64": [
"mount_windows_test.go",
],
"//conditions:default": [],
}),
library = ":go_default_library",

25
pkg/util/mount/mount.go
View File

@ -278,3 +278,28 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) {
}
return notMnt, nil
}
// isBind detects whether a bind mount is being requested and makes the remount options to
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
// The list equals:
// options - 'bind' + 'remount' (no duplicate)
func isBind(options []string) (bool, []string) {
bindRemountOpts := []string{"remount"}
bind := false
if len(options) != 0 {
for _, option := range options {
switch option {
case "bind":
bind = true
break
case "remount":
break
default:
bindRemountOpts = append(bindRemountOpts, option)
}
}
}
return bind, bindRemountOpts
}

25
pkg/util/mount/mount_linux.go
View File

@ -94,31 +94,6 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
}
// isBind detects whether a bind mount is being requested and makes the remount options to
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
// The list equals:
// options - 'bind' + 'remount' (no duplicate)
func isBind(options []string) (bool, []string) {
bindRemountOpts := []string{"remount"}
bind := false
if len(options) != 0 {
for _, option := range options {
switch option {
case "bind":
bind = true
break
case "remount":
break
default:
bindRemountOpts = append(bindRemountOpts, option)
}
}
}
return bind, bindRemountOpts
}
// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error {
mountArgs := makeMountArgs(source, target, fstype, options)

2
pkg/util/mount/mount_unsupported.go
View File

@ -1,4 +1,4 @@
// +build !linux
// +build !linux,!windows
/*
Copyright 2014 The Kubernetes Authors.

206
pkg/util/mount/mount_windows.go Normal file
View File

@ -0,0 +1,206 @@
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"github.com/golang/glog"
)
// Mounter provides the default implementation of mount.Interface
// for the windows platform. This implementation assumes that the
// kubelet is running in the host's root mount namespace.
type Mounter struct {
mounterPath string
}
// New returns a mount.Interface for the current system.
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
return &Mounter{
mounterPath: mounterPath,
}
}
// Mount : mounts source to target as NTFS with given options.
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
target = normalizeWindowsPath(target)
if source == "tmpfs" {
glog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options)
return os.MkdirAll(target, 0755)
}
parentDir := filepath.Dir(target)
if err := os.MkdirAll(parentDir, 0755); err != nil {
return err
}
glog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
options, source, target, fstype)
bindSource := ""
// tell it's going to mount azure disk or azure file according to options
if bind, _ := isBind(options); bind {
// mount azure disk
bindSource = normalizeWindowsPath(source)
} else {
if len(options) < 2 {
glog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
options, len(options), source, target)
return nil
}
// empty implementation for mounting azure file
return os.MkdirAll(target, 0755)
}
if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil {
glog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output))
return err
}
return nil
}
// Unmount unmounts the target.
func (mounter *Mounter) Unmount(target string) error {
glog.V(4).Infof("azureMount: Unmount target (%q)", target)
target = normalizeWindowsPath(target)
if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
glog.Errorf("rmdir failed: %v, output: %q", err, string(output))
return err
}
return nil
}
// List returns a list of all mounted filesystems. todo
func (mounter *Mounter) List() ([]MountPoint, error) {
return []MountPoint{}, nil
}
// IsMountPointMatch determines if the mountpoint matches the dir
func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
return mp.Path == dir
}
// IsNotMountPoint determines if a directory is a mountpoint.
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
return IsNotMountPoint(mounter, dir)
}
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
stat, err := os.Lstat(file)
if err != nil {
return true, err
}
// If current file is a symlink, then it is a mountpoint.
if stat.Mode()&os.ModeSymlink != 0 {
return false, nil
}
return true, nil
}
// GetDeviceNameFromMount given a mnt point, find the device
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return getDeviceNameFromMount(mounter, mountPath, pluginDir)
}
// DeviceOpened determines if the device is in use elsewhere
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
return false, nil
}
// PathIsDevice determines if a path is a device.
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
return false, nil
}
// MakeRShared checks that given path is on a mount with 'rshared' mount
// propagation. Empty implementation here.
func (mounter *Mounter) MakeRShared(path string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
if err := ValidateDiskNumber(source); err != nil {
glog.Errorf("azureMount: formatAndMount failed, err: %v\n", err)
return err
}
driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec)
if err != nil {
return err
}
driverPath := driveLetter + ":"
target = normalizeWindowsPath(target)
glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target)
if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil {
glog.Errorf("mklink failed: %v, output: %q", err, string(output))
return err
}
return nil
}
func normalizeWindowsPath(path string) string {
normalizedPath := strings.Replace(path, "/", "\\", -1)
if strings.HasPrefix(normalizedPath, "\\") {
normalizedPath = "c:" + normalizedPath
}
return normalizedPath
}
// ValidateDiskNumber : disk number should be a number in [0, 99]
func ValidateDiskNumber(disk string) error {
diskNum, err := strconv.Atoi(disk)
if err != nil {
return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err)
}
if diskNum < 0 || diskNum > 99 {
return fmt.Errorf("disk number out of range: %q", disk)
}
return nil
}
// Get drive letter according to windows disk number
func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) {
cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum)
output, err := exec.Run("powershell", "/c", cmd)
if err != nil {
return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output))
}
if len(string(output)) < 1 {
return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty")
}
return string(output)[:1], nil
}

71
pkg/util/mount/mount_windows_test.go Normal file
View File

@ -0,0 +1,71 @@
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"testing"
)
func TestGetAvailableDriveLetter(t *testing.T) {
if _, err := getAvailableDriveLetter(); err != nil {
t.Errorf("getAvailableDriveLetter test failed : %v", err)
}
}
func TestNormalizeWindowsPath(t *testing.T) {
path := `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4/volumes/kubernetes.io~azure-disk`
normalizedPath := normalizeWindowsPath(path)
if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` {
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
}
path = `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk`
normalizedPath = normalizeWindowsPath(path)
if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` {
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
}
path = `/`
normalizedPath = normalizeWindowsPath(path)
if normalizedPath != `c:\` {
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
}
}
func TestValidateDiskNumber(t *testing.T) {
diskNum := "0"
if err := ValidateDiskNumber(diskNum); err != nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
diskNum = "99"
if err := ValidateDiskNumber(diskNum); err != nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
diskNum = "ab"
if err := ValidateDiskNumber(diskNum); err == nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
diskNum = "100"
if err := ValidateDiskNumber(diskNum); err == nil {
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
}
}

12
pkg/volume/azure_dd/BUILD
View File

@ -11,10 +11,19 @@ go_library(
srcs = [
"attacher.go",
"azure_common.go",
"azure_common_unsupported.go",
"azure_dd.go",
"azure_mounter.go",
"azure_provision.go",
],
] + select({
"@io_bazel_rules_go//go/platform:linux_amd64": [
"azure_common_linux.go",
],
"@io_bazel_rules_go//go/platform:windows_amd64": [
"azure_common_windows.go",
],
"//conditions:default": [],
}),
deps = [
"//pkg/api:go_default_library",
"//pkg/cloudprovider:go_default_library",
@ -58,6 +67,7 @@ go_test(
],
library = ":go_default_library",
deps = [
"//pkg/util/mount:go_default_library",
"//pkg/volume:go_default_library",
"//pkg/volume/testing:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",

17
pkg/volume/azure_dd/attacher.go
View File

@ -20,6 +20,8 @@ import (
"fmt"
"os"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
@ -162,15 +164,17 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string,
return "", err
}
exec := a.plugin.host.GetExec(a.plugin.GetPluginName())
io := &osIOHandler{}
scsiHostRescan(io)
scsiHostRescan(io, exec)
diskName := volumeSource.DiskName
nodeName := a.plugin.host.GetHostName()
newDevicePath := ""
err = wait.Poll(1*time.Second, timeout, func() (bool, error) {
if newDevicePath, err = findDiskByLun(lun, io); err != nil {
if newDevicePath, err = findDiskByLun(lun, io, exec); err != nil {
return false, fmt.Errorf("azureDisk - WaitForAttach ticker failed node (%s) disk (%s) lun(%v) err(%s)", nodeName, diskName, lun, err)
}
@ -179,7 +183,7 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string,
// the curent sequence k8s uses for unformated disk (check-disk, mount, fail, mkfs.extX) hangs on
// Azure Managed disk scsi interface. this is a hack and will be replaced once we identify and solve
// the root case on Azure.
formatIfNotFormatted(newDevicePath, *volumeSource.FSType, a.plugin.host.GetExec(a.plugin.GetPluginName()))
formatIfNotFormatted(newDevicePath, *volumeSource.FSType, exec)
return true, nil
}
@ -214,7 +218,12 @@ func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath str
if err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
dir := deviceMountPath
if runtime.GOOS == "windows" {
// in windows, as we use mklink, only need to MkdirAll for parent directory
dir = filepath.Dir(deviceMountPath)
}
if err := os.MkdirAll(dir, 0750); err != nil {
return fmt.Errorf("azureDisk - mountDevice:CreateDirectory failed with %s", err)
}
notMnt = true

149
pkg/volume/azure_dd/azure_common.go
View File

@ -21,11 +21,9 @@ import (
"io/ioutil"
"os"
"path"
"strconv"
libstrings "strings"
storage "github.com/Azure/azure-sdk-for-go/arm/storage"
"github.com/golang/glog"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
@ -179,153 +177,6 @@ func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) {
return ioutil.ReadFile(filename)
}
// exclude those used by azure as resource and OS root in /dev/disk/azure
func listAzureDiskPath(io ioHandler) []string {
azureDiskPath := "/dev/disk/azure/"
var azureDiskList []string
if dirs, err := io.ReadDir(azureDiskPath); err == nil {
for _, f := range dirs {
name := f.Name()
diskPath := azureDiskPath + name
if link, linkErr := io.Readlink(diskPath); linkErr == nil {
sd := link[(libstrings.LastIndex(link, "/") + 1):]
azureDiskList = append(azureDiskList, sd)
}
}
}
glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList)
return azureDiskList
}
func scsiHostRescan(io ioHandler) {
scsi_path := "/sys/class/scsi_host/"
if dirs, err := io.ReadDir(scsi_path); err == nil {
for _, f := range dirs {
name := scsi_path + f.Name() + "/scan"
data := []byte("- - -")
if err = io.WriteFile(name, data, 0666); err != nil {
glog.Warningf("failed to rescan scsi host %s", name)
}
}
} else {
glog.Warningf("failed to read %s, err %v", scsi_path, err)
}
}
func findDiskByLun(lun int, io ioHandler) (string, error) {
azureDisks := listAzureDiskPath(io)
return findDiskByLunWithConstraint(lun, io, azureDisks)
}
// finds a device mounted to "current" node
func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) {
var err error
sys_path := "/sys/bus/scsi/devices"
if dirs, err := io.ReadDir(sys_path); err == nil {
for _, f := range dirs {
name := f.Name()
// look for path like /sys/bus/scsi/devices/3:0:0:1
arr := libstrings.Split(name, ":")
if len(arr) < 4 {
continue
}
// extract LUN from the path.
// LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1
l, err := strconv.Atoi(arr[3])
if err != nil {
// unknown path format, continue to read the next one
glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err)
continue
}
if lun == l {
// find the matching LUN
// read vendor and model to ensure it is a VHD disk
vendorPath := path.Join(sys_path, name, "vendor")
vendorBytes, err := io.ReadFile(vendorPath)
if err != nil {
glog.Errorf("failed to read device vendor, err: %v", err)
continue
}
vendor := libstrings.TrimSpace(string(vendorBytes))
if libstrings.ToUpper(vendor) != "MSFT" {
glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor)
continue
}
modelPath := path.Join(sys_path, name, "model")
modelBytes, err := io.ReadFile(modelPath)
if err != nil {
glog.Errorf("failed to read device model, err: %v", err)
continue
}
model := libstrings.TrimSpace(string(modelBytes))
if libstrings.ToUpper(model) != "VIRTUAL DISK" {
glog.V(4).Infof("model doesn't match VHD, got %s", model)
continue
}
// find a disk, validate name
dir := path.Join(sys_path, name, "block")
if dev, err := io.ReadDir(dir); err == nil {
found := false
for _, diskName := range azureDisks {
glog.V(12).Infof("azure disk - validating disk %q with sys disk %q", dev[0].Name(), diskName)
if string(dev[0].Name()) == diskName {
found = true
break
}
}
if !found {
return "/dev/" + dev[0].Name(), nil
}
}
}
}
}
return "", err
}
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
notFormatted, err := diskLooksUnformatted(disk, exec)
if err == nil && notFormatted {
args := []string{disk}
// Disk is unformatted so format it.
// Use 'ext4' as the default
if len(fstype) == 0 {
fstype = "ext4"
}
if fstype == "ext4" || fstype == "ext3" {
args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk}
}
glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args)
_, err := exec.Run("mkfs."+fstype, args...)
if err == nil {
// the disk has been formatted successfully try to mount it again.
glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt")
}
glog.Warningf("azureDisk - format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", disk, fstype, "tt", "o", err)
} else {
if err != nil {
glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err)
} else {
glog.Infof("azureDisk - Disk %s already formatted, will not format", disk)
}
}
}
func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) {
args := []string{"-nd", "-o", "FSTYPE", disk}
glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args)
dataOut, err := exec.Run("lsblk", args...)
if err != nil {
glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
return false, err
}
output := libstrings.TrimSpace(string(dataOut))
return output == "", nil
}
func getDiskController(host volume.VolumeHost) (DiskController, error) {
cloudProvider := host.GetCloudProvider()
az, ok := cloudProvider.(*azure.Cloud)

175
pkg/volume/azure_dd/azure_common_linux.go Normal file
View File

@ -0,0 +1,175 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure_dd
import (
"path"
"strconv"
libstrings "strings"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/mount"
)
// exclude those used by azure as resource and OS root in /dev/disk/azure
func listAzureDiskPath(io ioHandler) []string {
azureDiskPath := "/dev/disk/azure/"
var azureDiskList []string
if dirs, err := io.ReadDir(azureDiskPath); err == nil {
for _, f := range dirs {
name := f.Name()
diskPath := azureDiskPath + name
if link, linkErr := io.Readlink(diskPath); linkErr == nil {
sd := link[(libstrings.LastIndex(link, "/") + 1):]
azureDiskList = append(azureDiskList, sd)
}
}
}
glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList)
return azureDiskList
}
func scsiHostRescan(io ioHandler, exec mount.Exec) {
scsi_path := "/sys/class/scsi_host/"
if dirs, err := io.ReadDir(scsi_path); err == nil {
for _, f := range dirs {
name := scsi_path + f.Name() + "/scan"
data := []byte("- - -")
if err = io.WriteFile(name, data, 0666); err != nil {
glog.Warningf("failed to rescan scsi host %s", name)
}
}
} else {
glog.Warningf("failed to read %s, err %v", scsi_path, err)
}
}
func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) {
azureDisks := listAzureDiskPath(io)
return findDiskByLunWithConstraint(lun, io, azureDisks)
}
// finds a device mounted to "current" node
func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) {
var err error
sys_path := "/sys/bus/scsi/devices"
if dirs, err := io.ReadDir(sys_path); err == nil {
for _, f := range dirs {
name := f.Name()
// look for path like /sys/bus/scsi/devices/3:0:0:1
arr := libstrings.Split(name, ":")
if len(arr) < 4 {
continue
}
// extract LUN from the path.
// LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1
l, err := strconv.Atoi(arr[3])
if err != nil {
// unknown path format, continue to read the next one
glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err)
continue
}
if lun == l {
// find the matching LUN
// read vendor and model to ensure it is a VHD disk
vendorPath := path.Join(sys_path, name, "vendor")
vendorBytes, err := io.ReadFile(vendorPath)
if err != nil {
glog.Errorf("failed to read device vendor, err: %v", err)
continue
}
vendor := libstrings.TrimSpace(string(vendorBytes))
if libstrings.ToUpper(vendor) != "MSFT" {
glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor)
continue
}
modelPath := path.Join(sys_path, name, "model")
modelBytes, err := io.ReadFile(modelPath)
if err != nil {
glog.Errorf("failed to read device model, err: %v", err)
continue
}
model := libstrings.TrimSpace(string(modelBytes))
if libstrings.ToUpper(model) != "VIRTUAL DISK" {
glog.V(4).Infof("model doesn't match VHD, got %s", model)
continue
}
// find a disk, validate name
dir := path.Join(sys_path, name, "block")
if dev, err := io.ReadDir(dir); err == nil {
found := false
for _, diskName := range azureDisks {
glog.V(12).Infof("azure disk - validating disk %q with sys disk %q", dev[0].Name(), diskName)
if string(dev[0].Name()) == diskName {
found = true
break
}
}
if !found {
return "/dev/" + dev[0].Name(), nil
}
}
}
}
}
return "", err
}
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
notFormatted, err := diskLooksUnformatted(disk, exec)
if err == nil && notFormatted {
args := []string{disk}
// Disk is unformatted so format it.
// Use 'ext4' as the default
if len(fstype) == 0 {
fstype = "ext4"
}
if fstype == "ext4" || fstype == "ext3" {
args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk}
}
glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args)
_, err := exec.Run("mkfs."+fstype, args...)
if err == nil {
// the disk has been formatted successfully try to mount it again.
glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt")
}
glog.Warningf("azureDisk - format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", disk, fstype, "tt", "o", err)
} else {
if err != nil {
glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err)
} else {
glog.Infof("azureDisk - Disk %s already formatted, will not format", disk)
}
}
}
func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) {
args := []string{"-nd", "-o", "FSTYPE", disk}
glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args)
dataOut, err := exec.Run("lsblk", args...)
if err != nil {
glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
return false, err
}
output := libstrings.TrimSpace(string(dataOut))
return output == "", nil
}

17
pkg/volume/azure_dd/azure_common_test.go
View File

@ -19,9 +19,12 @@ package azure_dd
import (
"fmt"
"os"
"runtime"
"strings"
"testing"
"time"
"k8s.io/kubernetes/pkg/util/mount"
)
type fakeFileInfo struct {
@ -116,9 +119,15 @@ func (handler *fakeIOHandler) ReadFile(filename string) ([]byte, error) {
}
func TestIoHandler(t *testing.T) {
disk, err := findDiskByLun(lun, &fakeIOHandler{})
// if no disk matches lun, exit
if disk != "/dev/"+devName || err != nil {
t.Errorf("no data disk found: disk %v err %v", disk, err)
disk, err := findDiskByLun(lun, &fakeIOHandler{}, mount.NewOsExec())
if runtime.GOOS == "windows" {
if err != nil {
t.Errorf("no data disk found: disk %v err %v", disk, err)
}
} else {
// if no disk matches lun, exit
if disk != "/dev/"+devName || err != nil {
t.Errorf("no data disk found: disk %v err %v", disk, err)
}
}
}

31
pkg/volume/azure_dd/azure_common_unsupported.go Normal file
View File

@ -0,0 +1,31 @@
// +build !linux,!windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure_dd
import "k8s.io/kubernetes/pkg/util/mount"
func scsiHostRescan(io ioHandler, exec mount.Exec) {
}
func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) {
return "", nil
}
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
}

114
pkg/volume/azure_dd/azure_common_windows.go Normal file
View File

@ -0,0 +1,114 @@
// +build windows
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package azure_dd
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/util/mount"
)
func scsiHostRescan(io ioHandler, exec mount.Exec) {
cmd := "Update-HostStorageCache"
output, err := exec.Run("powershell", "/c", cmd)
if err != nil {
glog.Errorf("Update-HostStorageCache failed in scsiHostRescan, error: %v, output: %q", err, string(output))
}
}
// search Windows disk number by LUN
func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error) {
cmd := `Get-Disk | select number, location | ConvertTo-Json`
output, err := exec.Run("powershell", "/c", cmd)
if err != nil {
glog.Errorf("Get-Disk failed in findDiskByLun, error: %v, output: %q", err, string(output))
return "", err
}
if len(output) < 10 {
return "", fmt.Errorf("Get-Disk output is too short, output: %q", string(output))
}
var data []map[string]interface{}
if err = json.Unmarshal(output, &data); err != nil {
glog.Errorf("Get-Disk output is not a json array, output: %q", string(output))
return "", err
}
for _, v := range data {
if jsonLocation, ok := v["location"]; ok {
if location, ok := jsonLocation.(string); ok {
if !strings.Contains(location, " LUN ") {
continue
}
arr := strings.Split(location, " ")
arrLen := len(arr)
if arrLen < 3 {
glog.Warningf("unexpected json structure from Get-Disk, location: %q", jsonLocation)
continue
}
glog.V(4).Infof("found a disk, locatin: %q, lun: %q", location, arr[arrLen-1])
//last element of location field is LUN number, e.g.
// "location": "Integrated : Adapter 3 : Port 0 : Target 0 : LUN 1"
l, err := strconv.Atoi(arr[arrLen-1])
if err != nil {
glog.Warningf("cannot parse element from data structure, location: %q, element: %q", location, arr[arrLen-1])
continue
}
if l == lun {
glog.V(4).Infof("found a disk and lun, locatin: %q, lun: %d", location, lun)
if d, ok := v["number"]; ok {
if diskNum, ok := d.(float64); ok {
glog.V(2).Infof("azureDisk Mount: got disk number(%d) by LUN(%d)", int(diskNum), lun)
return strconv.Itoa(int(diskNum)), nil
}
glog.Warningf("LUN(%d) found, but could not get disk number(%q), location: %q", lun, d, location)
}
return "", fmt.Errorf("LUN(%d) found, but could not get disk number, location: %q", lun, location)
}
}
}
}
return "", nil
}
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
if err := mount.ValidateDiskNumber(disk); err != nil {
glog.Errorf("azureDisk Mount: formatIfNotFormatted failed, err: %v\n", err)
return
}
cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru", disk)
cmd += " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem NTFS -Confirm:$false"
output, err := exec.Run("powershell", "/c", cmd)
if err != nil {
glog.Errorf("azureDisk Mount: Get-Disk failed, error: %v, output: %q", err, string(output))
} else {
glog.Infof("azureDisk Mount: Disk successfully formatted, disk: %q, fstype: %q\n", disk, fstype)
}
}

2
pkg/volume/azure_dd/azure_dd.go
View File

@ -116,7 +116,7 @@ func (plugin *azureDataDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessM
func (plugin *azureDataDiskPlugin) NewAttacher() (volume.Attacher, error) {
azure, err := getCloud(plugin.host)
if err != nil {
glog.V(4).Infof("failed to get azure cloud in NewAttacher, plugin.host : %s", plugin.host.GetHostName())
glog.Errorf("failed to get azure cloud in NewAttacher, plugin.host : %s, err:%v", plugin.host.GetHostName(), err)
return nil, err
}

10
pkg/volume/azure_dd/azure_mounter.go
View File

@ -19,6 +19,7 @@ package azure_dd
import (
"fmt"
"os"
"runtime"
"github.com/golang/glog"
"k8s.io/api/core/v1"
@ -82,9 +83,12 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
return fmt.Errorf("azureDisk - Not a mounting point for disk %s on %s", diskName, dir)
}
if err := os.MkdirAll(dir, 0750); err != nil {
glog.Infof("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err)
return err
if runtime.GOOS != "windows" {
// in windows, we will use mklink to mount, will MkdirAll in Mount func
if err := os.MkdirAll(dir, 0750); err != nil {
glog.Errorf("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err)
return err
}
}
options := []string{"bind"}

24
pkg/volume/azure_file/azure_file.go
View File

@ -19,6 +19,7 @@ package azure_file
import (
"fmt"
"os"
"runtime"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -210,15 +211,24 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error {
if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.secretNamespace, b.secretName); err != nil {
return err
}
os.MkdirAll(dir, 0700)
source := fmt.Sprintf("//%s.file.%s/%s", accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), b.shareName)
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)}
if b.readOnly {
options = append(options, "ro")
mountOptions := []string{}
source := ""
osSeparator := string(os.PathSeparator)
source = fmt.Sprintf("%s%s%s.file.%s%s%s", osSeparator, osSeparator, accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), osSeparator, b.shareName)
if runtime.GOOS == "windows" {
mountOptions = []string{accountName, accountKey}
} else {
os.MkdirAll(dir, 0700)
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)}
if b.readOnly {
options = append(options, "ro")
}
mountOptions = volume.JoinMountOptions(b.mountOptions, options)
}
mountOptions := volume.JoinMountOptions(b.mountOptions, options)
err = b.mounter.Mount(source, dir, "cifs", mountOptions)
if err != nil {
notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)