mirror of https://github.com/k3s-io/k3s
enable azure disk mount on windows node
add initial work for mount azure file on windows fix review comments full implementation for attach azure file on windows node working azure file mount remove useless functions add a workable implementation about mounting azure file on windows node fix review comments and make the pod creating successful even azure file mount failed fix according to review comments add mount_windows_test add implementation for IsLikelyNotMountPoint func remove mount_windows_test.go temporaly add back unit test for mount_windows.go add normalizeWindowsPath func fix normalizeWindowsPath func issue implment azure disk on windows update bazel BUILD revert validation.go change as it's another PR fix merge issue and compiling issue fix windows compiling issue fix according to review comments fix according to review comments fix cross-build failure fix according to review comments fix test build failure temporalily fix darwin build failure fix azure windows test failure add empty implementation of MakeRShared on windows fix gofmt errorspull/6/head
xiazhang
andyzhangx
parent
a51eb2ac4e
commit
82c909cc99
|
|
@ -20,6 +20,9 @@ go_library(
|
|||
"mount_linux.go",
|
||||
"nsenter_mount.go",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:windows_amd64": [
|
||||
"mount_windows.go",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = [
|
||||
|
|
@ -44,6 +47,9 @@ go_test(
|
|||
"mount_linux_test.go",
|
||||
"nsenter_mount_test.go",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:windows_amd64": [
|
||||
"mount_windows_test.go",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
library = ":go_default_library",
|
||||
|
|
|
|||
|
|
@ -278,3 +278,28 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) {
|
|||
}
|
||||
return notMnt, nil
|
||||
}
|
||||
|
||||
// isBind detects whether a bind mount is being requested and makes the remount options to
|
||||
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
|
||||
// The list equals:
|
||||
// options - 'bind' + 'remount' (no duplicate)
|
||||
func isBind(options []string) (bool, []string) {
|
||||
bindRemountOpts := []string{"remount"}
|
||||
bind := false
|
||||
|
||||
if len(options) != 0 {
|
||||
for _, option := range options {
|
||||
switch option {
|
||||
case "bind":
|
||||
bind = true
|
||||
break
|
||||
case "remount":
|
||||
break
|
||||
default:
|
||||
bindRemountOpts = append(bindRemountOpts, option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bind, bindRemountOpts
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,31 +94,6 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
|
|||
return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
|
||||
}
|
||||
|
||||
// isBind detects whether a bind mount is being requested and makes the remount options to
|
||||
// use in case of bind mount, due to the fact that bind mount doesn't respect mount options.
|
||||
// The list equals:
|
||||
// options - 'bind' + 'remount' (no duplicate)
|
||||
func isBind(options []string) (bool, []string) {
|
||||
bindRemountOpts := []string{"remount"}
|
||||
bind := false
|
||||
|
||||
if len(options) != 0 {
|
||||
for _, option := range options {
|
||||
switch option {
|
||||
case "bind":
|
||||
bind = true
|
||||
break
|
||||
case "remount":
|
||||
break
|
||||
default:
|
||||
bindRemountOpts = append(bindRemountOpts, option)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bind, bindRemountOpts
|
||||
}
|
||||
|
||||
// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
|
||||
func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error {
|
||||
mountArgs := makeMountArgs(source, target, fstype, options)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// +build !linux
|
||||
// +build !linux,!windows
|
||||
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,206 @@
|
|||
// +build windows
|
||||
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
|
||||
// Mounter provides the default implementation of mount.Interface
|
||||
// for the windows platform. This implementation assumes that the
|
||||
// kubelet is running in the host's root mount namespace.
|
||||
type Mounter struct {
|
||||
mounterPath string
|
||||
}
|
||||
|
||||
// New returns a mount.Interface for the current system.
|
||||
// It provides options to override the default mounter behavior.
|
||||
// mounterPath allows using an alternative to `/bin/mount` for mounting.
|
||||
func New(mounterPath string) Interface {
|
||||
return &Mounter{
|
||||
mounterPath: mounterPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Mount : mounts source to target as NTFS with given options.
|
||||
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
|
||||
target = normalizeWindowsPath(target)
|
||||
|
||||
if source == "tmpfs" {
|
||||
glog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options)
|
||||
return os.MkdirAll(target, 0755)
|
||||
}
|
||||
|
||||
parentDir := filepath.Dir(target)
|
||||
if err := os.MkdirAll(parentDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
glog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount",
|
||||
options, source, target, fstype)
|
||||
bindSource := ""
|
||||
|
||||
// tell it's going to mount azure disk or azure file according to options
|
||||
if bind, _ := isBind(options); bind {
|
||||
// mount azure disk
|
||||
bindSource = normalizeWindowsPath(source)
|
||||
} else {
|
||||
if len(options) < 2 {
|
||||
glog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting",
|
||||
options, len(options), source, target)
|
||||
return nil
|
||||
}
|
||||
|
||||
// empty implementation for mounting azure file
|
||||
return os.MkdirAll(target, 0755)
|
||||
}
|
||||
|
||||
if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil {
|
||||
glog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unmount unmounts the target.
|
||||
func (mounter *Mounter) Unmount(target string) error {
|
||||
glog.V(4).Infof("azureMount: Unmount target (%q)", target)
|
||||
target = normalizeWindowsPath(target)
|
||||
if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil {
|
||||
glog.Errorf("rmdir failed: %v, output: %q", err, string(output))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns a list of all mounted filesystems. todo
|
||||
func (mounter *Mounter) List() ([]MountPoint, error) {
|
||||
return []MountPoint{}, nil
|
||||
}
|
||||
|
||||
// IsMountPointMatch determines if the mountpoint matches the dir
|
||||
func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
|
||||
return mp.Path == dir
|
||||
}
|
||||
|
||||
// IsNotMountPoint determines if a directory is a mountpoint.
|
||||
func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
|
||||
return IsNotMountPoint(mounter, dir)
|
||||
}
|
||||
|
||||
// IsLikelyNotMountPoint determines if a directory is not a mountpoint.
|
||||
func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
|
||||
stat, err := os.Lstat(file)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
// If current file is a symlink, then it is a mountpoint.
|
||||
if stat.Mode()&os.ModeSymlink != 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// GetDeviceNameFromMount given a mnt point, find the device
|
||||
func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
|
||||
return getDeviceNameFromMount(mounter, mountPath, pluginDir)
|
||||
}
|
||||
|
||||
// DeviceOpened determines if the device is in use elsewhere
|
||||
func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// PathIsDevice determines if a path is a device.
|
||||
func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// MakeRShared checks that given path is on a mount with 'rshared' mount
|
||||
// propagation. Empty implementation here.
|
||||
func (mounter *Mounter) MakeRShared(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
|
||||
// Try to mount the disk
|
||||
glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
|
||||
|
||||
if err := ValidateDiskNumber(source); err != nil {
|
||||
glog.Errorf("azureMount: formatAndMount failed, err: %v\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
driverPath := driveLetter + ":"
|
||||
target = normalizeWindowsPath(target)
|
||||
glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target)
|
||||
if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil {
|
||||
glog.Errorf("mklink failed: %v, output: %q", err, string(output))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeWindowsPath(path string) string {
|
||||
normalizedPath := strings.Replace(path, "/", "\\", -1)
|
||||
if strings.HasPrefix(normalizedPath, "\\") {
|
||||
normalizedPath = "c:" + normalizedPath
|
||||
}
|
||||
return normalizedPath
|
||||
}
|
||||
|
||||
// ValidateDiskNumber : disk number should be a number in [0, 99]
|
||||
func ValidateDiskNumber(disk string) error {
|
||||
diskNum, err := strconv.Atoi(disk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err)
|
||||
}
|
||||
|
||||
if diskNum < 0 || diskNum > 99 {
|
||||
return fmt.Errorf("disk number out of range: %q", disk)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get drive letter according to windows disk number
|
||||
func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) {
|
||||
cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum)
|
||||
output, err := exec.Run("powershell", "/c", cmd)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output))
|
||||
}
|
||||
if len(string(output)) < 1 {
|
||||
return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty")
|
||||
}
|
||||
return string(output)[:1], nil
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
// +build windows
|
||||
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetAvailableDriveLetter(t *testing.T) {
|
||||
if _, err := getAvailableDriveLetter(); err != nil {
|
||||
t.Errorf("getAvailableDriveLetter test failed : %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWindowsPath(t *testing.T) {
|
||||
path := `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4/volumes/kubernetes.io~azure-disk`
|
||||
normalizedPath := normalizeWindowsPath(path)
|
||||
if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` {
|
||||
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
|
||||
}
|
||||
|
||||
path = `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk`
|
||||
normalizedPath = normalizeWindowsPath(path)
|
||||
if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` {
|
||||
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
|
||||
}
|
||||
|
||||
path = `/`
|
||||
normalizedPath = normalizeWindowsPath(path)
|
||||
if normalizedPath != `c:\` {
|
||||
t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDiskNumber(t *testing.T) {
|
||||
diskNum := "0"
|
||||
if err := ValidateDiskNumber(diskNum); err != nil {
|
||||
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
|
||||
}
|
||||
|
||||
diskNum = "99"
|
||||
if err := ValidateDiskNumber(diskNum); err != nil {
|
||||
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
|
||||
}
|
||||
|
||||
diskNum = "ab"
|
||||
if err := ValidateDiskNumber(diskNum); err == nil {
|
||||
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
|
||||
}
|
||||
|
||||
diskNum = "100"
|
||||
if err := ValidateDiskNumber(diskNum); err == nil {
|
||||
t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum)
|
||||
}
|
||||
}
|
||||
|
|
@ -11,10 +11,19 @@ go_library(
|
|||
srcs = [
|
||||
"attacher.go",
|
||||
"azure_common.go",
|
||||
"azure_common_unsupported.go",
|
||||
"azure_dd.go",
|
||||
"azure_mounter.go",
|
||||
"azure_provision.go",
|
||||
],
|
||||
] + select({
|
||||
"@io_bazel_rules_go//go/platform:linux_amd64": [
|
||||
"azure_common_linux.go",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:windows_amd64": [
|
||||
"azure_common_windows.go",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
deps = [
|
||||
"//pkg/api:go_default_library",
|
||||
"//pkg/cloudprovider:go_default_library",
|
||||
|
|
@ -58,6 +67,7 @@ go_test(
|
|||
],
|
||||
library = ":go_default_library",
|
||||
deps = [
|
||||
"//pkg/util/mount:go_default_library",
|
||||
"//pkg/volume:go_default_library",
|
||||
"//pkg/volume/testing:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -162,15 +164,17 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string,
|
|||
return "", err
|
||||
}
|
||||
|
||||
exec := a.plugin.host.GetExec(a.plugin.GetPluginName())
|
||||
|
||||
io := &osIOHandler{}
|
||||
scsiHostRescan(io)
|
||||
scsiHostRescan(io, exec)
|
||||
|
||||
diskName := volumeSource.DiskName
|
||||
nodeName := a.plugin.host.GetHostName()
|
||||
newDevicePath := ""
|
||||
|
||||
err = wait.Poll(1*time.Second, timeout, func() (bool, error) {
|
||||
if newDevicePath, err = findDiskByLun(lun, io); err != nil {
|
||||
if newDevicePath, err = findDiskByLun(lun, io, exec); err != nil {
|
||||
return false, fmt.Errorf("azureDisk - WaitForAttach ticker failed node (%s) disk (%s) lun(%v) err(%s)", nodeName, diskName, lun, err)
|
||||
}
|
||||
|
||||
|
|
@ -179,7 +183,7 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string,
|
|||
// the curent sequence k8s uses for unformated disk (check-disk, mount, fail, mkfs.extX) hangs on
|
||||
// Azure Managed disk scsi interface. this is a hack and will be replaced once we identify and solve
|
||||
// the root case on Azure.
|
||||
formatIfNotFormatted(newDevicePath, *volumeSource.FSType, a.plugin.host.GetExec(a.plugin.GetPluginName()))
|
||||
formatIfNotFormatted(newDevicePath, *volumeSource.FSType, exec)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +218,12 @@ func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath str
|
|||
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(deviceMountPath, 0750); err != nil {
|
||||
dir := deviceMountPath
|
||||
if runtime.GOOS == "windows" {
|
||||
// in windows, as we use mklink, only need to MkdirAll for parent directory
|
||||
dir = filepath.Dir(deviceMountPath)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
||||
return fmt.Errorf("azureDisk - mountDevice:CreateDirectory failed with %s", err)
|
||||
}
|
||||
notMnt = true
|
||||
|
|
|
|||
|
|
@ -21,11 +21,9 @@ import (
|
|||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
libstrings "strings"
|
||||
|
||||
storage "github.com/Azure/azure-sdk-for-go/arm/storage"
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
|
|
@ -179,153 +177,6 @@ func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) {
|
|||
return ioutil.ReadFile(filename)
|
||||
}
|
||||
|
||||
// exclude those used by azure as resource and OS root in /dev/disk/azure
|
||||
func listAzureDiskPath(io ioHandler) []string {
|
||||
azureDiskPath := "/dev/disk/azure/"
|
||||
var azureDiskList []string
|
||||
if dirs, err := io.ReadDir(azureDiskPath); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := f.Name()
|
||||
diskPath := azureDiskPath + name
|
||||
if link, linkErr := io.Readlink(diskPath); linkErr == nil {
|
||||
sd := link[(libstrings.LastIndex(link, "/") + 1):]
|
||||
azureDiskList = append(azureDiskList, sd)
|
||||
}
|
||||
}
|
||||
}
|
||||
glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList)
|
||||
return azureDiskList
|
||||
}
|
||||
|
||||
func scsiHostRescan(io ioHandler) {
|
||||
scsi_path := "/sys/class/scsi_host/"
|
||||
if dirs, err := io.ReadDir(scsi_path); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := scsi_path + f.Name() + "/scan"
|
||||
data := []byte("- - -")
|
||||
if err = io.WriteFile(name, data, 0666); err != nil {
|
||||
glog.Warningf("failed to rescan scsi host %s", name)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("failed to read %s, err %v", scsi_path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func findDiskByLun(lun int, io ioHandler) (string, error) {
|
||||
azureDisks := listAzureDiskPath(io)
|
||||
return findDiskByLunWithConstraint(lun, io, azureDisks)
|
||||
}
|
||||
|
||||
// finds a device mounted to "current" node
|
||||
func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) {
|
||||
var err error
|
||||
sys_path := "/sys/bus/scsi/devices"
|
||||
if dirs, err := io.ReadDir(sys_path); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := f.Name()
|
||||
// look for path like /sys/bus/scsi/devices/3:0:0:1
|
||||
arr := libstrings.Split(name, ":")
|
||||
if len(arr) < 4 {
|
||||
continue
|
||||
}
|
||||
// extract LUN from the path.
|
||||
// LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1
|
||||
l, err := strconv.Atoi(arr[3])
|
||||
if err != nil {
|
||||
// unknown path format, continue to read the next one
|
||||
glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err)
|
||||
continue
|
||||
}
|
||||
if lun == l {
|
||||
// find the matching LUN
|
||||
// read vendor and model to ensure it is a VHD disk
|
||||
vendorPath := path.Join(sys_path, name, "vendor")
|
||||
vendorBytes, err := io.ReadFile(vendorPath)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to read device vendor, err: %v", err)
|
||||
continue
|
||||
}
|
||||
vendor := libstrings.TrimSpace(string(vendorBytes))
|
||||
if libstrings.ToUpper(vendor) != "MSFT" {
|
||||
glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor)
|
||||
continue
|
||||
}
|
||||
|
||||
modelPath := path.Join(sys_path, name, "model")
|
||||
modelBytes, err := io.ReadFile(modelPath)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to read device model, err: %v", err)
|
||||
continue
|
||||
}
|
||||
model := libstrings.TrimSpace(string(modelBytes))
|
||||
if libstrings.ToUpper(model) != "VIRTUAL DISK" {
|
||||
glog.V(4).Infof("model doesn't match VHD, got %s", model)
|
||||
continue
|
||||
}
|
||||
|
||||
// find a disk, validate name
|
||||
dir := path.Join(sys_path, name, "block")
|
||||
if dev, err := io.ReadDir(dir); err == nil {
|
||||
found := false
|
||||
for _, diskName := range azureDisks {
|
||||
glog.V(12).Infof("azure disk - validating disk %q with sys disk %q", dev[0].Name(), diskName)
|
||||
if string(dev[0].Name()) == diskName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return "/dev/" + dev[0].Name(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
|
||||
notFormatted, err := diskLooksUnformatted(disk, exec)
|
||||
if err == nil && notFormatted {
|
||||
args := []string{disk}
|
||||
// Disk is unformatted so format it.
|
||||
// Use 'ext4' as the default
|
||||
if len(fstype) == 0 {
|
||||
fstype = "ext4"
|
||||
}
|
||||
if fstype == "ext4" || fstype == "ext3" {
|
||||
args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk}
|
||||
}
|
||||
glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args)
|
||||
|
||||
_, err := exec.Run("mkfs."+fstype, args...)
|
||||
if err == nil {
|
||||
// the disk has been formatted successfully try to mount it again.
|
||||
glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt")
|
||||
}
|
||||
glog.Warningf("azureDisk - format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", disk, fstype, "tt", "o", err)
|
||||
} else {
|
||||
if err != nil {
|
||||
glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err)
|
||||
} else {
|
||||
glog.Infof("azureDisk - Disk %s already formatted, will not format", disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) {
|
||||
args := []string{"-nd", "-o", "FSTYPE", disk}
|
||||
glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args)
|
||||
dataOut, err := exec.Run("lsblk", args...)
|
||||
if err != nil {
|
||||
glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
|
||||
return false, err
|
||||
}
|
||||
output := libstrings.TrimSpace(string(dataOut))
|
||||
return output == "", nil
|
||||
}
|
||||
|
||||
func getDiskController(host volume.VolumeHost) (DiskController, error) {
|
||||
cloudProvider := host.GetCloudProvider()
|
||||
az, ok := cloudProvider.(*azure.Cloud)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
// +build linux
|
||||
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package azure_dd
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strconv"
|
||||
libstrings "strings"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
)
|
||||
|
||||
// exclude those used by azure as resource and OS root in /dev/disk/azure
|
||||
func listAzureDiskPath(io ioHandler) []string {
|
||||
azureDiskPath := "/dev/disk/azure/"
|
||||
var azureDiskList []string
|
||||
if dirs, err := io.ReadDir(azureDiskPath); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := f.Name()
|
||||
diskPath := azureDiskPath + name
|
||||
if link, linkErr := io.Readlink(diskPath); linkErr == nil {
|
||||
sd := link[(libstrings.LastIndex(link, "/") + 1):]
|
||||
azureDiskList = append(azureDiskList, sd)
|
||||
}
|
||||
}
|
||||
}
|
||||
glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList)
|
||||
return azureDiskList
|
||||
}
|
||||
|
||||
func scsiHostRescan(io ioHandler, exec mount.Exec) {
|
||||
scsi_path := "/sys/class/scsi_host/"
|
||||
if dirs, err := io.ReadDir(scsi_path); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := scsi_path + f.Name() + "/scan"
|
||||
data := []byte("- - -")
|
||||
if err = io.WriteFile(name, data, 0666); err != nil {
|
||||
glog.Warningf("failed to rescan scsi host %s", name)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("failed to read %s, err %v", scsi_path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) {
|
||||
azureDisks := listAzureDiskPath(io)
|
||||
return findDiskByLunWithConstraint(lun, io, azureDisks)
|
||||
}
|
||||
|
||||
// finds a device mounted to "current" node
|
||||
func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) {
|
||||
var err error
|
||||
sys_path := "/sys/bus/scsi/devices"
|
||||
if dirs, err := io.ReadDir(sys_path); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := f.Name()
|
||||
// look for path like /sys/bus/scsi/devices/3:0:0:1
|
||||
arr := libstrings.Split(name, ":")
|
||||
if len(arr) < 4 {
|
||||
continue
|
||||
}
|
||||
// extract LUN from the path.
|
||||
// LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1
|
||||
l, err := strconv.Atoi(arr[3])
|
||||
if err != nil {
|
||||
// unknown path format, continue to read the next one
|
||||
glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err)
|
||||
continue
|
||||
}
|
||||
if lun == l {
|
||||
// find the matching LUN
|
||||
// read vendor and model to ensure it is a VHD disk
|
||||
vendorPath := path.Join(sys_path, name, "vendor")
|
||||
vendorBytes, err := io.ReadFile(vendorPath)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to read device vendor, err: %v", err)
|
||||
continue
|
||||
}
|
||||
vendor := libstrings.TrimSpace(string(vendorBytes))
|
||||
if libstrings.ToUpper(vendor) != "MSFT" {
|
||||
glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor)
|
||||
continue
|
||||
}
|
||||
|
||||
modelPath := path.Join(sys_path, name, "model")
|
||||
modelBytes, err := io.ReadFile(modelPath)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to read device model, err: %v", err)
|
||||
continue
|
||||
}
|
||||
model := libstrings.TrimSpace(string(modelBytes))
|
||||
if libstrings.ToUpper(model) != "VIRTUAL DISK" {
|
||||
glog.V(4).Infof("model doesn't match VHD, got %s", model)
|
||||
continue
|
||||
}
|
||||
|
||||
// find a disk, validate name
|
||||
dir := path.Join(sys_path, name, "block")
|
||||
if dev, err := io.ReadDir(dir); err == nil {
|
||||
found := false
|
||||
for _, diskName := range azureDisks {
|
||||
glog.V(12).Infof("azure disk - validating disk %q with sys disk %q", dev[0].Name(), diskName)
|
||||
if string(dev[0].Name()) == diskName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return "/dev/" + dev[0].Name(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
|
||||
notFormatted, err := diskLooksUnformatted(disk, exec)
|
||||
if err == nil && notFormatted {
|
||||
args := []string{disk}
|
||||
// Disk is unformatted so format it.
|
||||
// Use 'ext4' as the default
|
||||
if len(fstype) == 0 {
|
||||
fstype = "ext4"
|
||||
}
|
||||
if fstype == "ext4" || fstype == "ext3" {
|
||||
args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk}
|
||||
}
|
||||
glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args)
|
||||
|
||||
_, err := exec.Run("mkfs."+fstype, args...)
|
||||
if err == nil {
|
||||
// the disk has been formatted successfully try to mount it again.
|
||||
glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt")
|
||||
}
|
||||
glog.Warningf("azureDisk - format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", disk, fstype, "tt", "o", err)
|
||||
} else {
|
||||
if err != nil {
|
||||
glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err)
|
||||
} else {
|
||||
glog.Infof("azureDisk - Disk %s already formatted, will not format", disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) {
|
||||
args := []string{"-nd", "-o", "FSTYPE", disk}
|
||||
glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args)
|
||||
dataOut, err := exec.Run("lsblk", args...)
|
||||
if err != nil {
|
||||
glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err)
|
||||
return false, err
|
||||
}
|
||||
output := libstrings.TrimSpace(string(dataOut))
|
||||
return output == "", nil
|
||||
}
|
||||
|
|
@ -19,9 +19,12 @@ package azure_dd
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
)
|
||||
|
||||
type fakeFileInfo struct {
|
||||
|
|
@ -116,9 +119,15 @@ func (handler *fakeIOHandler) ReadFile(filename string) ([]byte, error) {
|
|||
}
|
||||
|
||||
func TestIoHandler(t *testing.T) {
|
||||
disk, err := findDiskByLun(lun, &fakeIOHandler{})
|
||||
// if no disk matches lun, exit
|
||||
if disk != "/dev/"+devName || err != nil {
|
||||
t.Errorf("no data disk found: disk %v err %v", disk, err)
|
||||
disk, err := findDiskByLun(lun, &fakeIOHandler{}, mount.NewOsExec())
|
||||
if runtime.GOOS == "windows" {
|
||||
if err != nil {
|
||||
t.Errorf("no data disk found: disk %v err %v", disk, err)
|
||||
}
|
||||
} else {
|
||||
// if no disk matches lun, exit
|
||||
if disk != "/dev/"+devName || err != nil {
|
||||
t.Errorf("no data disk found: disk %v err %v", disk, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
// +build !linux,!windows
|
||||
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package azure_dd
|
||||
|
||||
import "k8s.io/kubernetes/pkg/util/mount"
|
||||
|
||||
func scsiHostRescan(io ioHandler, exec mount.Exec) {
|
||||
}
|
||||
|
||||
func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
// +build windows
|
||||
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package azure_dd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
)
|
||||
|
||||
func scsiHostRescan(io ioHandler, exec mount.Exec) {
|
||||
cmd := "Update-HostStorageCache"
|
||||
output, err := exec.Run("powershell", "/c", cmd)
|
||||
if err != nil {
|
||||
glog.Errorf("Update-HostStorageCache failed in scsiHostRescan, error: %v, output: %q", err, string(output))
|
||||
}
|
||||
}
|
||||
|
||||
// search Windows disk number by LUN
|
||||
func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error) {
|
||||
cmd := `Get-Disk | select number, location | ConvertTo-Json`
|
||||
output, err := exec.Run("powershell", "/c", cmd)
|
||||
if err != nil {
|
||||
glog.Errorf("Get-Disk failed in findDiskByLun, error: %v, output: %q", err, string(output))
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(output) < 10 {
|
||||
return "", fmt.Errorf("Get-Disk output is too short, output: %q", string(output))
|
||||
}
|
||||
|
||||
var data []map[string]interface{}
|
||||
if err = json.Unmarshal(output, &data); err != nil {
|
||||
glog.Errorf("Get-Disk output is not a json array, output: %q", string(output))
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, v := range data {
|
||||
if jsonLocation, ok := v["location"]; ok {
|
||||
if location, ok := jsonLocation.(string); ok {
|
||||
if !strings.Contains(location, " LUN ") {
|
||||
continue
|
||||
}
|
||||
|
||||
arr := strings.Split(location, " ")
|
||||
arrLen := len(arr)
|
||||
if arrLen < 3 {
|
||||
glog.Warningf("unexpected json structure from Get-Disk, location: %q", jsonLocation)
|
||||
continue
|
||||
}
|
||||
|
||||
glog.V(4).Infof("found a disk, locatin: %q, lun: %q", location, arr[arrLen-1])
|
||||
//last element of location field is LUN number, e.g.
|
||||
// "location": "Integrated : Adapter 3 : Port 0 : Target 0 : LUN 1"
|
||||
l, err := strconv.Atoi(arr[arrLen-1])
|
||||
if err != nil {
|
||||
glog.Warningf("cannot parse element from data structure, location: %q, element: %q", location, arr[arrLen-1])
|
||||
continue
|
||||
}
|
||||
|
||||
if l == lun {
|
||||
glog.V(4).Infof("found a disk and lun, locatin: %q, lun: %d", location, lun)
|
||||
if d, ok := v["number"]; ok {
|
||||
if diskNum, ok := d.(float64); ok {
|
||||
glog.V(2).Infof("azureDisk Mount: got disk number(%d) by LUN(%d)", int(diskNum), lun)
|
||||
return strconv.Itoa(int(diskNum)), nil
|
||||
}
|
||||
glog.Warningf("LUN(%d) found, but could not get disk number(%q), location: %q", lun, d, location)
|
||||
}
|
||||
return "", fmt.Errorf("LUN(%d) found, but could not get disk number, location: %q", lun, location)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) {
|
||||
if err := mount.ValidateDiskNumber(disk); err != nil {
|
||||
glog.Errorf("azureDisk Mount: formatIfNotFormatted failed, err: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru", disk)
|
||||
cmd += " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem NTFS -Confirm:$false"
|
||||
output, err := exec.Run("powershell", "/c", cmd)
|
||||
if err != nil {
|
||||
glog.Errorf("azureDisk Mount: Get-Disk failed, error: %v, output: %q", err, string(output))
|
||||
} else {
|
||||
glog.Infof("azureDisk Mount: Disk successfully formatted, disk: %q, fstype: %q\n", disk, fstype)
|
||||
}
|
||||
}
|
||||
|
|
@ -116,7 +116,7 @@ func (plugin *azureDataDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessM
|
|||
func (plugin *azureDataDiskPlugin) NewAttacher() (volume.Attacher, error) {
|
||||
azure, err := getCloud(plugin.host)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("failed to get azure cloud in NewAttacher, plugin.host : %s", plugin.host.GetHostName())
|
||||
glog.Errorf("failed to get azure cloud in NewAttacher, plugin.host : %s, err:%v", plugin.host.GetHostName(), err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package azure_dd
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/api/core/v1"
|
||||
|
|
@ -82,9 +83,12 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
|
|||
return fmt.Errorf("azureDisk - Not a mounting point for disk %s on %s", diskName, dir)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
||||
glog.Infof("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err)
|
||||
return err
|
||||
if runtime.GOOS != "windows" {
|
||||
// in windows, we will use mklink to mount, will MkdirAll in Mount func
|
||||
if err := os.MkdirAll(dir, 0750); err != nil {
|
||||
glog.Errorf("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
options := []string{"bind"}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package azure_file
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
|
@ -210,15 +211,24 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error {
|
|||
if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.secretNamespace, b.secretName); err != nil {
|
||||
return err
|
||||
}
|
||||
os.MkdirAll(dir, 0700)
|
||||
|
||||
source := fmt.Sprintf("//%s.file.%s/%s", accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), b.shareName)
|
||||
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
|
||||
options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)}
|
||||
if b.readOnly {
|
||||
options = append(options, "ro")
|
||||
mountOptions := []string{}
|
||||
source := ""
|
||||
osSeparator := string(os.PathSeparator)
|
||||
source = fmt.Sprintf("%s%s%s.file.%s%s%s", osSeparator, osSeparator, accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), osSeparator, b.shareName)
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
mountOptions = []string{accountName, accountKey}
|
||||
} else {
|
||||
os.MkdirAll(dir, 0700)
|
||||
// parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/
|
||||
options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)}
|
||||
if b.readOnly {
|
||||
options = append(options, "ro")
|
||||
}
|
||||
mountOptions = volume.JoinMountOptions(b.mountOptions, options)
|
||||
}
|
||||
mountOptions := volume.JoinMountOptions(b.mountOptions, options)
|
||||
|
||||
err = b.mounter.Mount(source, dir, "cifs", mountOptions)
|
||||
if err != nil {
|
||||
notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)
|
||||
|
|
|
|||
Loading…
Reference in New Issue