diff --git a/pkg/util/mount/BUILD b/pkg/util/mount/BUILD index 026e82c169..4c87dbd2a4 100644 --- a/pkg/util/mount/BUILD +++ b/pkg/util/mount/BUILD @@ -20,6 +20,9 @@ go_library( "mount_linux.go", "nsenter_mount.go", ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "mount_windows.go", + ], "//conditions:default": [], }), deps = [ @@ -44,6 +47,9 @@ go_test( "mount_linux_test.go", "nsenter_mount_test.go", ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "mount_windows_test.go", + ], "//conditions:default": [], }), library = ":go_default_library", diff --git a/pkg/util/mount/mount.go b/pkg/util/mount/mount.go index f5bf4454f8..5be3ca40af 100644 --- a/pkg/util/mount/mount.go +++ b/pkg/util/mount/mount.go @@ -278,3 +278,28 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) { } return notMnt, nil } + +// isBind detects whether a bind mount is being requested and makes the remount options to +// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. +// The list equals: +// options - 'bind' + 'remount' (no duplicate) +func isBind(options []string) (bool, []string) { + bindRemountOpts := []string{"remount"} + bind := false + + if len(options) != 0 { + for _, option := range options { + switch option { + case "bind": + bind = true + break + case "remount": + break + default: + bindRemountOpts = append(bindRemountOpts, option) + } + } + } + + return bind, bindRemountOpts +} diff --git a/pkg/util/mount/mount_linux.go b/pkg/util/mount/mount_linux.go index da7b47912f..0efc5d6801 100644 --- a/pkg/util/mount/mount_linux.go +++ b/pkg/util/mount/mount_linux.go @@ -94,31 +94,6 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options) } -// isBind detects whether a bind mount is being requested and makes the remount options to -// use in case of bind mount, due to the fact that bind mount doesn't respect mount options. -// The list equals: -// options - 'bind' + 'remount' (no duplicate) -func isBind(options []string) (bool, []string) { - bindRemountOpts := []string{"remount"} - bind := false - - if len(options) != 0 { - for _, option := range options { - switch option { - case "bind": - bind = true - break - case "remount": - break - default: - bindRemountOpts = append(bindRemountOpts, option) - } - } - } - - return bind, bindRemountOpts -} - // doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used. func (m *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error { mountArgs := makeMountArgs(source, target, fstype, options) diff --git a/pkg/util/mount/mount_unsupported.go b/pkg/util/mount/mount_unsupported.go index 383a9d4d9a..936c30aba5 100644 --- a/pkg/util/mount/mount_unsupported.go +++ b/pkg/util/mount/mount_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux +// +build !linux,!windows /* Copyright 2014 The Kubernetes Authors. diff --git a/pkg/util/mount/mount_windows.go b/pkg/util/mount/mount_windows.go new file mode 100644 index 0000000000..63294df9f0 --- /dev/null +++ b/pkg/util/mount/mount_windows.go @@ -0,0 +1,206 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/golang/glog" +) + +// Mounter provides the default implementation of mount.Interface +// for the windows platform. This implementation assumes that the +// kubelet is running in the host's root mount namespace. +type Mounter struct { + mounterPath string +} + +// New returns a mount.Interface for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) Interface { + return &Mounter{ + mounterPath: mounterPath, + } +} + +// Mount : mounts source to target as NTFS with given options. +func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { + target = normalizeWindowsPath(target) + + if source == "tmpfs" { + glog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options) + return os.MkdirAll(target, 0755) + } + + parentDir := filepath.Dir(target) + if err := os.MkdirAll(parentDir, 0755); err != nil { + return err + } + + glog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", + options, source, target, fstype) + bindSource := "" + + // tell it's going to mount azure disk or azure file according to options + if bind, _ := isBind(options); bind { + // mount azure disk + bindSource = normalizeWindowsPath(source) + } else { + if len(options) < 2 { + glog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", + options, len(options), source, target) + return nil + } + + // empty implementation for mounting azure file + return os.MkdirAll(target, 0755) + } + + if output, err := exec.Command("cmd", "/c", "mklink", "/D", target, bindSource).CombinedOutput(); err != nil { + glog.Errorf("mklink failed: %v, source(%q) target(%q) output: %q", err, bindSource, target, string(output)) + return err + } + + return nil +} + +// Unmount unmounts the target. +func (mounter *Mounter) Unmount(target string) error { + glog.V(4).Infof("azureMount: Unmount target (%q)", target) + target = normalizeWindowsPath(target) + if output, err := exec.Command("cmd", "/c", "rmdir", target).CombinedOutput(); err != nil { + glog.Errorf("rmdir failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +// List returns a list of all mounted filesystems. todo +func (mounter *Mounter) List() ([]MountPoint, error) { + return []MountPoint{}, nil +} + +// IsMountPointMatch determines if the mountpoint matches the dir +func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool { + return mp.Path == dir +} + +// IsNotMountPoint determines if a directory is a mountpoint. +func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) { + return IsNotMountPoint(mounter, dir) +} + +// IsLikelyNotMountPoint determines if a directory is not a mountpoint. +func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { + stat, err := os.Lstat(file) + if err != nil { + return true, err + } + // If current file is a symlink, then it is a mountpoint. + if stat.Mode()&os.ModeSymlink != 0 { + return false, nil + } + + return true, nil +} + +// GetDeviceNameFromMount given a mnt point, find the device +func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return getDeviceNameFromMount(mounter, mountPath, pluginDir) +} + +// DeviceOpened determines if the device is in use elsewhere +func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) { + return false, nil +} + +// PathIsDevice determines if a path is a device. +func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) { + return false, nil +} + +// MakeRShared checks that given path is on a mount with 'rshared' mount +// propagation. Empty implementation here. +func (mounter *Mounter) MakeRShared(path string) error { + return nil +} + +func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error { + // Try to mount the disk + glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target) + + if err := ValidateDiskNumber(source); err != nil { + glog.Errorf("azureMount: formatAndMount failed, err: %v\n", err) + return err + } + + driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec) + if err != nil { + return err + } + driverPath := driveLetter + ":" + target = normalizeWindowsPath(target) + glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, driverPath, target) + if output, err := mounter.Exec.Run("cmd", "/c", "mklink", "/D", target, driverPath); err != nil { + glog.Errorf("mklink failed: %v, output: %q", err, string(output)) + return err + } + return nil +} + +func normalizeWindowsPath(path string) string { + normalizedPath := strings.Replace(path, "/", "\\", -1) + if strings.HasPrefix(normalizedPath, "\\") { + normalizedPath = "c:" + normalizedPath + } + return normalizedPath +} + +// ValidateDiskNumber : disk number should be a number in [0, 99] +func ValidateDiskNumber(disk string) error { + diskNum, err := strconv.Atoi(disk) + if err != nil { + return fmt.Errorf("wrong disk number format: %q, err:%v", disk, err) + } + + if diskNum < 0 || diskNum > 99 { + return fmt.Errorf("disk number out of range: %q", disk) + } + + return nil +} + +// Get drive letter according to windows disk number +func getDriveLetterByDiskNumber(diskNum string, exec Exec) (string, error) { + cmd := fmt.Sprintf("(Get-Partition -DiskNumber %s).DriveLetter", diskNum) + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + return "", fmt.Errorf("azureMount: Get Drive Letter failed: %v, output: %q", err, string(output)) + } + if len(string(output)) < 1 { + return "", fmt.Errorf("azureMount: Get Drive Letter failed, output is empty") + } + return string(output)[:1], nil +} diff --git a/pkg/util/mount/mount_windows_test.go b/pkg/util/mount/mount_windows_test.go new file mode 100644 index 0000000000..6be0fc43e1 --- /dev/null +++ b/pkg/util/mount/mount_windows_test.go @@ -0,0 +1,71 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mount + +import ( + "testing" +) + +func TestGetAvailableDriveLetter(t *testing.T) { + if _, err := getAvailableDriveLetter(); err != nil { + t.Errorf("getAvailableDriveLetter test failed : %v", err) + } +} + +func TestNormalizeWindowsPath(t *testing.T) { + path := `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4/volumes/kubernetes.io~azure-disk` + normalizedPath := normalizeWindowsPath(path) + if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` { + t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) + } + + path = `/var/lib/kubelet/pods/146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` + normalizedPath = normalizeWindowsPath(path) + if normalizedPath != `c:\var\lib\kubelet\pods\146f8428-83e7-11e7-8dd4-000d3a31dac4\volumes\kubernetes.io~azure-disk` { + t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) + } + + path = `/` + normalizedPath = normalizeWindowsPath(path) + if normalizedPath != `c:\` { + t.Errorf("normizeWindowsPath test failed, normalizedPath : %q", normalizedPath) + } +} + +func TestValidateDiskNumber(t *testing.T) { + diskNum := "0" + if err := ValidateDiskNumber(diskNum); err != nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } + + diskNum = "99" + if err := ValidateDiskNumber(diskNum); err != nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } + + diskNum = "ab" + if err := ValidateDiskNumber(diskNum); err == nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } + + diskNum = "100" + if err := ValidateDiskNumber(diskNum); err == nil { + t.Errorf("TestValidateDiskNumber test failed, disk number : %s", diskNum) + } +} diff --git a/pkg/volume/azure_dd/BUILD b/pkg/volume/azure_dd/BUILD index ba57ee1ea6..88f1a017e5 100644 --- a/pkg/volume/azure_dd/BUILD +++ b/pkg/volume/azure_dd/BUILD @@ -11,10 +11,19 @@ go_library( srcs = [ "attacher.go", "azure_common.go", + "azure_common_unsupported.go", "azure_dd.go", "azure_mounter.go", "azure_provision.go", - ], + ] + select({ + "@io_bazel_rules_go//go/platform:linux_amd64": [ + "azure_common_linux.go", + ], + "@io_bazel_rules_go//go/platform:windows_amd64": [ + "azure_common_windows.go", + ], + "//conditions:default": [], + }), deps = [ "//pkg/api:go_default_library", "//pkg/cloudprovider:go_default_library", @@ -58,6 +67,7 @@ go_test( ], library = ":go_default_library", deps = [ + "//pkg/util/mount:go_default_library", "//pkg/volume:go_default_library", "//pkg/volume/testing:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", diff --git a/pkg/volume/azure_dd/attacher.go b/pkg/volume/azure_dd/attacher.go index 0ef1c6c047..11377b0b0e 100644 --- a/pkg/volume/azure_dd/attacher.go +++ b/pkg/volume/azure_dd/attacher.go @@ -20,6 +20,8 @@ import ( "fmt" "os" "path" + "path/filepath" + "runtime" "strconv" "strings" "time" @@ -162,15 +164,17 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, return "", err } + exec := a.plugin.host.GetExec(a.plugin.GetPluginName()) + io := &osIOHandler{} - scsiHostRescan(io) + scsiHostRescan(io, exec) diskName := volumeSource.DiskName nodeName := a.plugin.host.GetHostName() newDevicePath := "" err = wait.Poll(1*time.Second, timeout, func() (bool, error) { - if newDevicePath, err = findDiskByLun(lun, io); err != nil { + if newDevicePath, err = findDiskByLun(lun, io, exec); err != nil { return false, fmt.Errorf("azureDisk - WaitForAttach ticker failed node (%s) disk (%s) lun(%v) err(%s)", nodeName, diskName, lun, err) } @@ -179,7 +183,7 @@ func (a *azureDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath string, // the curent sequence k8s uses for unformated disk (check-disk, mount, fail, mkfs.extX) hangs on // Azure Managed disk scsi interface. this is a hack and will be replaced once we identify and solve // the root case on Azure. - formatIfNotFormatted(newDevicePath, *volumeSource.FSType, a.plugin.host.GetExec(a.plugin.GetPluginName())) + formatIfNotFormatted(newDevicePath, *volumeSource.FSType, exec) return true, nil } @@ -214,7 +218,12 @@ func (attacher *azureDiskAttacher) MountDevice(spec *volume.Spec, devicePath str if err != nil { if os.IsNotExist(err) { - if err := os.MkdirAll(deviceMountPath, 0750); err != nil { + dir := deviceMountPath + if runtime.GOOS == "windows" { + // in windows, as we use mklink, only need to MkdirAll for parent directory + dir = filepath.Dir(deviceMountPath) + } + if err := os.MkdirAll(dir, 0750); err != nil { return fmt.Errorf("azureDisk - mountDevice:CreateDirectory failed with %s", err) } notMnt = true diff --git a/pkg/volume/azure_dd/azure_common.go b/pkg/volume/azure_dd/azure_common.go index ec3d69fb76..157be0d422 100644 --- a/pkg/volume/azure_dd/azure_common.go +++ b/pkg/volume/azure_dd/azure_common.go @@ -21,11 +21,9 @@ import ( "io/ioutil" "os" "path" - "strconv" libstrings "strings" storage "github.com/Azure/azure-sdk-for-go/arm/storage" - "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -179,153 +177,6 @@ func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) { return ioutil.ReadFile(filename) } -// exclude those used by azure as resource and OS root in /dev/disk/azure -func listAzureDiskPath(io ioHandler) []string { - azureDiskPath := "/dev/disk/azure/" - var azureDiskList []string - if dirs, err := io.ReadDir(azureDiskPath); err == nil { - for _, f := range dirs { - name := f.Name() - diskPath := azureDiskPath + name - if link, linkErr := io.Readlink(diskPath); linkErr == nil { - sd := link[(libstrings.LastIndex(link, "/") + 1):] - azureDiskList = append(azureDiskList, sd) - } - } - } - glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList) - return azureDiskList -} - -func scsiHostRescan(io ioHandler) { - scsi_path := "/sys/class/scsi_host/" - if dirs, err := io.ReadDir(scsi_path); err == nil { - for _, f := range dirs { - name := scsi_path + f.Name() + "/scan" - data := []byte("- - -") - if err = io.WriteFile(name, data, 0666); err != nil { - glog.Warningf("failed to rescan scsi host %s", name) - } - } - } else { - glog.Warningf("failed to read %s, err %v", scsi_path, err) - } -} - -func findDiskByLun(lun int, io ioHandler) (string, error) { - azureDisks := listAzureDiskPath(io) - return findDiskByLunWithConstraint(lun, io, azureDisks) -} - -// finds a device mounted to "current" node -func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) { - var err error - sys_path := "/sys/bus/scsi/devices" - if dirs, err := io.ReadDir(sys_path); err == nil { - for _, f := range dirs { - name := f.Name() - // look for path like /sys/bus/scsi/devices/3:0:0:1 - arr := libstrings.Split(name, ":") - if len(arr) < 4 { - continue - } - // extract LUN from the path. - // LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1 - l, err := strconv.Atoi(arr[3]) - if err != nil { - // unknown path format, continue to read the next one - glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err) - continue - } - if lun == l { - // find the matching LUN - // read vendor and model to ensure it is a VHD disk - vendorPath := path.Join(sys_path, name, "vendor") - vendorBytes, err := io.ReadFile(vendorPath) - if err != nil { - glog.Errorf("failed to read device vendor, err: %v", err) - continue - } - vendor := libstrings.TrimSpace(string(vendorBytes)) - if libstrings.ToUpper(vendor) != "MSFT" { - glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor) - continue - } - - modelPath := path.Join(sys_path, name, "model") - modelBytes, err := io.ReadFile(modelPath) - if err != nil { - glog.Errorf("failed to read device model, err: %v", err) - continue - } - model := libstrings.TrimSpace(string(modelBytes)) - if libstrings.ToUpper(model) != "VIRTUAL DISK" { - glog.V(4).Infof("model doesn't match VHD, got %s", model) - continue - } - - // find a disk, validate name - dir := path.Join(sys_path, name, "block") - if dev, err := io.ReadDir(dir); err == nil { - found := false - for _, diskName := range azureDisks { - glog.V(12).Infof("azure disk - validating disk %q with sys disk %q", dev[0].Name(), diskName) - if string(dev[0].Name()) == diskName { - found = true - break - } - } - if !found { - return "/dev/" + dev[0].Name(), nil - } - } - } - } - } - return "", err -} - -func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { - notFormatted, err := diskLooksUnformatted(disk, exec) - if err == nil && notFormatted { - args := []string{disk} - // Disk is unformatted so format it. - // Use 'ext4' as the default - if len(fstype) == 0 { - fstype = "ext4" - } - if fstype == "ext4" || fstype == "ext3" { - args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk} - } - glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args) - - _, err := exec.Run("mkfs."+fstype, args...) - if err == nil { - // the disk has been formatted successfully try to mount it again. - glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt") - } - glog.Warningf("azureDisk - format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", disk, fstype, "tt", "o", err) - } else { - if err != nil { - glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err) - } else { - glog.Infof("azureDisk - Disk %s already formatted, will not format", disk) - } - } -} - -func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) { - args := []string{"-nd", "-o", "FSTYPE", disk} - glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) - dataOut, err := exec.Run("lsblk", args...) - if err != nil { - glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) - return false, err - } - output := libstrings.TrimSpace(string(dataOut)) - return output == "", nil -} - func getDiskController(host volume.VolumeHost) (DiskController, error) { cloudProvider := host.GetCloudProvider() az, ok := cloudProvider.(*azure.Cloud) diff --git a/pkg/volume/azure_dd/azure_common_linux.go b/pkg/volume/azure_dd/azure_common_linux.go new file mode 100644 index 0000000000..d5d0caa1e0 --- /dev/null +++ b/pkg/volume/azure_dd/azure_common_linux.go @@ -0,0 +1,175 @@ +// +build linux + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure_dd + +import ( + "path" + "strconv" + libstrings "strings" + + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/util/mount" +) + +// exclude those used by azure as resource and OS root in /dev/disk/azure +func listAzureDiskPath(io ioHandler) []string { + azureDiskPath := "/dev/disk/azure/" + var azureDiskList []string + if dirs, err := io.ReadDir(azureDiskPath); err == nil { + for _, f := range dirs { + name := f.Name() + diskPath := azureDiskPath + name + if link, linkErr := io.Readlink(diskPath); linkErr == nil { + sd := link[(libstrings.LastIndex(link, "/") + 1):] + azureDiskList = append(azureDiskList, sd) + } + } + } + glog.V(12).Infof("Azure sys disks paths: %v", azureDiskList) + return azureDiskList +} + +func scsiHostRescan(io ioHandler, exec mount.Exec) { + scsi_path := "/sys/class/scsi_host/" + if dirs, err := io.ReadDir(scsi_path); err == nil { + for _, f := range dirs { + name := scsi_path + f.Name() + "/scan" + data := []byte("- - -") + if err = io.WriteFile(name, data, 0666); err != nil { + glog.Warningf("failed to rescan scsi host %s", name) + } + } + } else { + glog.Warningf("failed to read %s, err %v", scsi_path, err) + } +} + +func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) { + azureDisks := listAzureDiskPath(io) + return findDiskByLunWithConstraint(lun, io, azureDisks) +} + +// finds a device mounted to "current" node +func findDiskByLunWithConstraint(lun int, io ioHandler, azureDisks []string) (string, error) { + var err error + sys_path := "/sys/bus/scsi/devices" + if dirs, err := io.ReadDir(sys_path); err == nil { + for _, f := range dirs { + name := f.Name() + // look for path like /sys/bus/scsi/devices/3:0:0:1 + arr := libstrings.Split(name, ":") + if len(arr) < 4 { + continue + } + // extract LUN from the path. + // LUN is the last index of the array, i.e. 1 in /sys/bus/scsi/devices/3:0:0:1 + l, err := strconv.Atoi(arr[3]) + if err != nil { + // unknown path format, continue to read the next one + glog.V(4).Infof("azure disk - failed to parse lun from %v (%v), err %v", arr[3], name, err) + continue + } + if lun == l { + // find the matching LUN + // read vendor and model to ensure it is a VHD disk + vendorPath := path.Join(sys_path, name, "vendor") + vendorBytes, err := io.ReadFile(vendorPath) + if err != nil { + glog.Errorf("failed to read device vendor, err: %v", err) + continue + } + vendor := libstrings.TrimSpace(string(vendorBytes)) + if libstrings.ToUpper(vendor) != "MSFT" { + glog.V(4).Infof("vendor doesn't match VHD, got %s", vendor) + continue + } + + modelPath := path.Join(sys_path, name, "model") + modelBytes, err := io.ReadFile(modelPath) + if err != nil { + glog.Errorf("failed to read device model, err: %v", err) + continue + } + model := libstrings.TrimSpace(string(modelBytes)) + if libstrings.ToUpper(model) != "VIRTUAL DISK" { + glog.V(4).Infof("model doesn't match VHD, got %s", model) + continue + } + + // find a disk, validate name + dir := path.Join(sys_path, name, "block") + if dev, err := io.ReadDir(dir); err == nil { + found := false + for _, diskName := range azureDisks { + glog.V(12).Infof("azure disk - validating disk %q with sys disk %q", dev[0].Name(), diskName) + if string(dev[0].Name()) == diskName { + found = true + break + } + } + if !found { + return "/dev/" + dev[0].Name(), nil + } + } + } + } + } + return "", err +} + +func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { + notFormatted, err := diskLooksUnformatted(disk, exec) + if err == nil && notFormatted { + args := []string{disk} + // Disk is unformatted so format it. + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + if fstype == "ext4" || fstype == "ext3" { + args = []string{"-E", "lazy_itable_init=0,lazy_journal_init=0", "-F", disk} + } + glog.Infof("azureDisk - Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", disk, fstype, args) + + _, err := exec.Run("mkfs."+fstype, args...) + if err == nil { + // the disk has been formatted successfully try to mount it again. + glog.Infof("azureDisk - Disk successfully formatted (mkfs): %s - %s %s", fstype, disk, "tt") + } + glog.Warningf("azureDisk - format of disk %q failed: type:(%q) target:(%q) options:(%q)error:(%v)", disk, fstype, "tt", "o", err) + } else { + if err != nil { + glog.Warningf("azureDisk - Failed to check if the disk %s formatted with error %s, will attach anyway", disk, err) + } else { + glog.Infof("azureDisk - Disk %s already formatted, will not format", disk) + } + } +} + +func diskLooksUnformatted(disk string, exec mount.Exec) (bool, error) { + args := []string{"-nd", "-o", "FSTYPE", disk} + glog.V(4).Infof("Attempting to determine if disk %q is formatted using lsblk with args: (%v)", disk, args) + dataOut, err := exec.Run("lsblk", args...) + if err != nil { + glog.Errorf("Could not determine if disk %q is formatted (%v)", disk, err) + return false, err + } + output := libstrings.TrimSpace(string(dataOut)) + return output == "", nil +} diff --git a/pkg/volume/azure_dd/azure_common_test.go b/pkg/volume/azure_dd/azure_common_test.go index 00a35b98ca..5c545257ff 100644 --- a/pkg/volume/azure_dd/azure_common_test.go +++ b/pkg/volume/azure_dd/azure_common_test.go @@ -19,9 +19,12 @@ package azure_dd import ( "fmt" "os" + "runtime" "strings" "testing" "time" + + "k8s.io/kubernetes/pkg/util/mount" ) type fakeFileInfo struct { @@ -116,9 +119,15 @@ func (handler *fakeIOHandler) ReadFile(filename string) ([]byte, error) { } func TestIoHandler(t *testing.T) { - disk, err := findDiskByLun(lun, &fakeIOHandler{}) - // if no disk matches lun, exit - if disk != "/dev/"+devName || err != nil { - t.Errorf("no data disk found: disk %v err %v", disk, err) + disk, err := findDiskByLun(lun, &fakeIOHandler{}, mount.NewOsExec()) + if runtime.GOOS == "windows" { + if err != nil { + t.Errorf("no data disk found: disk %v err %v", disk, err) + } + } else { + // if no disk matches lun, exit + if disk != "/dev/"+devName || err != nil { + t.Errorf("no data disk found: disk %v err %v", disk, err) + } } } diff --git a/pkg/volume/azure_dd/azure_common_unsupported.go b/pkg/volume/azure_dd/azure_common_unsupported.go new file mode 100644 index 0000000000..6710af8fcb --- /dev/null +++ b/pkg/volume/azure_dd/azure_common_unsupported.go @@ -0,0 +1,31 @@ +// +build !linux,!windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure_dd + +import "k8s.io/kubernetes/pkg/util/mount" + +func scsiHostRescan(io ioHandler, exec mount.Exec) { +} + +func findDiskByLun(lun int, io ioHandler, exec mount.Exec) (string, error) { + return "", nil +} + +func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { +} diff --git a/pkg/volume/azure_dd/azure_common_windows.go b/pkg/volume/azure_dd/azure_common_windows.go new file mode 100644 index 0000000000..e22aff4523 --- /dev/null +++ b/pkg/volume/azure_dd/azure_common_windows.go @@ -0,0 +1,114 @@ +// +build windows + +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure_dd + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/golang/glog" + + "k8s.io/kubernetes/pkg/util/mount" +) + +func scsiHostRescan(io ioHandler, exec mount.Exec) { + cmd := "Update-HostStorageCache" + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + glog.Errorf("Update-HostStorageCache failed in scsiHostRescan, error: %v, output: %q", err, string(output)) + } +} + +// search Windows disk number by LUN +func findDiskByLun(lun int, iohandler ioHandler, exec mount.Exec) (string, error) { + cmd := `Get-Disk | select number, location | ConvertTo-Json` + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + glog.Errorf("Get-Disk failed in findDiskByLun, error: %v, output: %q", err, string(output)) + return "", err + } + + if len(output) < 10 { + return "", fmt.Errorf("Get-Disk output is too short, output: %q", string(output)) + } + + var data []map[string]interface{} + if err = json.Unmarshal(output, &data); err != nil { + glog.Errorf("Get-Disk output is not a json array, output: %q", string(output)) + return "", err + } + + for _, v := range data { + if jsonLocation, ok := v["location"]; ok { + if location, ok := jsonLocation.(string); ok { + if !strings.Contains(location, " LUN ") { + continue + } + + arr := strings.Split(location, " ") + arrLen := len(arr) + if arrLen < 3 { + glog.Warningf("unexpected json structure from Get-Disk, location: %q", jsonLocation) + continue + } + + glog.V(4).Infof("found a disk, locatin: %q, lun: %q", location, arr[arrLen-1]) + //last element of location field is LUN number, e.g. + // "location": "Integrated : Adapter 3 : Port 0 : Target 0 : LUN 1" + l, err := strconv.Atoi(arr[arrLen-1]) + if err != nil { + glog.Warningf("cannot parse element from data structure, location: %q, element: %q", location, arr[arrLen-1]) + continue + } + + if l == lun { + glog.V(4).Infof("found a disk and lun, locatin: %q, lun: %d", location, lun) + if d, ok := v["number"]; ok { + if diskNum, ok := d.(float64); ok { + glog.V(2).Infof("azureDisk Mount: got disk number(%d) by LUN(%d)", int(diskNum), lun) + return strconv.Itoa(int(diskNum)), nil + } + glog.Warningf("LUN(%d) found, but could not get disk number(%q), location: %q", lun, d, location) + } + return "", fmt.Errorf("LUN(%d) found, but could not get disk number, location: %q", lun, location) + } + } + } + } + + return "", nil +} + +func formatIfNotFormatted(disk string, fstype string, exec mount.Exec) { + if err := mount.ValidateDiskNumber(disk); err != nil { + glog.Errorf("azureDisk Mount: formatIfNotFormatted failed, err: %v\n", err) + return + } + + cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru", disk) + cmd += " | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem NTFS -Confirm:$false" + output, err := exec.Run("powershell", "/c", cmd) + if err != nil { + glog.Errorf("azureDisk Mount: Get-Disk failed, error: %v, output: %q", err, string(output)) + } else { + glog.Infof("azureDisk Mount: Disk successfully formatted, disk: %q, fstype: %q\n", disk, fstype) + } +} diff --git a/pkg/volume/azure_dd/azure_dd.go b/pkg/volume/azure_dd/azure_dd.go index 80d9b98b17..bb45bf3b4d 100644 --- a/pkg/volume/azure_dd/azure_dd.go +++ b/pkg/volume/azure_dd/azure_dd.go @@ -116,7 +116,7 @@ func (plugin *azureDataDiskPlugin) GetAccessModes() []v1.PersistentVolumeAccessM func (plugin *azureDataDiskPlugin) NewAttacher() (volume.Attacher, error) { azure, err := getCloud(plugin.host) if err != nil { - glog.V(4).Infof("failed to get azure cloud in NewAttacher, plugin.host : %s", plugin.host.GetHostName()) + glog.Errorf("failed to get azure cloud in NewAttacher, plugin.host : %s, err:%v", plugin.host.GetHostName(), err) return nil, err } diff --git a/pkg/volume/azure_dd/azure_mounter.go b/pkg/volume/azure_dd/azure_mounter.go index 2af044c369..bf2d59ebe2 100644 --- a/pkg/volume/azure_dd/azure_mounter.go +++ b/pkg/volume/azure_dd/azure_mounter.go @@ -19,6 +19,7 @@ package azure_dd import ( "fmt" "os" + "runtime" "github.com/golang/glog" "k8s.io/api/core/v1" @@ -82,9 +83,12 @@ func (m *azureDiskMounter) SetUpAt(dir string, fsGroup *int64) error { return fmt.Errorf("azureDisk - Not a mounting point for disk %s on %s", diskName, dir) } - if err := os.MkdirAll(dir, 0750); err != nil { - glog.Infof("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err) - return err + if runtime.GOOS != "windows" { + // in windows, we will use mklink to mount, will MkdirAll in Mount func + if err := os.MkdirAll(dir, 0750); err != nil { + glog.Errorf("azureDisk - mkdir failed on disk %s on dir: %s (%v)", diskName, dir, err) + return err + } } options := []string{"bind"} diff --git a/pkg/volume/azure_file/azure_file.go b/pkg/volume/azure_file/azure_file.go index 2e51e46853..e418877c07 100644 --- a/pkg/volume/azure_file/azure_file.go +++ b/pkg/volume/azure_file/azure_file.go @@ -19,6 +19,7 @@ package azure_file import ( "fmt" "os" + "runtime" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -210,15 +211,24 @@ func (b *azureFileMounter) SetUpAt(dir string, fsGroup *int64) error { if accountName, accountKey, err = b.util.GetAzureCredentials(b.plugin.host, b.secretNamespace, b.secretName); err != nil { return err } - os.MkdirAll(dir, 0700) - source := fmt.Sprintf("//%s.file.%s/%s", accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), b.shareName) - // parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/ - options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)} - if b.readOnly { - options = append(options, "ro") + mountOptions := []string{} + source := "" + osSeparator := string(os.PathSeparator) + source = fmt.Sprintf("%s%s%s.file.%s%s%s", osSeparator, osSeparator, accountName, getStorageEndpointSuffix(b.plugin.host.GetCloudProvider()), osSeparator, b.shareName) + + if runtime.GOOS == "windows" { + mountOptions = []string{accountName, accountKey} + } else { + os.MkdirAll(dir, 0700) + // parameters suggested by https://azure.microsoft.com/en-us/documentation/articles/storage-how-to-use-files-linux/ + options := []string{fmt.Sprintf("vers=3.0,username=%s,password=%s,dir_mode=0700,file_mode=0700", accountName, accountKey)} + if b.readOnly { + options = append(options, "ro") + } + mountOptions = volume.JoinMountOptions(b.mountOptions, options) } - mountOptions := volume.JoinMountOptions(b.mountOptions, options) + err = b.mounter.Mount(source, dir, "cifs", mountOptions) if err != nil { notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir)