diff --git a/pkg/cloudprovider/providers/vsphere/BUILD b/pkg/cloudprovider/providers/vsphere/BUILD index 45e49e2a73..a1a173ab35 100644 --- a/pkg/cloudprovider/providers/vsphere/BUILD +++ b/pkg/cloudprovider/providers/vsphere/BUILD @@ -32,6 +32,7 @@ go_library( "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/node/helpers:go_default_library", "//staging/src/k8s.io/cloud-provider/volume/helpers:go_default_library", + "//vendor/github.com/vmware/govmomi/find:go_default_library", "//vendor/github.com/vmware/govmomi/object:go_default_library", "//vendor/github.com/vmware/govmomi/property:go_default_library", "//vendor/github.com/vmware/govmomi/vapi/rest:go_default_library", diff --git a/pkg/cloudprovider/providers/vsphere/nodemanager.go b/pkg/cloudprovider/providers/vsphere/nodemanager.go index f245e336f6..ceb23e4094 100644 --- a/pkg/cloudprovider/providers/vsphere/nodemanager.go +++ b/pkg/cloudprovider/providers/vsphere/nodemanager.go @@ -39,6 +39,11 @@ type NodeInfo struct { zone *cloudprovider.Zone } +func (n NodeInfo) String() string { + return fmt.Sprintf("{datacenter: %v, vm: %v, vcServer: %s, vmUUID: %s, zone: %v}", + *n.dataCenter, n.vm.Reference(), n.vcServer, n.vmUUID, *n.zone) +} + type NodeManager struct { // TODO: replace map with concurrent map when k8s supports go v1.9 diff --git a/pkg/cloudprovider/providers/vsphere/vclib/constants.go b/pkg/cloudprovider/providers/vsphere/vclib/constants.go index a8a4e5f420..2209898c6f 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/constants.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/constants.go @@ -52,6 +52,7 @@ const ( DatacenterType = "Datacenter" ClusterComputeResourceType = "ClusterComputeResource" HostSystemType = "HostSystem" + NameProperty = "name" ) // Test Constants diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go index bbd60cc620..c757be1d5d 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datacenter.go @@ -187,6 +187,28 @@ func (dc *Datacenter) GetDatastoreByName(ctx context.Context, name string) (*Dat return &datastore, nil } +// GetDatastoreInfoByName gets the Datastore object for the given datastore name +func (dc *Datacenter) GetDatastoreInfoByName(ctx context.Context, name string) (*DatastoreInfo, error) { + finder := getFinder(dc) + ds, err := finder.Datastore(ctx, name) + if err != nil { + klog.Errorf("Failed while searching for datastore: %s. err: %+v", name, err) + return nil, err + } + datastore := Datastore{ds, dc} + var dsMo mo.Datastore + pc := property.DefaultCollector(dc.Client()) + properties := []string{DatastoreInfoProperty} + err = pc.RetrieveOne(ctx, ds.Reference(), properties, &dsMo) + if err != nil { + klog.Errorf("Failed to get Datastore managed objects from datastore reference."+ + " dsRef: %+v, err: %+v", ds.Reference(), err) + return nil, err + } + klog.V(9).Infof("Result dsMo: %+v", dsMo) + return &DatastoreInfo{Datastore: &datastore, Info: dsMo.Info.GetDatastoreInfo()}, nil +} + // GetResourcePool gets the resource pool for the given path func (dc *Datacenter) GetResourcePool(ctx context.Context, resourcePoolPath string) (*object.ResourcePool, error) { finder := getFinder(dc) diff --git a/pkg/cloudprovider/providers/vsphere/vclib/datastore.go b/pkg/cloudprovider/providers/vsphere/vclib/datastore.go index 859e56239f..00fc1f42e2 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/datastore.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/datastore.go @@ -96,8 +96,14 @@ func (ds *Datastore) GetDatastoreHostMounts(ctx context.Context) ([]types.Manage return nil, err } hosts := make([]types.ManagedObjectReference, len(dsMo.Host)) - for _, dsHostMount := range dsMo.Host { - hosts = append(hosts, dsHostMount.Key) + for i, dsHostMount := range dsMo.Host { + hosts[i] = dsHostMount.Key } return hosts, nil } + +// Exists returns whether the given file exists in this datastore +func (ds *Datastore) Exists(ctx context.Context, file string) bool { + _, err := ds.Datastore.Stat(ctx, file) + return err == nil +} diff --git a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go index 637ac514bf..b48e303f36 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers/vmdm.go @@ -211,7 +211,7 @@ func (vmdisk vmDiskManager) createDummyVM(ctx context.Context, datacenter *vclib } // CleanUpDummyVMs deletes stale dummyVM's -func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datacenter) error { +func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder) error { vmList, err := folder.GetVirtualMachines(ctx) if err != nil { klog.V(4).Infof("Failed to get virtual machines in the kubernetes cluster: %s, err: %+v", folder.InventoryPath, err) @@ -230,7 +230,7 @@ func CleanUpDummyVMs(ctx context.Context, folder *vclib.Folder, dc *vclib.Datace continue } if strings.HasPrefix(vmName, vclib.DummyVMPrefixName) { - vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(dc.Client(), vm.Reference()), Datacenter: dc} + vmObj := vclib.VirtualMachine{VirtualMachine: object.NewVirtualMachine(folder.Client(), vm.Reference())} dummyVMList = append(dummyVMList, &vmObj) } } diff --git a/pkg/cloudprovider/providers/vsphere/vclib/pbm.go b/pkg/cloudprovider/providers/vsphere/vclib/pbm.go index 8070f20042..05eff9eb3c 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/pbm.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/pbm.go @@ -85,7 +85,7 @@ func (pbmClient *PbmClient) IsDatastoreCompatible(ctx context.Context, storagePo // GetCompatibleDatastores filters and returns compatible list of datastores for given storage policy id // For Non Compatible Datastores, fault message with the Datastore Name is also returned -func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, dc *Datacenter, storagePolicyID string, datastores []*DatastoreInfo) ([]*DatastoreInfo, string, error) { +func (pbmClient *PbmClient) GetCompatibleDatastores(ctx context.Context, storagePolicyID string, datastores []*DatastoreInfo) ([]*DatastoreInfo, string, error) { var ( dsMorNameMap = getDsMorNameMap(ctx, datastores) localizedMessagesForNotCompatibleDatastores = "" diff --git a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go index 878822c32d..07f1f28144 100644 --- a/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go +++ b/pkg/cloudprovider/providers/vsphere/vclib/virtualmachine.go @@ -235,7 +235,7 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da var dsMoList []mo.Datastore pc := property.DefaultCollector(vm.Client()) - properties := []string{DatastoreInfoProperty} + properties := []string{DatastoreInfoProperty, NameProperty} err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList) if err != nil { klog.Errorf("Failed to get Datastore managed objects from datastore objects."+ @@ -243,12 +243,19 @@ func (vm *VirtualMachine) GetAllAccessibleDatastores(ctx context.Context) ([]*Da return nil, err } klog.V(9).Infof("Result dsMoList: %+v", dsMoList) + finder := getFinder(vm.Datacenter) var dsObjList []*DatastoreInfo for _, dsMo := range dsMoList { + // use the finder so that InventoryPath is set correctly in ds + ds, err := finder.Datastore(ctx, dsMo.Name) + if err != nil { + klog.Errorf("Failed finding datastore: %s. err: %+v", dsMo.Name, err) + return nil, err + } + datastore := Datastore{ds, vm.Datacenter} dsObjList = append(dsObjList, &DatastoreInfo{ - &Datastore{object.NewDatastore(vm.Client(), dsMo.Reference()), - vm.Datacenter}, + &datastore, dsMo.Info.GetDatastoreInfo()}) } return dsObjList, nil diff --git a/pkg/cloudprovider/providers/vsphere/vsphere.go b/pkg/cloudprovider/providers/vsphere/vsphere.go index ff72c36d3c..54ab036dc4 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere.go @@ -1164,15 +1164,9 @@ func (vs *VSphere) DisksAreAttached(nodeVolumes map[k8stypes.NodeName][]string) func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) { klog.V(1).Infof("Starting to create a vSphere volume with volumeOptions: %+v", volumeOptions) createVolumeInternal := func(volumeOptions *vclib.VolumeOptions) (canonicalVolumePath string, err error) { - var datastore string + var datastoreInfo *vclib.DatastoreInfo var dsList []*vclib.DatastoreInfo - // If datastore not specified, then use default datastore - if volumeOptions.Datastore == "" { - datastore = vs.cfg.Workspace.DefaultDatastore - } else { - datastore = volumeOptions.Datastore - } - datastore = strings.TrimSpace(datastore) + // Create context ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1180,16 +1174,29 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo if err != nil { return "", err } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) + // If datastore not specified, then use default datastore + datastoreName := strings.TrimSpace(volumeOptions.Datastore) + if datastoreName == "" { + datastoreName = strings.TrimSpace(vs.cfg.Workspace.DefaultDatastore) + } + // The given datastoreName may be present in more than one datacenter + candidateDatastoreInfos, err := vs.FindDatastoreByName(ctx, datastoreName) if err != nil { return "", err } + // Each of the datastores found is a candidate for Volume creation. + // One of these will be selected based on given policy and/or zone. + candidateDatastores := make(map[string]*vclib.DatastoreInfo) + for _, dsInfo := range candidateDatastoreInfos { + candidateDatastores[dsInfo.Info.Url] = dsInfo + } + var vmOptions *vclib.VMOptions if volumeOptions.VSANStorageProfileData != "" || volumeOptions.StoragePolicyName != "" { // If datastore and zone are specified, first validate if the datastore is in the provided zone. if len(volumeOptions.Zone) != 0 && volumeOptions.Datastore != "" { klog.V(4).Infof("Specified zone : %s, datastore : %s", volumeOptions.Zone, volumeOptions.Datastore) - dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { return "", err } @@ -1197,8 +1204,7 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo // Validate if the datastore provided belongs to the zone. If not, fail the operation. found := false for _, ds := range dsList { - if ds.Info.Name == volumeOptions.Datastore { - found = true + if datastoreInfo, found = candidateDatastores[ds.Info.Url]; found { break } } @@ -1220,19 +1226,14 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo cleanUpRoutineInitialized = true } cleanUpRoutineInitLock.Unlock() - vmOptions, err = vs.setVMOptions(ctx, dc, vs.cfg.Workspace.ResourcePoolPath) - if err != nil { - klog.Errorf("Failed to set VM options requires to create a vsphere volume. err: %+v", err) - return "", err - } } if volumeOptions.StoragePolicyName != "" && volumeOptions.Datastore == "" { if len(volumeOptions.Zone) == 0 { klog.V(4).Infof("Selecting a shared datastore as per the storage policy %s", volumeOptions.StoragePolicyName) - datastore, err = getPbmCompatibleDatastore(ctx, dc, volumeOptions.StoragePolicyName, vs.nodeManager) + datastoreInfo, err = getPbmCompatibleDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, vs.nodeManager) } else { // If zone is specified, first get the datastores in the zone. - dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err) @@ -1248,18 +1249,18 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo klog.V(4).Infof("Specified zone : %s. Picking a datastore as per the storage policy %s among the zoned datastores : %s", volumeOptions.Zone, volumeOptions.StoragePolicyName, dsList) // Among the compatible datastores, select the one based on the maximum free space. - datastore, err = getPbmCompatibleZonedDatastore(ctx, dc, volumeOptions.StoragePolicyName, dsList) + datastoreInfo, err = getPbmCompatibleZonedDatastore(ctx, vsi.conn.Client, volumeOptions.StoragePolicyName, dsList) } - klog.V(1).Infof("Datastore selected as per policy : %s", datastore) if err != nil { klog.Errorf("Failed to get pbm compatible datastore with storagePolicy: %s. err: %+v", volumeOptions.StoragePolicyName, err) return "", err } + klog.V(1).Infof("Datastore selected as per policy : %s", datastoreInfo.Info.Name) } else { // If zone is specified, pick the datastore in the zone with maximum free space within the zone. if volumeOptions.Datastore == "" && len(volumeOptions.Zone) != 0 { klog.V(4).Infof("Specified zone : %s", volumeOptions.Zone) - dsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + dsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err) @@ -1272,40 +1273,40 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo return "", err } - datastore, err = getMostFreeDatastoreName(ctx, nil, dsList) + datastoreInfo, err = getMostFreeDatastore(ctx, nil, dsList) if err != nil { klog.Errorf("Failed to get shared datastore: %+v", err) return "", err } - klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", volumeOptions.StoragePolicyName, datastore) + klog.V(1).Infof("Specified zone : %s. Selected datastore : %s", volumeOptions.Zone, datastoreInfo.Info.Name) } else { var sharedDsList []*vclib.DatastoreInfo var err error if len(volumeOptions.Zone) == 0 { // If zone is not provided, get the shared datastore across all node VMs. - klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastore) - sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, dc, vs.nodeManager) + klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastoreName) + sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, vs.nodeManager) if err != nil { klog.Errorf("Failed to get shared datastore: %+v", err) return "", err } // Prepare error msg to be used later, if required. - err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastore) + err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastoreName) } else { // If zone is provided, get the shared datastores in that zone. - klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastore, volumeOptions.Zone) - sharedDsList, err = getDatastoresForZone(ctx, dc, vs.nodeManager, volumeOptions.Zone) + klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastoreName, volumeOptions.Zone) + sharedDsList, err = getDatastoresForZone(ctx, vs.nodeManager, volumeOptions.Zone) if err != nil { klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", volumeOptions.Zone, err) return "", err } // Prepare error msg to be used later, if required. - err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastore, volumeOptions.Zone) + err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastoreName, volumeOptions.Zone) } found := false // Check if the selected datastore belongs to the list of shared datastores computed. for _, sharedDs := range sharedDsList { - if datastore == sharedDs.Info.Name { + if datastoreInfo, found = candidateDatastores[sharedDs.Info.Url]; found { klog.V(4).Infof("Datastore validation succeeded") found = true break @@ -1317,11 +1318,19 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo } } } - ds, err := dc.GetDatastoreByName(ctx, datastore) + + // if datastoreInfo is still not determined, it is an error condition + if datastoreInfo == nil { + klog.Errorf("Ambigous datastore name %s, cannot be found among: %v", datastoreName, candidateDatastoreInfos) + return "", fmt.Errorf("Ambigous datastore name %s", datastoreName) + } + ds := datastoreInfo.Datastore + volumeOptions.Datastore = datastoreInfo.Info.Name + vmOptions, err = vs.setVMOptions(ctx, vsi.conn, ds) if err != nil { + klog.Errorf("Failed to set VM options required to create a vsphere volume. err: %+v", err) return "", err } - volumeOptions.Datastore = datastore kubeVolsPath := filepath.Clean(ds.Path(VolDir)) + "/" err = ds.CreateDirectory(ctx, kubeVolsPath, false) if err != nil && err != vclib.ErrFileAlreadyExist { @@ -1336,18 +1345,18 @@ func (vs *VSphere) CreateVolume(volumeOptions *vclib.VolumeOptions) (canonicalVo } volumePath, err = disk.Create(ctx, ds) if err != nil { - klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, datastore, err) + klog.Errorf("Failed to create a vsphere volume with volumeOptions: %+v on datastore: %s. err: %+v", volumeOptions, ds, err) return "", err } // Get the canonical path for the volume path. - canonicalVolumePath, err = getcanonicalVolumePath(ctx, dc, volumePath) + canonicalVolumePath, err = getcanonicalVolumePath(ctx, datastoreInfo.Datacenter, volumePath) if err != nil { - klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, datastore, err) + klog.Errorf("Failed to get canonical vsphere volume path for volume: %s with volumeOptions: %+v on datastore: %s. err: %+v", volumePath, volumeOptions, ds, err) return "", err } - if filepath.Base(datastore) != datastore { + if filepath.Base(datastoreName) != datastoreName { // If datastore is within cluster, add cluster path to the volumePath - canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastore), datastore, 1) + canonicalVolumePath = strings.Replace(canonicalVolumePath, filepath.Base(datastoreName), datastoreName, 1) } return canonicalVolumePath, nil } @@ -1576,12 +1585,29 @@ func (vs *VSphere) GetVolumeLabels(volumePath string) (map[string]string, error) return nil, nil } + // Find the datastore on which this volume resides datastorePathObj, err := vclib.GetDatastorePathObjFromVMDiskPath(volumePath) if err != nil { klog.Errorf("Failed to get datastore for volume: %v: %+v", volumePath, err) return nil, err } - dsZones, err := vs.GetZonesForDatastore(ctx, datastorePathObj.Datastore) + dsInfos, err := vs.FindDatastoreByName(ctx, datastorePathObj.Datastore) + if err != nil { + klog.Errorf("Failed to get datastore by name: %v: %+v", datastorePathObj.Datastore, err) + return nil, err + } + var datastore *vclib.Datastore + for _, dsInfo := range dsInfos { + if dsInfo.Datastore.Exists(ctx, datastorePathObj.Path) { + datastore = dsInfo.Datastore + } + } + if datastore == nil { + klog.Errorf("Could not find %s among %v", volumePath, dsInfos) + return nil, fmt.Errorf("Could not find the datastore for volume: %s", volumePath) + } + + dsZones, err := vs.GetZonesForDatastore(ctx, datastore) if err != nil { klog.Errorf("Failed to get zones for datastore %v: %+v", datastorePathObj.Datastore, err) return nil, err @@ -1619,25 +1645,16 @@ func (vs *VSphere) collapseZonesInRegion(ctx context.Context, zones []cloudprovi } // GetZonesForDatastore returns all the zones from which this datastore is visible -func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore string) ([]cloudprovider.Zone, error) { +func (vs *VSphere) GetZonesForDatastore(ctx context.Context, datastore *vclib.Datastore) ([]cloudprovider.Zone, error) { vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) if err != nil { klog.Errorf("Failed to get vSphere instance: %+v", err) return nil, err } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) - if err != nil { - klog.Errorf("Failed to get datacenter: %+v", err) - return nil, err - } + // get the hosts mounted on this datastore // datastore -> ["host-1", "host-2", "host-3", ...] - ds, err := dc.GetDatastoreByName(ctx, datastore) - if err != nil { - klog.Errorf("Failed to get datastore by name: %v: %+v", datastore, err) - return nil, err - } - dsHosts, err := ds.GetDatastoreHostMounts(ctx) + dsHosts, err := datastore.GetDatastoreHostMounts(ctx) if err != nil { klog.Errorf("Failed to get datastore host mounts for %v: %+v", datastore, err) return nil, err diff --git a/pkg/cloudprovider/providers/vsphere/vsphere_util.go b/pkg/cloudprovider/providers/vsphere/vsphere_util.go index ff0677a80d..615cd84cd8 100644 --- a/pkg/cloudprovider/providers/vsphere/vsphere_util.go +++ b/pkg/cloudprovider/providers/vsphere/vsphere_util.go @@ -20,20 +20,22 @@ import ( "context" "errors" "fmt" + "math/rand" "os" "path/filepath" "strings" "time" + "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + v1 "k8s.io/api/core/v1" "k8s.io/klog" - "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/version" "k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib" @@ -43,6 +45,7 @@ import ( const ( DatastoreProperty = "datastore" DatastoreInfoProperty = "info" + DatastoreNameProperty = "name" Folder = "Folder" VirtualMachine = "VirtualMachine" DummyDiskName = "kube-dummyDisk.vmdk" @@ -121,7 +124,7 @@ func getAccessibleDatastores(ctx context.Context, nodeVmDetail *NodeDetails, nod } // Get all datastores accessible for the virtual machine object. -func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) { +func getSharedDatastoresInK8SCluster(ctx context.Context, nodeManager *NodeManager) ([]*vclib.DatastoreInfo, error) { nodeVmDetails, err := nodeManager.GetNodeDetails() if err != nil { klog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err) @@ -155,12 +158,6 @@ func getSharedDatastoresInK8SCluster(ctx context.Context, dc *vclib.Datacenter, } } klog.V(9).Infof("sharedDatastores : %+v", sharedDatastores) - sharedDatastores, err = getDatastoresForEndpointVC(ctx, dc, sharedDatastores) - if err != nil { - klog.Errorf("Failed to get shared datastores from endpoint VC. err: %+v", err) - return nil, err - } - klog.V(9).Infof("sharedDatastores at endpoint VC: %+v", sharedDatastores) return sharedDatastores, nil } @@ -182,7 +179,7 @@ func intersect(list1 []*vclib.DatastoreInfo, list2 []*vclib.DatastoreInfo) []*vc } // getMostFreeDatastore gets the best fit compatible datastore by free space. -func getMostFreeDatastoreName(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (string, error) { +func getMostFreeDatastore(ctx context.Context, client *vim25.Client, dsInfoList []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) { var curMax int64 curMax = -1 var index int @@ -193,65 +190,46 @@ func getMostFreeDatastoreName(ctx context.Context, client *vim25.Client, dsInfoL index = i } } - return dsInfoList[index].Info.GetDatastoreInfo().Name, nil + return dsInfoList[index], nil } -// Returns the datastores in the given datacenter by performing lookup based on datastore URL. -func getDatastoresForEndpointVC(ctx context.Context, dc *vclib.Datacenter, sharedDsInfos []*vclib.DatastoreInfo) ([]*vclib.DatastoreInfo, error) { - var datastores []*vclib.DatastoreInfo - allDsInfoMap, err := dc.GetAllDatastores(ctx) +func getPbmCompatibleDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, nodeManager *NodeManager) (*vclib.DatastoreInfo, error) { + pbmClient, err := vclib.NewPbmClient(ctx, vcClient) if err != nil { return nil, err } - for _, sharedDsInfo := range sharedDsInfos { - dsInfo, ok := allDsInfoMap[sharedDsInfo.Info.Url] - if ok { - datastores = append(datastores, dsInfo) - } else { - klog.V(4).Infof("Warning: Shared datastore with URL %s does not exist in endpoint VC", sharedDsInfo.Info.Url) - } - } - klog.V(9).Infof("Datastore from endpoint VC: %+v", datastores) - return datastores, nil -} - -func getPbmCompatibleDatastore(ctx context.Context, dc *vclib.Datacenter, storagePolicyName string, nodeManager *NodeManager) (string, error) { - pbmClient, err := vclib.NewPbmClient(ctx, dc.Client()) - if err != nil { - return "", err - } storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName) if err != nil { klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err) - return "", err + return nil, err } - sharedDs, err := getSharedDatastoresInK8SCluster(ctx, dc, nodeManager) + sharedDs, err := getSharedDatastoresInK8SCluster(ctx, nodeManager) if err != nil { klog.Errorf("Failed to get shared datastores. err: %+v", err) - return "", err + return nil, err } if len(sharedDs) == 0 { msg := "No shared datastores found in the endpoint virtual center" klog.Errorf(msg) - return "", errors.New(msg) + return nil, errors.New(msg) } - compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, sharedDs) + compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, sharedDs) if err != nil { klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v", sharedDs, storagePolicyID, err) - return "", err + return nil, err } klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores) - datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores) + datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores) if err != nil { klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err) - return "", err + return nil, err } - klog.V(4).Infof("Most free datastore : %+s", datastore) + klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name) return datastore, err } -func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) { +func getDatastoresForZone(ctx context.Context, nodeManager *NodeManager, selectedZones []string) ([]*vclib.DatastoreInfo, error) { var sharedDatastores []*vclib.DatastoreInfo @@ -279,7 +257,7 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager var dsMoList []mo.Datastore pc := property.DefaultCollector(host.Client()) - properties := []string{DatastoreInfoProperty} + properties := []string{DatastoreInfoProperty, DatastoreNameProperty} err = pc.Retrieve(ctx, dsRefList, properties, &dsMoList) if err != nil { klog.Errorf("Failed to get Datastore managed objects from datastore objects."+ @@ -288,12 +266,37 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager } klog.V(9).Infof("Datastore mo details: %+v", dsMoList) + // find the Datacenter parent for this host + mes, err := mo.Ancestors(ctx, host.Client(), pc.Reference(), host.Reference()) + if err != nil { + return nil, err + } + var dcMoref *types.ManagedObjectReference + for i := len(mes) - 1; i > 0; i-- { + if mes[i].Self.Type == "Datacenter" { + dcMoref = &mes[i].Self + break + } + } + if dcMoref == nil { + return nil, fmt.Errorf("Failed to find the Datacenter of host %s", host) + } + + dc := object.NewDatacenter(host.Client(), *dcMoref) + finder := find.NewFinder(host.Client(), false) + finder.SetDatacenter(dc) var dsObjList []*vclib.DatastoreInfo for _, dsMo := range dsMoList { + // use the finder so that InventoryPath is set correctly in dsObj + dsObj, err := finder.Datastore(ctx, dsMo.Name) + if err != nil { + klog.Errorf("Failed to find datastore named %s in datacenter %s", dsMo.Name, dc) + return nil, err + } dsObjList = append(dsObjList, &vclib.DatastoreInfo{ - Datastore: &vclib.Datastore{Datastore: object.NewDatastore(host.Client(), dsMo.Reference()), - Datacenter: nil}, + Datastore: &vclib.Datastore{Datastore: dsObj, + Datacenter: &vclib.Datacenter{Datacenter: dc}}, Info: dsMo.Info.GetDatastoreInfo()}) } @@ -317,40 +320,48 @@ func getDatastoresForZone(ctx context.Context, dc *vclib.Datacenter, nodeManager return sharedDatastores, nil } -func getPbmCompatibleZonedDatastore(ctx context.Context, dc *vclib.Datacenter, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (string, error) { - pbmClient, err := vclib.NewPbmClient(ctx, dc.Client()) +func getPbmCompatibleZonedDatastore(ctx context.Context, vcClient *vim25.Client, storagePolicyName string, zonedDatastores []*vclib.DatastoreInfo) (*vclib.DatastoreInfo, error) { + pbmClient, err := vclib.NewPbmClient(ctx, vcClient) if err != nil { - return "", err + return nil, err } storagePolicyID, err := pbmClient.ProfileIDByName(ctx, storagePolicyName) if err != nil { klog.Errorf("Failed to get Profile ID by name: %s. err: %+v", storagePolicyName, err) - return "", err + return nil, err } - compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, dc, storagePolicyID, zonedDatastores) + compatibleDatastores, _, err := pbmClient.GetCompatibleDatastores(ctx, storagePolicyID, zonedDatastores) if err != nil { klog.Errorf("Failed to get compatible datastores from datastores : %+v with storagePolicy: %s. err: %+v", zonedDatastores, storagePolicyID, err) - return "", err + return nil, err } klog.V(9).Infof("compatibleDatastores : %+v", compatibleDatastores) - datastore, err := getMostFreeDatastoreName(ctx, dc.Client(), compatibleDatastores) + datastore, err := getMostFreeDatastore(ctx, vcClient, compatibleDatastores) if err != nil { klog.Errorf("Failed to get most free datastore from compatible datastores: %+v. err: %+v", compatibleDatastores, err) - return "", err + return nil, err } - klog.V(4).Infof("Most free datastore : %+s", datastore) + klog.V(4).Infof("Most free datastore : %+s", datastore.Info.Name) return datastore, err } -func (vs *VSphere) setVMOptions(ctx context.Context, dc *vclib.Datacenter, resourcePoolPath string) (*vclib.VMOptions, error) { +func (vs *VSphere) setVMOptions(ctx context.Context, connection *vclib.VSphereConnection, ds *vclib.Datastore) (*vclib.VMOptions, error) { var vmOptions vclib.VMOptions - resourcePool, err := dc.GetResourcePool(ctx, resourcePoolPath) + dsHosts, err := ds.GetDatastoreHostMounts(ctx) if err != nil { + klog.Errorf("Failed to get datastore host mounts for %v: %+v", ds, err) return nil, err } - klog.V(9).Infof("Resource pool path %s, resourcePool %+v", resourcePoolPath, resourcePool) - folder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) + // pick a host at random to use for Volume creation + dsHostMoref := dsHosts[rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(dsHosts))] + dummyVMHost := object.NewHostSystem(connection.Client, dsHostMoref) + resourcePool, err := dummyVMHost.ResourcePool(ctx) + if err != nil { + klog.Errorf("Failed to get resource pool from host %v", dummyVMHost) + return nil, err + } + folder, err := ds.Datacenter.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) if err != nil { return nil, err } @@ -366,32 +377,30 @@ func (vs *VSphere) cleanUpDummyVMs(dummyVMPrefix string) { defer cancel() for { time.Sleep(CleanUpDummyVMRoutineInterval * time.Minute) - vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) + datacenters, err := vs.GetWorkspaceDatacenters(ctx) if err != nil { - klog.V(4).Infof("Failed to get VSphere instance with err: %+v. Retrying again...", err) + klog.V(4).Infof("Failed to get datacenters from VC. err: %+v", err) continue } - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) - if err != nil { - klog.V(4).Infof("Failed to get the datacenter: %s from VC. err: %+v", vs.cfg.Workspace.Datacenter, err) - continue - } - // Get the folder reference for global working directory where the dummy VM needs to be created. - vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) - if err != nil { - klog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err) - continue - } - // A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests. - cleanUpDummyVMs := func() { - cleanUpDummyVMLock.Lock() - defer cleanUpDummyVMLock.Unlock() - err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder, dc) + // Clean up dummy VMs in each datacenter + for _, dc := range datacenters { + // Get the folder reference for global working directory where the dummy VM needs to be created. + vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) if err != nil { - klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err) + klog.V(4).Infof("Unable to get the kubernetes folder: %q reference. err: %+v", vs.cfg.Workspace.Folder, err) + continue } + // A write lock is acquired to make sure the cleanUp routine doesn't delete any VM's created by ongoing PVC requests. + cleanUpDummyVMs := func() { + cleanUpDummyVMLock.Lock() + defer cleanUpDummyVMLock.Unlock() + err = diskmanagers.CleanUpDummyVMs(ctx, vmFolder) + if err != nil { + klog.V(4).Infof("Unable to clean up dummy VM's in the kubernetes cluster: %q. err: %+v", vs.cfg.Workspace.Folder, err) + } + } + cleanUpDummyVMs() } - cleanUpDummyVMs() } } @@ -586,43 +595,6 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN return nodesToRetry, nil } -func (vs *VSphere) IsDummyVMPresent(vmName string) (bool, error) { - isDummyVMPresent := false - - // Create context - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - vsi, err := vs.getVSphereInstanceForServer(vs.cfg.Workspace.VCenterIP, ctx) - if err != nil { - return isDummyVMPresent, err - } - - dc, err := vclib.GetDatacenter(ctx, vsi.conn, vs.cfg.Workspace.Datacenter) - if err != nil { - return isDummyVMPresent, err - } - - vmFolder, err := dc.GetFolderByPath(ctx, vs.cfg.Workspace.Folder) - if err != nil { - return isDummyVMPresent, err - } - - vms, err := vmFolder.GetVirtualMachines(ctx) - if err != nil { - return isDummyVMPresent, err - } - - for _, vm := range vms { - if vm.Name() == vmName { - isDummyVMPresent = true - break - } - } - - return isDummyVMPresent, nil -} - func (vs *VSphere) GetNodeNameFromProviderID(providerID string) (string, error) { var nodeName string nodes, err := vs.nodeManager.GetNodeDetails() @@ -698,3 +670,63 @@ func GetVMUUID() (string, error) { uuid = fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:32]) return uuid, nil } + +// GetWorkspaceDatacenters returns the Datacenter objects that VCP has access to. +// User can configure the list of datacenters in vsphere.conf. Otherwise all the +// Datacenters in the configured list of VCs are returned. +func (vs *VSphere) GetWorkspaceDatacenters(ctx context.Context) ([]*vclib.Datacenter, error) { + var datacenterObjs []*vclib.Datacenter + for vc, vsi := range vs.vsphereInstanceMap { + // ensure connection to VC + err := vs.nodeManager.vcConnect(ctx, vsi) + if err != nil { + return nil, err + } + if vsi.cfg.Datacenters == "" { + vcDatacenterObjs, err := vclib.GetAllDatacenter(ctx, vsi.conn) + if err != nil { + klog.Errorf("Error fetching list of datacenters from VC %s: %+v", vc, err) + return nil, err + } + datacenterObjs = append(datacenterObjs, vcDatacenterObjs...) + } else { + datacenters := strings.Split(vsi.cfg.Datacenters, ",") + for _, dc := range datacenters { + dc = strings.TrimSpace(dc) + if dc == "" { + continue + } + datacenterObj, err := vclib.GetDatacenter(ctx, vsi.conn, dc) + if err != nil { + klog.Errorf("Error fetching datacenter %s from VC %s: %+v", dc, vc, err) + return nil, err + } + datacenterObjs = append(datacenterObjs, datacenterObj) + } + } + } + return datacenterObjs, nil +} + +// FindDatastoreByName looks for the given datastore by name across all available datacenters. +// If more than one Datacenter has a Datastore with the given name, then returns reference to all of them. +func (vs *VSphere) FindDatastoreByName(ctx context.Context, datastoreName string) ([]*vclib.DatastoreInfo, error) { + datacenters, err := vs.GetWorkspaceDatacenters(ctx) + if err != nil { + return nil, err + } + var datastoreInfos []*vclib.DatastoreInfo + for _, dc := range datacenters { + datastoreInfo, err := dc.GetDatastoreInfoByName(ctx, datastoreName) + if err != nil { + klog.V(9).Infof("Did not find datastore %s in datacenter %s, still looking.", datastoreName, dc.Name()) + continue + } + datastoreInfos = append(datastoreInfos, datastoreInfo) + } + if len(datastoreInfos) == 0 { + return nil, fmt.Errorf("Datastore '%s' not found", datastoreName) + } + klog.V(4).Infof("Found datastore infos %v for datastore %s", datastoreInfos, datastoreName) + return datastoreInfos, nil +} diff --git a/test/e2e/storage/vsphere/vsphere_volume_datastore.go b/test/e2e/storage/vsphere/vsphere_volume_datastore.go index dfd497fa55..137852636d 100644 --- a/test/e2e/storage/vsphere/vsphere_volume_datastore.go +++ b/test/e2e/storage/vsphere/vsphere_volume_datastore.go @@ -70,7 +70,7 @@ var _ = utils.SIGDescribe("Volume Provisioning on Datastore [Feature:vsphere]", scParameters[DiskFormat] = ThinDisk err := invokeInvalidDatastoreTestNeg(client, namespace, scParameters) Expect(err).To(HaveOccurred()) - errorMsg := `Failed to provision volume with StorageClass \"` + DatastoreSCName + `\": The specified datastore ` + InvalidDatastore + ` is not a shared datastore across node VMs` + errorMsg := `Failed to provision volume with StorageClass \"` + DatastoreSCName + `\": Datastore ` + InvalidDatastore + ` not found` if !strings.Contains(err.Error(), errorMsg) { Expect(err).NotTo(HaveOccurred(), errorMsg) } diff --git a/test/e2e/storage/vsphere/vsphere_zone_support.go b/test/e2e/storage/vsphere/vsphere_zone_support.go index 5ad8cf1b49..0d270f3310 100644 --- a/test/e2e/storage/vsphere/vsphere_zone_support.go +++ b/test/e2e/storage/vsphere/vsphere_zone_support.go @@ -176,6 +176,13 @@ var _ = utils.SIGDescribe("Zone Support", func() { verifyPVCAndPodCreationSucceeds(client, namespace, scParameters, zones) }) + It("Verify a pod is created on a non-Workspace zone and attached to a dynamically created PV, based on the allowed zones and storage policy specified in storage class", func() { + By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneB, compatPolicy)) + scParameters[SpbmStoragePolicy] = compatPolicy + zones = append(zones, zoneB) + verifyPVCAndPodCreationSucceeds(client, namespace, scParameters, zones) + }) + It("Verify PVC creation with incompatible storagePolicy and zone combination specified in storage class fails", func() { By(fmt.Sprintf("Creating storage class with zone :%s and storage policy :%s", zoneA, nonCompatPolicy)) scParameters[SpbmStoragePolicy] = nonCompatPolicy