github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #59519 from vmware/vm_uuid_provider_id 

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Report InstanceID for vSphere Cloud Provider as UUID obtained from product_serial file 

**What this PR does / why we need it**:
vSphere Cloud Provider is not able to find the nodes for VMs created on vSphere v1.6.5. Kubelet fetches SystemUUID from file ```/sys/class/dmi/id/product_uuid```. vSphere Cloud Provider uses this uuid as VM identifier to get node information from vCenter. vCenter v1.6.5 doesn't recognize this uuids, as a result, nodes are not found. 

UUID present in file ```/sys/class/dmi/id/product_serial``` is recognized by vCenter. Yet,  Kubelet doesn't report this. Therefore, in this PR InstanceID is reported as UUID which is fetched from file 
```/sys/class/dmi/id/product_serial```.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes https://github.com/kubernetes/kubernetes/issues/58927

**Special notes for your reviewer**:
Internally review here: https://github.com/vmware/kubernetes/pull/452

Tested:
Launched K8s cluster using kubeadm (Used Ubuntu VM compatible with vSphere version 6.5.)
_**Note: Installed Ubuntu from ISO**_
Observed following:
```
Master
> cat /sys/class/dmi/id/product_uuid
743F0E42-84EA-A2F9-7736-6106BB5DBF6B

> cat /sys/class/dmi/id/product_serial
VMware-42 0e 3f 74 ea 84 f9 a2-77 36 61 06 bb 5d bf 6b

Node
> cat /sys/class/dmi/id/product_uuid
956E0E42-CC9D-3D89-9757-F27CEB539B76

> cat /sys/class/dmi/id/product_serial
VMware-42 0e 6e 95 9d cc 89 3d-97 57 f2 7c eb 53 9b 76
```
With this fix controller manager was able to find the nodes.
**controller manager logs**
```
{"log":"I0205 22:43:00.106416       1 nodemanager.go:183] Found node ubuntu-node as vm=VirtualMachine:vm-95 in vc=10.161.120.115 and datacenter=vcqaDC\n","stream":"stderr","time":"2018-02-05T22:43:00.421010375Z"}
```


**Release note**:

```release-note
vSphere Cloud Provider supports VMs provisioned on vSphere v1.6.5
```
pull/6/head
Kubernetes Submit Queue 2018-02-08 09:43:08 -08:00 committed by GitHub
parent af3ea72cda 918e47c438
commit c0a337d4cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 72 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/cloudprovider/providers/vsphere/nodemanager.go
View File

@ -33,6 +33,7 @@ type NodeInfo struct {
dataCenter *vclib.Datacenter
vm *vclib.VirtualMachine
vcServer string
vmUUID string
}
type NodeManager struct {
@ -53,6 +54,7 @@ type NodeManager struct {
type NodeDetails struct {
NodeName string
vm *vclib.VirtualMachine
VMUUID string
}
// TODO: Make it configurable in vsphere.conf
@ -74,7 +76,10 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error {
var globalErr *error
queueChannel = make(chan *VmSearch, QUEUE_SIZE)
nodeUUID := node.Status.NodeInfo.SystemUUID
nodeUUID := GetUUIDFromProviderID(node.Spec.ProviderID)
glog.V(4).Infof("Discovering node %s with uuid %s", node.ObjectMeta.Name, nodeUUID)
vmFound := false
globalErr = nil
@ -178,7 +183,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error {
glog.V(4).Infof("Found node %s as vm=%+v in vc=%s and datacenter=%s",
node.Name, vm, res.vc, res.datacenter.Name())
nodeInfo := &NodeInfo{dataCenter: res.datacenter, vm: vm, vcServer: res.vc}
nodeInfo := &NodeInfo{dataCenter: res.datacenter, vm: vm, vcServer: res.vc, vmUUID: nodeUUID}
nm.addNodeInfo(node.ObjectMeta.Name, nodeInfo)
for range queueChannel {
}
@ -311,7 +316,7 @@ func (nm *NodeManager) GetNodeDetails() ([]NodeDetails, error) {
}
nm.nodeInfoMap[nodeName] = n
glog.V(4).Infof("Updated NodeInfo %q for node %q.", nodeInfo, nodeName)
nodeDetails = append(nodeDetails, NodeDetails{nodeName, n.vm})
nodeDetails = append(nodeDetails, NodeDetails{nodeName, n.vm, n.vmUUID})
}
return nodeDetails, nil
}

36
pkg/cloudprovider/providers/vsphere/vsphere.go
View File

@ -54,8 +54,6 @@ const (
MacOuiVC = "00:50:56"
MacOuiEsx = "00:0c:29"
CleanUpDummyVMRoutineInterval = 5
UUIDPath = "/sys/class/dmi/id/product_serial"
UUIDPrefix = "VMware-"
)
var cleanUpRoutineInitialized = false
@ -72,6 +70,7 @@ type VSphere struct {
vsphereInstanceMap map[string]*VSphereInstance
// Responsible for managing discovery of k8s node, their location etc.
nodeManager *NodeManager
vmUUID string
}
// Represents a vSphere instance where one or more kubernetes nodes are running.
@ -237,7 +236,11 @@ func newWorkerNode() (*VSphere, error) {
glog.Errorf("Failed to get hostname. err: %+v", err)
return nil, err
}
vs.vmUUID, err = GetVMUUID()
if err != nil {
glog.Errorf("Failed to get uuid. err: %+v", err)
return nil, err
}
return &vs, nil
}
@ -395,6 +398,11 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) {
glog.Errorf("Failed to get hostname. err: %+v", err)
return nil, err
}
vs.vmUUID, err = GetVMUUID()
if err != nil {
glog.Errorf("Failed to get uuid. err: %+v", err)
return nil, err
}
runtime.SetFinalizer(&vs, logout)
return &vs, nil
}
@ -576,7 +584,23 @@ func (vs *VSphere) ExternalID(ctx context.Context, nodeName k8stypes.NodeName) (
// InstanceExistsByProviderID returns true if the instance with the given provider id still exists and is running.
// If false is returned with no error, the instance will be immediately deleted by the cloud controller manager.
func (vs *VSphere) InstanceExistsByProviderID(ctx context.Context, providerID string) (bool, error) {
_, err := vs.InstanceID(ctx, convertToK8sType(providerID))
var nodeName string
nodes, err := vs.nodeManager.GetNodeDetails()
if err != nil {
glog.Errorf("Error while obtaining Kubernetes node nodeVmDetail details. error : %+v", err)
return false, err
}
for _, node := range nodes {
if node.VMUUID == GetUUIDFromProviderID(providerID) {
nodeName = node.NodeName
break
}
}
if nodeName == "" {
msg := fmt.Sprintf("Error while obtaining Kubernetes nodename for providerID %s.", providerID)
return false, errors.New(msg)
}
_, err = vs.InstanceID(ctx, convertToK8sType(nodeName))
if err == nil {
return true, nil
}
@ -589,7 +613,7 @@ func (vs *VSphere) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) (
instanceIDInternal := func() (string, error) {
if vs.hostName == convertToString(nodeName) {
return vs.hostName, nil
return vs.vmUUID, nil
}
// Below logic can be performed only on master node where VC details are preset.
@ -623,7 +647,7 @@ func (vs *VSphere) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) (
return "", err
}
if isActive {
return convertToString(nodeName), nil
return vs.vmUUID, nil
}
glog.Warningf("The VM: %s is not in %s state", convertToString(nodeName), vclib.ActivePowerState)
return "", cloudprovider.InstanceNotFound

36
pkg/cloudprovider/providers/vsphere/vsphere_util.go
View File

@ -32,6 +32,7 @@ import (
"path/filepath"
"github.com/vmware/govmomi/vim25/mo"
"io/ioutil"
k8stypes "k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib"
"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere/vclib/diskmanagers"
@ -43,6 +44,9 @@ const (
Folder = "Folder"
VirtualMachine = "VirtualMachine"
DummyDiskName = "kube-dummyDisk.vmdk"
UUIDPath = "/sys/class/dmi/id/product_serial"
UUIDPrefix = "VMware-"
ProviderPrefix = "vsphere://"
vSphereConfFileEnvVar = "VSPHERE_CONF_FILE"
)
@ -468,8 +472,9 @@ func (vs *VSphere) checkDiskAttached(ctx context.Context, nodes []k8stypes.NodeN
if err != nil {
return nodesToRetry, err
}
glog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %s", nodeName, node.Status.NodeInfo.SystemUUID, vmMoMap)
vclib.VerifyVolumePathsForVM(vmMoMap[strings.ToLower(node.Status.NodeInfo.SystemUUID)], nodeVolumes[nodeName], convertToString(nodeName), attached)
nodeUUID := strings.ToLower(GetUUIDFromProviderID(node.Spec.ProviderID))
glog.V(9).Infof("Verifying volume for node %s with nodeuuid %q: %s", nodeName, nodeUUID, vmMoMap)
vclib.VerifyVolumePathsForVM(vmMoMap[nodeUUID], nodeVolumes[nodeName], convertToString(nodeName), attached)
}
return nodesToRetry, nil
}
@ -510,3 +515,30 @@ func (vs *VSphere) IsDummyVMPresent(vmName string) (bool, error) {
return isDummyVMPresent, nil
}
func GetVMUUID() (string, error) {
id, err := ioutil.ReadFile(UUIDPath)
if err != nil {
return "", fmt.Errorf("error retrieving vm uuid: %s", err)
}
uuidFromFile := string(id[:])
//strip leading and trailing white space and new line char
uuid := strings.TrimSpace(uuidFromFile)
// check the uuid starts with "VMware-"
if !strings.HasPrefix(uuid, UUIDPrefix) {
return "", fmt.Errorf("Failed to match Prefix, UUID read from the file is %v", uuidFromFile)
}
// Strip the prefix and white spaces and -
uuid = strings.Replace(uuid[len(UUIDPrefix):(len(uuid))], " ", "", -1)
uuid = strings.Replace(uuid, "-", "", -1)
if len(uuid) != 32 {
return "", fmt.Errorf("Length check failed, UUID read from the file is %v", uuidFromFile)
}
// need to add dashes, e.g. "564d395e-d807-e18a-cb25-b79f65eb2b9f"
uuid = fmt.Sprintf("%s-%s-%s-%s-%s", uuid[0:8], uuid[8:12], uuid[12:16], uuid[16:20], uuid[20:32])
return uuid, nil
}
func GetUUIDFromProviderID(providerID string) string {
return strings.TrimPrefix(providerID, ProviderPrefix)
}