github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #63902 from vmware/vcp_secrets 

Automatic merge from submit-queue (batch tested with PRs 63969, 63902, 63689, 63973, 63978). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Adds a mechanism in vSphere Cloud Provider to get credentials from Kubernetes secrets

**What this PR does / why we need it**:
Currently, vCenter credentials are stored in plain text in vsphere.conf. This PR adds a mechanism in vSphere Cloud Provider to get vCenter credentials from Kubernetes secrets.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:
Internally review here: https://github.com/vmware/kubernetes/pull/484
**Workflow:**
1. Create vsphere.conf file with ```secret-name``` and ```secret-namespace```.
	```
	[Global]
	insecure-flag = 1
	secret-name = "vcconf"
	secret-namespace = "kube-system"

	[VirtualCenter "10.160.45.119"]
	port = 443
	datacenters = k8s-dc-1

	[Workspace]
	server = 10.160.45.119
	datacenter = k8s-dc-1
	default-datastore = sharedVMFS-0
	folder = Discovered virtual machine
	```
2. Launch Kubernetes cluster with vSphere Cloud Provider Configured.
3. Create secret with vCenter credentials.
	a. Create base64 encoding for username and password:
	username:
	```	
		> echo -n 'admin' | base64
		YWRtaW4= 
	```
	password:
	```
		> echo -n 'vsphere' | base64
		dnNwaGVyZQ==
	```

	b. kubectl create -f vccredentials.yaml
	```
		#vccredentials.yaml
		apiVersion: v1
		kind: Secret
		metadata:
			name: vcconf
		type: Opaque
		data:
			10.192.44.199.username: YWRtaW4=
			10.192.44.199.password: dnNwaGVyZQ==
	```
4. vSphere Cloud Provider can be used now.

**Note:**
Secrets info can be provided with both (old and new) vSphere Cloud provider configuration formats.


**Tests Done:**
- [x] vSphere Cloud Provider unit test.
- [x] Volume lifecyle with Username and Password in vsphere.conf (for backward compability)
- [x] Volume lifecyle with secrets information in vsphere.conf.
- [x] Update secrets workflow

**Release note**:

```release-note
Adds a mechanism in vSphere Cloud Provider to get credentials from Kubernetes secrets
```
pull/8/head
Kubernetes Submit Queue 2018-05-18 15:59:15 -07:00 committed by GitHub
parent d15985798e 99076ce32a
commit 2d1f42e0b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 969 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/cloudprovider/providers/vsphere/BUILD
View File

@ -9,6 +9,7 @@ load(
go_library(
name = "go_default_library",
srcs = [
"credentialmanager.go",
"nodemanager.go",
"vsphere.go",
"vsphere_util.go",
@ -26,25 +27,36 @@ go_library(
"//vendor/github.com/vmware/govmomi/vim25/mo:go_default_library",
"//vendor/gopkg.in/gcfg.v1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = ["vsphere_test.go"],
srcs = [
"credentialmanager_test.go",
"vsphere_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/cloudprovider:go_default_library",
"//pkg/cloudprovider/providers/vsphere/vclib:go_default_library",
"//pkg/controller:go_default_library",
"//vendor/github.com/vmware/govmomi/lookup/simulator:go_default_library",
"//vendor/github.com/vmware/govmomi/simulator:go_default_library",
"//vendor/github.com/vmware/govmomi/simulator/vpx:go_default_library",
"//vendor/github.com/vmware/govmomi/sts/simulator:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

164
pkg/cloudprovider/providers/vsphere/credentialmanager.go Normal file
View File

@ -0,0 +1,164 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vsphere
import (
"errors"
"fmt"
"github.com/golang/glog"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/listers/core/v1"
"net/http"
"strings"
"sync"
)
// Error Messages
const (
CredentialsNotFoundErrMsg = "Credentials not found"
CredentialMissingErrMsg = "Username/Password is missing"
UnknownSecretKeyErrMsg = "Unknown secret key"
)
// Error constants
var (
ErrCredentialsNotFound = errors.New(CredentialsNotFoundErrMsg)
ErrCredentialMissing = errors.New(CredentialMissingErrMsg)
ErrUnknownSecretKey = errors.New(UnknownSecretKeyErrMsg)
)
type SecretCache struct {
cacheLock sync.Mutex
VirtualCenter map[string]*Credential
Secret *corev1.Secret
}
type Credential struct {
User string `gcfg:"user"`
Password string `gcfg:"password"`
}
type SecretCredentialManager struct {
SecretName string
SecretNamespace string
SecretLister v1.SecretLister
Cache *SecretCache
}
// GetCredential returns credentials for the given vCenter Server.
// GetCredential returns error if Secret is not added.
// GetCredential return error is the secret doesn't contain any credentials.
func (secretCredentialManager *SecretCredentialManager) GetCredential(server string) (*Credential, error) {
err := secretCredentialManager.updateCredentialsMap()
if err != nil {
statusErr, ok := err.(*apierrors.StatusError)
if (ok && statusErr.ErrStatus.Code != http.StatusNotFound) || !ok {
return nil, err
}
// Handle secrets deletion by finding credentials from cache
glog.Warningf("secret %q not found in namespace %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace)
}
credential, found := secretCredentialManager.Cache.GetCredential(server)
if !found {
glog.Errorf("credentials not found for server %q", server)
return nil, ErrCredentialsNotFound
}
return &credential, nil
}
func (secretCredentialManager *SecretCredentialManager) updateCredentialsMap() error {
if secretCredentialManager.SecretLister == nil {
return fmt.Errorf("SecretLister is not initialized")
}
secret, err := secretCredentialManager.SecretLister.Secrets(secretCredentialManager.SecretNamespace).Get(secretCredentialManager.SecretName)
if err != nil {
glog.Errorf("Cannot get secret %s in namespace %s. error: %q", secretCredentialManager.SecretName, secretCredentialManager.SecretNamespace, err)
return err
}
cacheSecret := secretCredentialManager.Cache.GetSecret()
if cacheSecret != nil &&
cacheSecret.GetResourceVersion() == secret.GetResourceVersion() {
glog.V(4).Infof("VCP SecretCredentialManager: Secret %q will not be updated in cache. Since, secrets have same resource version %q", secretCredentialManager.SecretName, cacheSecret.GetResourceVersion())
return nil
}
secretCredentialManager.Cache.UpdateSecret(secret)
return secretCredentialManager.Cache.parseSecret()
}
func (cache *SecretCache) GetSecret() *corev1.Secret {
cache.cacheLock.Lock()
defer cache.cacheLock.Unlock()
return cache.Secret
}
func (cache *SecretCache) UpdateSecret(secret *corev1.Secret) {
cache.cacheLock.Lock()
defer cache.cacheLock.Unlock()
cache.Secret = secret
}
func (cache *SecretCache) GetCredential(server string) (Credential, bool) {
cache.cacheLock.Lock()
defer cache.cacheLock.Unlock()
credential, found := cache.VirtualCenter[server]
if !found {
return Credential{}, found
}
return *credential, found
}
func (cache *SecretCache) parseSecret() error {
cache.cacheLock.Lock()
defer cache.cacheLock.Unlock()
return parseConfig(cache.Secret.Data, cache.VirtualCenter)
}
// parseConfig returns vCenter ip/fdqn mapping to its credentials viz. Username and Password.
func parseConfig(data map[string][]byte, config map[string]*Credential) error {
if len(data) == 0 {
return ErrCredentialMissing
}
for credentialKey, credentialValue := range data {
credentialKey = strings.ToLower(credentialKey)
vcServer := ""
if strings.HasSuffix(credentialKey, "password") {
vcServer = strings.Split(credentialKey, ".password")[0]
if _, ok := config[vcServer]; !ok {
config[vcServer] = &Credential{}
}
config[vcServer].Password = string(credentialValue)
} else if strings.HasSuffix(credentialKey, "username") {
vcServer = strings.Split(credentialKey, ".username")[0]
if _, ok := config[vcServer]; !ok {
config[vcServer] = &Credential{}
}
config[vcServer].User = string(credentialValue)
} else {
glog.Errorf("Unknown secret key %s", credentialKey)
return ErrUnknownSecretKey
}
}
for vcServer, credential := range config {
if credential.User == "" || credential.Password == "" {
glog.Errorf("Username/Password is missing for server %s", vcServer)
return ErrCredentialMissing
}
}
return nil
}

338
pkg/cloudprovider/providers/vsphere/credentialmanager_test.go Normal file
View File

@ -0,0 +1,338 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vsphere
import (
"reflect"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/pkg/controller"
)
func TestSecretCredentialManager_GetCredential(t *testing.T) {
var (
userKey = "username"
passwordKey = "password"
testUser = "user"
testPassword = "password"
testServer = "0.0.0.0"
testServer2 = "0.0.1.1"
testUserServer2 = "user1"
testPasswordServer2 = "password1"
testIncorrectServer = "1.1.1.1"
)
var (
secretName = "vsconf"
secretNamespace = "kube-system"
)
var (
addSecretOp = "ADD_SECRET_OP"
getCredentialsOp = "GET_CREDENTIAL_OP"
deleteSecretOp = "DELETE_SECRET_OP"
)
type GetCredentialsTest struct {
server string
username string
password string
err error
}
type OpSecretTest struct {
secret *corev1.Secret
}
type testEnv struct {
testName string
ops []string
expectedValues []interface{}
}
client := &fake.Clientset{}
metaObj := metav1.ObjectMeta{
Name: secretName,
Namespace: secretNamespace,
}
defaultSecret := &corev1.Secret{
ObjectMeta: metaObj,
Data: map[string][]byte{
testServer + "." + userKey: []byte(testUser),
testServer + "." + passwordKey: []byte(testPassword),
},
}
multiVCSecret := &corev1.Secret{
ObjectMeta: metaObj,
Data: map[string][]byte{
testServer + "." + userKey: []byte(testUser),
testServer + "." + passwordKey: []byte(testPassword),
testServer2 + "." + userKey: []byte(testUserServer2),
testServer2 + "." + passwordKey: []byte(testPasswordServer2),
},
}
emptySecret := &corev1.Secret{
ObjectMeta: metaObj,
Data: map[string][]byte{},
}
tests := []testEnv{
{
testName: "Deleting secret should give the credentials from cache",
ops: []string{addSecretOp, getCredentialsOp, deleteSecretOp, getCredentialsOp},
expectedValues: []interface{}{
OpSecretTest{
secret: defaultSecret,
},
GetCredentialsTest{
username: testUser,
password: testPassword,
server: testServer,
},
OpSecretTest{
secret: defaultSecret,
},
GetCredentialsTest{
username: testUser,
password: testPassword,
server: testServer,
},
},
},
{
testName: "Add secret and get credentials",
ops: []string{addSecretOp, getCredentialsOp},
expectedValues: []interface{}{
OpSecretTest{
secret: defaultSecret,
},
GetCredentialsTest{
username: testUser,
password: testPassword,
server: testServer,
},
},
},
{
testName: "Getcredentials should fail by not adding at secret at first time",
ops: []string{getCredentialsOp},
expectedValues: []interface{}{
GetCredentialsTest{
username: testUser,
password: testPassword,
server: testServer,
err: ErrCredentialsNotFound,
},
},
},
{
testName: "GetCredential should fail to get credentials from empty secrets",
ops: []string{addSecretOp, getCredentialsOp},
expectedValues: []interface{}{
OpSecretTest{
secret: emptySecret,
},
GetCredentialsTest{
server: testServer,
err: ErrCredentialMissing,
},
},
},
{
testName: "GetCredential should fail to get credentials for invalid server",
ops: []string{addSecretOp, getCredentialsOp},
expectedValues: []interface{}{
OpSecretTest{
secret: defaultSecret,
},
GetCredentialsTest{
server: testIncorrectServer,
err: ErrCredentialsNotFound,
},
},
},
{
testName: "GetCredential for multi-vc",
ops: []string{addSecretOp, getCredentialsOp},
expectedValues: []interface{}{
OpSecretTest{
secret: multiVCSecret,
},
GetCredentialsTest{
server: testServer2,
username: testUserServer2,
password: testPasswordServer2,
},
},
},
}
informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
secretInformer := informerFactory.Core().V1().Secrets()
secretCredentialManager := &SecretCredentialManager{
SecretName: secretName,
SecretNamespace: secretNamespace,
SecretLister: secretInformer.Lister(),
Cache: &SecretCache{
VirtualCenter: make(map[string]*Credential),
},
}
cleanupSecretCredentialManager := func() {
secretCredentialManager.Cache.Secret = nil
for key := range secretCredentialManager.Cache.VirtualCenter {
delete(secretCredentialManager.Cache.VirtualCenter, key)
}
secrets, err := secretCredentialManager.SecretLister.List(labels.Everything())
if err != nil {
t.Fatal("Failed to get all secrets from sharedInformer. error: ", err)
}
for _, secret := range secrets {
secretInformer.Informer().GetIndexer().Delete(secret)
}
}
for _, test := range tests {
t.Logf("Executing Testcase: %s", test.testName)
for ntest, op := range test.ops {
switch op {
case addSecretOp:
expected := test.expectedValues[ntest].(OpSecretTest)
t.Logf("Adding secret: %s", expected.secret)
err := secretInformer.Informer().GetIndexer().Add(expected.secret)
if err != nil {
t.Fatalf("Failed to add secret to internal cache: %v", err)
}
case getCredentialsOp:
expected := test.expectedValues[ntest].(GetCredentialsTest)
credential, err := secretCredentialManager.GetCredential(expected.server)
t.Logf("Retrieving credentials for server %s", expected.server)
if err != expected.err {
t.Fatalf("Fail to get credentials with error: %v", err)
}
if expected.err == nil {
if expected.username != credential.User ||
expected.password != credential.Password {
t.Fatalf("Received credentials %v "+
"are different than actual credential user:%s password:%s", credential, expected.username,
expected.password)
}
}
case deleteSecretOp:
expected := test.expectedValues[ntest].(OpSecretTest)
t.Logf("Deleting secret: %s", expected.secret)
err := secretInformer.Informer().GetIndexer().Delete(expected.secret)
if err != nil {
t.Fatalf("Failed to delete secret to internal cache: %v", err)
}
}
}
cleanupSecretCredentialManager()
}
}
func TestParseSecretConfig(t *testing.T) {
var (
testUsername = "Admin"
testPassword = "Password"
testIP = "10.20.30.40"
)
var testcases = []struct {
testName string
data map[string][]byte
config map[string]*Credential
expectedError error
}{
{
testName: "Valid username and password",
data: map[string][]byte{
"10.20.30.40.username": []byte(testUsername),
"10.20.30.40.password": []byte(testPassword),
},
config: map[string]*Credential{
testIP: {
User: testUsername,
Password: testPassword,
},
},
expectedError: nil,
},
{
testName: "Invalid username key with valid password key",
data: map[string][]byte{
"10.20.30.40.usernam": []byte(testUsername),
"10.20.30.40.password": []byte(testPassword),
},
config: nil,
expectedError: ErrUnknownSecretKey,
},
{
testName: "Missing username",
data: map[string][]byte{
"10.20.30.40.password": []byte(testPassword),
},
config: map[string]*Credential{
testIP: {
Password: testPassword,
},
},
expectedError: ErrCredentialMissing,
},
{
testName: "Missing password",
data: map[string][]byte{
"10.20.30.40.username": []byte(testUsername),
},
config: map[string]*Credential{
testIP: {
User: testUsername,
},
},
expectedError: ErrCredentialMissing,
},
{
testName: "IP with unknown key",
data: map[string][]byte{
"10.20.30.40": []byte(testUsername),
},
config: nil,
expectedError: ErrUnknownSecretKey,
},
}
resultConfig := make(map[string]*Credential)
cleanupResultConfig := func(config map[string]*Credential) {
for k := range config {
delete(config, k)
}
}
for _, testcase := range testcases {
err := parseConfig(testcase.data, resultConfig)
t.Logf("Executing Testcase: %s", testcase.testName)
if err != testcase.expectedError {
t.Fatalf("Parsing Secret failed for data %+v: %s", testcase.data, err)
}
if testcase.config != nil && !reflect.DeepEqual(testcase.config, resultConfig) {
t.Fatalf("Parsing Secret failed for data %+v expected config %+v and actual config %+v",
testcase.data, resultConfig, testcase.config)
}
cleanupResultConfig(resultConfig)
}
}

113
pkg/cloudprovider/providers/vsphere/nodemanager.go
View File

@ -45,10 +45,13 @@ type NodeManager struct {
nodeInfoMap map[string]*NodeInfo
// Maps node name to node structure
registeredNodes map[string]*v1.Node
//CredentialsManager
credentialManager *SecretCredentialManager
// Mutexes
registeredNodesLock sync.RWMutex
nodeInfoLock sync.RWMutex
registeredNodesLock sync.RWMutex
nodeInfoLock sync.RWMutex
credentialManagerLock sync.Mutex
}
type NodeDetails struct {
@ -119,7 +122,7 @@ func (nm *NodeManager) DiscoverNode(node *v1.Node) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := vsi.conn.Connect(ctx)
err := nm.vcConnect(ctx, vsi)
if err != nil {
glog.V(4).Info("Discovering node error vc:", err)
setGlobalErr(err)
@ -297,30 +300,17 @@ func (nm *NodeManager) GetNodeInfo(nodeName k8stypes.NodeName) (NodeInfo, error)
//
// This method is a getter but it can cause side-effect of updating NodeInfo objects.
func (nm *NodeManager) GetNodeDetails() ([]NodeDetails, error) {
nm.nodeInfoLock.RLock()
defer nm.nodeInfoLock.RUnlock()
nm.registeredNodesLock.Lock()
defer nm.registeredNodesLock.Unlock()
var nodeDetails []NodeDetails
vsphereSessionRefreshMap := make(map[string]bool)
for nodeName, nodeInfo := range nm.nodeInfoMap {
var n *NodeInfo
var err error
if vsphereSessionRefreshMap[nodeInfo.vcServer] {
// vSphere connection already refreshed. Just refresh VM and Datacenter.
glog.V(4).Infof("Renewing NodeInfo %+v for node %q. No new connection needed.", nodeInfo, nodeName)
n, err = nm.renewNodeInfo(nodeInfo, false)
} else {
// Refresh vSphere connection, VM and Datacenter.
glog.V(4).Infof("Renewing NodeInfo %+v for node %q with new vSphere connection.", nodeInfo, nodeName)
n, err = nm.renewNodeInfo(nodeInfo, true)
vsphereSessionRefreshMap[nodeInfo.vcServer] = true
}
for nodeName, nodeObj := range nm.registeredNodes {
nodeInfo, err := nm.GetNodeInfoWithNodeObject(nodeObj)
if err != nil {
return nil, err
}
nm.nodeInfoMap[nodeName] = n
glog.V(4).Infof("Updated NodeInfo %q for node %q.", nodeInfo, nodeName)
nodeDetails = append(nodeDetails, NodeDetails{nodeName, n.vm, n.vmUUID})
nodeDetails = append(nodeDetails, NodeDetails{nodeName, nodeInfo.vm, nodeInfo.vmUUID})
}
return nodeDetails, nil
}
@ -355,7 +345,7 @@ func (nm *NodeManager) renewNodeInfo(nodeInfo *NodeInfo, reconnect bool) (*NodeI
return nil, err
}
if reconnect {
err := vsphereInstance.conn.Connect(ctx)
err := nm.vcConnect(ctx, vsphereInstance)
if err != nil {
return nil, err
}
@ -370,3 +360,82 @@ func (nodeInfo *NodeInfo) VM() *vclib.VirtualMachine {
}
return nodeInfo.vm
}
// vcConnect connects to vCenter with existing credentials
// If credentials are invalid:
// 1. It will fetch credentials from credentialManager
// 2. Update the credentials
// 3. Connects again to vCenter with fetched credentials
func (nm *NodeManager) vcConnect(ctx context.Context, vsphereInstance *VSphereInstance) error {
err := vsphereInstance.conn.Connect(ctx)
if err == nil {
return nil
}
credentialManager := nm.CredentialManager()
if !vclib.IsInvalidCredentialsError(err) || credentialManager == nil {
glog.Errorf("Cannot connect to vCenter with err: %v", err)
return err
}
glog.V(4).Infof("Invalid credentials. Cannot connect to server %q. "+
"Fetching credentials from secrets.", vsphereInstance.conn.Hostname)
// Get latest credentials from SecretCredentialManager
credentials, err := credentialManager.GetCredential(vsphereInstance.conn.Hostname)
if err != nil {
glog.Errorf("Failed to get credentials from Secret Credential Manager with err: %v", err)
return err
}
vsphereInstance.conn.UpdateCredentials(credentials.User, credentials.Password)
return vsphereInstance.conn.Connect(ctx)
}
// GetNodeInfoWithNodeObject returns a NodeInfo which datacenter, vm and vc server ip address.
// This method returns an error if it is unable find node VCs and DCs listed in vSphere.conf
// NodeInfo returned may not be updated to reflect current VM location.
//
// This method is a getter but it can cause side-effect of updating NodeInfo object.
func (nm *NodeManager) GetNodeInfoWithNodeObject(node *v1.Node) (NodeInfo, error) {
nodeName := node.Name
getNodeInfo := func(nodeName string) *NodeInfo {
nm.nodeInfoLock.RLock()
nodeInfo := nm.nodeInfoMap[nodeName]
nm.nodeInfoLock.RUnlock()
return nodeInfo
}
nodeInfo := getNodeInfo(nodeName)
var err error
if nodeInfo == nil {
// Rediscover node if no NodeInfo found.
glog.V(4).Infof("No VM found for node %q. Initiating rediscovery.", nodeName)
err = nm.DiscoverNode(node)
if err != nil {
glog.Errorf("Error %q node info for node %q not found", err, nodeName)
return NodeInfo{}, err
}
nodeInfo = getNodeInfo(nodeName)
} else {
// Renew the found NodeInfo to avoid stale vSphere connection.
glog.V(4).Infof("Renewing NodeInfo %+v for node %q", nodeInfo, nodeName)
nodeInfo, err = nm.renewNodeInfo(nodeInfo, true)
if err != nil {
glog.Errorf("Error %q occurred while renewing NodeInfo for %q", err, nodeName)
return NodeInfo{}, err
}
nm.addNodeInfo(nodeName, nodeInfo)
}
return *nodeInfo, nil
}
func (nm *NodeManager) CredentialManager() *SecretCredentialManager {
nm.credentialManagerLock.Lock()
defer nm.credentialManagerLock.Unlock()
return nm.credentialManager
}
func (nm *NodeManager) UpdateCredentialManager(credentialManager *SecretCredentialManager) {
nm.credentialManagerLock.Lock()
defer nm.credentialManagerLock.Unlock()
nm.credentialManager = credentialManager
}

12
pkg/cloudprovider/providers/vsphere/vclib/connection.go
View File

@ -40,6 +40,7 @@ type VSphereConnection struct {
Port string
Insecure bool
RoundTripperCount uint
credentialsLock sync.Mutex
}
var (
@ -85,6 +86,8 @@ func (connection *VSphereConnection) Connect(ctx context.Context) error {
// otherwise calls SessionManager.Login with user and password.
func (connection *VSphereConnection) login(ctx context.Context, client *vim25.Client) error {
m := session.NewManager(client)
connection.credentialsLock.Lock()
defer connection.credentialsLock.Unlock()
// TODO: Add separate fields for certificate and private-key.
// For now we can leave the config structs and validation as-is and
@ -163,3 +166,12 @@ func (connection *VSphereConnection) NewClient(ctx context.Context) (*vim25.Clie
client.RoundTripper = vim25.Retry(client.RoundTripper, vim25.TemporaryNetworkError(int(connection.RoundTripperCount)))
return client, nil
}
// UpdateCredentials updates username and password.
// Note: Updated username and password will be used when there is no session active
func (connection *VSphereConnection) UpdateCredentials(username string, password string) {
connection.credentialsLock.Lock()
defer connection.credentialsLock.Unlock()
connection.Username = username
connection.Password = password
}

9
pkg/cloudprovider/providers/vsphere/vclib/utils.go
View File

@ -172,6 +172,15 @@ func IsManagedObjectNotFoundError(err error) bool {
return isManagedObjectNotFoundError
}
// IsInvalidCredentialsError returns true if error is of type InvalidLogin
func IsInvalidCredentialsError(err error) bool {
isInvalidCredentialsError := false
if soap.IsSoapFault(err) {
_, isInvalidCredentialsError = soap.ToSoapFault(err).VimFault().(types.InvalidLogin)
}
return isInvalidCredentialsError
}
// VerifyVolumePathsForVM verifies if the volume paths (volPaths) are attached to VM.
func VerifyVolumePathsForVM(vmMo mo.VirtualMachine, volPaths []string, nodeName string, nodeVolumeMap map[string]map[string]bool) {
// Verify if the volume paths are present on the VM backing virtual disk devices

168
pkg/cloudprovider/providers/vsphere/vsphere.go
View File

@ -61,6 +61,18 @@ var datastoreFolderIDMap = make(map[string]map[string]string)
var cleanUpRoutineInitLock sync.Mutex
var cleanUpDummyVMLock sync.RWMutex
// Error Messages
const (
MissingUsernameErrMsg = "Username is missing"
MissingPasswordErrMsg = "Password is missing"
)
// Error constants
var (
ErrUsernameMissing = errors.New(MissingUsernameErrMsg)
ErrPasswordMissing = errors.New(MissingPasswordErrMsg)
)
// VSphere is an implementation of cloud provider Interface for VSphere.
type VSphere struct {
cfg *VSphereConfig
@ -68,8 +80,9 @@ type VSphere struct {
// Maps the VSphere IP address to VSphereInstance
vsphereInstanceMap map[string]*VSphereInstance
// Responsible for managing discovery of k8s node, their location etc.
nodeManager *NodeManager
vmUUID string
nodeManager *NodeManager
vmUUID string
isSecretInfoProvided bool
}
// Represents a vSphere instance where one or more kubernetes nodes are running.
@ -131,6 +144,10 @@ type VSphereConfig struct {
// Combining the WorkingDir and VMName can form a unique InstanceID.
// When vm-name is set, no username/password is required on worker nodes.
VMName string `gcfg:"vm-name"`
// Name of the secret were vCenter credentials are present.
SecretName string `gcfg:"secret-name"`
// Secret Namespace where secret will be present that has vCenter credentials.
SecretNamespace string `gcfg:"secret-namespace"`
}
VirtualCenter map[string]*VirtualCenterConfig
@ -217,6 +234,18 @@ func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory)
return
}
if vs.isSecretInfoProvided {
secretCredentialManager := &SecretCredentialManager{
SecretName: vs.cfg.Global.SecretName,
SecretNamespace: vs.cfg.Global.SecretNamespace,
SecretLister: informerFactory.Core().V1().Secrets().Lister(),
Cache: &SecretCache{
VirtualCenter: make(map[string]*Credential),
},
}
vs.nodeManager.UpdateCredentialManager(secretCredentialManager)
}
// Only on controller node it is required to register listeners.
// Register callbacks for node updates
glog.V(4).Infof("Setting up node informers for vSphere Cloud Provider")
@ -226,6 +255,7 @@ func (vs *VSphere) SetInformers(informerFactory informers.SharedInformerFactory)
DeleteFunc: vs.NodeDeleted,
})
glog.V(4).Infof("Node informers in vSphere cloud provider initialized")
}
// Creates new worker node interface and returns
@ -247,19 +277,40 @@ func newWorkerNode() (*VSphere, error) {
func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance, error) {
vsphereInstanceMap := make(map[string]*VSphereInstance)
isSecretInfoProvided := true
if cfg.Global.SecretName == "" || cfg.Global.SecretNamespace == "" {
glog.Warningf("SecretName and/or SecretNamespace is not provided. " +
"VCP will use username and password from config file")
isSecretInfoProvided = false
}
if isSecretInfoProvided {
if cfg.Global.User != "" {
glog.Warning("Global.User and Secret info provided. VCP will use secret to get credentials")
cfg.Global.User = ""
}
if cfg.Global.Password != "" {
glog.Warning("Global.Password and Secret info provided. VCP will use secret to get credentials")
cfg.Global.Password = ""
}
}
// Check if the vsphere.conf is in old format. In this
// format the cfg.VirtualCenter will be nil or empty.
if cfg.VirtualCenter == nil || len(cfg.VirtualCenter) == 0 {
glog.V(4).Infof("Config is not per virtual center and is in old format.")
if cfg.Global.User == "" {
glog.Error("Global.User is empty!")
return nil, errors.New("Global.User is empty!")
}
if cfg.Global.Password == "" {
glog.Error("Global.Password is empty!")
return nil, errors.New("Global.Password is empty!")
if !isSecretInfoProvided {
if cfg.Global.User == "" {
glog.Error("Global.User is empty!")
return nil, ErrUsernameMissing
}
if cfg.Global.Password == "" {
glog.Error("Global.Password is empty!")
return nil, ErrPasswordMissing
}
}
if cfg.Global.WorkingDir == "" {
glog.Error("Global.WorkingDir is empty!")
return nil, errors.New("Global.WorkingDir is empty!")
@ -285,6 +336,8 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance
RoundTripperCount: cfg.Global.RoundTripperCount,
}
// Note: If secrets info is provided username and password will be populated
// once secret is created.
vSphereConn := vclib.VSphereConnection{
Username: vcConfig.User,
Password: vcConfig.Password,
@ -305,31 +358,44 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance
glog.Error(msg)
return nil, errors.New(msg)
}
for vcServer, vcConfig := range cfg.VirtualCenter {
glog.V(4).Infof("Initializing vc server %s", vcServer)
if vcServer == "" {
glog.Error("vsphere.conf does not have the VirtualCenter IP address specified")
return nil, errors.New("vsphere.conf does not have the VirtualCenter IP address specified")
}
if vcConfig.User == "" {
vcConfig.User = cfg.Global.User
}
if vcConfig.Password == "" {
vcConfig.Password = cfg.Global.Password
}
if vcConfig.User == "" {
msg := fmt.Sprintf("vcConfig.User is empty for vc %s!", vcServer)
glog.Error(msg)
return nil, errors.New(msg)
}
if vcConfig.Password == "" {
msg := fmt.Sprintf("vcConfig.Password is empty for vc %s!", vcServer)
glog.Error(msg)
return nil, errors.New(msg)
if !isSecretInfoProvided {
if vcConfig.User == "" {
vcConfig.User = cfg.Global.User
if vcConfig.User == "" {
glog.Errorf("vcConfig.User is empty for vc %s!", vcServer)
return nil, ErrUsernameMissing
}
}
if vcConfig.Password == "" {
vcConfig.Password = cfg.Global.Password
if vcConfig.Password == "" {
glog.Errorf("vcConfig.Password is empty for vc %s!", vcServer)
return nil, ErrPasswordMissing
}
}
} else {
if vcConfig.User != "" {
glog.Warningf("vcConfig.User for server %s and Secret info provided. VCP will use secret to get credentials", vcServer)
vcConfig.User = ""
}
if vcConfig.Password != "" {
glog.Warningf("vcConfig.Password for server %s and Secret info provided. VCP will use secret to get credentials", vcServer)
vcConfig.Password = ""
}
}
if vcConfig.VCenterPort == "" {
vcConfig.VCenterPort = cfg.Global.VCenterPort
}
if vcConfig.Datacenters == "" {
if cfg.Global.Datacenters != "" {
vcConfig.Datacenters = cfg.Global.Datacenters
@ -342,6 +408,8 @@ func populateVsphereInstanceMap(cfg *VSphereConfig) (map[string]*VSphereInstance
vcConfig.RoundTripperCount = cfg.Global.RoundTripperCount
}
// Note: If secrets info is provided username and password will be populated
// once secret is created.
vSphereConn := vclib.VSphereConnection{
Username: vcConfig.User,
Password: vcConfig.Password,
@ -365,7 +433,30 @@ var getVMUUID = GetVMUUID
// Creates new Controller node interface and returns
func newControllerNode(cfg VSphereConfig) (*VSphere, error) {
var err error
vs, err := buildVSphereFromConfig(cfg)
if err != nil {
return nil, err
}
vs.hostName, err = os.Hostname()
if err != nil {
glog.Errorf("Failed to get hostname. err: %+v", err)
return nil, err
}
vs.vmUUID, err = getVMUUID()
if err != nil {
glog.Errorf("Failed to get uuid. err: %+v", err)
return nil, err
}
runtime.SetFinalizer(vs, logout)
return vs, nil
}
// Initializes vSphere from vSphere CloudProvider Configuration
func buildVSphereFromConfig(cfg VSphereConfig) (*VSphere, error) {
isSecretInfoProvided := false
if cfg.Global.SecretName != "" && cfg.Global.SecretNamespace != "" {
isSecretInfoProvided = true
}
if cfg.Disk.SCSIControllerType == "" {
cfg.Disk.SCSIControllerType = vclib.PVSCSIControllerType
@ -394,20 +485,9 @@ func newControllerNode(cfg VSphereConfig) (*VSphere, error) {
nodeInfoMap: make(map[string]*NodeInfo),
registeredNodes: make(map[string]*v1.Node),
},
cfg: &cfg,
isSecretInfoProvided: isSecretInfoProvided,
cfg: &cfg,
}
vs.hostName, err = os.Hostname()
if err != nil {
glog.Errorf("Failed to get hostname. err: %+v", err)
return nil, err
}
vs.vmUUID, err = getVMUUID()
if err != nil {
glog.Errorf("Failed to get uuid. err: %+v", err)
return nil, err
}
runtime.SetFinalizer(&vs, logout)
return &vs, nil
}
@ -480,7 +560,7 @@ func (vs *VSphere) getVSphereInstanceForServer(vcServer string, ctx context.Cont
return nil, errors.New(fmt.Sprintf("Cannot find node %q in vsphere configuration map", vcServer))
}
// Ensure client is logged in and session is valid
err := vsphereIns.conn.Connect(ctx)
err := vs.nodeManager.vcConnect(ctx, vsphereIns)
if err != nil {
glog.Errorf("failed connecting to vcServer %q with error %+v", vcServer, err)
return nil, err
@ -519,7 +599,7 @@ func (vs *VSphere) NodeAddresses(ctx context.Context, nodeName k8stypes.NodeName
return nil, err
}
// Ensure client is logged in and session is valid
err = vsi.conn.Connect(ctx)
err = vs.nodeManager.vcConnect(ctx, vsi)
if err != nil {
return nil, err
}
@ -634,7 +714,7 @@ func (vs *VSphere) InstanceID(ctx context.Context, nodeName k8stypes.NodeName) (
return "", err
}
// Ensure client is logged in and session is valid
err = vsi.conn.Connect(ctx)
err = vs.nodeManager.vcConnect(ctx, vsi)
if err != nil {
return "", err
}
@ -725,7 +805,7 @@ func (vs *VSphere) AttachDisk(vmDiskPath string, storagePolicyName string, nodeN
return "", err
}
// Ensure client is logged in and session is valid
err = vsi.conn.Connect(ctx)
err = vs.nodeManager.vcConnect(ctx, vsi)
if err != nil {
return "", err
}
@ -792,7 +872,7 @@ func (vs *VSphere) DetachDisk(volPath string, nodeName k8stypes.NodeName) error
return err
}
// Ensure client is logged in and session is valid
err = vsi.conn.Connect(ctx)
err = vs.nodeManager.vcConnect(ctx, vsi)
if err != nil {
return err
}
@ -847,7 +927,7 @@ func (vs *VSphere) DiskIsAttached(volPath string, nodeName k8stypes.NodeName) (b
return false, err
}
// Ensure client is logged in and session is valid
err = vsi.conn.Connect(ctx)
err = vs.nodeManager.vcConnect(ctx, vsi)
if err != nil {
return false, err
}

218
pkg/cloudprovider/providers/vsphere/vsphere_test.go
View File

@ -352,3 +352,221 @@ func TestVolumes(t *testing.T) {
// t.Fatalf("Cannot delete VMDK volume %s: %v", volPath, err)
// }
}
func TestSecretVSphereConfig(t *testing.T) {
var vs *VSphere
var (
username = "user"
password = "password"
)
var testcases = []struct {
testName string
conf string
expectedIsSecretProvided bool
expectedUsername string
expectedPassword string
expectedError error
}{
{
testName: "Username and password with old configuration",
conf: `[Global]
server = 0.0.0.0
user = user
password = password
datacenter = us-west
working-dir = kubernetes
`,
expectedUsername: username,
expectedPassword: password,
expectedError: nil,
},
{
testName: "SecretName and SecretNamespace in old configuration",
conf: `[Global]
server = 0.0.0.0
datacenter = us-west
secret-name = "vccreds"
secret-namespace = "kube-system"
working-dir = kubernetes
`,
expectedIsSecretProvided: true,
expectedError: nil,
},
{
testName: "SecretName and SecretNamespace with Username and Password in old configuration",
conf: `[Global]
server = 0.0.0.0
user = user
password = password
datacenter = us-west
secret-name = "vccreds"
secret-namespace = "kube-system"
working-dir = kubernetes
`,
expectedIsSecretProvided: true,
expectedError: nil,
},
{
testName: "SecretName and SecretNamespace with Username missing in old configuration",
conf: `[Global]
server = 0.0.0.0
password = password
datacenter = us-west
secret-name = "vccreds"
secret-namespace = "kube-system"
working-dir = kubernetes
`,
expectedIsSecretProvided: true,
expectedError: nil,
},
{
testName: "SecretNamespace missing with Username and Password in old configuration",
conf: `[Global]
server = 0.0.0.0
user = user
password = password
datacenter = us-west
secret-name = "vccreds"
working-dir = kubernetes
`,
expectedUsername: username,
expectedPassword: password,
expectedError: nil,
},
{
testName: "SecretNamespace and Username missing in old configuration",
conf: `[Global]
server = 0.0.0.0
password = password
datacenter = us-west
secret-name = "vccreds"
working-dir = kubernetes
`,
expectedError: ErrUsernameMissing,
},
{
testName: "SecretNamespace and Password missing in old configuration",
conf: `[Global]
server = 0.0.0.0
user = user
datacenter = us-west
secret-name = "vccreds"
working-dir = kubernetes
`,
expectedError: ErrPasswordMissing,
},
{
testName: "SecretNamespace, Username and Password missing in old configuration",
conf: `[Global]
server = 0.0.0.0
datacenter = us-west
secret-name = "vccreds"
working-dir = kubernetes
`,
expectedError: ErrUsernameMissing,
},
{
testName: "Username and password with new configuration but username and password in global section",
conf: `[Global]
user = user
password = password
datacenter = us-west
[VirtualCenter "0.0.0.0"]
[Workspace]
server = 0.0.0.0
datacenter = us-west
folder = kubernetes
`,
expectedUsername: username,
expectedPassword: password,
expectedError: nil,
},
{
testName: "Username and password with new configuration, username and password in virtualcenter section",
conf: `[Global]
server = 0.0.0.0
port = 443
insecure-flag = true
datacenter = us-west
[VirtualCenter "0.0.0.0"]
user = user
password = password
[Workspace]
server = 0.0.0.0
datacenter = us-west
folder = kubernetes
`,
expectedUsername: username,
expectedPassword: password,
expectedError: nil,
},
{
testName: "SecretName and SecretNamespace with new configuration",
conf: `[Global]
server = 0.0.0.0
secret-name = "vccreds"
secret-namespace = "kube-system"
datacenter = us-west
[VirtualCenter "0.0.0.0"]
[Workspace]
server = 0.0.0.0
datacenter = us-west
folder = kubernetes
`,
expectedIsSecretProvided: true,
expectedError: nil,
},
{
testName: "SecretName and SecretNamespace with Username missing in new configuration",
conf: `[Global]
server = 0.0.0.0
port = 443
insecure-flag = true
datacenter = us-west
secret-name = "vccreds"
secret-namespace = "kube-system"
[VirtualCenter "0.0.0.0"]
password = password
[Workspace]
server = 0.0.0.0
datacenter = us-west
folder = kubernetes
`,
expectedIsSecretProvided: true,
expectedError: nil,
},
}
for _, testcase := range testcases {
t.Logf("Executing Testcase: %s", testcase.testName)
cfg, err := readConfig(strings.NewReader(testcase.conf))
if err != nil {
t.Fatalf("Should succeed when a valid config is provided: %s", err)
}
vs, err = buildVSphereFromConfig(cfg)
if err != testcase.expectedError {
t.Fatalf("Should succeed when a valid config is provided: %s", err)
}
if err != nil {
continue
}
if vs.isSecretInfoProvided != testcase.expectedIsSecretProvided {
t.Fatalf("SecretName and SecretNamespace was expected in config %s. error: %s",
testcase.conf, err)
}
if !testcase.expectedIsSecretProvided {
for _, vsInstance := range vs.vsphereInstanceMap {
if vsInstance.conn.Username != testcase.expectedUsername {
t.Fatalf("Expected username %s doesn't match actual username %s in config %s. error: %s",
testcase.expectedUsername, vsInstance.conn.Username, testcase.conf, err)
}
if vsInstance.conn.Password != testcase.expectedPassword {
t.Fatalf("Expected password %s doesn't match actual password %s in config %s. error: %s",
testcase.expectedPassword, vsInstance.conn.Password, testcase.conf, err)
}
}
}
}
}