add LocalZone into gce.conf and refactor gce cloud provider configuration to allow avoiding external communication

pull/6/head
Minhan Xia 2017-07-31 17:39:13 -07:00
parent 942f030ff0
commit 1cad829b6e
4 changed files with 331 additions and 94 deletions

View File

@ -91,6 +91,7 @@ go_test(
deps = [
"//pkg/cloudprovider:go_default_library",
"//pkg/kubelet/apis:go_default_library",
"//vendor/golang.org/x/oauth2/google:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
"//vendor/google.golang.org/api/googleapi:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",

View File

@ -133,7 +133,7 @@ type GCEServiceManager struct {
gce *GCECloud
}
type Config struct {
type ConfigFile struct {
Global struct {
TokenURL string `gcfg:"token-url"`
TokenBody string `gcfg:"token-body"`
@ -145,9 +145,25 @@ type Config struct {
Multizone bool `gcfg:"multizone"`
// Specifying ApiEndpoint will override the default GCE compute API endpoint.
ApiEndpoint string `gcfg:"api-endpoint"`
LocalZone string `gcfg:"local-zone"`
}
}
// CloudConfig includes all the necessary configuration for creating GCECloud
type CloudConfig struct {
ApiEndpoint string
ProjectID string
Region string
Zone string
ManagedZones []string
NetworkURL string
SubnetworkURL string
NodeTags []string
NodeInstancePrefix string
TokenSource oauth2.TokenSource
UseMetadataServer bool
}
func init() {
cloudprovider.RegisterCloudProvider(
ProviderName,
@ -172,77 +188,28 @@ func (g *GCECloud) GetProjectID() string {
}
// newGCECloud creates a new instance of GCECloud.
func newGCECloud(config io.Reader) (*GCECloud, error) {
apiEndpoint := ""
projectID, zone, err := getProjectAndZone()
if err != nil {
return nil, err
}
func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) {
var cloudConfig *CloudConfig
var configFile *ConfigFile
region, err := GetGCERegion(zone)
if err != nil {
return nil, err
}
networkName, err := getNetworkNameViaMetadata()
if err != nil {
return nil, err
}
networkURL := gceNetworkURL("", projectID, networkName)
subnetworkURL := ""
// By default, Kubernetes clusters only run against one zone
managedZones := []string{zone}
tokenSource := google.ComputeTokenSource("")
var nodeTags []string
var nodeInstancePrefix string
if config != nil {
cfg, err := readConfig(config)
configFile, err = readConfig(config)
if err != nil {
return nil, err
}
glog.Infof("Using GCE provider config %+v", cfg)
if cfg.Global.ApiEndpoint != "" {
apiEndpoint = cfg.Global.ApiEndpoint
}
if cfg.Global.ProjectID != "" {
projectID = cfg.Global.ProjectID
glog.Infof("Using GCE provider config %+v", configFile)
}
if cfg.Global.NetworkName != "" {
if strings.Contains(cfg.Global.NetworkName, "/") {
networkURL = cfg.Global.NetworkName
} else {
networkURL = gceNetworkURL(apiEndpoint, projectID, cfg.Global.NetworkName)
cloudConfig, err = generateCloudConfig(configFile)
if err != nil {
return nil, err
}
return CreateGCECloud(cloudConfig)
}
if cfg.Global.SubnetworkName != "" {
if strings.Contains(cfg.Global.SubnetworkName, "/") {
subnetworkURL = cfg.Global.SubnetworkName
} else {
subnetworkURL = gceSubnetworkURL(apiEndpoint, projectID, region, cfg.Global.SubnetworkName)
}
}
if cfg.Global.TokenURL != "" {
tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
}
nodeTags = cfg.Global.NodeTags
nodeInstancePrefix = cfg.Global.NodeInstancePrefix
if cfg.Global.Multizone {
managedZones = nil // Use all zones in region
}
}
return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL,
nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */)
}
func readConfig(reader io.Reader) (*Config, error) {
cfg := &Config{}
func readConfig(reader io.Reader) (*ConfigFile, error) {
cfg := &ConfigFile{}
if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
glog.Errorf("Couldn't read config: %v", err)
return nil, err
@ -250,14 +217,92 @@ func readConfig(reader io.Reader) (*Config, error) {
return cfg, nil
}
func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err error) {
cloudConfig = &CloudConfig{}
// By default, fetch token from GCE metadata server
cloudConfig.TokenSource = google.ComputeTokenSource("")
cloudConfig.UseMetadataServer = true
if configFile != nil {
if configFile.Global.ApiEndpoint != "" {
cloudConfig.ApiEndpoint = configFile.Global.ApiEndpoint
}
if configFile.Global.TokenURL != "" {
// if tokenURL is nil, set tokenSource to nil. This will force the OAuth client to fall
// back to use DefaultTokenSource. This allows running gceCloud remotely.
if configFile.Global.TokenURL == "nil" {
cloudConfig.TokenSource = nil
} else {
cloudConfig.TokenSource = NewAltTokenSource(configFile.Global.TokenURL, configFile.Global.TokenBody)
}
}
cloudConfig.NodeTags = configFile.Global.NodeTags
cloudConfig.NodeInstancePrefix = configFile.Global.NodeInstancePrefix
}
// retrieve projectID and zone
if configFile == nil || configFile.Global.ProjectID == "" || configFile.Global.LocalZone == "" {
cloudConfig.ProjectID, cloudConfig.Zone, err = getProjectAndZone()
if err != nil {
return nil, err
}
}
if configFile != nil {
if configFile.Global.ProjectID != "" {
cloudConfig.ProjectID = configFile.Global.ProjectID
}
if configFile.Global.LocalZone != "" {
cloudConfig.Zone = configFile.Global.LocalZone
}
}
// retrieve region
cloudConfig.Region, err = GetGCERegion(cloudConfig.Zone)
if err != nil {
return nil, err
}
// generate managedZones
cloudConfig.ManagedZones = []string{cloudConfig.Zone}
if configFile != nil && configFile.Global.Multizone {
cloudConfig.ManagedZones = nil // Use all zones in region
}
// generate networkURL
if configFile != nil && configFile.Global.NetworkName != "" {
if strings.Contains(configFile.Global.NetworkName, "/") {
cloudConfig.NetworkURL = configFile.Global.NetworkName
} else {
cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName)
}
} else {
networkName, err := getNetworkNameViaMetadata()
if err != nil {
return nil, err
}
cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName)
}
// generate subnetworkURL
if configFile != nil && configFile.Global.SubnetworkName != "" {
if strings.Contains(configFile.Global.SubnetworkName, "/") {
cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName
} else {
cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName)
}
}
return cloudConfig, err
}
// Creates a GCECloud object using the specified parameters.
// If no networkUrl is specified, loads networkName via rest call.
// If no tokenSource is specified, uses oauth2.DefaultTokenSource.
// If managedZones is nil / empty all zones in the region will be managed.
func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string,
nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) {
func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
client, err := newOauthClient(tokenSource)
client, err := newOauthClient(config.TokenSource)
if err != nil {
return nil, err
}
@ -266,7 +311,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
return nil, err
}
client, err = newOauthClient(tokenSource)
client, err = newOauthClient(config.TokenSource)
if err != nil {
return nil, err
}
@ -275,7 +320,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
return nil, err
}
client, err = newOauthClient(tokenSource)
client, err = newOauthClient(config.TokenSource)
if err != nil {
return nil, err
}
@ -288,10 +333,10 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
// Generate alpha and beta api endpoints based on override v1 api endpoint.
// For example,
// staging API endpoint: https://www.googleapis.com/compute/staging_v1/
if apiEndpoint != "" {
service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint)
serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "beta", -1))
serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "alpha", -1))
if config.ApiEndpoint != "" {
service.BasePath = fmt.Sprintf("%sprojects/", config.ApiEndpoint)
serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "beta", -1))
serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "alpha", -1))
}
containerService, err := container.New(client)
@ -304,28 +349,28 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
return nil, err
}
if networkURL == "" {
networkName, err := getNetworkNameViaAPICall(service, projectID)
if config.NetworkURL == "" {
networkName, err := getNetworkNameViaAPICall(service, config.ProjectID)
if err != nil {
return nil, err
}
networkURL = gceNetworkURL(apiEndpoint, projectID, networkName)
config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName)
}
networkProjectID, err := getProjectIDInURL(networkURL)
networkProjectID, err := getProjectIDInURL(config.NetworkURL)
if err != nil {
return nil, err
}
onXPN := networkProjectID != projectID
onXPN := networkProjectID != config.ProjectID
if len(managedZones) == 0 {
managedZones, err = getZonesForRegion(service, projectID, region)
if len(config.ManagedZones) == 0 {
config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region)
if err != nil {
return nil, err
}
}
if len(managedZones) != 1 {
glog.Infof("managing multiple zones: %v", managedZones)
if len(config.ManagedZones) != 1 {
glog.Infof("managing multiple zones: %v", config.ManagedZones)
}
operationPollRateLimiter := flowcontrol.NewTokenBucketRateLimiter(10, 100) // 10 qps, 100 bucket size.
@ -336,17 +381,17 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
serviceBeta: serviceBeta,
containerService: containerService,
cloudkmsService: cloudkmsService,
projectID: projectID,
projectID: config.ProjectID,
networkProjectID: networkProjectID,
onXPN: onXPN,
region: region,
localZone: zone,
managedZones: managedZones,
networkURL: networkURL,
subnetworkURL: subnetworkURL,
nodeTags: nodeTags,
nodeInstancePrefix: nodeInstancePrefix,
useMetadataServer: useMetadataServer,
region: config.Region,
localZone: config.Zone,
managedZones: config.ManagedZones,
networkURL: config.NetworkURL,
subnetworkURL: config.SubnetworkURL,
nodeTags: config.NodeTags,
nodeInstancePrefix: config.NodeInstancePrefix,
useMetadataServer: config.UseMetadataServer,
operationPollRateLimiter: operationPollRateLimiter,
}

View File

@ -17,6 +17,7 @@ limitations under the License.
package gce
import (
"golang.org/x/oauth2/google"
"reflect"
"strings"
"testing"
@ -255,3 +256,184 @@ func TestSplitProviderID(t *testing.T) {
}
}
}
func TestGenerateCloudConfigs(t *testing.T) {
testCases := []struct {
TokenURL string
TokenBody string
ProjectID string
NetworkName string
SubnetworkName string
NodeTags []string
NodeInstancePrefix string
Multizone bool
ApiEndpoint string
LocalZone string
cloudConfig *CloudConfig
}{
{
TokenURL: "",
TokenBody: "",
ProjectID: "project-id",
NetworkName: "network-name",
SubnetworkName: "subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
Multizone: false,
ApiEndpoint: "",
LocalZone: "us-central1-a",
cloudConfig: &CloudConfig{
ApiEndpoint: "",
ProjectID: "project-id",
Region: "us-central1",
Zone: "us-central1-a",
ManagedZones: []string{"us-central1-a"},
NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
TokenSource: google.ComputeTokenSource(""),
UseMetadataServer: true,
},
},
// nil token source
{
TokenURL: "nil",
TokenBody: "",
ProjectID: "project-id",
NetworkName: "network-name",
SubnetworkName: "subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
Multizone: false,
ApiEndpoint: "",
LocalZone: "us-central1-a",
cloudConfig: &CloudConfig{
ApiEndpoint: "",
ProjectID: "project-id",
Region: "us-central1",
Zone: "us-central1-a",
ManagedZones: []string{"us-central1-a"},
NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
TokenSource: nil,
UseMetadataServer: true,
},
},
// specified api endpoint
{
TokenURL: "",
TokenBody: "",
ProjectID: "project-id",
NetworkName: "network-name",
SubnetworkName: "subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
Multizone: false,
ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/",
LocalZone: "us-central1-a",
cloudConfig: &CloudConfig{
ApiEndpoint: "https://www.googleapis.com/compute/staging_v1/",
ProjectID: "project-id",
Region: "us-central1",
Zone: "us-central1-a",
ManagedZones: []string{"us-central1-a"},
NetworkURL: "https://www.googleapis.com/compute/staging_v1/projects/project-id/global/networks/network-name",
SubnetworkURL: "https://www.googleapis.com/compute/staging_v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
TokenSource: google.ComputeTokenSource(""),
UseMetadataServer: true,
},
},
// empty subnet-name
{
TokenURL: "",
TokenBody: "",
ProjectID: "project-id",
NetworkName: "network-name",
SubnetworkName: "",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
Multizone: false,
ApiEndpoint: "",
LocalZone: "us-central1-a",
cloudConfig: &CloudConfig{
ApiEndpoint: "",
ProjectID: "project-id",
Region: "us-central1",
Zone: "us-central1-a",
ManagedZones: []string{"us-central1-a"},
NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
SubnetworkURL: "",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
TokenSource: google.ComputeTokenSource(""),
UseMetadataServer: true,
},
},
// multi zone
{
TokenURL: "",
TokenBody: "",
ProjectID: "project-id",
NetworkName: "network-name",
SubnetworkName: "subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
Multizone: true,
ApiEndpoint: "",
LocalZone: "us-central1-a",
cloudConfig: &CloudConfig{
ApiEndpoint: "",
ProjectID: "project-id",
Region: "us-central1",
Zone: "us-central1-a",
ManagedZones: nil,
NetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
SubnetworkURL: "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
NodeTags: []string{"node-tag"},
NodeInstancePrefix: "node-prefix",
TokenSource: google.ComputeTokenSource(""),
UseMetadataServer: true,
},
},
}
for _, tc := range testCases {
cloudConfig, err := generateCloudConfig(&ConfigFile{
Global: struct {
TokenURL string `gcfg:"token-url"`
TokenBody string `gcfg:"token-body"`
ProjectID string `gcfg:"project-id"`
NetworkName string `gcfg:"network-name"`
SubnetworkName string `gcfg:"subnetwork-name"`
NodeTags []string `gcfg:"node-tags"`
NodeInstancePrefix string `gcfg:"node-instance-prefix"`
Multizone bool `gcfg:"multizone"`
ApiEndpoint string `gcfg:"api-endpoint"`
LocalZone string `gcfg:"local-zone"`
}{
TokenURL: tc.TokenURL,
TokenBody: tc.TokenBody,
ProjectID: tc.ProjectID,
NetworkName: tc.NetworkName,
SubnetworkName: tc.SubnetworkName,
NodeTags: tc.NodeTags,
NodeInstancePrefix: tc.NodeInstancePrefix,
Multizone: tc.Multizone,
ApiEndpoint: tc.ApiEndpoint,
LocalZone: tc.LocalZone,
},
})
if err != nil {
t.Fatalf("Unexpect error: %v", err)
}
if !reflect.DeepEqual(cloudConfig, tc.cloudConfig) {
t.Errorf("Expecting cloud config: %v, but got %v", tc.cloudConfig, cloudConfig)
}
}
}

View File

@ -78,10 +78,19 @@ func setupProviderConfig() error {
managedZones = []string{zone}
}
gceCloud, err := gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint,
framework.TestContext.CloudConfig.ProjectID,
region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */
"" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */)
gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
ApiEndpoint: framework.TestContext.CloudConfig.ApiEndpoint,
ProjectID: framework.TestContext.CloudConfig.ProjectID,
Region: region,
Zone: zone,
ManagedZones: managedZones,
NetworkURL: "",
SubnetworkURL: "",
NodeTags: nil,
NodeInstancePrefix: "",
TokenSource: nil,
UseMetadataServer: false})
if err != nil {
return fmt.Errorf("Error building GCE/GKE provider: %v", err)
}