add LocalZone into gce.conf and refactor gce cloud provider configuration to allow avoiding external communication

pkg/cloudprovider/providers/gce/BUILD

@@ -91,6 +91,7 @@ go_test(
     deps = [
         "//pkg/cloudprovider:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
+        "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

pkg/cloudprovider/providers/gce/gce.go

@@ -133,7 +133,7 @@ type GCEServiceManager struct {
 	gce *GCECloud
 }
 
-type Config struct {
+type ConfigFile struct {
 	Global struct {
 		TokenURL           string   `gcfg:"token-url"`
 		TokenBody          string   `gcfg:"token-body"`
@@ -145,9 +145,25 @@ type Config struct {
 		Multizone          bool     `gcfg:"multizone"`
 		// Specifying ApiEndpoint will override the default GCE compute API endpoint.
 		ApiEndpoint string `gcfg:"api-endpoint"`
+		LocalZone   string `gcfg:"local-zone"`
 	}
 }
 
+// CloudConfig includes all the necessary configuration for creating GCECloud
+type CloudConfig struct {
+	ApiEndpoint        string
+	ProjectID          string
+	Region             string
+	Zone               string
+	ManagedZones       []string
+	NetworkURL         string
+	SubnetworkURL      string
+	NodeTags           []string
+	NodeInstancePrefix string
+	TokenSource        oauth2.TokenSource
+	UseMetadataServer  bool
+}
+
 func init() {
 	cloudprovider.RegisterCloudProvider(
 		ProviderName,
@@ -172,77 +188,28 @@ func (g *GCECloud) GetProjectID() string {
 }
 
 // newGCECloud creates a new instance of GCECloud.
-func newGCECloud(config io.Reader) (*GCECloud, error) {
+func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) {
-	apiEndpoint := ""
+	var cloudConfig *CloudConfig
-	projectID, zone, err := getProjectAndZone()
+	var configFile *ConfigFile
-	if err != nil {
-		return nil, err
-	}
 
-	region, err := GetGCERegion(zone)
-	if err != nil {
-		return nil, err
-	}
-
-	networkName, err := getNetworkNameViaMetadata()
-	if err != nil {
-		return nil, err
-	}
-	networkURL := gceNetworkURL("", projectID, networkName)
-	subnetworkURL := ""
-
-	// By default, Kubernetes clusters only run against one zone
-	managedZones := []string{zone}
-
-	tokenSource := google.ComputeTokenSource("")
-	var nodeTags []string
-	var nodeInstancePrefix string
 	if config != nil {
-		cfg, err := readConfig(config)
+		configFile, err = readConfig(config)
 		if err != nil {
 			return nil, err
 		}
-
+		glog.Infof("Using GCE provider config %+v", configFile)
-		glog.Infof("Using GCE provider config %+v", cfg)
-		if cfg.Global.ApiEndpoint != "" {
-			apiEndpoint = cfg.Global.ApiEndpoint
-		}
-		if cfg.Global.ProjectID != "" {
-			projectID = cfg.Global.ProjectID
 	}
 
-		if cfg.Global.NetworkName != "" {
+	cloudConfig, err = generateCloudConfig(configFile)
-			if strings.Contains(cfg.Global.NetworkName, "/") {
+	if err != nil {
-				networkURL = cfg.Global.NetworkName
+		return nil, err
-			} else {
-				networkURL = gceNetworkURL(apiEndpoint, projectID, cfg.Global.NetworkName)
 	}
+	return CreateGCECloud(cloudConfig)
+
 }
 
-		if cfg.Global.SubnetworkName != "" {
+func readConfig(reader io.Reader) (*ConfigFile, error) {
-			if strings.Contains(cfg.Global.SubnetworkName, "/") {
+	cfg := &ConfigFile{}
-				subnetworkURL = cfg.Global.SubnetworkName
-			} else {
-				subnetworkURL = gceSubnetworkURL(apiEndpoint, projectID, region, cfg.Global.SubnetworkName)
-			}
-		}
-
-		if cfg.Global.TokenURL != "" {
-			tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
-		}
-		nodeTags = cfg.Global.NodeTags
-		nodeInstancePrefix = cfg.Global.NodeInstancePrefix
-		if cfg.Global.Multizone {
-			managedZones = nil // Use all zones in region
-		}
-	}
-
-	return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL,
-		nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */)
-}
-
-func readConfig(reader io.Reader) (*Config, error) {
-	cfg := &Config{}
 	if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
 		glog.Errorf("Couldn't read config: %v", err)
 		return nil, err
@@ -250,14 +217,92 @@ func readConfig(reader io.Reader) (*Config, error) {
 	return cfg, nil
 }
 
+func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err error) {
+	cloudConfig = &CloudConfig{}
+	// By default, fetch token from GCE metadata server
+	cloudConfig.TokenSource = google.ComputeTokenSource("")
+	cloudConfig.UseMetadataServer = true
+
+	if configFile != nil {
+		if configFile.Global.ApiEndpoint != "" {
+			cloudConfig.ApiEndpoint = configFile.Global.ApiEndpoint
+		}
+
+		if configFile.Global.TokenURL != "" {
+			// if tokenURL is nil, set tokenSource to nil. This will force the OAuth client to fall
+			// back to use DefaultTokenSource. This allows running gceCloud remotely.
+			if configFile.Global.TokenURL == "nil" {
+				cloudConfig.TokenSource = nil
+			} else {
+				cloudConfig.TokenSource = NewAltTokenSource(configFile.Global.TokenURL, configFile.Global.TokenBody)
+			}
+		}
+
+		cloudConfig.NodeTags = configFile.Global.NodeTags
+		cloudConfig.NodeInstancePrefix = configFile.Global.NodeInstancePrefix
+	}
+
+	// retrieve projectID and zone
+	if configFile == nil || configFile.Global.ProjectID == "" || configFile.Global.LocalZone == "" {
+		cloudConfig.ProjectID, cloudConfig.Zone, err = getProjectAndZone()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if configFile != nil {
+		if configFile.Global.ProjectID != "" {
+			cloudConfig.ProjectID = configFile.Global.ProjectID
+		}
+		if configFile.Global.LocalZone != "" {
+			cloudConfig.Zone = configFile.Global.LocalZone
+		}
+	}
+
+	// retrieve region
+	cloudConfig.Region, err = GetGCERegion(cloudConfig.Zone)
+	if err != nil {
+		return nil, err
+	}
+
+	// generate managedZones
+	cloudConfig.ManagedZones = []string{cloudConfig.Zone}
+	if configFile != nil && configFile.Global.Multizone {
+		cloudConfig.ManagedZones = nil // Use all zones in region
+	}
+
+	// generate networkURL
+	if configFile != nil && configFile.Global.NetworkName != "" {
+		if strings.Contains(configFile.Global.NetworkName, "/") {
+			cloudConfig.NetworkURL = configFile.Global.NetworkName
+		} else {
+			cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName)
+		}
+	} else {
+		networkName, err := getNetworkNameViaMetadata()
+		if err != nil {
+			return nil, err
+		}
+		cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName)
+	}
+
+	// generate subnetworkURL
+	if configFile != nil && configFile.Global.SubnetworkName != "" {
+		if strings.Contains(configFile.Global.SubnetworkName, "/") {
+			cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName
+		} else {
+			cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName)
+		}
+	}
+	return cloudConfig, err
+}
+
 // Creates a GCECloud object using the specified parameters.
 // If no networkUrl is specified, loads networkName via rest call.
 // If no tokenSource is specified, uses oauth2.DefaultTokenSource.
 // If managedZones is nil / empty all zones in the region will be managed.
-func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string,
+func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
-	nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) {
 
-	client, err := newOauthClient(tokenSource)
+	client, err := newOauthClient(config.TokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -266,7 +311,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		return nil, err
 	}
 
-	client, err = newOauthClient(tokenSource)
+	client, err = newOauthClient(config.TokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -275,7 +320,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		return nil, err
 	}
 
-	client, err = newOauthClient(tokenSource)
+	client, err = newOauthClient(config.TokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -288,10 +333,10 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 	// Generate alpha and beta api endpoints based on override v1 api endpoint.
 	// For example,
 	// staging API endpoint: https://www.googleapis.com/compute/staging_v1/
-	if apiEndpoint != "" {
+	if config.ApiEndpoint != "" {
-		service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint)
+		service.BasePath = fmt.Sprintf("%sprojects/", config.ApiEndpoint)
-		serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "beta", -1))
+		serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "beta", -1))
-		serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "alpha", -1))
+		serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "alpha", -1))
 	}
 
 	containerService, err := container.New(client)
@@ -304,28 +349,28 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		return nil, err
 	}
 
-	if networkURL == "" {
+	if config.NetworkURL == "" {
-		networkName, err := getNetworkNameViaAPICall(service, projectID)
+		networkName, err := getNetworkNameViaAPICall(service, config.ProjectID)
 		if err != nil {
 			return nil, err
 		}
-		networkURL = gceNetworkURL(apiEndpoint, projectID, networkName)
+		config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName)
 	}
 
-	networkProjectID, err := getProjectIDInURL(networkURL)
+	networkProjectID, err := getProjectIDInURL(config.NetworkURL)
 	if err != nil {
 		return nil, err
 	}
-	onXPN := networkProjectID != projectID
+	onXPN := networkProjectID != config.ProjectID
 
-	if len(managedZones) == 0 {
+	if len(config.ManagedZones) == 0 {
-		managedZones, err = getZonesForRegion(service, projectID, region)
+		config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region)
 		if err != nil {
 			return nil, err
 		}
 	}
-	if len(managedZones) != 1 {
+	if len(config.ManagedZones) != 1 {
-		glog.Infof("managing multiple zones: %v", managedZones)
+		glog.Infof("managing multiple zones: %v", config.ManagedZones)
 	}
 
 	operationPollRateLimiter := flowcontrol.NewTokenBucketRateLimiter(10, 100) // 10 qps, 100 bucket size.
@@ -336,17 +381,17 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		serviceBeta:              serviceBeta,
 		containerService:         containerService,
 		cloudkmsService:          cloudkmsService,
-		projectID:                projectID,
+		projectID:                config.ProjectID,
 		networkProjectID:         networkProjectID,
 		onXPN:                    onXPN,
-		region:                   region,
+		region:                   config.Region,
-		localZone:                zone,
+		localZone:                config.Zone,
-		managedZones:             managedZones,
+		managedZones:             config.ManagedZones,
-		networkURL:               networkURL,
+		networkURL:               config.NetworkURL,
-		subnetworkURL:            subnetworkURL,
+		subnetworkURL:            config.SubnetworkURL,
-		nodeTags:                 nodeTags,
+		nodeTags:                 config.NodeTags,
-		nodeInstancePrefix:       nodeInstancePrefix,
+		nodeInstancePrefix:       config.NodeInstancePrefix,
-		useMetadataServer:        useMetadataServer,
+		useMetadataServer:        config.UseMetadataServer,
 		operationPollRateLimiter: operationPollRateLimiter,
 	}
 

pkg/cloudprovider/providers/gce/gce_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package gce
 
 import (
+	"golang.org/x/oauth2/google"
 	"reflect"
 	"strings"
 	"testing"
@@ -255,3 +256,184 @@ func TestSplitProviderID(t *testing.T) {
 		}
 	}
 }
+
+func TestGenerateCloudConfigs(t *testing.T) {
+	testCases := []struct {
+		TokenURL           string
+		TokenBody          string
+		ProjectID          string
+		NetworkName        string
+		SubnetworkName     string
+		NodeTags           []string
+		NodeInstancePrefix string
+		Multizone          bool
+		ApiEndpoint        string
+		LocalZone          string
+		cloudConfig        *CloudConfig
+	}{
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+		// nil token source
+		{
+			TokenURL:           "nil",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        nil,
+				UseMetadataServer:  true,
+			},
+		},
+		// specified api endpoint
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "https://www.googleapis.com/compute/staging_v1/",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "https://www.googleapis.com/compute/staging_v1/",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/staging_v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/staging_v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+		// empty subnet-name
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+		// multi zone
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          true,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       nil,
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		cloudConfig, err := generateCloudConfig(&ConfigFile{
+			Global: struct {
+				TokenURL           string   `gcfg:"token-url"`
+				TokenBody          string   `gcfg:"token-body"`
+				ProjectID          string   `gcfg:"project-id"`
+				NetworkName        string   `gcfg:"network-name"`
+				SubnetworkName     string   `gcfg:"subnetwork-name"`
+				NodeTags           []string `gcfg:"node-tags"`
+				NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
+				Multizone          bool     `gcfg:"multizone"`
+				ApiEndpoint        string   `gcfg:"api-endpoint"`
+				LocalZone          string   `gcfg:"local-zone"`
+			}{
+				TokenURL:           tc.TokenURL,
+				TokenBody:          tc.TokenBody,
+				ProjectID:          tc.ProjectID,
+				NetworkName:        tc.NetworkName,
+				SubnetworkName:     tc.SubnetworkName,
+				NodeTags:           tc.NodeTags,
+				NodeInstancePrefix: tc.NodeInstancePrefix,
+				Multizone:          tc.Multizone,
+				ApiEndpoint:        tc.ApiEndpoint,
+				LocalZone:          tc.LocalZone,
+			},
+		})
+		if err != nil {
+			t.Fatalf("Unexpect error: %v", err)
+		}
+
+		if !reflect.DeepEqual(cloudConfig, tc.cloudConfig) {
+			t.Errorf("Expecting cloud config: %v, but got %v", tc.cloudConfig, cloudConfig)
+		}
+	}
+}

test/e2e/e2e.go

@@ -78,10 +78,19 @@ func setupProviderConfig() error {
 			managedZones = []string{zone}
 		}
 
-		gceCloud, err := gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint,
+		gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
-			framework.TestContext.CloudConfig.ProjectID,
+			ApiEndpoint:        framework.TestContext.CloudConfig.ApiEndpoint,
-			region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */
+			ProjectID:          framework.TestContext.CloudConfig.ProjectID,
-			"" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */)
+			Region:             region,
+			Zone:               zone,
+			ManagedZones:       managedZones,
+			NetworkURL:         "",
+			SubnetworkURL:      "",
+			NodeTags:           nil,
+			NodeInstancePrefix: "",
+			TokenSource:        nil,
+			UseMetadataServer:  false})
+
 		if err != nil {
 			return fmt.Errorf("Error building GCE/GKE provider: %v", err)
 		}