Merge pull request #49912 from freehan/local-cloud-provider

Automatic merge from submit-queue (batch tested with PRs 50418, 49830, 49206, 49061, 49912)

add LocalZone into gce.conf and refactor gce cloud provider configura…

The main goal of this PR is to make gce cloud provider able to run locally. 

1. added a LocalZone parameter into gce.conf. 
2. refactor `newGCECloud` to avoid contacting metadata server if configuration is already available. 

```release-note
None
```

pkg/cloudprovider/providers/gce/BUILD

@@ -91,6 +91,7 @@ go_test(
     deps = [
         "//pkg/cloudprovider:go_default_library",
         "//pkg/kubelet/apis:go_default_library",
+        "//vendor/golang.org/x/oauth2/google:go_default_library",
         "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/google.golang.org/api/googleapi:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

pkg/cloudprovider/providers/gce/gce.go

@@ -133,7 +133,7 @@ type GCEServiceManager struct {
 	gce *GCECloud
 }
 
-type Config struct {
+type ConfigFile struct {
 	Global struct {
 		TokenURL           string   `gcfg:"token-url"`
 		TokenBody          string   `gcfg:"token-body"`
@@ -145,9 +145,25 @@ type Config struct {
 		Multizone          bool     `gcfg:"multizone"`
 		// Specifying ApiEndpoint will override the default GCE compute API endpoint.
 		ApiEndpoint string `gcfg:"api-endpoint"`
+		LocalZone   string `gcfg:"local-zone"`
 	}
 }
 
+// CloudConfig includes all the necessary configuration for creating GCECloud
+type CloudConfig struct {
+	ApiEndpoint        string
+	ProjectID          string
+	Region             string
+	Zone               string
+	ManagedZones       []string
+	NetworkURL         string
+	SubnetworkURL      string
+	NodeTags           []string
+	NodeInstancePrefix string
+	TokenSource        oauth2.TokenSource
+	UseMetadataServer  bool
+}
+
 func init() {
 	cloudprovider.RegisterCloudProvider(
 		ProviderName,
@@ -172,77 +188,28 @@ func (g *GCECloud) GetProjectID() string {
 }
 
 // newGCECloud creates a new instance of GCECloud.
-func newGCECloud(config io.Reader) (*GCECloud, error) {
-	apiEndpoint := ""
-	projectID, zone, err := getProjectAndZone()
-	if err != nil {
-		return nil, err
-	}
+func newGCECloud(config io.Reader) (gceCloud *GCECloud, err error) {
+	var cloudConfig *CloudConfig
+	var configFile *ConfigFile
 
-	region, err := GetGCERegion(zone)
-	if err != nil {
-		return nil, err
-	}
-
-	networkName, err := getNetworkNameViaMetadata()
-	if err != nil {
-		return nil, err
-	}
-	networkURL := gceNetworkURL("", projectID, networkName)
-	subnetworkURL := ""
-
-	// By default, Kubernetes clusters only run against one zone
-	managedZones := []string{zone}
-
-	tokenSource := google.ComputeTokenSource("")
-	var nodeTags []string
-	var nodeInstancePrefix string
 	if config != nil {
-		cfg, err := readConfig(config)
+		configFile, err = readConfig(config)
 		if err != nil {
 			return nil, err
 		}
-
-		glog.Infof("Using GCE provider config %+v", cfg)
-		if cfg.Global.ApiEndpoint != "" {
-			apiEndpoint = cfg.Global.ApiEndpoint
-		}
-		if cfg.Global.ProjectID != "" {
-			projectID = cfg.Global.ProjectID
-		}
-
-		if cfg.Global.NetworkName != "" {
-			if strings.Contains(cfg.Global.NetworkName, "/") {
-				networkURL = cfg.Global.NetworkName
-			} else {
-				networkURL = gceNetworkURL(apiEndpoint, projectID, cfg.Global.NetworkName)
-			}
-		}
-
-		if cfg.Global.SubnetworkName != "" {
-			if strings.Contains(cfg.Global.SubnetworkName, "/") {
-				subnetworkURL = cfg.Global.SubnetworkName
-			} else {
-				subnetworkURL = gceSubnetworkURL(apiEndpoint, projectID, region, cfg.Global.SubnetworkName)
-			}
-		}
-
-		if cfg.Global.TokenURL != "" {
-			tokenSource = NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
-		}
-		nodeTags = cfg.Global.NodeTags
-		nodeInstancePrefix = cfg.Global.NodeInstancePrefix
-		if cfg.Global.Multizone {
-			managedZones = nil // Use all zones in region
-		}
+		glog.Infof("Using GCE provider config %+v", configFile)
 	}
 
-	return CreateGCECloud(apiEndpoint, projectID, region, zone, managedZones, networkURL, subnetworkURL,
-		nodeTags, nodeInstancePrefix, tokenSource, true /* useMetadataServer */)
+	cloudConfig, err = generateCloudConfig(configFile)
+	if err != nil {
+		return nil, err
+	}
+	return CreateGCECloud(cloudConfig)
+
 }
 
-func readConfig(reader io.Reader) (*Config, error) {
-	cfg := &Config{}
+func readConfig(reader io.Reader) (*ConfigFile, error) {
+	cfg := &ConfigFile{}
 	if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, reader)); err != nil {
 		glog.Errorf("Couldn't read config: %v", err)
 		return nil, err
@@ -250,14 +217,92 @@ func readConfig(reader io.Reader) (*Config, error) {
 	return cfg, nil
 }
 
+func generateCloudConfig(configFile *ConfigFile) (cloudConfig *CloudConfig, err error) {
+	cloudConfig = &CloudConfig{}
+	// By default, fetch token from GCE metadata server
+	cloudConfig.TokenSource = google.ComputeTokenSource("")
+	cloudConfig.UseMetadataServer = true
+
+	if configFile != nil {
+		if configFile.Global.ApiEndpoint != "" {
+			cloudConfig.ApiEndpoint = configFile.Global.ApiEndpoint
+		}
+
+		if configFile.Global.TokenURL != "" {
+			// if tokenURL is nil, set tokenSource to nil. This will force the OAuth client to fall
+			// back to use DefaultTokenSource. This allows running gceCloud remotely.
+			if configFile.Global.TokenURL == "nil" {
+				cloudConfig.TokenSource = nil
+			} else {
+				cloudConfig.TokenSource = NewAltTokenSource(configFile.Global.TokenURL, configFile.Global.TokenBody)
+			}
+		}
+
+		cloudConfig.NodeTags = configFile.Global.NodeTags
+		cloudConfig.NodeInstancePrefix = configFile.Global.NodeInstancePrefix
+	}
+
+	// retrieve projectID and zone
+	if configFile == nil || configFile.Global.ProjectID == "" || configFile.Global.LocalZone == "" {
+		cloudConfig.ProjectID, cloudConfig.Zone, err = getProjectAndZone()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if configFile != nil {
+		if configFile.Global.ProjectID != "" {
+			cloudConfig.ProjectID = configFile.Global.ProjectID
+		}
+		if configFile.Global.LocalZone != "" {
+			cloudConfig.Zone = configFile.Global.LocalZone
+		}
+	}
+
+	// retrieve region
+	cloudConfig.Region, err = GetGCERegion(cloudConfig.Zone)
+	if err != nil {
+		return nil, err
+	}
+
+	// generate managedZones
+	cloudConfig.ManagedZones = []string{cloudConfig.Zone}
+	if configFile != nil && configFile.Global.Multizone {
+		cloudConfig.ManagedZones = nil // Use all zones in region
+	}
+
+	// generate networkURL
+	if configFile != nil && configFile.Global.NetworkName != "" {
+		if strings.Contains(configFile.Global.NetworkName, "/") {
+			cloudConfig.NetworkURL = configFile.Global.NetworkName
+		} else {
+			cloudConfig.NetworkURL = gceNetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, configFile.Global.NetworkName)
+		}
+	} else {
+		networkName, err := getNetworkNameViaMetadata()
+		if err != nil {
+			return nil, err
+		}
+		cloudConfig.NetworkURL = gceNetworkURL("", cloudConfig.ProjectID, networkName)
+	}
+
+	// generate subnetworkURL
+	if configFile != nil && configFile.Global.SubnetworkName != "" {
+		if strings.Contains(configFile.Global.SubnetworkName, "/") {
+			cloudConfig.SubnetworkURL = configFile.Global.SubnetworkName
+		} else {
+			cloudConfig.SubnetworkURL = gceSubnetworkURL(cloudConfig.ApiEndpoint, cloudConfig.ProjectID, cloudConfig.Region, configFile.Global.SubnetworkName)
+		}
+	}
+	return cloudConfig, err
+}
+
 // Creates a GCECloud object using the specified parameters.
 // If no networkUrl is specified, loads networkName via rest call.
 // If no tokenSource is specified, uses oauth2.DefaultTokenSource.
 // If managedZones is nil / empty all zones in the region will be managed.
-func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []string, networkURL, subnetworkURL string, nodeTags []string,
-	nodeInstancePrefix string, tokenSource oauth2.TokenSource, useMetadataServer bool) (*GCECloud, error) {
+func CreateGCECloud(config *CloudConfig) (*GCECloud, error) {
 
-	client, err := newOauthClient(tokenSource)
+	client, err := newOauthClient(config.TokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -266,7 +311,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		return nil, err
 	}
 
-	client, err = newOauthClient(tokenSource)
+	client, err = newOauthClient(config.TokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -275,7 +320,7 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		return nil, err
 	}
 
-	client, err = newOauthClient(tokenSource)
+	client, err = newOauthClient(config.TokenSource)
 	if err != nil {
 		return nil, err
 	}
@@ -288,10 +333,10 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 	// Generate alpha and beta api endpoints based on override v1 api endpoint.
 	// For example,
 	// staging API endpoint: https://www.googleapis.com/compute/staging_v1/
-	if apiEndpoint != "" {
-		service.BasePath = fmt.Sprintf("%sprojects/", apiEndpoint)
-		serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "beta", -1))
-		serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(apiEndpoint, "v1", "alpha", -1))
+	if config.ApiEndpoint != "" {
+		service.BasePath = fmt.Sprintf("%sprojects/", config.ApiEndpoint)
+		serviceBeta.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "beta", -1))
+		serviceAlpha.BasePath = fmt.Sprintf("%sprojects/", strings.Replace(config.ApiEndpoint, "v1", "alpha", -1))
 	}
 
 	containerService, err := container.New(client)
@@ -304,28 +349,28 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		return nil, err
 	}
 
-	if networkURL == "" {
-		networkName, err := getNetworkNameViaAPICall(service, projectID)
+	if config.NetworkURL == "" {
+		networkName, err := getNetworkNameViaAPICall(service, config.ProjectID)
 		if err != nil {
 			return nil, err
 		}
-		networkURL = gceNetworkURL(apiEndpoint, projectID, networkName)
+		config.NetworkURL = gceNetworkURL(config.ApiEndpoint, config.ProjectID, networkName)
 	}
 
-	networkProjectID, err := getProjectIDInURL(networkURL)
+	networkProjectID, err := getProjectIDInURL(config.NetworkURL)
 	if err != nil {
 		return nil, err
 	}
-	onXPN := networkProjectID != projectID
+	onXPN := networkProjectID != config.ProjectID
 
-	if len(managedZones) == 0 {
-		managedZones, err = getZonesForRegion(service, projectID, region)
+	if len(config.ManagedZones) == 0 {
+		config.ManagedZones, err = getZonesForRegion(service, config.ProjectID, config.Region)
 		if err != nil {
 			return nil, err
 		}
 	}
-	if len(managedZones) != 1 {
-		glog.Infof("managing multiple zones: %v", managedZones)
+	if len(config.ManagedZones) != 1 {
+		glog.Infof("managing multiple zones: %v", config.ManagedZones)
 	}
 
 	operationPollRateLimiter := flowcontrol.NewTokenBucketRateLimiter(10, 100) // 10 qps, 100 bucket size.
@@ -336,17 +381,17 @@ func CreateGCECloud(apiEndpoint, projectID, region, zone string, managedZones []
 		serviceBeta:              serviceBeta,
 		containerService:         containerService,
 		cloudkmsService:          cloudkmsService,
-		projectID:                projectID,
+		projectID:                config.ProjectID,
 		networkProjectID:         networkProjectID,
 		onXPN:                    onXPN,
-		region:                   region,
-		localZone:                zone,
-		managedZones:             managedZones,
-		networkURL:               networkURL,
-		subnetworkURL:            subnetworkURL,
-		nodeTags:                 nodeTags,
-		nodeInstancePrefix:       nodeInstancePrefix,
-		useMetadataServer:        useMetadataServer,
+		region:                   config.Region,
+		localZone:                config.Zone,
+		managedZones:             config.ManagedZones,
+		networkURL:               config.NetworkURL,
+		subnetworkURL:            config.SubnetworkURL,
+		nodeTags:                 config.NodeTags,
+		nodeInstancePrefix:       config.NodeInstancePrefix,
+		useMetadataServer:        config.UseMetadataServer,
 		operationPollRateLimiter: operationPollRateLimiter,
 	}
 

pkg/cloudprovider/providers/gce/gce_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package gce
 
 import (
+	"golang.org/x/oauth2/google"
 	"reflect"
 	"strings"
 	"testing"
@@ -255,3 +256,184 @@ func TestSplitProviderID(t *testing.T) {
 		}
 	}
 }
+
+func TestGenerateCloudConfigs(t *testing.T) {
+	testCases := []struct {
+		TokenURL           string
+		TokenBody          string
+		ProjectID          string
+		NetworkName        string
+		SubnetworkName     string
+		NodeTags           []string
+		NodeInstancePrefix string
+		Multizone          bool
+		ApiEndpoint        string
+		LocalZone          string
+		cloudConfig        *CloudConfig
+	}{
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+		// nil token source
+		{
+			TokenURL:           "nil",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        nil,
+				UseMetadataServer:  true,
+			},
+		},
+		// specified api endpoint
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "https://www.googleapis.com/compute/staging_v1/",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "https://www.googleapis.com/compute/staging_v1/",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/staging_v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/staging_v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+		// empty subnet-name
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          false,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       []string{"us-central1-a"},
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+		// multi zone
+		{
+			TokenURL:           "",
+			TokenBody:          "",
+			ProjectID:          "project-id",
+			NetworkName:        "network-name",
+			SubnetworkName:     "subnetwork-name",
+			NodeTags:           []string{"node-tag"},
+			NodeInstancePrefix: "node-prefix",
+			Multizone:          true,
+			ApiEndpoint:        "",
+			LocalZone:          "us-central1-a",
+			cloudConfig: &CloudConfig{
+				ApiEndpoint:        "",
+				ProjectID:          "project-id",
+				Region:             "us-central1",
+				Zone:               "us-central1-a",
+				ManagedZones:       nil,
+				NetworkURL:         "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/network-name",
+				SubnetworkURL:      "https://www.googleapis.com/compute/v1/projects/project-id/regions/us-central1/subnetworks/subnetwork-name",
+				NodeTags:           []string{"node-tag"},
+				NodeInstancePrefix: "node-prefix",
+				TokenSource:        google.ComputeTokenSource(""),
+				UseMetadataServer:  true,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		cloudConfig, err := generateCloudConfig(&ConfigFile{
+			Global: struct {
+				TokenURL           string   `gcfg:"token-url"`
+				TokenBody          string   `gcfg:"token-body"`
+				ProjectID          string   `gcfg:"project-id"`
+				NetworkName        string   `gcfg:"network-name"`
+				SubnetworkName     string   `gcfg:"subnetwork-name"`
+				NodeTags           []string `gcfg:"node-tags"`
+				NodeInstancePrefix string   `gcfg:"node-instance-prefix"`
+				Multizone          bool     `gcfg:"multizone"`
+				ApiEndpoint        string   `gcfg:"api-endpoint"`
+				LocalZone          string   `gcfg:"local-zone"`
+			}{
+				TokenURL:           tc.TokenURL,
+				TokenBody:          tc.TokenBody,
+				ProjectID:          tc.ProjectID,
+				NetworkName:        tc.NetworkName,
+				SubnetworkName:     tc.SubnetworkName,
+				NodeTags:           tc.NodeTags,
+				NodeInstancePrefix: tc.NodeInstancePrefix,
+				Multizone:          tc.Multizone,
+				ApiEndpoint:        tc.ApiEndpoint,
+				LocalZone:          tc.LocalZone,
+			},
+		})
+		if err != nil {
+			t.Fatalf("Unexpect error: %v", err)
+		}
+
+		if !reflect.DeepEqual(cloudConfig, tc.cloudConfig) {
+			t.Errorf("Expecting cloud config: %v, but got %v", tc.cloudConfig, cloudConfig)
+		}
+	}
+}

test/e2e/e2e.go

@@ -78,10 +78,19 @@ func setupProviderConfig() error {
 			managedZones = []string{zone}
 		}
 
-		gceCloud, err := gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ApiEndpoint,
-			framework.TestContext.CloudConfig.ProjectID,
-			region, zone, managedZones, "" /* networkUrl */, "" /* subnetworkUrl */, nil, /* nodeTags */
-			"" /* nodeInstancePerfix */, nil /* tokenSource */, false /* useMetadataServer */)
+		gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
+			ApiEndpoint:        framework.TestContext.CloudConfig.ApiEndpoint,
+			ProjectID:          framework.TestContext.CloudConfig.ProjectID,
+			Region:             region,
+			Zone:               zone,
+			ManagedZones:       managedZones,
+			NetworkURL:         "",
+			SubnetworkURL:      "",
+			NodeTags:           nil,
+			NodeInstancePrefix: "",
+			TokenSource:        nil,
+			UseMetadataServer:  false})
+
 		if err != nil {
 			return fmt.Errorf("Error building GCE/GKE provider: %v", err)
 		}