Merge pull request #24369 from Clarifai/ecr

Automatic merge from submit-queue

AWS: Allow cross-region image pulling with ECR

Fixes #23298
Definitely should be in the release notes; should maybe get merged in 1.2 along with #23594 after some soaking. Documentation changes to follow.

cc @justinsb @erictune @rata @miguelfrde

This is step two. We now create long-lived, lazy ECR providers in all regions.
When first used, they will create the actual ECR providers doing the work
behind the scenes, namely talking to ECR in the region where the image lives,
rather than the one our instance is running in.

Also:
- moved the list of AWS regions out of the AWS cloudprovider and into the
credentialprovider, then exported it from there.
- improved logging

Behold, running in us-east-1:

```
aws_credentials.go:127] Creating ecrProvider for us-west-2
aws_credentials.go:63] AWS request: ecr:GetAuthorizationToken in us-west-2
aws_credentials.go:217] Adding credentials for user AWS in us-west-2
Successfully pulled image "123456789012.dkr.ecr.us-west-2.amazonaws.com/test:latest"
```

*"One small step for a pod, one giant leap for Kube-kind."*

<!-- Reviewable:start -->
---
This change is [<img src="http://reviewable.k8s.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](http://reviewable.k8s.io/reviews/kubernetes/kubernetes/24369)
<!-- Reviewable:end -->

pkg/cloudprovider/providers/aws/aws.go

@@ -42,7 +42,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/cloudprovider"
-	"k8s.io/kubernetes/pkg/credentialprovider/aws"
+	aws_credentials "k8s.io/kubernetes/pkg/credentialprovider/aws"
 	"k8s.io/kubernetes/pkg/types"
 
 	"github.com/golang/glog"
@@ -585,21 +585,7 @@ func getAvailabilityZone(metadata EC2Metadata) (string, error) {
 }
 
 func isRegionValid(region string) bool {
-	regions := [...]string{
-		"us-east-1",
-		"us-west-1",
-		"us-west-2",
-		"eu-west-1",
-		"eu-central-1",
-		"ap-southeast-1",
-		"ap-southeast-2",
-		"ap-northeast-1",
-		"ap-northeast-2",
-		"cn-north-1",
-		"us-gov-west-1",
-		"sa-east-1",
-	}
-	for _, r := range regions {
+	for _, r := range aws_credentials.AWSRegions {
 		if r == region {
 			return true
 		}

pkg/credentialprovider/aws/aws_credentials.go

@@ -14,10 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package aws_credentials
+package credentials
 
 import (
 	"encoding/base64"
+	"fmt"
 	"strings"
 	"time"
 
@@ -26,23 +27,40 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/credentialprovider"
 )
 
-var registryUrls = []string{"*.dkr.ecr.*.amazonaws.com"}
+// AWSRegions is the complete list of regions known to the AWS cloudprovider
+// and credentialprovider.
+var AWSRegions = [...]string{
+	"us-east-1",
+	"us-west-1",
+	"us-west-2",
+	"eu-west-1",
+	"eu-central-1",
+	"ap-southeast-1",
+	"ap-southeast-2",
+	"ap-northeast-1",
+	"ap-northeast-2",
+	"cn-north-1",
+	"us-gov-west-1",
+	"sa-east-1",
+}
+
+const registryURLTemplate = "*.dkr.ecr.%s.amazonaws.com"
 
 // awsHandlerLogger is a handler that logs all AWS SDK requests
 // Copied from cloudprovider/aws/log_handler.go
 func awsHandlerLogger(req *request.Request) {
 	service := req.ClientInfo.ServiceName
+	region := req.Config.Region
 
 	name := "?"
 	if req.Operation != nil {
 		name = req.Operation.Name
 	}
 
-	glog.V(4).Infof("AWS request: %s %s", service, name)
+	glog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region)
 }
 
 // An interface for testing purposes.
@@ -59,22 +77,86 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI
 	return p.svc.GetAuthorizationToken(input)
 }
 
+// lazyEcrProvider is a DockerConfigProvider that creates on demand an
+// ecrProvider for a given region and then proxies requests to it.
+type lazyEcrProvider struct {
+	region         string
+	regionURL      string
+	actualProvider *credentialprovider.CachingDockerConfigProvider
+}
+
+var _ credentialprovider.DockerConfigProvider = &lazyEcrProvider{}
+
 // ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens
 // from AWS to access ECR.
 type ecrProvider struct {
+	region    string
+	regionURL string
 	getter    tokenGetter
 }
 
+var _ credentialprovider.DockerConfigProvider = &ecrProvider{}
+
+// Init creates a lazy provider for each AWS region, in order to support
+// cross-region ECR access. They have to be lazy because it's unlikely, but not
+// impossible, that we'll use more than one.
 // Not using the package init() function: this module should be initialized only
 // if using the AWS cloud provider. This way, we avoid timeouts waiting for a
 // non-existent provider.
 func Init() {
-	credentialprovider.RegisterCredentialProvider("aws-ecr-key",
-		&credentialprovider.CachingDockerConfigProvider{
-			Provider: &ecrProvider{},
-			// Refresh credentials a little earlier before they expire
-			Lifetime: 11*time.Hour + 55*time.Minute,
+	for _, region := range AWSRegions {
+		credentialprovider.RegisterCredentialProvider("aws-ecr-"+region,
+			&lazyEcrProvider{
+				region:    region,
+				regionURL: fmt.Sprintf(registryURLTemplate, region),
 			})
+	}
+
+}
+
+// Enabled implements DockerConfigProvider.Enabled for the lazy provider.
+// Since we perform no checks/work of our own and actualProvider is only created
+// later at image pulling time (if ever), always return true.
+func (p *lazyEcrProvider) Enabled() bool {
+	return true
+}
+
+// LazyProvide implements DockerConfigProvider.LazyProvide. It will be called
+// by the client when attempting to pull an image and it will create the actual
+// provider only when we actually need it the first time.
+func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry {
+	if p.actualProvider == nil {
+		glog.V(2).Infof("Creating ecrProvider for %s", p.region)
+		p.actualProvider = &credentialprovider.CachingDockerConfigProvider{
+			Provider: newEcrProvider(p.region, nil),
+			// Refresh credentials a little earlier than expiration time
+			Lifetime: 11*time.Hour + 55*time.Minute,
+		}
+		if !p.actualProvider.Enabled() {
+			return nil
+		}
+	}
+	entry := p.actualProvider.Provide()[p.regionURL]
+	return &entry
+}
+
+// Provide implements DockerConfigProvider.Provide, creating dummy credentials.
+// Client code will call Provider.LazyProvide() at image pulling time.
+func (p *lazyEcrProvider) Provide() credentialprovider.DockerConfig {
+	entry := credentialprovider.DockerConfigEntry{
+		Provider: p,
+	}
+	cfg := credentialprovider.DockerConfig{}
+	cfg[p.regionURL] = entry
+	return cfg
+}
+
+func newEcrProvider(region string, getter tokenGetter) *ecrProvider {
+	return &ecrProvider{
+		region:    region,
+		regionURL: fmt.Sprintf(registryURLTemplate, region),
+		getter:    getter,
+	}
 }
 
 // Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation.
@@ -82,33 +164,14 @@ func Init() {
 // TODO: figure how to enable it manually for deployments that are not on AWS but still
 // use ECR somehow?
 func (p *ecrProvider) Enabled() bool {
-	provider, err := cloudprovider.GetCloudProvider("aws", nil)
-	if err != nil {
-		glog.Errorf("while initializing AWS cloud provider %v", err)
-		return false
-	}
-	if provider == nil {
-		return false
-	}
-
-	zones, ok := provider.Zones()
-	if !ok {
-		glog.Errorf("couldn't get Zones() interface")
-		return false
-	}
-	zone, err := zones.GetZone()
-	if err != nil {
-		glog.Errorf("while getting zone %v", err)
-		return false
-	}
-	if zone.Region == "" {
-		glog.Errorf("Region information is empty")
+	if p.region == "" {
+		glog.Errorf("Called ecrProvider.Enabled() with no region set")
 		return false
 	}
 
 	getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{
 		Credentials: nil,
-		Region:      &zone.Region,
+		Region:      &p.region,
 	}))}
 	getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{
 		Name: "k8s/logger",
@@ -158,10 +221,10 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig {
 				Email: "not@val.id",
 			}
 
-			// Add our entry for each of the supported container registry URLs
-			for _, k := range registryUrls {
-				cfg[k] = entry
-			}
+			glog.V(3).Infof("Adding credentials for user %s in %s", user, p.region)
+			// Add our config entry for this region's registry URLs
+			cfg[p.regionURL] = entry
+
 		}
 	}
 	return cfg

pkg/credentialprovider/aws/aws_credentials_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package aws_credentials
+package credentials
 
 import (
 	"encoding/base64"
@@ -58,17 +58,18 @@ func (p *testTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationToken
 
 func TestEcrProvide(t *testing.T) {
 	registry := "123456789012.dkr.ecr.lala-land-1.amazonaws.com"
-	otherRegistries := []string{"private.registry.com",
+	otherRegistries := []string{
+		"private.registry.com",
 		"gcr.io",
 	}
 	image := "foo/bar"
 
-	provider := &ecrProvider{
-		getter: &testTokenGetter{
+	provider := newEcrProvider("lala-land-1",
+		&testTokenGetter{
 			user:     user,
 			password: password,
-			endpoint: registry},
-	}
+			endpoint: registry,
+		})
 
 	keyring := &credentialprovider.BasicDockerKeyring{}
 	keyring.Add(provider.Provide())