From eea29e88518a36d12b3d3e97d7027232c49baa03 Mon Sep 17 00:00:00 2001 From: Rudi Chiarito Date: Mon, 2 May 2016 13:22:11 -0400 Subject: [PATCH 1/3] Allow cross-region image pulling with AWS' ECR This is step two. We now create long-lived, lazy ECR providers in all regions. When first used, they will create the actual ECR providers doing the work behind the scenes, namely talking to ECR in the region where the image lives, rather than the one our instance is running in. Also: - moved the list of AWS regions out of the AWS cloudprovider and into the credentialprovider, then exported it from there. - improved logging Behold, running in us-east-1: ``` aws_credentials.go:127] Creating ecrProvider for us-west-2 aws_credentials.go:63] AWS request: ecr:GetAuthorizationToken in us-west-2 aws_credentials.go:217] Adding credentials for user AWS in us-west-2 Successfully pulled image 123456789012.dkr.ecr.us-west-2.amazonaws.com/test:latest" ``` *"One small step for a pod, one giant leap for Kube-kind."* --- pkg/cloudprovider/providers/aws/aws.go | 16 +-- pkg/credentialprovider/aws/aws_credentials.go | 127 +++++++++++++----- .../aws/aws_credentials_test.go | 5 +- 3 files changed, 97 insertions(+), 51 deletions(-) diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index e0cae84b4c..c273a60c57 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -566,21 +566,7 @@ func getAvailabilityZone(metadata EC2Metadata) (string, error) { } func isRegionValid(region string) bool { - regions := [...]string{ - "us-east-1", - "us-west-1", - "us-west-2", - "eu-west-1", - "eu-central-1", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - "ap-northeast-2", - "cn-north-1", - "us-gov-west-1", - "sa-east-1", - } - for _, r := range regions { + for _, r := range aws_credentials.AWSRegions { if r == region { return true } diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index aeb7316b32..7c44bd67c2 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -18,6 +18,7 @@ package aws_credentials import ( "encoding/base64" + "fmt" "strings" "time" @@ -26,23 +27,39 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ecr" "github.com/golang/glog" - "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/credentialprovider" ) -var registryUrls = []string{"*.dkr.ecr.*.amazonaws.com"} +// AWSRegions is the complete list of regions known to the AWS cloudprovider +// and credentialprovider. +var AWSRegions = [...]string{ + "us-east-1", + "us-west-1", + "us-west-2", + "eu-west-1", + "eu-central-1", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + "cn-north-1", + "us-gov-west-1", + "sa-east-1", +} + +const registryURLTemplate = "*.dkr.ecr.%s.amazonaws.com" // awsHandlerLogger is a handler that logs all AWS SDK requests // Copied from cloudprovider/aws/log_handler.go func awsHandlerLogger(req *request.Request) { service := req.ClientInfo.ServiceName + region := req.Config.Region name := "?" if req.Operation != nil { name = req.Operation.Name } - glog.V(4).Infof("AWS request: %s %s", service, name) + glog.V(3).Infof("AWS request: %s:%s in %s", service, name, *region) } // An interface for testing purposes. @@ -59,22 +76,81 @@ func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenI return p.svc.GetAuthorizationToken(input) } +// lazyEcrProvider is a DockerConfigProvider that creates on demand an +// ecrProvider for a given region and then proxies requests to it. +type lazyEcrProvider struct { + region string + regionURL string + actualProvider *credentialprovider.CachingDockerConfigProvider +} + // ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens // from AWS to access ECR. type ecrProvider struct { - getter tokenGetter + region string + regionURL string + getter tokenGetter } +// Init creates a lazy provider for each AWS region, in order to support +// cross-region ECR access. They have to be lazy because it's unlikely, but not +// impossible, that we'll use more than one. // Not using the package init() function: this module should be initialized only // if using the AWS cloud provider. This way, we avoid timeouts waiting for a // non-existent provider. func Init() { - credentialprovider.RegisterCredentialProvider("aws-ecr-key", - &credentialprovider.CachingDockerConfigProvider{ - Provider: &ecrProvider{}, - // Refresh credentials a little earlier before they expire + for _, region := range AWSRegions { + credentialprovider.RegisterCredentialProvider("aws-ecr-"+region, + &credentialprovider.CachingDockerConfigProvider{ + Provider: &lazyEcrProvider{ + region: region, + regionURL: fmt.Sprintf(registryURLTemplate, region), + }, + // This is going to be just a lazy proxy to the real ecrProvider. + // It holds no real credentials, so refresh practically never. + Lifetime: 365 * 24 * time.Hour, + }) + } + +} + +// Enabled implements DockerConfigProvider.Enabled for the lazy provider. +func (p *lazyEcrProvider) Enabled() bool { + return true +} + +// LazyProvide implements DockerConfigProvider.LazyProvide. It will be called +// by the client when attempting to pull an image and it will create the actual +// provider only when we actually need it the first time. +func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry { + if p.actualProvider == nil { + glog.V(2).Infof("Creating ecrProvider for %s", p.region) + p.actualProvider = &credentialprovider.CachingDockerConfigProvider{ + Provider: &ecrProvider{ + region: p.region, + regionURL: p.regionURL, + }, + // Refresh credentials a little earlier than expiration time Lifetime: 11*time.Hour + 55*time.Minute, - }) + } + if !p.actualProvider.Enabled() { + + return nil + } + } + entry := p.actualProvider.Provide()[p.regionURL] + return &entry +} + +// Provide implements DockerConfigProvider.Provide, creating dummy credentials. +// Client code will call Provider.LazyProvide() at image pulling time. +func (p *lazyEcrProvider) Provide() credentialprovider.DockerConfig { + entry := credentialprovider.DockerConfigEntry{ + Provider: p, + } + cfg := credentialprovider.DockerConfig{} + cfg[p.regionURL] = entry + return cfg } // Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation. @@ -82,33 +158,14 @@ func Init() { // TODO: figure how to enable it manually for deployments that are not on AWS but still // use ECR somehow? func (p *ecrProvider) Enabled() bool { - provider, err := cloudprovider.GetCloudProvider("aws", nil) - if err != nil { - glog.Errorf("while initializing AWS cloud provider %v", err) - return false - } - if provider == nil { - return false - } - - zones, ok := provider.Zones() - if !ok { - glog.Errorf("couldn't get Zones() interface") - return false - } - zone, err := zones.GetZone() - if err != nil { - glog.Errorf("while getting zone %v", err) - return false - } - if zone.Region == "" { - glog.Errorf("Region information is empty") + if p.region == "" { + glog.Errorf("Called ecrProvider.Enabled() with no region set") return false } getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{ Credentials: nil, - Region: &zone.Region, + Region: &p.region, }))} getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{ Name: "k8s/logger", @@ -158,10 +215,10 @@ func (p *ecrProvider) Provide() credentialprovider.DockerConfig { Email: "not@val.id", } - // Add our entry for each of the supported container registry URLs - for _, k := range registryUrls { - cfg[k] = entry - } + glog.V(3).Infof("Adding credentials for user %s in %s", user, p.region) + // Add our config entry for this region's registry URLs + cfg[p.regionURL] = entry + } } return cfg diff --git a/pkg/credentialprovider/aws/aws_credentials_test.go b/pkg/credentialprovider/aws/aws_credentials_test.go index a07493993f..7a9596519e 100644 --- a/pkg/credentialprovider/aws/aws_credentials_test.go +++ b/pkg/credentialprovider/aws/aws_credentials_test.go @@ -58,12 +58,15 @@ func (p *testTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationToken func TestEcrProvide(t *testing.T) { registry := "123456789012.dkr.ecr.lala-land-1.amazonaws.com" - otherRegistries := []string{"private.registry.com", + otherRegistries := []string{ + "private.registry.com", "gcr.io", } image := "foo/bar" provider := &ecrProvider{ + region: "lala-land-1", + regionURL: "*.dkr.ecr.lala-land-1.amazonaws.com", getter: &testTokenGetter{ user: user, password: password, From 6e6ea46182d6d8de2d0d4ebfae95d0269205c86e Mon Sep 17 00:00:00 2001 From: Rudi Chiarito Date: Sun, 17 Apr 2016 17:22:52 -0400 Subject: [PATCH 2/3] Include changes from feedback Use constructor for ecrProvider Rename package to "credentials" like golint requests Don't wrap the lazy provider with a caching provider Add immedita compile-time interface conformance checks for the interfaces Added comments --- pkg/cloudprovider/providers/aws/aws.go | 2 +- pkg/credentialprovider/aws/aws_credentials.go | 33 +++++++++++-------- .../aws/aws_credentials_test.go | 12 +++---- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/pkg/cloudprovider/providers/aws/aws.go b/pkg/cloudprovider/providers/aws/aws.go index c273a60c57..5ea442e8e3 100644 --- a/pkg/cloudprovider/providers/aws/aws.go +++ b/pkg/cloudprovider/providers/aws/aws.go @@ -42,7 +42,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/cloudprovider" - "k8s.io/kubernetes/pkg/credentialprovider/aws" + aws_credentials "k8s.io/kubernetes/pkg/credentialprovider/aws" "k8s.io/kubernetes/pkg/types" "github.com/golang/glog" diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index 7c44bd67c2..a27a224a7a 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package aws_credentials +package credentials import ( "encoding/base64" @@ -84,6 +84,8 @@ type lazyEcrProvider struct { actualProvider *credentialprovider.CachingDockerConfigProvider } +var _ credentialprovider.DockerConfigProvider = &lazyEcrProvider{} + // ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens // from AWS to access ECR. type ecrProvider struct { @@ -92,6 +94,8 @@ type ecrProvider struct { getter tokenGetter } +var _ credentialprovider.DockerConfigProvider = &ecrProvider{} + // Init creates a lazy provider for each AWS region, in order to support // cross-region ECR access. They have to be lazy because it's unlikely, but not // impossible, that we'll use more than one. @@ -101,20 +105,17 @@ type ecrProvider struct { func Init() { for _, region := range AWSRegions { credentialprovider.RegisterCredentialProvider("aws-ecr-"+region, - &credentialprovider.CachingDockerConfigProvider{ - Provider: &lazyEcrProvider{ - region: region, - regionURL: fmt.Sprintf(registryURLTemplate, region), - }, - // This is going to be just a lazy proxy to the real ecrProvider. - // It holds no real credentials, so refresh practically never. - Lifetime: 365 * 24 * time.Hour, + &lazyEcrProvider{ + region: region, + regionURL: fmt.Sprintf(registryURLTemplate, region), }) } } // Enabled implements DockerConfigProvider.Enabled for the lazy provider. +// Since we perform no checks/work of our own and actualProvider is only created +// later at image pulling time (if ever), always return true. func (p *lazyEcrProvider) Enabled() bool { return true } @@ -126,15 +127,11 @@ func (p *lazyEcrProvider) LazyProvide() *credentialprovider.DockerConfigEntry { if p.actualProvider == nil { glog.V(2).Infof("Creating ecrProvider for %s", p.region) p.actualProvider = &credentialprovider.CachingDockerConfigProvider{ - Provider: &ecrProvider{ - region: p.region, - regionURL: p.regionURL, - }, + Provider: newEcrProvider(p.region, nil), // Refresh credentials a little earlier than expiration time Lifetime: 11*time.Hour + 55*time.Minute, } if !p.actualProvider.Enabled() { - return nil } } @@ -153,6 +150,14 @@ func (p *lazyEcrProvider) Provide() credentialprovider.DockerConfig { return cfg } +func newEcrProvider(region string, getter tokenGetter) *ecrProvider { + return &ecrProvider{ + region: region, + regionURL: fmt.Sprintf(registryURLTemplate, region), + getter: getter, + } +} + // Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation. // For now, it gets activated only if AWS was chosen as the cloud provider. // TODO: figure how to enable it manually for deployments that are not on AWS but still diff --git a/pkg/credentialprovider/aws/aws_credentials_test.go b/pkg/credentialprovider/aws/aws_credentials_test.go index 7a9596519e..b286c7d61d 100644 --- a/pkg/credentialprovider/aws/aws_credentials_test.go +++ b/pkg/credentialprovider/aws/aws_credentials_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package aws_credentials +package credentials import ( "encoding/base64" @@ -64,14 +64,12 @@ func TestEcrProvide(t *testing.T) { } image := "foo/bar" - provider := &ecrProvider{ - region: "lala-land-1", - regionURL: "*.dkr.ecr.lala-land-1.amazonaws.com", - getter: &testTokenGetter{ + provider := newEcrProvider("lala-land-1", + &testTokenGetter{ user: user, password: password, - endpoint: registry}, - } + endpoint: registry, + }) keyring := &credentialprovider.BasicDockerKeyring{} keyring.Add(provider.Provide()) From 684517f74ffc4cdd9c33a196525f5cd1dc10aa25 Mon Sep 17 00:00:00 2001 From: Rudi Chiarito Date: Mon, 2 May 2016 13:23:11 -0400 Subject: [PATCH 3/3] Add ap-northeast-2 --- pkg/credentialprovider/aws/aws_credentials.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go index a27a224a7a..3b9b5c9820 100644 --- a/pkg/credentialprovider/aws/aws_credentials.go +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -41,6 +41,7 @@ var AWSRegions = [...]string{ "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", + "ap-northeast-2", "cn-north-1", "us-gov-west-1", "sa-east-1",