diff --git a/cluster/aws/templates/iam/kubernetes-minion-policy.json b/cluster/aws/templates/iam/kubernetes-minion-policy.json index 32453443a4..0a7ba67849 100644 --- a/cluster/aws/templates/iam/kubernetes-minion-policy.json +++ b/cluster/aws/templates/iam/kubernetes-minion-policy.json @@ -22,6 +22,19 @@ "Effect": "Allow", "Action": "ec2:DetachVolume", "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" } ] } diff --git a/cmd/kubelet/app/plugins.go b/cmd/kubelet/app/plugins.go index 3343da0a78..9dc450a584 100644 --- a/cmd/kubelet/app/plugins.go +++ b/cmd/kubelet/app/plugins.go @@ -19,6 +19,7 @@ package app // This file exists to force the desired plugin implementations to be linked. import ( // Credential providers + _ "k8s.io/kubernetes/pkg/credentialprovider/aws" _ "k8s.io/kubernetes/pkg/credentialprovider/gcp" // Network plugins "k8s.io/kubernetes/pkg/kubelet/network" diff --git a/docs/design/aws_under_the_hood.md b/docs/design/aws_under_the_hood.md index a551c07c3f..019b07d621 100644 --- a/docs/design/aws_under_the_hood.md +++ b/docs/design/aws_under_the_hood.md @@ -171,7 +171,11 @@ The nodes do not need a lot of access to the AWS APIs. They need to download a distribution file, and then are responsible for attaching and detaching EBS volumes from itself. -The node policy is relatively minimal. The master policy is probably overly +The node policy is relatively minimal. In 1.2 and later, nodes can retrieve ECR +authorization tokens, refresh them every 12 hours if needed, and fetch Docker +images from it, as long as the appropriate permissions are enabled. Those in +[AmazonEC2ContainerRegistryReadOnly](http://docs.aws.amazon.com/AmazonECR/latest/userguide/ecr_managed_policies.html#AmazonEC2ContainerRegistryReadOnly), +without write access, should suffice. The master policy is probably overly permissive. The security conscious may want to lock-down the IAM policies further ([#11936](http://issues.k8s.io/11936)). @@ -180,7 +184,7 @@ are correctly configured ([#14226](http://issues.k8s.io/14226)). ### Tagging -All AWS resources are tagged with a tag named "KuberentesCluster", with a value +All AWS resources are tagged with a tag named "KubernetesCluster", with a value that is the unique cluster-id. This tag is used to identify a particular 'instance' of Kubernetes, even if two clusters are deployed into the same VPC. Resources are considered to belong to the same cluster if and only if they have diff --git a/docs/user-guide/images.md b/docs/user-guide/images.md index a4ce92bf13..7c2db77c2c 100644 --- a/docs/user-guide/images.md +++ b/docs/user-guide/images.md @@ -47,6 +47,7 @@ The `image` property of a container supports the same syntax as the `docker` com - [Updating Images](#updating-images) - [Using a Private Registry](#using-a-private-registry) - [Using Google Container Registry](#using-google-container-registry) + - [Using AWS EC2 Container Registry](#using-aws-ec2-container-registry) - [Configuring Nodes to Authenticate to a Private Repository](#configuring-nodes-to-authenticate-to-a-private-repository) - [Pre-pulling Images](#pre-pulling-images) - [Specifying ImagePullSecrets on a Pod](#specifying-imagepullsecrets-on-a-pod) @@ -97,6 +98,21 @@ Google service account. The service account on the instance will have a `https://www.googleapis.com/auth/devstorage.read_only`, so it can pull from the project's GCR, but not push. +### Using AWS EC2 Container Registry + +Kubernetes has native support for the [AWS EC2 Container +Registry](https://aws.amazon.com/ecr/), when nodes are AWS instances. + +Simply use the full image name (e.g. `ACCOUNT.dkr.ecr.REGION.amazonaws.com/imagename:tag`) +in the Pod definition. + +All users of the cluster who can create pods will be able to run pods that use any of the +images in the ECR registry. + +The kubelet will fetch and periodically refresh ECR credentials. It needs the +`ecr:GetAuthorizationToken` permission to do this. + + ### Configuring Nodes to Authenticate to a Private Repository **Note:** if you are running on Google Container Engine (GKE), there will already be a `.dockercfg` on each node diff --git a/pkg/credentialprovider/aws/aws_credentials.go b/pkg/credentialprovider/aws/aws_credentials.go new file mode 100644 index 0000000000..dd349a58f4 --- /dev/null +++ b/pkg/credentialprovider/aws/aws_credentials.go @@ -0,0 +1,163 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws_credentials + +import ( + "encoding/base64" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ecr" + "github.com/golang/glog" + "k8s.io/kubernetes/pkg/cloudprovider" + aws_cloud "k8s.io/kubernetes/pkg/cloudprovider/providers/aws" + "k8s.io/kubernetes/pkg/credentialprovider" +) + +var registryUrls = []string{"*.dkr.ecr.*.amazonaws.com"} + +// awsHandlerLogger is a handler that logs all AWS SDK requests +// Copied from cloudprovider/aws/log_handler.go +func awsHandlerLogger(req *request.Request) { + service := req.ClientInfo.ServiceName + + name := "?" + if req.Operation != nil { + name = req.Operation.Name + } + + glog.V(4).Infof("AWS request: %s %s", service, name) +} + +// An interface for testing purposes. +type tokenGetter interface { + GetAuthorizationToken(input *ecr.GetAuthorizationTokenInput) (*ecr.GetAuthorizationTokenOutput, error) +} + +// The canonical implementation +type ecrTokenGetter struct { + svc *ecr.ECR +} + +func (p *ecrTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenInput) (*ecr.GetAuthorizationTokenOutput, error) { + return p.svc.GetAuthorizationToken(input) +} + +// ecrProvider is a DockerConfigProvider that gets and refreshes 12-hour tokens +// from AWS to access ECR. +type ecrProvider struct { + getter tokenGetter +} + +// init registers the various means by which ECR credentials may +// be resolved. +func init() { + credentialprovider.RegisterCredentialProvider("aws-ecr-key", + &credentialprovider.CachingDockerConfigProvider{ + Provider: &ecrProvider{}, + // Refresh credentials a little earlier before they expire + Lifetime: 11*time.Hour + 55*time.Minute, + }) +} + +// Enabled implements DockerConfigProvider.Enabled for the AWS token-based implementation. +// For now, it gets activated only if AWS was chosen as the cloud provider. +// TODO: figure how to enable it manually for deployments that are not on AWS but still +// use ECR somehow? +func (p *ecrProvider) Enabled() bool { + provider, err := cloudprovider.GetCloudProvider(aws_cloud.ProviderName, nil) + if err != nil { + glog.Errorf("while initializing AWS cloud provider %v", err) + return false + } + if provider == nil { + return false + } + + zones, ok := provider.Zones() + if !ok { + glog.Errorf("couldn't get Zones() interface") + return false + } + zone, err := zones.GetZone() + if err != nil { + glog.Errorf("while getting zone %v", err) + return false + } + if zone.Region == "" { + glog.Errorf("Region information is empty") + return false + } + + getter := &ecrTokenGetter{svc: ecr.New(session.New(&aws.Config{ + Credentials: nil, + Region: &zone.Region, + }))} + getter.svc.Handlers.Sign.PushFrontNamed(request.NamedHandler{ + Name: "k8s/logger", + Fn: awsHandlerLogger, + }) + p.getter = getter + + return true +} + +// Provide implements DockerConfigProvider.Provide, refreshing ECR tokens on demand +func (p *ecrProvider) Provide() credentialprovider.DockerConfig { + cfg := credentialprovider.DockerConfig{} + + // TODO: fill in RegistryIds? + params := &ecr.GetAuthorizationTokenInput{} + output, err := p.getter.GetAuthorizationToken(params) + if err != nil { + glog.Errorf("while requesting ECR authorization token %v", err) + return cfg + } + if output == nil { + glog.Errorf("Got back no ECR token") + return cfg + } + + for _, data := range output.AuthorizationData { + if data.ProxyEndpoint != nil && + data.AuthorizationToken != nil { + decodedToken, err := base64.StdEncoding.DecodeString(aws.StringValue(data.AuthorizationToken)) + if err != nil { + glog.Errorf("while decoding token for endpoint %s %v", data.ProxyEndpoint, err) + return cfg + } + parts := strings.SplitN(string(decodedToken), ":", 2) + user := parts[0] + password := parts[1] + entry := credentialprovider.DockerConfigEntry{ + Username: user, + Password: password, + // ECR doesn't care and Docker is about to obsolete it + Email: "not@val.id", + } + + // Add our entry for each of the supported container registry URLs + for _, k := range registryUrls { + cfg[k] = entry + } + } + } + return cfg +} diff --git a/pkg/credentialprovider/aws/aws_credentials_test.go b/pkg/credentialprovider/aws/aws_credentials_test.go new file mode 100644 index 0000000000..a07493993f --- /dev/null +++ b/pkg/credentialprovider/aws/aws_credentials_test.go @@ -0,0 +1,108 @@ +/* +Copyright 2014 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws_credentials + +import ( + "encoding/base64" + "fmt" + "path" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ecr" + + "k8s.io/kubernetes/pkg/credentialprovider" +) + +const user = "foo" +const password = "1234567890abcdef" +const email = "not@val.id" + +// Mock implementation +type testTokenGetter struct { + user string + password string + endpoint string +} + +func (p *testTokenGetter) GetAuthorizationToken(input *ecr.GetAuthorizationTokenInput) (*ecr.GetAuthorizationTokenOutput, error) { + + expiration := time.Now().Add(1 * time.Hour) + creds := []byte(fmt.Sprintf("%s:%s", p.user, p.password)) + data := &ecr.AuthorizationData{ + AuthorizationToken: aws.String(base64.StdEncoding.EncodeToString(creds)), + ExpiresAt: &expiration, + ProxyEndpoint: aws.String(p.endpoint), + } + output := &ecr.GetAuthorizationTokenOutput{ + AuthorizationData: []*ecr.AuthorizationData{data}, + } + + return output, nil //p.svc.GetAuthorizationToken(input) +} + +func TestEcrProvide(t *testing.T) { + registry := "123456789012.dkr.ecr.lala-land-1.amazonaws.com" + otherRegistries := []string{"private.registry.com", + "gcr.io", + } + image := "foo/bar" + + provider := &ecrProvider{ + getter: &testTokenGetter{ + user: user, + password: password, + endpoint: registry}, + } + + keyring := &credentialprovider.BasicDockerKeyring{} + keyring.Add(provider.Provide()) + + // Verify that we get the expected username/password combo for + // an ECR image name. + fullImage := path.Join(registry, image) + creds, ok := keyring.Lookup(fullImage) + if !ok { + t.Errorf("Didn't find expected URL: %s", fullImage) + return + } + if len(creds) > 1 { + t.Errorf("Got more hits than expected: %s", creds) + } + val := creds[0] + + if user != val.Username { + t.Errorf("Unexpected username value, want: _token, got: %s", val.Username) + } + if password != val.Password { + t.Errorf("Unexpected password value, want: %s, got: %s", password, val.Password) + } + if email != val.Email { + t.Errorf("Unexpected email value, want: %s, got: %s", email, val.Email) + } + + // Verify that we get an error for other images. + for _, otherRegistry := range otherRegistries { + fullImage = path.Join(otherRegistry, image) + creds, ok = keyring.Lookup(fullImage) + if ok { + t.Errorf("Unexpectedly found image: %s", fullImage) + return + } + } +}