Merge pull request #48981 from colemickens/acr

Automatic merge from submit-queue (batch tested with PRs 48981, 47316, 49180)

azure: acr: support MSI with preview ACR with AAD auth

**What this PR does / why we need it**:

The recently added support for Managed Identity in Azure (#48854) was incompatible with automatic ACR docker credential integration (#48980).

This PR resolves that, by leveraging a feature available in Preview regions, on new managed clusters with support for AAD `access_token` authentication.

Notes:
* This includes code copied from [Azure/acr-docker-credential-helper](https://github.com/Azure/acr-docker-credential-helper). I copied the MIT license from that project and added a copyright line for Microsoft on it. (but one of the hack/verify-* scripts requires the Kubernetes copyright header. So there are two copyright headers in the file now...)
* Eventually this should vendor  [Azure/acr-docker-credential-helper](https://github.com/Azure/acr-docker-credential-helper) when it exposes the right functionality.
* This includes a small, non-function-impacting workaround for a temporary service-side bug.


**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #48980

**Special notes for your reviewer**:
Please don't LGTM it without reviewing the `azure_acr_helper.go` file's license header...

**Release note**:
```release-note
azure: acr: support MSI with preview ACR with AAD auth
```

pkg/credentialprovider/azure/BUILD

@@ -10,14 +10,19 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["azure_credentials.go"],
+    srcs = [
+        "azure_acr_helper.go",
+        "azure_credentials.go",
+    ],
     tags = ["automanaged"],
     deps = [
         "//pkg/cloudprovider/providers/azure:go_default_library",
         "//pkg/credentialprovider:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/arm/containerregistry:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library",
+        "//vendor/github.com/dgrijalva/jwt-go:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
     ],

pkg/credentialprovider/azure/azure_acr_helper.go

@@ -0,0 +1,300 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+Copyright 2017 Microsoft Corporation
+
+MIT License
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE
+*/
+
+// Source: https://github.com/Azure/acr-docker-credential-helper/blob/a79b541f3ee761f6cc4511863ed41fb038c19464/src/docker-credential-acr/acr_login.go
+
+package azure
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"unicode"
+
+	jwt "github.com/dgrijalva/jwt-go"
+)
+
+type authDirective struct {
+	service string
+	realm   string
+}
+
+type accessTokenPayload struct {
+	TenantID string `json:"tid"`
+}
+
+type acrTokenPayload struct {
+	Expiration int64  `json:"exp"`
+	TenantID   string `json:"tenant"`
+	Credential string `json:"credential"`
+}
+
+type acrAuthResponse struct {
+	RefreshToken string `json:"refresh_token"`
+}
+
+// 5 minutes buffer time to allow timeshift between local machine and AAD
+const timeShiftBuffer = 300
+const userAgentHeader = "User-Agent"
+const userAgent = "kubernetes-credentialprovider-acr"
+
+const dockerTokenLoginUsernameGUID = "00000000-0000-0000-0000-000000000000"
+
+var client = &http.Client{}
+
+func receiveChallengeFromLoginServer(serverAddress string) (*authDirective, error) {
+	challengeURL := url.URL{
+		Scheme: "https",
+		Host:   serverAddress,
+		Path:   "v2/",
+	}
+	var err error
+	var r *http.Request
+	r, _ = http.NewRequest("GET", challengeURL.String(), nil)
+	r.Header.Add(userAgentHeader, userAgent)
+
+	var challenge *http.Response
+	if challenge, err = client.Do(r); err != nil {
+		return nil, fmt.Errorf("Error reaching registry endpoint %s, error: %s", challengeURL.String(), err)
+	}
+	defer challenge.Body.Close()
+
+	if challenge.StatusCode != 401 {
+		return nil, fmt.Errorf("Registry did not issue a valid AAD challenge, status: %d", challenge.StatusCode)
+	}
+
+	var authHeader []string
+	var ok bool
+	if authHeader, ok = challenge.Header["Www-Authenticate"]; !ok {
+		return nil, fmt.Errorf("Challenge response does not contain header 'Www-Authenticate'")
+	}
+
+	if len(authHeader) != 1 {
+		return nil, fmt.Errorf("Registry did not issue a valid AAD challenge, authenticate header [%s]",
+			strings.Join(authHeader, ", "))
+	}
+
+	authSections := strings.SplitN(authHeader[0], " ", 2)
+	authType := strings.ToLower(authSections[0])
+	var authParams *map[string]string
+	if authParams, err = parseAssignments(authSections[1]); err != nil {
+		return nil, fmt.Errorf("Unable to understand the contents of Www-Authenticate header %s", authSections[1])
+	}
+
+	// verify headers
+	if !strings.EqualFold("Bearer", authType) {
+		return nil, fmt.Errorf("Www-Authenticate: expected realm: Bearer, actual: %s", authType)
+	}
+	if len((*authParams)["service"]) == 0 {
+		return nil, fmt.Errorf("Www-Authenticate: missing header \"service\"")
+	}
+	if len((*authParams)["realm"]) == 0 {
+		return nil, fmt.Errorf("Www-Authenticate: missing header \"realm\"")
+	}
+
+	return &authDirective{
+		service: (*authParams)["service"],
+		realm:   (*authParams)["realm"],
+	}, nil
+}
+
+func parseAcrToken(identityToken string) (token *acrTokenPayload, err error) {
+	tokenSegments := strings.Split(identityToken, ".")
+	if len(tokenSegments) < 2 {
+		return nil, fmt.Errorf("Invalid existing refresh token length: %d", len(tokenSegments))
+	}
+	payloadSegmentEncoded := tokenSegments[1]
+	var payloadBytes []byte
+	if payloadBytes, err = jwt.DecodeSegment(payloadSegmentEncoded); err != nil {
+		return nil, fmt.Errorf("Error decoding payload segment from refresh token, error: %s", err)
+	}
+	var payload acrTokenPayload
+	if err = json.Unmarshal(payloadBytes, &payload); err != nil {
+		return nil, fmt.Errorf("Error unmarshalling acr payload, error: %s", err)
+	}
+	return &payload, nil
+}
+
+func performTokenExchange(
+	serverAddress string,
+	directive *authDirective,
+	tenant string,
+	accessToken string) (string, error) {
+	var err error
+	data := url.Values{
+		"service":       []string{directive.service},
+		"grant_type":    []string{"access_token_refresh_token"},
+		"access_token":  []string{accessToken},
+		"refresh_token": []string{accessToken},
+		"tenant":        []string{tenant},
+	}
+
+	var realmURL *url.URL
+	if realmURL, err = url.Parse(directive.realm); err != nil {
+		return "", fmt.Errorf("Www-Authenticate: invalid realm %s", directive.realm)
+	}
+	authEndpoint := fmt.Sprintf("%s://%s/oauth2/exchange", realmURL.Scheme, realmURL.Host)
+
+	datac := data.Encode()
+	var r *http.Request
+	r, _ = http.NewRequest("POST", authEndpoint, bytes.NewBufferString(datac))
+	r.Header.Add(userAgentHeader, userAgent)
+	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	r.Header.Add("Content-Length", strconv.Itoa(len(datac)))
+
+	var exchange *http.Response
+	if exchange, err = client.Do(r); err != nil {
+		return "", fmt.Errorf("Www-Authenticate: failed to reach auth url %s", authEndpoint)
+	}
+
+	defer exchange.Body.Close()
+	if exchange.StatusCode != 200 {
+		return "", fmt.Errorf("Www-Authenticate: auth url %s responded with status code %d", authEndpoint, exchange.StatusCode)
+	}
+
+	var content []byte
+	if content, err = ioutil.ReadAll(exchange.Body); err != nil {
+		return "", fmt.Errorf("Www-Authenticate: error reading response from %s", authEndpoint)
+	}
+
+	var authResp acrAuthResponse
+	if err = json.Unmarshal(content, &authResp); err != nil {
+		return "", fmt.Errorf("Www-Authenticate: unable to read response %s", content)
+	}
+
+	return authResp.RefreshToken, nil
+}
+
+// Try and parse a string of assignments in the form of:
+// key1 = value1, key2 = "value 2", key3 = ""
+// Note: this method and handle quotes but does not handle escaping of quotes
+func parseAssignments(statements string) (*map[string]string, error) {
+	var cursor int
+	result := make(map[string]string)
+	var errorMsg = fmt.Errorf("malformed header value: %s", statements)
+	for {
+		// parse key
+		equalIndex := nextOccurrence(statements, cursor, "=")
+		if equalIndex == -1 {
+			return nil, errorMsg
+		}
+		key := strings.TrimSpace(statements[cursor:equalIndex])
+
+		// parse value
+		cursor = nextNoneSpace(statements, equalIndex+1)
+		if cursor == -1 {
+			return nil, errorMsg
+		}
+		// case: value is quoted
+		if statements[cursor] == '"' {
+			cursor = cursor + 1
+			// like I said, not handling escapes, but this will skip any comma that's
+			// within the quotes which is somewhat more likely
+			closeQuoteIndex := nextOccurrence(statements, cursor, "\"")
+			if closeQuoteIndex == -1 {
+				return nil, errorMsg
+			}
+			value := statements[cursor:closeQuoteIndex]
+			result[key] = value
+
+			commaIndex := nextNoneSpace(statements, closeQuoteIndex+1)
+			if commaIndex == -1 {
+				// no more comma, done
+				return &result, nil
+			} else if statements[commaIndex] != ',' {
+				// expect comma immediately after close quote
+				return nil, errorMsg
+			} else {
+				cursor = commaIndex + 1
+			}
+		} else {
+			commaIndex := nextOccurrence(statements, cursor, ",")
+			endStatements := commaIndex == -1
+			var untrimmed string
+			if endStatements {
+				untrimmed = statements[cursor:commaIndex]
+			} else {
+				untrimmed = statements[cursor:]
+			}
+			value := strings.TrimSpace(untrimmed)
+
+			if len(value) == 0 {
+				// disallow empty value without quote
+				return nil, errorMsg
+			}
+
+			result[key] = value
+
+			if endStatements {
+				return &result, nil
+			}
+			cursor = commaIndex + 1
+		}
+	}
+}
+
+func nextOccurrence(str string, start int, sep string) int {
+	if start >= len(str) {
+		return -1
+	}
+	offset := strings.Index(str[start:], sep)
+	if offset == -1 {
+		return -1
+	}
+	return offset + start
+}
+
+func nextNoneSpace(str string, start int) int {
+	if start >= len(str) {
+		return -1
+	}
+	offset := strings.IndexFunc(str[start:], func(c rune) bool { return !unicode.IsSpace(c) })
+	if offset == -1 {
+		return -1
+	}
+	return offset + start
+}

pkg/credentialprovider/azure/azure_credentials.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/arm/containerregistry"
 	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/adal"
 	azureapi "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/golang/glog"
 	"github.com/spf13/pflag"
@@ -58,10 +59,11 @@ func NewACRProvider(configFile *string) credentialprovider.DockerConfigProvider
 }
 
 type acrProvider struct {
-	file           *string
-	config         *azure.Config
-	environment    *azureapi.Environment
-	registryClient RegistriesClient
+	file                  *string
+	config                *azure.Config
+	environment           *azureapi.Environment
+	registryClient        RegistriesClient
+	servicePrincipalToken *adal.ServicePrincipalToken
 }
 
 func (a *acrProvider) loadConfig(rdr io.Reader) error {
@@ -92,7 +94,7 @@ func (a *acrProvider) Enabled() bool {
 		return false
 	}
 
-	servicePrincipalToken, err := azure.GetServicePrincipalToken(a.config, a.environment)
+	a.servicePrincipalToken, err = azure.GetServicePrincipalToken(a.config, a.environment)
 	if err != nil {
 		glog.Errorf("Failed to create service principal token: %v", err)
 		return false
@@ -100,7 +102,7 @@ func (a *acrProvider) Enabled() bool {
 
 	registryClient := containerregistry.NewRegistriesClient(a.config.SubscriptionID)
 	registryClient.BaseURI = a.environment.ResourceManagerEndpoint
-	registryClient.Authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken)
+	registryClient.Authorizer = autorest.NewBearerAuthorizer(a.servicePrincipalToken)
 	a.registryClient = registryClient
 
 	return true
@@ -108,21 +110,32 @@ func (a *acrProvider) Enabled() bool {
 
 func (a *acrProvider) Provide() credentialprovider.DockerConfig {
 	cfg := credentialprovider.DockerConfig{}
-	entry := credentialprovider.DockerConfigEntry{
-		Username: a.config.AADClientID,
-		Password: a.config.AADClientSecret,
-		Email:    dummyRegistryEmail,
-	}
 
+	glog.V(4).Infof("listing registries")
 	res, err := a.registryClient.List()
 	if err != nil {
 		glog.Errorf("Failed to list registries: %v", err)
 		return cfg
 	}
+
 	for ix := range *res.Value {
 		loginServer := getLoginServer((*res.Value)[ix])
-		glog.V(4).Infof("Adding Azure Container Registry docker credential for %s", loginServer)
-		cfg[loginServer] = entry
+		var cred *credentialprovider.DockerConfigEntry
+
+		if a.config.UseManagedIdentityExtension {
+			cred, err = getACRDockerEntryFromARMToken(a, loginServer)
+			if err != nil {
+				continue
+			}
+		} else {
+			cred = &credentialprovider.DockerConfigEntry{
+				Username: a.config.AADClientID,
+				Password: a.config.AADClientSecret,
+				Email:    dummyRegistryEmail,
+			}
+		}
+
+		cfg[loginServer] = *cred
 	}
 	return cfg
 }
@@ -131,6 +144,32 @@ func getLoginServer(registry containerregistry.Registry) string {
 	return *(*registry.RegistryProperties).LoginServer
 }
 
+func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialprovider.DockerConfigEntry, error) {
+	armAccessToken := a.servicePrincipalToken.AccessToken
+
+	glog.V(4).Infof("discovering auth redirects for: %s", loginServer)
+	directive, err := receiveChallengeFromLoginServer(loginServer)
+	if err != nil {
+		glog.Errorf("failed to receive challenge: %s", err)
+		return nil, err
+	}
+
+	glog.V(4).Infof("exchanging an acr refresh_token")
+	registryRefreshToken, err := performTokenExchange(
+		loginServer, directive, a.config.TenantID, armAccessToken)
+	if err != nil {
+		glog.Errorf("failed to perform token exchange: %s", err)
+		return nil, err
+	}
+
+	glog.V(4).Infof("adding ACR docker config entry for: %s", loginServer)
+	return &credentialprovider.DockerConfigEntry{
+		Username: dockerTokenLoginUsernameGUID,
+		Password: registryRefreshToken,
+		Email:    dummyRegistryEmail,
+	}, nil
+}
+
 func (a *acrProvider) LazyProvide() *credentialprovider.DockerConfigEntry {
 	return nil
 }