Contribute grafana/agent sigv4 code (#8509)

* Contribute grafana/agent sigv4 code
* address review feedback
  - move validation logic for RemoteWrite into unmarshal
  - copy configuration fields from ec2 SD config
  - remove enabled field, use pointer for enabling sigv4
* Update config/config.go
* Don't provide credentials if secret key / access key left blank
* Add SigV4 headers to the list of unchangeable headers.
* sigv4: don't include all headers in signature
* only test for equality in the authorization header, not the signed date
* address review feedback
  1. s/httpClientConfigEnabled/httpClientConfigAuthEnabled
  2. bearer_token tuples to "authorization"
  3. Un-export NewSigV4RoundTripper
* add x-amz-content-sha256 to list of unchangeable headers
* Document sigv4 configuration
* add suggestion for using default AWS SDK credentials

Signed-off-by: Robert Fratto <robertfratto@gmail.com>
Co-authored-by: Julien Pivotto <roidelapluie@gmail.com>

config/config.go

@@ -51,6 +51,11 @@ var (
 		"accept-encoding":                   {},
 		"x-prometheus-remote-write-version": {},
 		"x-prometheus-remote-read-version":  {},
+
+		// Added by SigV4.
+		"x-amz-date":           {},
+		"x-amz-security-token": {},
+		"x-amz-content-sha256": {},
 	}
 )
 
@@ -601,6 +606,7 @@ type RemoteWriteConfig struct {
 	HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
 	QueueConfig      QueueConfig             `yaml:"queue_config,omitempty"`
 	MetadataConfig   MetadataConfig          `yaml:"metadata_config,omitempty"`
+	SigV4Config      *SigV4Config            `yaml:"sigv4,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -630,7 +636,18 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
 	// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
 	// We cannot make it a pointer as the parser panics for inlined pointer structs.
 	// Thus we just do its validation here.
-	return c.HTTPClientConfig.Validate()
+	if err := c.HTTPClientConfig.Validate(); err != nil {
+		return err
+	}
+
+	httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil ||
+		c.HTTPClientConfig.Authorization != nil
+
+	if httpClientConfigAuthEnabled && c.SigV4Config != nil {
+		return fmt.Errorf("at most one of basic_auth, authorization, & sigv4 must be configured")
+	}
+
+	return nil
 }
 
 func validateHeaders(headers map[string]string) error {
@@ -679,6 +696,17 @@ type MetadataConfig struct {
 	SendInterval model.Duration `yaml:"send_interval"`
 }
 
+// SigV4Config is the configuration for signing remote write requests with
+// AWS's SigV4 verification process. Empty values will be retrieved using the
+// AWS default credentials chain.
+type SigV4Config struct {
+	Region    string        `yaml:"region,omitempty"`
+	AccessKey string        `yaml:"access_key,omitempty"`
+	SecretKey config.Secret `yaml:"secret_key,omitempty"`
+	Profile   string        `yaml:"profile,omitempty"`
+	RoleARN   string        `yaml:"role_arn,omitempty"`
+}
+
 // RemoteReadConfig is the configuration for reading from remote storage.
 type RemoteReadConfig struct {
 	URL           *config.URL       `yaml:"url"`

docs/configuration/configuration.md

@@ -1222,13 +1222,13 @@ namespaces:
   names:
     [ - <string> ]
 
-# Optional label and field selectors to limit the discovery process to a subset of available resources. 
+# Optional label and field selectors to limit the discovery process to a subset of available resources.
 # See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/
-# and https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ to learn more about the possible 
+# and https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ to learn more about the possible
 # filters that can be used. Endpoints role supports pod, service and endpoints selectors, other roles
 # only support selectors matching the role itself (e.g. node role can only contain node selectors).
 
-# Note: When making decision about using field/label selector make sure that this 
+# Note: When making decision about using field/label selector make sure that this
 # is the best approach - it will prevent Prometheus from reusing single list/watch
 # for all scrape configs. This might result in a bigger load on the Kubernetes API,
 # because per each selector combination there will be additional LIST/WATCH. On the other hand,
@@ -1788,7 +1788,7 @@ headers:
 write_relabel_configs:
   [ - <relabel_config> ... ]
 
-# Name of the remote write config, which if specified must be unique among remote write configs. 
+# Name of the remote write config, which if specified must be unique among remote write configs.
 # The name will be used in metrics and logging in place of a generated value to help users distinguish between
 # remote write configs.
 [ name: <string> ]
@@ -1812,6 +1812,25 @@ authorization:
   # It is mutually exclusive with `credentials`.
   [ credentials_file: <filename> ]
 
+# Optionally configures AWS's Signature Verification 4 signing process to
+# sign requests. Cannot be set at the same time as basic_auth or authorization.
+# To use the default credentials from the AWS SDK, use `sigv4: {}`.
+sigv4:
+  # The AWS region. If blank, the region from the default credentials chain
+  # is used.
+  [ region: <string> ]
+
+  # The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID`
+  # and `AWS_SECRET_ACCESS_KEY` are used.
+  [ access_key: <string> ]
+  [ secret_key: <secret> ]
+
+  # Named AWS profile used to authenticate.
+  [ profile: <string> ]
+
+  # AWS Role ARN, an alternative to using AWS API keys.
+  [ role_arn: <string> ]
+
 # Configures the remote write request's TLS settings.
 tls_config:
   [ <tls_config> ]
@@ -1865,7 +1884,7 @@ with this feature.
 # The URL of the endpoint to query from.
 url: <string>
 
-# Name of the remote read config, which if specified must be unique among remote read configs. 
+# Name of the remote read config, which if specified must be unique among remote read configs.
 # The name will be used in metrics and logging in place of a generated value to help users distinguish between
 # remote read configs.
 [ name: <string> ]

storage/remote/client.go

@@ -35,6 +35,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 )
 
@@ -96,6 +97,7 @@ type ClientConfig struct {
 	URL              *config_util.URL
 	Timeout          model.Duration
 	HTTPClientConfig config_util.HTTPClientConfig
+	SigV4Config      *config.SigV4Config
 	Headers          map[string]string
 	RetryOnRateLimit bool
 }
@@ -138,11 +140,19 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	t := httpClient.Transport
+
+	if conf.SigV4Config != nil {
+		t, err = newSigV4RoundTripper(conf.SigV4Config, httpClient.Transport)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	if len(conf.Headers) > 0 {
 		t = newInjectHeadersRoundTripper(conf.Headers, t)
 	}
+
 	httpClient.Transport = &nethttp.Transport{
 		RoundTripper: t,
 	}

storage/remote/sigv4.go

@@ -0,0 +1,138 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remote
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/textproto"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
+	"github.com/aws/aws-sdk-go/aws/session"
+	signer "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/prometheus/prometheus/config"
+)
+
+var sigv4HeaderDenylist = []string{
+	"uber-trace-id",
+}
+
+type sigV4RoundTripper struct {
+	region string
+	next   http.RoundTripper
+	pool   sync.Pool
+
+	signer *signer.Signer
+}
+
+// newSigV4RoundTripper returns a new http.RoundTripper that will sign requests
+// using Amazon's Signature Verification V4 signing procedure. The request will
+// then be handed off to the next RoundTripper provided by next. If next is nil,
+// http.DefaultTransport will be used.
+//
+// Credentials for signing are retrieved using the the default AWS credential
+// chain. If credentials cannot be found, an error will be returned.
+func newSigV4RoundTripper(cfg *config.SigV4Config, next http.RoundTripper) (http.RoundTripper, error) {
+	if next == nil {
+		next = http.DefaultTransport
+	}
+
+	creds := credentials.NewStaticCredentials(cfg.AccessKey, string(cfg.SecretKey), "")
+	if cfg.AccessKey == "" && cfg.SecretKey == "" {
+		creds = nil
+	}
+
+	sess, err := session.NewSessionWithOptions(session.Options{
+		Config: aws.Config{
+			Region:      aws.String(cfg.Region),
+			Credentials: creds,
+		},
+		Profile: cfg.Profile,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not create new AWS session: %w", err)
+	}
+	if _, err := sess.Config.Credentials.Get(); err != nil {
+		return nil, fmt.Errorf("could not get SigV4 credentials: %w", err)
+	}
+	if aws.StringValue(sess.Config.Region) == "" {
+		return nil, fmt.Errorf("region not configured in sigv4 or in default credentials chain")
+	}
+
+	signerCreds := sess.Config.Credentials
+	if cfg.RoleARN != "" {
+		signerCreds = stscreds.NewCredentials(sess, cfg.RoleARN)
+	}
+
+	rt := &sigV4RoundTripper{
+		region: cfg.Region,
+		next:   next,
+		signer: signer.NewSigner(signerCreds),
+	}
+	rt.pool.New = rt.newBuf
+	return rt, nil
+}
+
+func (rt *sigV4RoundTripper) newBuf() interface{} {
+	return bytes.NewBuffer(make([]byte, 0, 1024))
+}
+
+func (rt *sigV4RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	// rt.signer.Sign needs a seekable body, so we replace the body with a
+	// buffered reader filled with the contents of original body.
+	buf := rt.pool.Get().(*bytes.Buffer)
+	defer func() {
+		buf.Reset()
+		rt.pool.Put(buf)
+	}()
+	if _, err := io.Copy(buf, req.Body); err != nil {
+		return nil, err
+	}
+	// Close the original body since we don't need it anymore.
+	_ = req.Body.Close()
+
+	// Ensure our seeker is back at the start of the buffer once we return.
+	var seeker io.ReadSeeker = bytes.NewReader(buf.Bytes())
+	defer func() {
+		_, _ = seeker.Seek(0, io.SeekStart)
+	}()
+	req.Body = ioutil.NopCloser(seeker)
+
+	// Clone the request and trim out headers that we don't want to sign.
+	signReq := req.Clone(req.Context())
+	for _, header := range sigv4HeaderDenylist {
+		signReq.Header.Del(header)
+	}
+
+	headers, err := rt.signer.Sign(signReq, seeker, "aps", rt.region, time.Now().UTC())
+	if err != nil {
+		return nil, fmt.Errorf("failed to sign request: %w", err)
+	}
+
+	// Copy over signed headers. Authorization header is not returned by
+	// rt.signer.Sign and needs to be copied separately.
+	for k, v := range headers {
+		req.Header[textproto.CanonicalMIMEHeaderKey(k)] = v
+	}
+	req.Header.Set("Authorization", signReq.Header.Get("Authorization"))
+
+	return rt.next.RoundTrip(req)
+}

storage/remote/sigv4_test.go

@@ -0,0 +1,92 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remote
+
+import (
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	signer "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSigV4_Inferred_Region(t *testing.T) {
+	os.Setenv("AWS_ACCESS_KEY_ID", "secret")
+	os.Setenv("AWS_SECRET_ACCESS_KEY", "token")
+	os.Setenv("AWS_REGION", "us-west-2")
+
+	sess, err := session.NewSession(&aws.Config{
+		// Setting to an empty string to demostrate the default value from the yaml
+		// won't override the environment's region.
+		Region: aws.String(""),
+	})
+	require.NoError(t, err)
+	_, err = sess.Config.Credentials.Get()
+	require.NoError(t, err)
+
+	require.NotNil(t, sess.Config.Region)
+	require.Equal(t, "us-west-2", *sess.Config.Region)
+}
+
+func TestSigV4RoundTripper(t *testing.T) {
+	var gotReq *http.Request
+
+	rt := &sigV4RoundTripper{
+		region: "us-east-2",
+		next: promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			gotReq = req
+			return &http.Response{StatusCode: http.StatusOK}, nil
+		}),
+		signer: signer.NewSigner(credentials.NewStaticCredentials(
+			"test-id",
+			"secret",
+			"token",
+		)),
+	}
+	rt.pool.New = rt.newBuf
+
+	cli := &http.Client{Transport: rt}
+
+	req, err := http.NewRequest(http.MethodPost, "google.com", strings.NewReader("Hello, world!"))
+	require.NoError(t, err)
+
+	_, err = cli.Do(req)
+	require.NoError(t, err)
+	require.NotNil(t, gotReq)
+
+	origReq := gotReq
+	require.NotEmpty(t, origReq.Header.Get("Authorization"))
+	require.NotEmpty(t, origReq.Header.Get("X-Amz-Date"))
+
+	// Perform the same request but with a header that shouldn't included in the
+	// signature; validate that the Authorization signature matches.
+	t.Run("Ignored Headers", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodPost, "google.com", strings.NewReader("Hello, world!"))
+		require.NoError(t, err)
+
+		req.Header.Add("Uber-Trace-Id", "some-trace-id")
+
+		_, err = cli.Do(req)
+		require.NoError(t, err)
+		require.NotNil(t, gotReq)
+
+		require.Equal(t, origReq.Header.Get("Authorization"), gotReq.Header.Get("Authorization"))
+	})
+}

storage/remote/write.go

@@ -134,6 +134,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error {
 			URL:              rwConf.URL,
 			Timeout:          rwConf.RemoteTimeout,
 			HTTPClientConfig: rwConf.HTTPClientConfig,
+			SigV4Config:      rwConf.SigV4Config,
 			Headers:          rwConf.Headers,
 			RetryOnRateLimit: rwConf.QueueConfig.RetryOnRateLimit,
 		})