Replace regex with Secret type and remarshal config to hide secrets (#2775)

cmd/prometheus/config.go

@@ -398,7 +398,7 @@ func parseAlertmanagerURLToConfig(us string) (*config.AlertmanagerConfig, error)
 		}
 
 		if password, isSet := u.User.Password(); isSet {
-			acfg.HTTPClientConfig.BasicAuth.Password = password
+			acfg.HTTPClientConfig.BasicAuth.Password = config.Secret(password)
 		}
 	}
 

cmd/prometheus/config_test.go

@@ -13,7 +13,11 @@
 
 package main
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/prometheus/prometheus/config"
+)
 
 func TestParse(t *testing.T) {
 	tests := []struct {
@@ -72,7 +76,7 @@ func TestParseAlertmanagerURLToConfig(t *testing.T) {
 	tests := []struct {
 		url      string
 		username string
-		password string
+		password config.Secret
 	}{
 		{
 			url:      "http://alertmanager.company.com",

config/config.go

@@ -30,7 +30,6 @@ import (
 var (
 	patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`)
 	patRulePath   = regexp.MustCompile(`^[^*]*(\*[^/]*)?$`)
-	patAuthLine   = regexp.MustCompile(`((?:password|bearer_token|secret_key|client_secret):\s+)(".+"|'.+'|[^\s]+)`)
 	relabelTarget = regexp.MustCompile(`^(?:(?:[a-zA-Z_]|\$(?:\{\w+\}|\w+))+\w*)+$`)
 )
 
@@ -219,6 +218,23 @@ type Config struct {
 	original string
 }
 
+// Secret special type for storing secrets.
+type Secret string
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface for Secrets.
+func (s *Secret) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type plain Secret
+	return unmarshal((*plain)(s))
+}
+
+// MarshalYAML implements the yaml.Marshaler interface for Secrets.
+func (s Secret) MarshalYAML() (interface{}, error) {
+	if s != "" {
+		return "<secret>", nil
+	}
+	return nil, nil
+}
+
 // resolveFilepaths joins all relative paths in a configuration
 // with a given base directory.
 func resolveFilepaths(baseDir string, cfg *Config) {
@@ -281,17 +297,11 @@ func checkOverflow(m map[string]interface{}, ctx string) error {
 }
 
 func (c Config) String() string {
-	var s string
-	if c.original != "" {
-		s = c.original
-	} else {
-		b, err := yaml.Marshal(c)
-		if err != nil {
-			return fmt.Sprintf("<error creating config string: %s>", err)
-		}
-		s = string(b)
+	b, err := yaml.Marshal(c)
+	if err != nil {
+		return fmt.Sprintf("<error creating config string: %s>", err)
 	}
-	return patAuthLine.ReplaceAllString(s, "${1}<hidden>")
+	return string(b)
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
@@ -480,7 +490,7 @@ type HTTPClientConfig struct {
 	// The HTTP basic authentication credentials for the targets.
 	BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"`
 	// The bearer token for the targets.
-	BearerToken string `yaml:"bearer_token,omitempty"`
+	BearerToken Secret `yaml:"bearer_token,omitempty"`
 	// The bearer token file for the targets.
 	BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
 	// HTTP proxy server to use to connect to the targets.
@@ -544,7 +554,7 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err != nil {
 		return err
 	}
-	if err := checkOverflow(c.XXX, "scrape_config"); err != nil {
+	if err = checkOverflow(c.XXX, "scrape_config"); err != nil {
 		return err
 	}
 	if len(c.JobName) == 0 {
@@ -554,7 +564,7 @@ func (c *ScrapeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	// The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer.
 	// We cannot make it a pointer as the parser panics for inlined pointer structs.
 	// Thus we just do its validation here.
-	if err := c.HTTPClientConfig.validate(); err != nil {
+	if err = c.HTTPClientConfig.validate(); err != nil {
 		return err
 	}
 
@@ -660,7 +670,7 @@ func CheckTargetAddress(address model.LabelValue) error {
 // BasicAuth contains basic HTTP authentication credentials.
 type BasicAuth struct {
 	Username string `yaml:"username"`
-	Password string `yaml:"password"`
+	Password Secret `yaml:"password"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -669,7 +679,7 @@ type BasicAuth struct {
 // ClientCert contains client cert credentials.
 type ClientCert struct {
 	Cert string `yaml:"cert"`
-	Key  string `yaml:"key"`
+	Key  Secret `yaml:"key"`
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`
@@ -830,7 +840,7 @@ type ConsulSDConfig struct {
 	TagSeparator string `yaml:"tag_separator,omitempty"`
 	Scheme       string `yaml:"scheme,omitempty"`
 	Username     string `yaml:"username,omitempty"`
-	Password     string `yaml:"password,omitempty"`
+	Password     Secret `yaml:"password,omitempty"`
 	// The list of services for which targets are discovered.
 	// Defaults to all services if empty.
 	Services []string `yaml:"services"`
@@ -933,7 +943,7 @@ type MarathonSDConfig struct {
 	Timeout         model.Duration `yaml:"timeout,omitempty"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 	TLSConfig       TLSConfig      `yaml:"tls_config,omitempty"`
-	BearerToken     string         `yaml:"bearer_token,omitempty"`
+	BearerToken     Secret         `yaml:"bearer_token,omitempty"`
 	BearerTokenFile string         `yaml:"bearer_token_file,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.
@@ -990,7 +1000,7 @@ type KubernetesSDConfig struct {
 	APIServer          URL                          `yaml:"api_server"`
 	Role               KubernetesRole               `yaml:"role"`
 	BasicAuth          *BasicAuth                   `yaml:"basic_auth,omitempty"`
-	BearerToken        string                       `yaml:"bearer_token,omitempty"`
+	BearerToken        Secret                       `yaml:"bearer_token,omitempty"`
 	BearerTokenFile    string                       `yaml:"bearer_token_file,omitempty"`
 	TLSConfig          TLSConfig                    `yaml:"tls_config,omitempty"`
 	NamespaceDiscovery KubernetesNamespaceDiscovery `yaml:"namespaces"`
@@ -1095,7 +1105,7 @@ func (c *GCESDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 type EC2SDConfig struct {
 	Region          string         `yaml:"region"`
 	AccessKey       string         `yaml:"access_key,omitempty"`
-	SecretKey       string         `yaml:"secret_key,omitempty"`
+	SecretKey       Secret         `yaml:"secret_key,omitempty"`
 	Profile         string         `yaml:"profile,omitempty"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 	Port            int            `yaml:"port"`
@@ -1127,7 +1137,7 @@ type AzureSDConfig struct {
 	SubscriptionID  string         `yaml:"subscription_id"`
 	TenantID        string         `yaml:"tenant_id,omitempty"`
 	ClientID        string         `yaml:"client_id,omitempty"`
-	ClientSecret    string         `yaml:"client_secret,omitempty"`
+	ClientSecret    Secret         `yaml:"client_secret,omitempty"`
 	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
 
 	// Catches all undefined fields and must be empty after parsing.

config/config_test.go

@@ -18,6 +18,7 @@ import (
 	"io/ioutil"
 	"net/url"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -144,6 +145,7 @@ var expectedConf = &Config{
 			},
 		},
 		{
+
 			JobName: "service-x",
 
 			ScrapeInterval: model.Duration(50 * time.Second),
@@ -153,7 +155,7 @@ var expectedConf = &Config{
 			HTTPClientConfig: HTTPClientConfig{
 				BasicAuth: &BasicAuth{
 					Username: "admin_name",
-					Password: "admin_password",
+					Password: "multiline\nmysecret\ntest",
 				},
 			},
 			MetricsPath: "/my_path",
@@ -284,7 +286,7 @@ var expectedConf = &Config{
 					KeyFile:  "testdata/valid_key_file",
 				},
 
-				BearerToken: "avalidtoken",
+				BearerToken: "mysecret",
 			},
 		},
 		{
@@ -303,7 +305,7 @@ var expectedConf = &Config{
 						Role:      KubernetesRoleEndpoint,
 						BasicAuth: &BasicAuth{
 							Username: "myusername",
-							Password: "mypassword",
+							Password: "mysecret",
 						},
 						NamespaceDiscovery: KubernetesNamespaceDiscovery{},
 					},
@@ -372,7 +374,7 @@ var expectedConf = &Config{
 					{
 						Region:          "us-east-1",
 						AccessKey:       "access",
-						SecretKey:       "secret",
+						SecretKey:       "mysecret",
 						Profile:         "profile",
 						RefreshInterval: model.Duration(60 * time.Second),
 						Port:            80,
@@ -395,7 +397,7 @@ var expectedConf = &Config{
 						SubscriptionID:  "11AAAA11-A11A-111A-A111-1111A1111A11",
 						TenantID:        "BBBB222B-B2B2-2B22-B222-2BB2222BB2B2",
 						ClientID:        "333333CC-3C33-3333-CCC3-33C3CCCCC33C",
-						ClientSecret:    "nAdvAK2oBuVym4IXix",
+						ClientSecret:    "mysecret",
 						RefreshInterval: model.Duration(5 * time.Minute),
 						Port:            9100,
 					},
@@ -538,9 +540,12 @@ func TestLoadConfig(t *testing.T) {
 
 	// String method must not reveal authentication credentials.
 	s := c.String()
-	if strings.Contains(s, "admin_password") {
+	secretRe := regexp.MustCompile("<secret>")
+	matches := secretRe.FindAllStringIndex(s, -1)
+	if len(matches) != 5 || strings.Contains(s, "mysecret") {
 		t.Fatalf("config's String method reveals authentication credentials.")
 	}
+
 }
 
 var expectedErrors = []struct {

config/testdata/conf.good.yml

@@ -67,7 +67,7 @@ scrape_configs:
 
   basic_auth:
     username: admin_name
-    password: admin_password
+    password: "multiline\nmysecret\ntest"
 
   scrape_interval: 50s
   scrape_timeout:  5s
@@ -134,7 +134,7 @@ scrape_configs:
     cert_file: valid_cert_file
     key_file: valid_key_file
 
-  bearer_token: avalidtoken
+  bearer_token: mysecret
 
 - job_name: service-kubernetes
 
@@ -144,7 +144,7 @@ scrape_configs:
 
     basic_auth:
       username: 'myusername'
-      password: 'mypassword'
+      password: 'mysecret'
 
 - job_name: service-kubernetes-namespaces
 
@@ -168,7 +168,7 @@ scrape_configs:
   ec2_sd_configs:
     - region: us-east-1
       access_key: access
-      secret_key: secret
+      secret_key: mysecret
       profile: profile
 
 - job_name: service-azure
@@ -176,7 +176,7 @@ scrape_configs:
     - subscription_id: 11AAAA11-A11A-111A-A111-1111A1111A11
       tenant_id: BBBB222B-B2B2-2B22-B222-2BB2222BB2B2
       client_id: 333333CC-3C33-3333-CCC3-33C3CCCCC33C
-      client_secret: nAdvAK2oBuVym4IXix
+      client_secret: mysecret
       port: 9100
 
 - job_name: service-nerve

discovery/azure/azure.go

@@ -120,7 +120,7 @@ func createAzureClient(cfg config.AzureSDConfig) (azureClient, error) {
 	if err != nil {
 		return azureClient{}, err
 	}
-	spt, err := azure.NewServicePrincipalToken(*oauthConfig, cfg.ClientID, cfg.ClientSecret, azure.PublicCloud.ResourceManagerEndpoint)
+	spt, err := azure.NewServicePrincipalToken(*oauthConfig, cfg.ClientID, string(cfg.ClientSecret), azure.PublicCloud.ResourceManagerEndpoint)
 	if err != nil {
 		return azureClient{}, err
 	}

discovery/consul/consul.go

@@ -107,7 +107,7 @@ func NewDiscovery(conf *config.ConsulSDConfig) (*Discovery, error) {
 		Token:      conf.Token,
 		HttpAuth: &consul.HttpBasicAuth{
 			Username: conf.Username,
-			Password: conf.Password,
+			Password: string(conf.Password),
 		},
 		HttpClient: wrapper,
 	}

discovery/ec2/ec2.go

@@ -76,7 +76,7 @@ type Discovery struct {
 
 // NewDiscovery returns a new EC2Discovery which periodically refreshes its targets.
 func NewDiscovery(conf *config.EC2SDConfig) *Discovery {
-	creds := credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, "")
+	creds := credentials.NewStaticCredentials(conf.AccessKey, string(conf.SecretKey), "")
 	if conf.AccessKey == "" && conf.SecretKey == "" {
 		creds = nil
 	}

discovery/kubernetes/kubernetes.go

@@ -124,7 +124,7 @@ func New(l log.Logger, conf *config.KubernetesSDConfig) (*Discovery, error) {
 				Insecure: conf.TLSConfig.InsecureSkipVerify,
 			},
 		}
-		token := conf.BearerToken
+		token := string(conf.BearerToken)
 		if conf.BearerTokenFile != "" {
 			bf, err := ioutil.ReadFile(conf.BearerTokenFile)
 			if err != nil {
@@ -136,7 +136,7 @@ func New(l log.Logger, conf *config.KubernetesSDConfig) (*Discovery, error) {
 
 		if conf.BasicAuth != nil {
 			kcfg.Username = conf.BasicAuth.Username
-			kcfg.Password = conf.BasicAuth.Password
+			kcfg.Password = string(conf.BasicAuth.Password)
 		}
 	}
 

discovery/marathon/marathon.go

@@ -94,7 +94,7 @@ func NewDiscovery(conf *config.MarathonSDConfig) (*Discovery, error) {
 		return nil, err
 	}
 
-	token := conf.BearerToken
+	token := string(conf.BearerToken)
 	if conf.BearerTokenFile != "" {
 		bf, err := ioutil.ReadFile(conf.BearerTokenFile)
 		if err != nil {

util/httputil/client.go

@@ -49,7 +49,7 @@ func NewClientFromConfig(cfg config.HTTPClientConfig) (*http.Client, error) {
 
 	// If a bearer token is provided, create a round tripper that will set the
 	// Authorization header correctly on each request.
-	bearerToken := cfg.BearerToken
+	bearerToken := string(cfg.BearerToken)
 	if len(bearerToken) == 0 && len(cfg.BearerTokenFile) > 0 {
 		b, err := ioutil.ReadFile(cfg.BearerTokenFile)
 		if err != nil {
@@ -63,7 +63,7 @@ func NewClientFromConfig(cfg config.HTTPClientConfig) (*http.Client, error) {
 	}
 
 	if cfg.BasicAuth != nil {
-		rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, rt)
+		rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, string(cfg.BasicAuth.Password), rt)
 	}
 
 	// Return a new client with the configured round tripper.