Added feature to get encrypt key from Vault

To use the feature we need to set the following options in config file: use_vault = true vault_address = http://<IP_ADDRESS:<IP_PORT> vault_role_id = <ROLE_ID> vault_secret_id = <SECRET_ID> vault_secret_path = <deployments/unit/dev/user/passwords_yml> vault_secret_mount_path = <secret_v2> credential_name_in_vault_secret = <consul_encrypt> https://github.com/hashicorp/consul/issues/2209 https://github.com/hashicorp/consul/issues/4685 https://github.com/hashicorp/consul/issues/5043
2 weeks ago · 7d9c3dbe7b
parent a2e69236a2
commit 7d9c3dbe7b
9 changed files with 91 additions and 3 deletions
--- a/agent/config/builder.go
+++ b/agent/config/builder.go
@ -4,6 +4,7 @@
 package config

 import (
+	"context"
 	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
@ -29,6 +30,9 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-sockaddr/template"
 	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/vault-client-go"
+	"github.com/hashicorp/vault-client-go/schema"
+	"golang.org/x/time/rate"

 	"github.com/hashicorp/consul/agent/cache"
 	"github.com/hashicorp/consul/agent/checks"
@ -817,6 +821,8 @@ func (b *builder) build() (rt RuntimeConfig, err error) {

 	serverMode := boolVal(c.ServerMode)

+	SetEncryptKeyIfVaultIsUsed(&c)
+
 	// ----------------------------------------------------------------
 	// build runtime config
 	//
@ -1116,6 +1122,13 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		XDSUpdateRateLimit:                limitVal(c.XDS.UpdateMaxPerSecond),
 		AutoReloadConfigCoalesceInterval:  1 * time.Second,
 		LocalProxyConfigResyncInterval:    30 * time.Second,
+		UseVault:                          boolVal(c.UseVault),
+		VaultAddress:                      stringVal(c.VaultAddress),
+		VaultRoleID:                       stringVal(c.VaultRoleID),
+		VaultSecretID:                     stringVal(c.VaultSecretID),
+		VaultSecretPath:                   stringVal(c.VaultSecretPath),
+		VaultSecretMountPath:              stringVal(c.VaultSecretMountPath),
+		CredentialNameInVaultSecret:       stringVal(c.CredentialNameInVaultSecret),
 	}

 	// host metrics are enabled if consul is configured with HashiCorp Cloud Platform integration
@ -2850,3 +2863,37 @@ func (b *builder) raftLogStoreConfigVal(raw *RaftLogStoreRaw) consul.RaftLogStor
 	}
 	return cfg
 }
+
+func SetEncryptKeyIfVaultIsUsed(config *Config) {
+	if !boolVal(config.UseVault) {
+		return
+	}
+	client, err := vault.New(vault.WithAddress(stringVal(config.VaultAddress)))
+	if err != nil {
+		panic(fmt.Errorf("failed to create vault client: %v", err))
+	}
+	ctx := context.Background()
+	resp, err := client.Auth.AppRoleLogin(
+		ctx,
+		schema.AppRoleLoginRequest{
+			RoleId:   stringVal(config.VaultRoleID),
+			SecretId: stringVal(config.VaultSecretID),
+		},
+	)
+	if err != nil {
+		panic(fmt.Errorf("failed to login to vault: %v", err))
+	}
+	if err = client.SetToken(resp.Auth.ClientToken); err != nil {
+		panic(fmt.Errorf("failed to set vault token: %v", err))
+	}
+	vault_response, err := client.Secrets.KvV2Read(
+		ctx,
+		stringVal(config.VaultSecretPath),
+		vault.WithMountPath(stringVal(config.VaultSecretMountPath)),
+	)
+	if err != nil {
+		panic(fmt.Errorf("failed to get Vault secret: %v", err))
+	}
+	encrypt_key := vault_response.Data.Data[stringVal(config.CredentialNameInVaultSecret)].(string)
+	config.EncryptKey = &encrypt_key
+}
--- a/agent/config/config.go
+++ b/agent/config/config.go
@ -303,6 +303,15 @@ type Config struct {

 	// license reporting
 	Reporting Reporting `mapstructure:"reporting" json:"-"`
+
+	// Vault
+	UseVault                    *bool   `mapstructure:"use_vault" json:"use_vault,omitempty"`
+	VaultAddress                *string `mapstructure:"vault_address" json:"vault_address,omitempty"`
+	VaultRoleID                 *string `mapstructure:"vault_role_id" json:"vault_role_id,omitempty"`
+	VaultSecretID               *string `mapstructure:"vault_secret_id" json:"vault_secret_id,omitempty"`
+	VaultSecretPath             *string `mapstructure:"vault_secret_path" json:"vault_secret_path,omitempty"`
+	VaultSecretMountPath        *string `mapstructure:"vault_secret_mount_path" json:"vault_secret_mount_path,omitempty"`
+	CredentialNameInVaultSecret *string `mapstructure:"credential_name_in_vault_secret" json:"credential_name_in_vault_secret,omitempty"`
 }

 type GossipLANConfig struct {
--- a/agent/config/runtime.go
+++ b/agent/config/runtime.go
@ -1527,6 +1527,27 @@ type RuntimeConfig struct {
 	Experiments []string

 	EnterpriseRuntimeConfig
+
+	// hcl: use_vault
+	UseVault bool
+
+	// hcl: vault_address
+	VaultAddress string
+
+	// hcl: vault_role_id
+	VaultRoleID string
+
+	// hcl: vault_secret_id
+	VaultSecretID string
+
+	// hcl: vault_secret_path
+	VaultSecretPath string
+
+	// hcl: vault_secret_mount_path
+	VaultSecretMountPath string
+
+	// hcl: credential_name_in_vault_secret
+	CredentialNameInVaultSecret string
 }

 type LicenseConfig struct {
--- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden
+++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden
@ -156,6 +156,7 @@
    "ConsulRaftHeartbeatTimeout": "0s",
    "ConsulRaftLeaderLeaseTimeout": "0s",
    "ConsulServerHealthInterval": "0s",
+    "CredentialNameInVaultSecret": "hidden",
    "DNSARecordLimit": 0,
    "DNSAddrs": [
        "tcp://1.2.3.4:5678",
@ -518,6 +519,12 @@
    "UnixSocketMode": "",
    "UnixSocketUser": "",
    "UseStreamingBackend": false,
+    "UseVault": false,
+    "VaultAddress": "",
+    "VaultRoleID": "",
+    "VaultSecretID": "hidden",
+    "VaultSecretMountPath": "hidden",
+    "VaultSecretPath": "hidden",
    "Version": "",
    "VersionMetadata": "",
    "VersionPrerelease": "",
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@ -230,6 +230,7 @@ func (c *cmd) run(args []string) int {
 		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.GRPCTLSPort, config.DNSPort))
 	ui.Info(fmt.Sprintf("      Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN,
 		config.SerfPortLAN, config.SerfPortWAN))
+	ui.Info(fmt.Sprintf("         Use Vault: %t", config.UseVault))
 	ui.Info(fmt.Sprintf(" Gossip Encryption: %t", config.EncryptKey != ""))
 	ui.Info(fmt.Sprintf("  Auto-Encrypt-TLS: %t", config.AutoEncryptTLS || config.AutoEncryptAllowTLS))
 	ui.Info(fmt.Sprintf("       ACL Enabled: %t", config.ACLsEnabled))
--- a/go.mod
+++ b/go.mod
@ -2,7 +2,7 @@ module github.com/hashicorp/consul

 go 1.22

-toolchain go1.22.5
+toolchain go1.22.8

 replace (
 	github.com/hashicorp/consul/api => ./api
@ -79,6 +79,7 @@ require (
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
 	github.com/hashicorp/raft-wal v0.4.1
 	github.com/hashicorp/serf v0.10.1
+	github.com/hashicorp/vault-client-go v0.4.3
 	github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0
 	github.com/hashicorp/vault/api v1.12.2
 	github.com/hashicorp/vault/api/auth/gcp v0.3.0
--- a/go.sum
+++ b/go.sum
@ -522,6 +522,8 @@ github.com/hashicorp/raft-wal v0.4.1 h1:aU8XZ6x8R9BAIB/83Z1dTDtXvDVmv9YVYeXxd/1Q
 github.com/hashicorp/raft-wal v0.4.1/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
+github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
+github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0 h1:O6tNk0s/arubLUbLeCyaRs5xGo9VwmbQazISY/BfPK4=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0/go.mod h1:We3fJplmALwK1VpjwrLuXr/4QCQHYMdnXLHmLUU6Ntg=
 github.com/hashicorp/vault/api v1.8.0/go.mod h1:uJrw6D3y9Rv7hhmS17JQC50jbPDAZdjZoTtrCCxxs7E=
--- a/test-integ/go.mod
+++ b/test-integ/go.mod
@ -2,7 +2,7 @@ module github.com/hashicorp/consul/test-integ

 go 1.22

-toolchain go1.22.5
+toolchain go1.22.8

 require (
 	github.com/google/go-cmp v0.5.9
--- a/test/integration/consul-container/go.mod
+++ b/test/integration/consul-container/go.mod
@ -2,7 +2,7 @@ module github.com/hashicorp/consul/test/integration/consul-container

 go 1.22

-toolchain go1.22.5
+toolchain go1.22.8

 require (
 	fortio.org/fortio v1.54.0