Added feature to get encrypt key from Vault

To use the feature we need to set the following options in config file:
use_vault = true
vault_address = http://<IP_ADDRESS:<IP_PORT>
vault_role_id = <ROLE_ID>
vault_secret_id = <SECRET_ID>
vault_secret_path = <deployments/unit/dev/user/passwords_yml>
vault_secret_mount_path = <secret_v2>
credential_name_in_vault_secret = <consul_encrypt>

https://github.com/hashicorp/consul/issues/2209
https://github.com/hashicorp/consul/issues/4685
https://github.com/hashicorp/consul/issues/5043

agent/config/builder.go

@@ -4,6 +4,7 @@
 package config
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
@@ -29,6 +30,9 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-sockaddr/template"
 	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/vault-client-go"
+	"github.com/hashicorp/vault-client-go/schema"
+	"golang.org/x/time/rate"
 
 	"github.com/hashicorp/consul/agent/cache"
 	"github.com/hashicorp/consul/agent/checks"
@@ -817,6 +821,8 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 
 	serverMode := boolVal(c.ServerMode)
 
+	SetEncryptKeyIfVaultIsUsed(&c)
+
 	// ----------------------------------------------------------------
 	// build runtime config
 	//
@@ -1116,6 +1122,13 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		XDSUpdateRateLimit:                limitVal(c.XDS.UpdateMaxPerSecond),
 		AutoReloadConfigCoalesceInterval:  1 * time.Second,
 		LocalProxyConfigResyncInterval:    30 * time.Second,
+		UseVault:                          boolVal(c.UseVault),
+		VaultAddress:                      stringVal(c.VaultAddress),
+		VaultRoleID:                       stringVal(c.VaultRoleID),
+		VaultSecretID:                     stringVal(c.VaultSecretID),
+		VaultSecretPath:                   stringVal(c.VaultSecretPath),
+		VaultSecretMountPath:              stringVal(c.VaultSecretMountPath),
+		CredentialNameInVaultSecret:       stringVal(c.CredentialNameInVaultSecret),
 	}
 
 	// host metrics are enabled if consul is configured with HashiCorp Cloud Platform integration
@@ -2850,3 +2863,37 @@ func (b *builder) raftLogStoreConfigVal(raw *RaftLogStoreRaw) consul.RaftLogStor
 	}
 	return cfg
 }
+
+func SetEncryptKeyIfVaultIsUsed(config *Config) {
+	if !boolVal(config.UseVault) {
+		return
+	}
+	client, err := vault.New(vault.WithAddress(stringVal(config.VaultAddress)))
+	if err != nil {
+		panic(fmt.Errorf("failed to create vault client: %v", err))
+	}
+	ctx := context.Background()
+	resp, err := client.Auth.AppRoleLogin(
+		ctx,
+		schema.AppRoleLoginRequest{
+			RoleId:   stringVal(config.VaultRoleID),
+			SecretId: stringVal(config.VaultSecretID),
+		},
+	)
+	if err != nil {
+		panic(fmt.Errorf("failed to login to vault: %v", err))
+	}
+	if err = client.SetToken(resp.Auth.ClientToken); err != nil {
+		panic(fmt.Errorf("failed to set vault token: %v", err))
+	}
+	vault_response, err := client.Secrets.KvV2Read(
+		ctx,
+		stringVal(config.VaultSecretPath),
+		vault.WithMountPath(stringVal(config.VaultSecretMountPath)),
+	)
+	if err != nil {
+		panic(fmt.Errorf("failed to get Vault secret: %v", err))
+	}
+	encrypt_key := vault_response.Data.Data[stringVal(config.CredentialNameInVaultSecret)].(string)
+	config.EncryptKey = &encrypt_key
+}

agent/config/config.go

@@ -303,6 +303,15 @@ type Config struct {
 
 	// license reporting
 	Reporting Reporting `mapstructure:"reporting" json:"-"`
+
+	// Vault
+	UseVault                    *bool   `mapstructure:"use_vault" json:"use_vault,omitempty"`
+	VaultAddress                *string `mapstructure:"vault_address" json:"vault_address,omitempty"`
+	VaultRoleID                 *string `mapstructure:"vault_role_id" json:"vault_role_id,omitempty"`
+	VaultSecretID               *string `mapstructure:"vault_secret_id" json:"vault_secret_id,omitempty"`
+	VaultSecretPath             *string `mapstructure:"vault_secret_path" json:"vault_secret_path,omitempty"`
+	VaultSecretMountPath        *string `mapstructure:"vault_secret_mount_path" json:"vault_secret_mount_path,omitempty"`
+	CredentialNameInVaultSecret *string `mapstructure:"credential_name_in_vault_secret" json:"credential_name_in_vault_secret,omitempty"`
 }
 
 type GossipLANConfig struct {

agent/config/runtime.go

@@ -1527,6 +1527,27 @@ type RuntimeConfig struct {
 	Experiments []string
 
 	EnterpriseRuntimeConfig
+
+	// hcl: use_vault
+	UseVault bool
+
+	// hcl: vault_address
+	VaultAddress string
+
+	// hcl: vault_role_id
+	VaultRoleID string
+
+	// hcl: vault_secret_id
+	VaultSecretID string
+
+	// hcl: vault_secret_path
+	VaultSecretPath string
+
+	// hcl: vault_secret_mount_path
+	VaultSecretMountPath string
+
+	// hcl: credential_name_in_vault_secret
+	CredentialNameInVaultSecret string
 }
 
 type LicenseConfig struct {

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -156,6 +156,7 @@
     "ConsulRaftHeartbeatTimeout": "0s",
     "ConsulRaftLeaderLeaseTimeout": "0s",
     "ConsulServerHealthInterval": "0s",
+    "CredentialNameInVaultSecret": "hidden",
     "DNSARecordLimit": 0,
     "DNSAddrs": [
         "tcp://1.2.3.4:5678",
@@ -518,6 +519,12 @@
     "UnixSocketMode": "",
     "UnixSocketUser": "",
     "UseStreamingBackend": false,
+    "UseVault": false,
+    "VaultAddress": "",
+    "VaultRoleID": "",
+    "VaultSecretID": "hidden",
+    "VaultSecretMountPath": "hidden",
+    "VaultSecretPath": "hidden",
     "Version": "",
     "VersionMetadata": "",
     "VersionPrerelease": "",

command/agent/agent.go

@@ -230,6 +230,7 @@ func (c *cmd) run(args []string) int {
 		config.HTTPPort, config.HTTPSPort, config.GRPCPort, config.GRPCTLSPort, config.DNSPort))
 	ui.Info(fmt.Sprintf("      Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddrLAN,
 		config.SerfPortLAN, config.SerfPortWAN))
+	ui.Info(fmt.Sprintf("         Use Vault: %t", config.UseVault))
 	ui.Info(fmt.Sprintf(" Gossip Encryption: %t", config.EncryptKey != ""))
 	ui.Info(fmt.Sprintf("  Auto-Encrypt-TLS: %t", config.AutoEncryptTLS || config.AutoEncryptAllowTLS))
 	ui.Info(fmt.Sprintf("       ACL Enabled: %t", config.ACLsEnabled))

go.mod

@@ -2,7 +2,7 @@ module github.com/hashicorp/consul
 
 go 1.22
 
-toolchain go1.22.5
+toolchain go1.22.8
 
 replace (
 	github.com/hashicorp/consul/api => ./api
@@ -79,6 +79,7 @@ require (
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
 	github.com/hashicorp/raft-wal v0.4.1
 	github.com/hashicorp/serf v0.10.1
+	github.com/hashicorp/vault-client-go v0.4.3
 	github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0
 	github.com/hashicorp/vault/api v1.12.2
 	github.com/hashicorp/vault/api/auth/gcp v0.3.0

go.sum

@@ -522,6 +522,8 @@ github.com/hashicorp/raft-wal v0.4.1 h1:aU8XZ6x8R9BAIB/83Z1dTDtXvDVmv9YVYeXxd/1Q
 github.com/hashicorp/raft-wal v0.4.1/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
+github.com/hashicorp/vault-client-go v0.4.3 h1:zG7STGVgn/VK6rnZc0k8PGbfv2x/sJExRKHSUg3ljWc=
+github.com/hashicorp/vault-client-go v0.4.3/go.mod h1:4tDw7Uhq5XOxS1fO+oMtotHL7j4sB9cp0T7U6m4FzDY=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0 h1:O6tNk0s/arubLUbLeCyaRs5xGo9VwmbQazISY/BfPK4=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0/go.mod h1:We3fJplmALwK1VpjwrLuXr/4QCQHYMdnXLHmLUU6Ntg=
 github.com/hashicorp/vault/api v1.8.0/go.mod h1:uJrw6D3y9Rv7hhmS17JQC50jbPDAZdjZoTtrCCxxs7E=

test-integ/go.mod

@@ -2,7 +2,7 @@ module github.com/hashicorp/consul/test-integ
 
 go 1.22
 
-toolchain go1.22.5
+toolchain go1.22.8
 
 require (
 	github.com/google/go-cmp v0.5.9

test/integration/consul-container/go.mod

@@ -2,7 +2,7 @@ module github.com/hashicorp/consul/test/integration/consul-container
 
 go 1.22
 
-toolchain go1.22.5
+toolchain go1.22.8
 
 require (
 	fortio.org/fortio v1.54.0