github
/
consul
mirror of https://github.com/hashicorp/consul
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[CC-7063] Fetch HCP agent bootstrap config in Link reconciler (#20306) 

* Move config-dependent methods to separate package

In order to reuse the fetching and file creation part of the
bootstrap package, move the code that would cause cyclical
dependencies to a different package.

* Export needed bootstrap methods and variables

Also add back validating persisted config and update tests.

* Add support to check for just management token

Add a new method that fetches the bootstrap configuration only if
there isn't a valid management token file instead of checking for
all the hcp-config files.

* Pass data dir as a dependency to link controller

The link controller needs to check the data directory for
the hcp-config files.

* Fetch bootstrap config for token in controller

Load the management token when reconciling a link resource, which will
fetch the agent boostrap configuration if the token is not already
persisted locally. Skip this step if the cluster is in read-only mode.

* Validate resource ID format in link creation

* Handle unauthorized and forbidden errors

Check for 401 and 403s when making GNM requests, exit bootstrap fetch
loop and return specific failure statuses for link.

* Move test function to a testing file

* Log load and status write errors
pull/20334/head
Melissa Kam 2024-01-24 09:51:43 -06:00 committed by GitHub
parent 3446eb3b1b
commit 7900544249
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 1107 additions and 616 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
agent/consul/server.go
View File

@ -994,6 +994,7 @@ func (s *Server) registerControllers(deps Deps, proxyUpdater ProxyUpdater) error
ResourceApisEnabled: s.useV2Resources,
HCPAllowV2ResourceApis: s.hcpAllowV2Resources,
CloudConfig: deps.HCP.Config,
DataDir: deps.HCP.DataDir,
})
// When not enabled, the v1 tenancy bridge is used by default.

272
agent/hcp/bootstrap/bootstrap.go
View File

@ -1,10 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
// Package bootstrap handles bootstrapping an agent's config from HCP. It must be a
// separate package from other HCP components because it has a dependency on
// agent/config while other components need to be imported and run within the
// server process in agent/consul and that would create a dependency cycle.
// Package bootstrap handles bootstrapping an agent's config from HCP.
package bootstrap
import (
@ -21,27 +18,25 @@ import (
"strings"
"time"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/agent/connect"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/lib/retry"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-uuid"
)
const (
subDir = "hcp-config"
SubDir = "hcp-config"
caFileName = "server-tls-cas.pem"
certFileName = "server-tls-cert.pem"
configFileName = "server-config.json"
keyFileName = "server-tls-key.pem"
tokenFileName = "hcp-management-token"
successFileName = "successful-bootstrap"
CAFileName = "server-tls-cas.pem"
CertFileName = "server-tls-cert.pem"
ConfigFileName = "server-config.json"
KeyFileName = "server-tls-key.pem"
TokenFileName = "hcp-management-token"
SuccessFileName = "successful-bootstrap"
)
type ConfigLoader func(source config.Source) (config.LoadResult, error)
// UI is a shim to allow the agent command to pass in it's mitchelh/cli.UI so we
// can output useful messages to the user during bootstrapping. For example if
// we have to retry several times to bootstrap we don't want the agent to just
@ -61,155 +56,27 @@ type RawBootstrapConfig struct {
ManagementToken string
}
// LoadConfig will attempt to load previously-fetched config from disk and fall back to
// fetch from HCP servers if the local data is incomplete.
// It must be passed a (CLI) UI implementation so it can deliver progress
// updates to the user, for example if it is waiting to retry for a long period.
func LoadConfig(ctx context.Context, client hcpclient.Client, dataDir string, loader ConfigLoader, ui UI) (ConfigLoader, error) {
ui.Output("Loading configuration from HCP")
// See if we have existing config on disk
//
// OPTIMIZE: We could probably be more intelligent about config loading.
// The currently implemented approach is:
// 1. Attempt to load data from disk
// 2. If that fails or the data is incomplete, block indefinitely fetching remote config.
//
// What if instead we had the following flow:
// 1. Attempt to fetch config from HCP.
// 2. If that fails, fall back to data on disk from last fetch.
// 3. If that fails, go into blocking loop to fetch remote config.
//
// This should allow us to more gracefully transition cases like when
// an existing cluster is linked, but then wants to receive TLS materials
// at a later time. Currently, if we observe the existing-cluster marker we
// don't attempt to fetch any additional configuration from HCP.
cfg, ok := loadPersistedBootstrapConfig(dataDir, ui)
if !ok {
ui.Info("Fetching configuration from HCP servers")
var err error
cfg, err = fetchBootstrapConfig(ctx, client, dataDir, ui)
if err != nil {
return nil, fmt.Errorf("failed to bootstrap from HCP: %w", err)
}
ui.Info("Configuration fetched from HCP and saved on local disk")
} else {
ui.Info("Loaded HCP configuration from local disk")
}
// Create a new loader func to return
newLoader := bootstrapConfigLoader(loader, cfg)
return newLoader, nil
}
func AddAclPolicyAccessControlHeader(baseLoader ConfigLoader) ConfigLoader {
return func(source config.Source) (config.LoadResult, error) {
res, err := baseLoader(source)
if err != nil {
return res, err
}
rc := res.RuntimeConfig
// HTTP response headers are modified for the HCP UI to work.
if rc.HTTPResponseHeaders == nil {
rc.HTTPResponseHeaders = make(map[string]string)
}
prevValue, ok := rc.HTTPResponseHeaders[accessControlHeaderName]
if !ok {
rc.HTTPResponseHeaders[accessControlHeaderName] = accessControlHeaderValue
} else {
rc.HTTPResponseHeaders[accessControlHeaderName] = prevValue + "," + accessControlHeaderValue
}
return res, nil
}
}
// bootstrapConfigLoader is a ConfigLoader for passing bootstrap JSON config received from HCP
// to the config.builder. ConfigLoaders are functions used to build an agent's RuntimeConfig
// from various sources like files and flags. This config is contained in the config.LoadResult.
//
// The flow to include bootstrap config from HCP as a loader's data source is as follows:
//
// 1. A base ConfigLoader function (baseLoader) is created on agent start, and it sets the input
// source argument as the DefaultConfig.
//
// 2. When a server agent can be configured by HCP that baseLoader is wrapped in this bootstrapConfigLoader.
//
// 3. The bootstrapConfigLoader calls that base loader with the bootstrap JSON config as the
// default source. This data will be merged with other valid sources in the config.builder.
//
// 4. The result of the call to baseLoader() below contains the resulting RuntimeConfig, and we do some
// additional modifications to attach data that doesn't get populated during the build in the config pkg.
//
// Note that since the ConfigJSON is stored as the baseLoader's DefaultConfig, its data is the first
// to be merged by the config.builder and could be overwritten by user-provided values in config files or
// CLI flags. However, values set to RuntimeConfig after the baseLoader call are final.
func bootstrapConfigLoader(baseLoader ConfigLoader, cfg *RawBootstrapConfig) ConfigLoader {
return func(source config.Source) (config.LoadResult, error) {
// Don't allow any further attempts to provide a DefaultSource. This should
// only ever be needed later in client agent AutoConfig code but that should
// be mutually exclusive from this bootstrapping mechanism since this is
// only for servers. If we ever try to change that, this clear failure
// should alert future developers that the assumptions are changing rather
// than quietly not applying the config they expect!
if source != nil {
return config.LoadResult{},
fmt.Errorf("non-nil config source provided to a loader after HCP bootstrap already provided a DefaultSource")
}
// Otherwise, just call to the loader we were passed with our own additional
// JSON as the source.
//
// OPTIMIZE: We could check/log whether any fields set by the remote config were overwritten by a user-provided flag.
res, err := baseLoader(config.FileSource{
Name: "HCP Bootstrap",
Format: "json",
Data: cfg.ConfigJSON,
})
if err != nil {
return res, fmt.Errorf("failed to load HCP Bootstrap config: %w", err)
}
finalizeRuntimeConfig(res.RuntimeConfig, cfg)
return res, nil
}
}
const (
accessControlHeaderName = "Access-Control-Expose-Headers"
accessControlHeaderValue = "x-consul-default-acl-policy"
)
// finalizeRuntimeConfig will set additional HCP-specific values that are not
// handled by the config.builder.
func finalizeRuntimeConfig(rc *config.RuntimeConfig, cfg *RawBootstrapConfig) {
rc.Cloud.ManagementToken = cfg.ManagementToken
}
// fetchBootstrapConfig will fetch boostrap configuration from remote servers and persist it to disk.
// FetchBootstrapConfig will fetch bootstrap configuration from remote servers and persist it to disk.
// It will retry until successful or a terminal error condition is found (e.g. permission denied).
func fetchBootstrapConfig(ctx context.Context, client hcpclient.Client, dataDir string, ui UI) (*RawBootstrapConfig, error) {
func FetchBootstrapConfig(ctx context.Context, client hcpclient.Client, dataDir string, ui UI) (*RawBootstrapConfig, error) {
w := retry.Waiter{
MinWait: 1 * time.Second,
MaxWait: 5 * time.Minute,
Jitter: retry.NewJitter(50),
}
var bsCfg *hcpclient.BootstrapConfig
for {
// Note we don't want to shadow `ctx` here since we need that for the Wait
// below.
reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
resp, err := client.FetchBootstrap(reqCtx)
cfg, err := fetchBootstrapConfig(reqCtx, client, dataDir)
if err != nil {
if errors.Is(err, hcpclient.ErrUnauthorized) || errors.Is(err, hcpclient.ErrForbidden) {
// Don't retry on terminal errors
return nil, err
}
ui.Error(fmt.Sprintf("Error: failed to fetch bootstrap config from HCP, will retry in %s: %s",
w.NextWait().Round(time.Second), err))
if err := w.Wait(ctx); err != nil {
@ -218,12 +85,22 @@ func fetchBootstrapConfig(ctx context.Context, client hcpclient.Client, dataDir
// Finished waiting, restart loop
continue
}
bsCfg = resp
break
return cfg, nil
}
}
// fetchBootstrapConfig will fetch the bootstrap configuration from remote servers and persist it to disk.
func fetchBootstrapConfig(ctx context.Context, client hcpclient.Client, dataDir string) (*RawBootstrapConfig, error) {
reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
resp, err := client.FetchBootstrap(reqCtx)
if err != nil {
return nil, fmt.Errorf("failed to fetch bootstrap config from HCP: %w", err)
}
bsCfg := resp
devMode := dataDir == ""
cfgJSON, err := persistAndProcessConfig(dataDir, devMode, bsCfg)
if err != nil {
return nil, fmt.Errorf("failed to persist config for existing cluster: %w", err)
@ -251,7 +128,7 @@ func persistAndProcessConfig(dataDir string, devMode bool, bsCfg *hcpclient.Boot
}
// Create subdir if it's not already there.
dir := filepath.Join(dataDir, subDir)
dir := filepath.Join(dataDir, SubDir)
if err := lib.EnsurePath(dir, true); err != nil {
return "", fmt.Errorf("failed to ensure directory %q: %w", dir, err)
}
@ -281,7 +158,7 @@ func persistAndProcessConfig(dataDir string, devMode bool, bsCfg *hcpclient.Boot
var cfgJSON string
if bsCfg.TLSCert != "" {
if err := validateTLSCerts(bsCfg.TLSCert, bsCfg.TLSCertKey, bsCfg.TLSCAs); err != nil {
if err := ValidateTLSCerts(bsCfg.TLSCert, bsCfg.TLSCertKey, bsCfg.TLSCAs); err != nil {
return "", fmt.Errorf("invalid certificates: %w", err)
}
@ -292,9 +169,9 @@ func persistAndProcessConfig(dataDir string, devMode bool, bsCfg *hcpclient.Boot
}
// Store paths to the persisted TLS cert files.
cfg["ca_file"] = filepath.Join(dir, caFileName)
cfg["cert_file"] = filepath.Join(dir, certFileName)
cfg["key_file"] = filepath.Join(dir, keyFileName)
cfg["ca_file"] = filepath.Join(dir, CAFileName)
cfg["cert_file"] = filepath.Join(dir, CertFileName)
cfg["key_file"] = filepath.Join(dir, KeyFileName)
// Convert the bootstrap config map back into a string
cfgJSONBytes, err := json.Marshal(cfg)
@ -330,7 +207,7 @@ func persistAndProcessConfig(dataDir string, devMode bool, bsCfg *hcpclient.Boot
}
func persistSuccessMarker(dir string) error {
name := filepath.Join(dir, successFileName)
name := filepath.Join(dir, SuccessFileName)
return os.WriteFile(name, []byte(""), 0600)
}
@ -343,7 +220,7 @@ func persistTLSCerts(dir string, serverCert, serverKey string, caCerts []string)
// Write out CA cert(s). We write them all to one file because Go's x509
// machinery will read as many certs as it finds from each PEM file provided
// and add them separaetly to the CertPool for validation
f, err := os.OpenFile(filepath.Join(dir, caFileName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
f, err := os.OpenFile(filepath.Join(dir, CAFileName), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
@ -358,11 +235,11 @@ func persistTLSCerts(dir string, serverCert, serverKey string, caCerts []string)
return err
}
if err := os.WriteFile(filepath.Join(dir, certFileName), []byte(serverCert), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, CertFileName), []byte(serverCert), 0600); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(dir, keyFileName), []byte(serverKey), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, KeyFileName), []byte(serverKey), 0600); err != nil {
return err
}
@ -379,26 +256,26 @@ func validateManagementToken(token string) error {
}
func persistManagementToken(dir, token string) error {
name := filepath.Join(dir, tokenFileName)
name := filepath.Join(dir, TokenFileName)
return os.WriteFile(name, []byte(token), 0600)
}
func persistBootstrapConfig(dir, cfgJSON string) error {
// Persist the important bits we got from bootstrapping. The TLS certs are
// already persisted, just need to persist the config we are going to add.
name := filepath.Join(dir, configFileName)
name := filepath.Join(dir, ConfigFileName)
return os.WriteFile(name, []byte(cfgJSON), 0600)
}
func loadPersistedBootstrapConfig(dataDir string, ui UI) (*RawBootstrapConfig, bool) {
func LoadPersistedBootstrapConfig(dataDir string, ui UI) (*RawBootstrapConfig, bool) {
if dataDir == "" {
// There's no files to load when in dev mode.
return nil, false
}
dir := filepath.Join(dataDir, subDir)
dir := filepath.Join(dataDir, SubDir)
_, err := os.Stat(filepath.Join(dir, successFileName))
_, err := os.Stat(filepath.Join(dir, SuccessFileName))
if os.IsNotExist(err) {
// Haven't bootstrapped from HCP.
return nil, false
@ -432,7 +309,7 @@ func loadPersistedBootstrapConfig(dataDir string, ui UI) (*RawBootstrapConfig, b
}
func loadBootstrapConfigJSON(dataDir string) (string, error) {
filename := filepath.Join(dataDir, subDir, configFileName)
filename := filepath.Join(dataDir, SubDir, ConfigFileName)
_, err := os.Stat(filename)
if os.IsNotExist(err) {
@ -442,21 +319,6 @@ func loadBootstrapConfigJSON(dataDir string) (string, error) {
return "", fmt.Errorf("failed to check for bootstrap config: %w", err)
}
// Attempt to load persisted config to check for errors and basic validity.
// Errors here will raise issues like referencing unsupported config fields.
_, err = config.Load(config.LoadOpts{
ConfigFiles: []string{filename},
HCL: []string{
"server = true",
`bind_addr = "127.0.0.1"`,
fmt.Sprintf("data_dir = %q", dataDir),
},
ConfigFormat: "json",
})
if err != nil {
return "", fmt.Errorf("failed to parse local bootstrap config: %w", err)
}
jsonBs, err := os.ReadFile(filename)
if err != nil {
return "", fmt.Errorf(fmt.Sprintf("failed to read local bootstrap config file: %s", err))
@ -465,7 +327,7 @@ func loadBootstrapConfigJSON(dataDir string) (string, error) {
}
func loadManagementToken(dir string) (string, error) {
name := filepath.Join(dir, tokenFileName)
name := filepath.Join(dir, TokenFileName)
bytes, err := os.ReadFile(name)
if os.IsNotExist(err) {
return "", errors.New("configuration files on disk are incomplete, missing: " + name)
@ -484,9 +346,9 @@ func loadManagementToken(dir string) (string, error) {
func checkCerts(dir string) error {
files := []string{
filepath.Join(dir, caFileName),
filepath.Join(dir, certFileName),
filepath.Join(dir, keyFileName),
filepath.Join(dir, CAFileName),
filepath.Join(dir, CertFileName),
filepath.Join(dir, KeyFileName),
}
missing := make([]string, 0)
@ -512,28 +374,28 @@ func checkCerts(dir string) error {
return fmt.Errorf("configuration files on disk are incomplete, missing: %v", missing)
}
cert, key, caCerts, err := loadCerts(dir)
cert, key, caCerts, err := LoadCerts(dir)
if err != nil {
return fmt.Errorf("failed to load certs from disk: %w", err)
}
if err = validateTLSCerts(cert, key, caCerts); err != nil {
if err = ValidateTLSCerts(cert, key, caCerts); err != nil {
return fmt.Errorf("invalid certs on disk: %w", err)
}
return nil
}
func loadCerts(dir string) (cert, key string, caCerts []string, err error) {
certPEMBlock, err := os.ReadFile(filepath.Join(dir, certFileName))
func LoadCerts(dir string) (cert, key string, caCerts []string, err error) {
certPEMBlock, err := os.ReadFile(filepath.Join(dir, CertFileName))
if err != nil {
return "", "", nil, err
}
keyPEMBlock, err := os.ReadFile(filepath.Join(dir, keyFileName))
keyPEMBlock, err := os.ReadFile(filepath.Join(dir, KeyFileName))
if err != nil {
return "", "", nil, err
}
caPEMs, err := os.ReadFile(filepath.Join(dir, caFileName))
caPEMs, err := os.ReadFile(filepath.Join(dir, CAFileName))
if err != nil {
return "", "", nil, err
}
@ -572,12 +434,12 @@ func splitCACerts(caPEMs []byte) ([]string, error) {
return out, nil
}
// validateTLSCerts checks that the CA cert, server cert, and key on disk are structurally valid.
// ValidateTLSCerts checks that the CA cert, server cert, and key on disk are structurally valid.
//
// OPTIMIZE: This could be improved by returning an error if certs are expired or close to expiration.
// However, that requires issuing new certs on bootstrap requests, since returning an error
// would trigger a re-fetch from HCP.
func validateTLSCerts(cert, key string, caCerts []string) error {
func ValidateTLSCerts(cert, key string, caCerts []string) error {
leaf, err := tls.X509KeyPair([]byte(cert), []byte(key))
if err != nil {
return errors.New("invalid server certificate or key")
@ -595,3 +457,25 @@ func validateTLSCerts(cert, key string, caCerts []string) error {
}
return nil
}
// LoadManagementToken returns the management token, either by loading it from the persisted
// token config file or by fetching it from HCP if the token file does not exist.
func LoadManagementToken(ctx context.Context, logger hclog.Logger, client hcpclient.Client, dataDir string) (string, error) {
hcpCfgDir := filepath.Join(dataDir, SubDir)
token, err := loadManagementToken(hcpCfgDir)
if err != nil {
logger.Debug("failed to load management token from local disk, fetching configuration from HCP", "error", err)
var err error
cfg, err := fetchBootstrapConfig(ctx, client, dataDir)
if err != nil {
return "", err
}
logger.Debug("configuration fetched from HCP and saved on local disk")
token = cfg.ManagementToken
} else {
logger.Trace("loaded HCP configuration from local disk")
}
return token, nil
}

487
agent/hcp/bootstrap/bootstrap_test.go
View File

@ -5,340 +5,23 @@ package bootstrap
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http/httptest"
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/agent/hcp"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/hashicorp/consul/tlsutil"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-uuid"
"github.com/mitchellh/cli"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestBootstrapConfigLoader(t *testing.T) {
baseLoader := func(source config.Source) (config.LoadResult, error) {
return config.Load(config.LoadOpts{
DefaultConfig: source,
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
`data_dir = "/tmp/consul-data"`,
},
})
}
bootstrapLoader := func(source config.Source) (config.LoadResult, error) {
return bootstrapConfigLoader(baseLoader, &RawBootstrapConfig{
ConfigJSON: `{"bootstrap_expect": 8}`,
ManagementToken: "test-token",
})(source)
}
result, err := bootstrapLoader(nil)
require.NoError(t, err)
// bootstrap_expect and management token are injected from bootstrap config received from HCP.
require.Equal(t, 8, result.RuntimeConfig.BootstrapExpect)
require.Equal(t, "test-token", result.RuntimeConfig.Cloud.ManagementToken)
}
func Test_finalizeRuntimeConfig(t *testing.T) {
type testCase struct {
rc *config.RuntimeConfig
cfg *RawBootstrapConfig
verifyFn func(t *testing.T, rc *config.RuntimeConfig)
}
run := func(t *testing.T, tc testCase) {
finalizeRuntimeConfig(tc.rc, tc.cfg)
tc.verifyFn(t, tc.rc)
}
tt := map[string]testCase{
"set management token": {
rc: &config.RuntimeConfig{},
cfg: &RawBootstrapConfig{
ManagementToken: "test-token",
},
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Equal(t, "test-token", rc.Cloud.ManagementToken)
},
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func Test_AddAclPolicyAccessControlHeader(t *testing.T) {
type testCase struct {
rc *config.RuntimeConfig
cfg *RawBootstrapConfig
baseLoader ConfigLoader
verifyFn func(t *testing.T, rc *config.RuntimeConfig)
}
run := func(t *testing.T, tc testCase) {
loader := AddAclPolicyAccessControlHeader(tc.baseLoader)
result, err := loader(nil)
require.NoError(t, err)
tc.verifyFn(t, result.RuntimeConfig)
}
tt := map[string]testCase{
"append to header if present": {
baseLoader: func(source config.Source) (config.LoadResult, error) {
return config.Load(config.LoadOpts{
DefaultConfig: config.DefaultSource(),
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
`data_dir = "/tmp/consul-data"`,
fmt.Sprintf(`http_config = { response_headers = { %s = "test" } }`, accessControlHeaderName),
},
})
},
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Equal(t, "test,x-consul-default-acl-policy", rc.HTTPResponseHeaders[accessControlHeaderName])
},
},
"set header if not present": {
baseLoader: func(source config.Source) (config.LoadResult, error) {
return config.Load(config.LoadOpts{
DefaultConfig: config.DefaultSource(),
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
`data_dir = "/tmp/consul-data"`,
},
})
},
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Equal(t, "x-consul-default-acl-policy", rc.HTTPResponseHeaders[accessControlHeaderName])
},
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func boolPtr(value bool) *bool {
return &value
}
func TestLoadConfig_Persistence(t *testing.T) {
type testCase struct {
// resourceID is the HCP resource ID. If set, a server is considered to be cloud-enabled.
resourceID string
// devMode indicates whether the loader should not have a data directory.
devMode bool
// verifyFn issues case-specific assertions.
verifyFn func(t *testing.T, rc *config.RuntimeConfig)
}
run := func(t *testing.T, tc testCase) {
dir, err := os.MkdirTemp(os.TempDir(), "bootstrap-test-")
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(dir) })
s := hcp.NewMockHCPServer()
s.AddEndpoint(TestEndpoint())
// Use an HTTPS server since that's what the HCP SDK expects for auth.
srv := httptest.NewTLSServer(s)
defer srv.Close()
caCert, err := x509.ParseCertificate(srv.TLS.Certificates[0].Certificate[0])
require.NoError(t, err)
pool := x509.NewCertPool()
pool.AddCert(caCert)
clientTLS := &tls.Config{RootCAs: pool}
baseOpts := config.LoadOpts{
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
fmt.Sprintf(`http_config = { response_headers = { %s = "Content-Encoding" } }`, accessControlHeaderName),
fmt.Sprintf(`cloud { client_id="test" client_secret="test" hostname=%q auth_url=%q resource_id=%q }`,
srv.Listener.Addr().String(), srv.URL, tc.resourceID),
},
}
if tc.devMode {
baseOpts.DevMode = boolPtr(true)
} else {
baseOpts.HCL = append(baseOpts.HCL, fmt.Sprintf(`data_dir = %q`, dir))
}
baseLoader := func(source config.Source) (config.LoadResult, error) {
baseOpts.DefaultConfig = source
return config.Load(baseOpts)
}
ui := cli.NewMockUi()
// Load initial config to check whether bootstrapping from HCP is enabled.
initial, err := baseLoader(nil)
require.NoError(t, err)
// Override the client TLS config so that the test server can be trusted.
initial.RuntimeConfig.Cloud.WithTLSConfig(clientTLS)
client, err := hcpclient.NewClient(initial.RuntimeConfig.Cloud)
require.NoError(t, err)
loader, err := LoadConfig(context.Background(), client, initial.RuntimeConfig.DataDir, baseLoader, ui)
require.NoError(t, err)
// Load the agent config with the potentially wrapped loader.
fromRemote, err := loader(nil)
require.NoError(t, err)
// HCP-enabled cases should fetch from HCP on the first run of LoadConfig.
require.Contains(t, ui.OutputWriter.String(), "Fetching configuration from HCP")
// Run case-specific verification.
tc.verifyFn(t, fromRemote.RuntimeConfig)
require.Empty(t, fromRemote.RuntimeConfig.ACLInitialManagementToken,
"initial_management token should have been sanitized")
if tc.devMode {
// Re-running the bootstrap func below isn't relevant to dev mode
// since they don't have a data directory to load data from.
return
}
// Run LoadConfig again to exercise the logic of loading config from disk.
loader, err = LoadConfig(context.Background(), client, initial.RuntimeConfig.DataDir, baseLoader, ui)
require.NoError(t, err)
fromDisk, err := loader(nil)
require.NoError(t, err)
// HCP-enabled cases should fetch from disk on the second run.
require.Contains(t, ui.OutputWriter.String(), "Loaded HCP configuration from local disk")
// Config loaded from disk should be the same as the one that was initially fetched from the HCP servers.
require.Equal(t, fromRemote.RuntimeConfig, fromDisk.RuntimeConfig)
}
tt := map[string]testCase{
"dev mode": {
devMode: true,
resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"consul.cluster/new-cluster-id",
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Empty(t, rc.DataDir)
// Dev mode should have persisted certs since they can't be inlined.
require.NotEmpty(t, rc.TLS.HTTPS.CertFile)
require.NotEmpty(t, rc.TLS.HTTPS.KeyFile)
require.NotEmpty(t, rc.TLS.HTTPS.CAFile)
// Find the temporary directory they got stored in.
dir := filepath.Dir(rc.TLS.HTTPS.CertFile)
// Ensure we only stored the TLS materials.
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, 3)
haveFiles := make([]string, 3)
for i, entry := range entries {
haveFiles[i] = entry.Name()
}
wantFiles := []string{caFileName, certFileName, keyFileName}
require.ElementsMatch(t, wantFiles, haveFiles)
},
},
"new cluster": {
resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"consul.cluster/new-cluster-id",
// New clusters should have received and persisted the whole suite of config.
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
dir := filepath.Join(rc.DataDir, subDir)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, 6)
files := []string{
filepath.Join(dir, configFileName),
filepath.Join(dir, caFileName),
filepath.Join(dir, certFileName),
filepath.Join(dir, keyFileName),
filepath.Join(dir, tokenFileName),
filepath.Join(dir, successFileName),
}
for _, name := range files {
_, err := os.Stat(name)
require.NoError(t, err)
}
require.Equal(t, filepath.Join(dir, certFileName), rc.TLS.HTTPS.CertFile)
require.Equal(t, filepath.Join(dir, keyFileName), rc.TLS.HTTPS.KeyFile)
require.Equal(t, filepath.Join(dir, caFileName), rc.TLS.HTTPS.CAFile)
cert, key, caCerts, err := loadCerts(dir)
require.NoError(t, err)
require.NoError(t, validateTLSCerts(cert, key, caCerts))
},
},
"existing cluster": {
resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"consul.cluster/" + TestExistingClusterID,
// Existing clusters should have only received and persisted the management token.
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
dir := filepath.Join(rc.DataDir, subDir)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, 3)
files := []string{
filepath.Join(dir, tokenFileName),
filepath.Join(dir, successFileName),
filepath.Join(dir, configFileName),
}
for _, name := range files {
_, err := os.Stat(name)
require.NoError(t, err)
}
},
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func Test_loadPersistedBootstrapConfig(t *testing.T) {
type expect struct {
loaded bool
@ -352,11 +35,9 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
}
run := func(t *testing.T, tc testCase) {
dataDir, err := os.MkdirTemp(os.TempDir(), "load-bootstrap-test-")
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(dataDir) })
dataDir := testutil.TempDir(t, "load-bootstrap-cfg")
dir := filepath.Join(dataDir, subDir)
dir := filepath.Join(dataDir, SubDir)
// Do some common setup as if we received config from HCP and persisted it to disk.
require.NoError(t, lib.EnsurePath(dir, true))
@ -376,6 +57,7 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
var token string
if !tc.disableManagementToken {
var err error
token, err = uuid.GenerateUUID()
require.NoError(t, err)
require.NoError(t, persistManagementToken(dir, token))
@ -387,7 +69,7 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
}
ui := cli.NewMockUi()
cfg, loaded := loadPersistedBootstrapConfig(dataDir, ui)
cfg, loaded := LoadPersistedBootstrapConfig(dataDir, ui)
require.Equal(t, tc.expect.loaded, loaded, ui.ErrorWriter.String())
if loaded {
require.Equal(t, token, cfg.ManagementToken)
@ -444,7 +126,7 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
"new cluster some files": {
mutateFn: func(t *testing.T, dir string) {
// Remove one of the required files
require.NoError(t, os.Remove(filepath.Join(dir, certFileName)))
require.NoError(t, os.Remove(filepath.Join(dir, CertFileName)))
},
expect: expect{
loaded: false,
@ -464,7 +146,7 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
},
"new cluster invalid cert": {
mutateFn: func(t *testing.T, dir string) {
name := filepath.Join(dir, certFileName)
name := filepath.Join(dir, CertFileName)
require.NoError(t, os.WriteFile(name, []byte("not-a-cert"), 0600))
},
expect: expect{
@ -474,7 +156,7 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
},
"new cluster invalid CA": {
mutateFn: func(t *testing.T, dir string) {
name := filepath.Join(dir, caFileName)
name := filepath.Join(dir, CAFileName)
require.NoError(t, os.WriteFile(name, []byte("not-a-ca-cert"), 0600))
},
expect: expect{
@ -482,20 +164,10 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
warning: "invalid CA certificate",
},
},
"new cluster invalid config flag": {
mutateFn: func(t *testing.T, dir string) {
name := filepath.Join(dir, configFileName)
require.NoError(t, os.WriteFile(name, []byte(`{"not_a_consul_agent_config_field" = "zap"}`), 0600))
},
expect: expect{
loaded: false,
warning: "failed to parse local bootstrap config",
},
},
"existing cluster invalid token": {
existingCluster: true,
mutateFn: func(t *testing.T, dir string) {
name := filepath.Join(dir, tokenFileName)
name := filepath.Join(dir, TokenFileName)
require.NoError(t, os.WriteFile(name, []byte("not-a-uuid"), 0600))
},
expect: expect{
@ -511,3 +183,136 @@ func Test_loadPersistedBootstrapConfig(t *testing.T) {
})
}
}
func TestFetchBootstrapConfig(t *testing.T) {
type testCase struct {
expectFetchErr error
expectRetry bool
}
run := func(t *testing.T, tc testCase) {
ui := cli.NewMockUi()
dataDir := testutil.TempDir(t, "fetch-bootstrap-cfg")
clientM := hcpclient.NewMockClient(t)
if tc.expectFetchErr != nil && tc.expectRetry {
clientM.On("FetchBootstrap", mock.Anything).
Return(nil, tc.expectFetchErr)
} else if tc.expectFetchErr != nil && !tc.expectRetry {
clientM.On("FetchBootstrap", mock.Anything).
Return(nil, tc.expectFetchErr).Once()
} else {
validToken, err := uuid.GenerateUUID()
require.NoError(t, err)
clientM.EXPECT().FetchBootstrap(mock.Anything).Return(&hcpclient.BootstrapConfig{
ManagementToken: validToken,
ConsulConfig: "{}",
}, nil).Once()
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
cfg, err := FetchBootstrapConfig(ctx, clientM, dataDir, ui)
if tc.expectFetchErr == nil {
require.NoError(t, err)
require.NotNil(t, cfg)
return
}
require.Error(t, err)
require.Nil(t, cfg)
if tc.expectRetry {
require.ErrorIs(t, err, context.DeadlineExceeded)
} else {
require.ErrorIs(t, err, tc.expectFetchErr)
}
}
tt := map[string]testCase{
"success": {},
"unauthorized": {
expectFetchErr: hcpclient.ErrUnauthorized,
},
"forbidden": {
expectFetchErr: hcpclient.ErrForbidden,
},
"retryable fetch error": {
expectFetchErr: errors.New("error"),
expectRetry: true,
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func TestLoadManagementToken(t *testing.T) {
type testCase struct {
skipHCPConfigDir bool
skipTokenFile bool
tokenFileContent string
skipBootstrap bool
}
validToken, err := uuid.GenerateUUID()
require.NoError(t, err)
run := func(t *testing.T, tc testCase) {
dataDir := testutil.TempDir(t, "load-management-token")
hcpCfgDir := filepath.Join(dataDir, SubDir)
if !tc.skipHCPConfigDir {
err := os.Mkdir(hcpCfgDir, 0755)
require.NoError(t, err)
}
tokenFilePath := filepath.Join(hcpCfgDir, TokenFileName)
if !tc.skipTokenFile {
err := os.WriteFile(tokenFilePath, []byte(tc.tokenFileContent), 0600)
require.NoError(t, err)
}
clientM := hcpclient.NewMockClient(t)
if !tc.skipBootstrap {
clientM.EXPECT().FetchBootstrap(mock.Anything).Return(&hcpclient.BootstrapConfig{
ManagementToken: validToken,
ConsulConfig: "{}",
}, nil).Once()
}
token, err := LoadManagementToken(context.Background(), hclog.NewNullLogger(), clientM, dataDir)
require.NoError(t, err)
require.Equal(t, validToken, token)
bytes, err := os.ReadFile(tokenFilePath)
require.NoError(t, err)
require.Equal(t, validToken, string(bytes))
}
tt := map[string]testCase{
"token configured": {
skipBootstrap: true,
tokenFileContent: validToken,
},
"no token configured": {
skipTokenFile: true,
},
"invalid token configured": {
tokenFileContent: "invalid",
},
"no hcp-config directory": {
skipHCPConfigDir: true,
skipTokenFile: true,
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}

178
agent/hcp/bootstrap/config-loader/loader.go Normal file
View File

@ -0,0 +1,178 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
// Package loader handles loading the bootstrap agent config fetched from HCP into
// the agent's config. It must be a separate package from other HCP components
// because it has a dependency on agent/config while other components need to be
// imported and run within the server process in agent/consul and that would create
// a dependency cycle.
package loader
import (
"context"
"fmt"
"path/filepath"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/agent/hcp/bootstrap"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
)
type ConfigLoader func(source config.Source) (config.LoadResult, error)
// LoadConfig will attempt to load previously-fetched config from disk and fall back to
// fetch from HCP servers if the local data is incomplete.
// It must be passed a (CLI) UI implementation so it can deliver progress
// updates to the user, for example if it is waiting to retry for a long period.
func LoadConfig(ctx context.Context, client hcpclient.Client, dataDir string, loader ConfigLoader, ui bootstrap.UI) (ConfigLoader, error) {
ui.Output("Loading configuration from HCP")
// See if we have existing config on disk
//
// OPTIMIZE: We could probably be more intelligent about config loading.
// The currently implemented approach is:
// 1. Attempt to load data from disk
// 2. If that fails or the data is incomplete, block indefinitely fetching remote config.
//
// What if instead we had the following flow:
// 1. Attempt to fetch config from HCP.
// 2. If that fails, fall back to data on disk from last fetch.
// 3. If that fails, go into blocking loop to fetch remote config.
//
// This should allow us to more gracefully transition cases like when
// an existing cluster is linked, but then wants to receive TLS materials
// at a later time. Currently, if we observe the existing-cluster marker we
// don't attempt to fetch any additional configuration from HCP.
cfg, ok := bootstrap.LoadPersistedBootstrapConfig(dataDir, ui)
if ok {
// Persisted bootstrap config exists, but needs to be validated
err := validatePersistedConfig(dataDir)
if err != nil {
ok = false
}
}
if !ok {
ui.Info("Fetching configuration from HCP servers")
var err error
cfg, err = bootstrap.FetchBootstrapConfig(ctx, client, dataDir, ui)
if err != nil {
return nil, fmt.Errorf("failed to bootstrap from HCP: %w", err)
}
ui.Info("Configuration fetched from HCP and saved on local disk")
} else {
ui.Info("Loaded HCP configuration from local disk")
}
// Create a new loader func to return
newLoader := bootstrapConfigLoader(loader, cfg)
return newLoader, nil
}
func AddAclPolicyAccessControlHeader(baseLoader ConfigLoader) ConfigLoader {
return func(source config.Source) (config.LoadResult, error) {
res, err := baseLoader(source)
if err != nil {
return res, err
}
rc := res.RuntimeConfig
// HTTP response headers are modified for the HCP UI to work.
if rc.HTTPResponseHeaders == nil {
rc.HTTPResponseHeaders = make(map[string]string)
}
prevValue, ok := rc.HTTPResponseHeaders[accessControlHeaderName]
if !ok {
rc.HTTPResponseHeaders[accessControlHeaderName] = accessControlHeaderValue
} else {
rc.HTTPResponseHeaders[accessControlHeaderName] = prevValue + "," + accessControlHeaderValue
}
return res, nil
}
}
// bootstrapConfigLoader is a ConfigLoader for passing bootstrap JSON config received from HCP
// to the config.builder. ConfigLoaders are functions used to build an agent's RuntimeConfig
// from various sources like files and flags. This config is contained in the config.LoadResult.
//
// The flow to include bootstrap config from HCP as a loader's data source is as follows:
//
// 1. A base ConfigLoader function (baseLoader) is created on agent start, and it sets the input
// source argument as the DefaultConfig.
//
// 2. When a server agent can be configured by HCP that baseLoader is wrapped in this bootstrapConfigLoader.
//
// 3. The bootstrapConfigLoader calls that base loader with the bootstrap JSON config as the
// default source. This data will be merged with other valid sources in the config.builder.
//
// 4. The result of the call to baseLoader() below contains the resulting RuntimeConfig, and we do some
// additional modifications to attach data that doesn't get populated during the build in the config pkg.
//
// Note that since the ConfigJSON is stored as the baseLoader's DefaultConfig, its data is the first
// to be merged by the config.builder and could be overwritten by user-provided values in config files or
// CLI flags. However, values set to RuntimeConfig after the baseLoader call are final.
func bootstrapConfigLoader(baseLoader ConfigLoader, cfg *bootstrap.RawBootstrapConfig) ConfigLoader {
return func(source config.Source) (config.LoadResult, error) {
// Don't allow any further attempts to provide a DefaultSource. This should
// only ever be needed later in client agent AutoConfig code but that should
// be mutually exclusive from this bootstrapping mechanism since this is
// only for servers. If we ever try to change that, this clear failure
// should alert future developers that the assumptions are changing rather
// than quietly not applying the config they expect!
if source != nil {
return config.LoadResult{},
fmt.Errorf("non-nil config source provided to a loader after HCP bootstrap already provided a DefaultSource")
}
// Otherwise, just call to the loader we were passed with our own additional
// JSON as the source.
//
// OPTIMIZE: We could check/log whether any fields set by the remote config were overwritten by a user-provided flag.
res, err := baseLoader(config.FileSource{
Name: "HCP Bootstrap",
Format: "json",
Data: cfg.ConfigJSON,
})
if err != nil {
return res, fmt.Errorf("failed to load HCP Bootstrap config: %w", err)
}
finalizeRuntimeConfig(res.RuntimeConfig, cfg)
return res, nil
}
}
const (
accessControlHeaderName = "Access-Control-Expose-Headers"
accessControlHeaderValue = "x-consul-default-acl-policy"
)
// finalizeRuntimeConfig will set additional HCP-specific values that are not
// handled by the config.builder.
func finalizeRuntimeConfig(rc *config.RuntimeConfig, cfg *bootstrap.RawBootstrapConfig) {
rc.Cloud.ManagementToken = cfg.ManagementToken
}
// validatePersistedConfig attempts to load persisted config to check for errors and basic validity.
// Errors here will raise issues like referencing unsupported config fields.
func validatePersistedConfig(dataDir string) error {
filename := filepath.Join(dataDir, bootstrap.SubDir, bootstrap.ConfigFileName)
_, err := config.Load(config.LoadOpts{
ConfigFiles: []string{filename},
HCL: []string{
"server = true",
`bind_addr = "127.0.0.1"`,
fmt.Sprintf("data_dir = %q", dataDir),
},
ConfigFormat: "json",
})
if err != nil {
return fmt.Errorf("failed to parse local bootstrap config: %w", err)
}
return nil
}

389
agent/hcp/bootstrap/config-loader/loader_test.go Normal file
View File

@ -0,0 +1,389 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package loader
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/agent/hcp"
"github.com/hashicorp/consul/agent/hcp/bootstrap"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/lib"
"github.com/mitchellh/cli"
"github.com/stretchr/testify/require"
)
func TestBootstrapConfigLoader(t *testing.T) {
baseLoader := func(source config.Source) (config.LoadResult, error) {
return config.Load(config.LoadOpts{
DefaultConfig: source,
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
`data_dir = "/tmp/consul-data"`,
},
})
}
bootstrapLoader := func(source config.Source) (config.LoadResult, error) {
return bootstrapConfigLoader(baseLoader, &bootstrap.RawBootstrapConfig{
ConfigJSON: `{"bootstrap_expect": 8}`,
ManagementToken: "test-token",
})(source)
}
result, err := bootstrapLoader(nil)
require.NoError(t, err)
// bootstrap_expect and management token are injected from bootstrap config received from HCP.
require.Equal(t, 8, result.RuntimeConfig.BootstrapExpect)
require.Equal(t, "test-token", result.RuntimeConfig.Cloud.ManagementToken)
}
func Test_finalizeRuntimeConfig(t *testing.T) {
type testCase struct {
rc *config.RuntimeConfig
cfg *bootstrap.RawBootstrapConfig
verifyFn func(t *testing.T, rc *config.RuntimeConfig)
}
run := func(t *testing.T, tc testCase) {
finalizeRuntimeConfig(tc.rc, tc.cfg)
tc.verifyFn(t, tc.rc)
}
tt := map[string]testCase{
"set management token": {
rc: &config.RuntimeConfig{},
cfg: &bootstrap.RawBootstrapConfig{
ManagementToken: "test-token",
},
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Equal(t, "test-token", rc.Cloud.ManagementToken)
},
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func Test_AddAclPolicyAccessControlHeader(t *testing.T) {
type testCase struct {
baseLoader ConfigLoader
verifyFn func(t *testing.T, rc *config.RuntimeConfig)
}
run := func(t *testing.T, tc testCase) {
loader := AddAclPolicyAccessControlHeader(tc.baseLoader)
result, err := loader(nil)
require.NoError(t, err)
tc.verifyFn(t, result.RuntimeConfig)
}
tt := map[string]testCase{
"append to header if present": {
baseLoader: func(source config.Source) (config.LoadResult, error) {
return config.Load(config.LoadOpts{
DefaultConfig: config.DefaultSource(),
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
`data_dir = "/tmp/consul-data"`,
fmt.Sprintf(`http_config = { response_headers = { %s = "test" } }`, accessControlHeaderName),
},
})
},
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Equal(t, "test,x-consul-default-acl-policy", rc.HTTPResponseHeaders[accessControlHeaderName])
},
},
"set header if not present": {
baseLoader: func(source config.Source) (config.LoadResult, error) {
return config.Load(config.LoadOpts{
DefaultConfig: config.DefaultSource(),
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
`data_dir = "/tmp/consul-data"`,
},
})
},
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Equal(t, "x-consul-default-acl-policy", rc.HTTPResponseHeaders[accessControlHeaderName])
},
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func boolPtr(value bool) *bool {
return &value
}
func TestLoadConfig_Persistence(t *testing.T) {
type testCase struct {
// resourceID is the HCP resource ID. If set, a server is considered to be cloud-enabled.
resourceID string
// devMode indicates whether the loader should not have a data directory.
devMode bool
// verifyFn issues case-specific assertions.
verifyFn func(t *testing.T, rc *config.RuntimeConfig)
}
run := func(t *testing.T, tc testCase) {
dir, err := os.MkdirTemp(os.TempDir(), "bootstrap-test-")
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(dir) })
s := hcp.NewMockHCPServer()
s.AddEndpoint(bootstrap.TestEndpoint())
// Use an HTTPS server since that's what the HCP SDK expects for auth.
srv := httptest.NewTLSServer(s)
defer srv.Close()
caCert, err := x509.ParseCertificate(srv.TLS.Certificates[0].Certificate[0])
require.NoError(t, err)
pool := x509.NewCertPool()
pool.AddCert(caCert)
clientTLS := &tls.Config{RootCAs: pool}
baseOpts := config.LoadOpts{
HCL: []string{
`server = true`,
`bind_addr = "127.0.0.1"`,
fmt.Sprintf(`http_config = { response_headers = { %s = "Content-Encoding" } }`, accessControlHeaderName),
fmt.Sprintf(`cloud { client_id="test" client_secret="test" hostname=%q auth_url=%q resource_id=%q }`,
srv.Listener.Addr().String(), srv.URL, tc.resourceID),
},
}
if tc.devMode {
baseOpts.DevMode = boolPtr(true)
} else {
baseOpts.HCL = append(baseOpts.HCL, fmt.Sprintf(`data_dir = %q`, dir))
}
baseLoader := func(source config.Source) (config.LoadResult, error) {
baseOpts.DefaultConfig = source
return config.Load(baseOpts)
}
ui := cli.NewMockUi()
// Load initial config to check whether bootstrapping from HCP is enabled.
initial, err := baseLoader(nil)
require.NoError(t, err)
// Override the client TLS config so that the test server can be trusted.
initial.RuntimeConfig.Cloud.WithTLSConfig(clientTLS)
client, err := hcpclient.NewClient(initial.RuntimeConfig.Cloud)
require.NoError(t, err)
loader, err := LoadConfig(context.Background(), client, initial.RuntimeConfig.DataDir, baseLoader, ui)
require.NoError(t, err)
// Load the agent config with the potentially wrapped loader.
fromRemote, err := loader(nil)
require.NoError(t, err)
// HCP-enabled cases should fetch from HCP on the first run of LoadConfig.
require.Contains(t, ui.OutputWriter.String(), "Fetching configuration from HCP")
// Run case-specific verification.
tc.verifyFn(t, fromRemote.RuntimeConfig)
require.Empty(t, fromRemote.RuntimeConfig.ACLInitialManagementToken,
"initial_management token should have been sanitized")
if tc.devMode {
// Re-running the bootstrap func below isn't relevant to dev mode
// since they don't have a data directory to load data from.
return
}
// Run LoadConfig again to exercise the logic of loading config from disk.
loader, err = LoadConfig(context.Background(), client, initial.RuntimeConfig.DataDir, baseLoader, ui)
require.NoError(t, err)
fromDisk, err := loader(nil)
require.NoError(t, err)
// HCP-enabled cases should fetch from disk on the second run.
require.Contains(t, ui.OutputWriter.String(), "Loaded HCP configuration from local disk")
// Config loaded from disk should be the same as the one that was initially fetched from the HCP servers.
require.Equal(t, fromRemote.RuntimeConfig, fromDisk.RuntimeConfig)
}
tt := map[string]testCase{
"dev mode": {
devMode: true,
resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"consul.cluster/new-cluster-id",
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
require.Empty(t, rc.DataDir)
// Dev mode should have persisted certs since they can't be inlined.
require.NotEmpty(t, rc.TLS.HTTPS.CertFile)
require.NotEmpty(t, rc.TLS.HTTPS.KeyFile)
require.NotEmpty(t, rc.TLS.HTTPS.CAFile)
// Find the temporary directory they got stored in.
dir := filepath.Dir(rc.TLS.HTTPS.CertFile)
// Ensure we only stored the TLS materials.
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, 3)
haveFiles := make([]string, 3)
for i, entry := range entries {
haveFiles[i] = entry.Name()
}
wantFiles := []string{bootstrap.CAFileName, bootstrap.CertFileName, bootstrap.KeyFileName}
require.ElementsMatch(t, wantFiles, haveFiles)
},
},
"new cluster": {
resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"consul.cluster/new-cluster-id",
// New clusters should have received and persisted the whole suite of config.
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
dir := filepath.Join(rc.DataDir, bootstrap.SubDir)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, 6)
files := []string{
filepath.Join(dir, bootstrap.ConfigFileName),
filepath.Join(dir, bootstrap.CAFileName),
filepath.Join(dir, bootstrap.CertFileName),
filepath.Join(dir, bootstrap.KeyFileName),
filepath.Join(dir, bootstrap.TokenFileName),
filepath.Join(dir, bootstrap.SuccessFileName),
}
for _, name := range files {
_, err := os.Stat(name)
require.NoError(t, err)
}
require.Equal(t, filepath.Join(dir, bootstrap.CertFileName), rc.TLS.HTTPS.CertFile)
require.Equal(t, filepath.Join(dir, bootstrap.KeyFileName), rc.TLS.HTTPS.KeyFile)
require.Equal(t, filepath.Join(dir, bootstrap.CAFileName), rc.TLS.HTTPS.CAFile)
cert, key, caCerts, err := bootstrap.LoadCerts(dir)
require.NoError(t, err)
require.NoError(t, bootstrap.ValidateTLSCerts(cert, key, caCerts))
},
},
"existing cluster": {
resourceID: "organization/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"project/0b9de9a3-8403-4ca6-aba8-fca752f42100/" +
"consul.cluster/" + bootstrap.TestExistingClusterID,
// Existing clusters should have only received and persisted the management token.
verifyFn: func(t *testing.T, rc *config.RuntimeConfig) {
dir := filepath.Join(rc.DataDir, bootstrap.SubDir)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, 3)
files := []string{
filepath.Join(dir, bootstrap.TokenFileName),
filepath.Join(dir, bootstrap.SuccessFileName),
filepath.Join(dir, bootstrap.ConfigFileName),
}
for _, name := range files {
_, err := os.Stat(name)
require.NoError(t, err)
}
},
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}
func TestValidatePersistedConfig(t *testing.T) {
type testCase struct {
configContents string
expectErr string
}
run := func(t *testing.T, tc testCase) {
dataDir, err := os.MkdirTemp(os.TempDir(), "load-bootstrap-test-")
require.NoError(t, err)
t.Cleanup(func() { os.RemoveAll(dataDir) })
dir := filepath.Join(dataDir, bootstrap.SubDir)
require.NoError(t, lib.EnsurePath(dir, true))
if tc.configContents != "" {
name := filepath.Join(dir, bootstrap.ConfigFileName)
require.NoError(t, os.WriteFile(name, []byte(tc.configContents), 0600))
}
err = validatePersistedConfig(dataDir)
if tc.expectErr != "" {
require.Error(t, err)
require.Contains(t, err.Error(), tc.expectErr)
} else {
require.NoError(t, err)
}
}
tt := map[string]testCase{
"valid": {
configContents: `{"bootstrap_expect": 1, "cloud": {"resource_id": "id"}}`,
},
"invalid config key": {
configContents: `{"not_a_consul_agent_config_field": "zap"}`,
expectErr: "invalid config key not_a_consul_agent_config_field",
},
"invalid format": {
configContents: `{"not_json" = "invalid"}`,
expectErr: "invalid character '=' after object key",
},
"missing configuration file": {
expectErr: "no such file or directory",
},
}
for name, tc := range tt {
t.Run(name, func(t *testing.T) {
run(t, tc)
})
}
}

35
agent/hcp/client/client.go
View File

@ -5,12 +5,16 @@ package client
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
httptransport "github.com/go-openapi/runtime/client"
"github.com/go-openapi/strfmt"
"golang.org/x/oauth2"
hcptelemetry "github.com/hashicorp/hcp-sdk-go/clients/cloud-consul-telemetry-gateway/preview/2023-04-14/client/consul_telemetry_service"
hcpgnm "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/client/global_network_manager_service"
@ -124,9 +128,8 @@ func (c *hcpClient) FetchBootstrap(ctx context.Context) (*BootstrapConfig, error
resp, err := c.gnm.AgentBootstrapConfig(params, nil)
if err != nil {
return nil, err
return nil, decodeError(err)
}
return bootstrapConfigFromHCP(resp.Payload), nil
}
@ -327,7 +330,7 @@ func (c *hcpClient) GetCluster(ctx context.Context) (*Cluster, error) {
resp, err := c.gnm.GetCluster(params, nil)
if err != nil {
return nil, err
return nil, decodeError(err)
}
return clusterFromHCP(resp.Payload), nil
@ -340,3 +343,29 @@ func clusterFromHCP(payload *gnmmod.HashicorpCloudGlobalNetworkManager20220215Ge
HCPPortalURL: payload.Cluster.HcpPortalURL,
}
}
func decodeError(err error) error {
// Determine the code from the type of error
var code int
switch e := err.(type) {
case *url.Error:
oauthErr, ok := errors.Unwrap(e.Err).(*oauth2.RetrieveError)
if ok {
code = oauthErr.Response.StatusCode
}
case *hcpgnm.AgentBootstrapConfigDefault:
code = e.Code()
case *hcpgnm.GetClusterDefault:
code = e.Code()
}
// Return specific error for codes if relevant
switch code {
case http.StatusUnauthorized:
return ErrUnauthorized
case http.StatusForbidden:
return ErrForbidden
}
return err
}

11
agent/hcp/client/errors.go Normal file
View File

@ -0,0 +1,11 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package client
import "errors"
var (
ErrUnauthorized = errors.New("unauthorized")
ErrForbidden = errors.New("forbidden")
)

4
agent/hcp/deps.go
View File

@ -23,9 +23,10 @@ type Deps struct {
Provider scada.Provider
Sink metrics.MetricSink
TelemetryProvider *hcpProviderImpl
DataDir string
}
func NewDeps(cfg config.CloudConfig, logger hclog.Logger) (Deps, error) {
func NewDeps(cfg config.CloudConfig, logger hclog.Logger, dataDir string) (Deps, error) {
ctx := context.Background()
ctx = hclog.WithContext(ctx, logger)
@ -59,6 +60,7 @@ func NewDeps(cfg config.CloudConfig, logger hclog.Logger) (Deps, error) {
Provider: provider,
Sink: sink,
TelemetryProvider: metricsProvider,
DataDir: dataDir,
}, nil
}

6
agent/setup.go
View File

@ -141,13 +141,17 @@ func NewBaseDeps(configLoader ConfigLoader, logOut io.Writer, providedLogger hcl
// This values is set late within newNodeIDFromConfig above
cfg.Cloud.NodeID = cfg.NodeID
d.HCP, err = hcp.NewDeps(cfg.Cloud, d.Logger.Named("hcp"))
d.HCP, err = hcp.NewDeps(cfg.Cloud, d.Logger.Named("hcp"), cfg.DataDir)
if err != nil {
return d, err
}
if d.HCP.Sink != nil {
extraSinks = append(extraSinks, d.HCP.Sink)
}
} else {
d.HCP = hcp.Deps{
DataDir: cfg.DataDir,
}
}
d.MetricsConfig, err = lib.InitTelemetry(cfg.Telemetry, d.Logger, extraSinks...)

2
command/agent/agent.go
View File

@ -21,7 +21,7 @@ import (
"github.com/hashicorp/consul/agent"
"github.com/hashicorp/consul/agent/config"
hcpbootstrap "github.com/hashicorp/consul/agent/hcp/bootstrap"
hcpbootstrap "github.com/hashicorp/consul/agent/hcp/bootstrap/config-loader"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/command/cli"
"github.com/hashicorp/consul/command/flags"

59
internal/hcp/internal/controllers/link/controller.go
View File

@ -9,19 +9,18 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/anypb"
gnmmod "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models"
"google.golang.org/protobuf/types/known/anypb"
"github.com/hashicorp/consul/agent/hcp/bootstrap"
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/agent/hcp/config"
"github.com/hashicorp/consul/internal/resource"
"github.com/hashicorp/consul/internal/storage"
"github.com/hashicorp/consul/proto-public/pbresource"
"github.com/hashicorp/consul/internal/controller"
"github.com/hashicorp/consul/internal/hcp/internal/types"
"github.com/hashicorp/consul/internal/storage"
pbhcp "github.com/hashicorp/consul/proto-public/pbhcp/v2"
"github.com/hashicorp/consul/proto-public/pbresource"
)
// HCPClientFn is a function that can be used to create an HCP client from a Link object.
@ -42,7 +41,13 @@ var DefaultHCPClientFn HCPClientFn = func(link *pbhcp.Link) (hcpclient.Client, e
return hcpClient, nil
}
func LinkController(resourceApisEnabled bool, hcpAllowV2ResourceApis bool, hcpClientFn HCPClientFn, cfg config.CloudConfig) *controller.Controller {
func LinkController(
resourceApisEnabled bool,
hcpAllowV2ResourceApis bool,
hcpClientFn HCPClientFn,
cfg config.CloudConfig,
dataDir string,
) *controller.Controller {
return controller.NewController("link", pbhcp.LinkType).
WithInitializer(&linkInitializer{
cloudConfig: cfg,
@ -51,6 +56,7 @@ func LinkController(resourceApisEnabled bool, hcpAllowV2ResourceApis bool, hcpCl
resourceApisEnabled: resourceApisEnabled,
hcpAllowV2ResourceApis: hcpAllowV2ResourceApis,
hcpClientFn: hcpClientFn,
dataDir: dataDir,
})
}
@ -58,26 +64,7 @@ type linkReconciler struct {
resourceApisEnabled bool
hcpAllowV2ResourceApis bool
hcpClientFn HCPClientFn
}
func (r *linkReconciler) writeStatusIfNotEqual(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, status *pbresource.Status) error {
if resource.EqualStatus(res.Status[StatusKey], status, false) {
return nil
}
_, err := rt.Client.WriteStatus(ctx, &pbresource.WriteStatusRequest{
Id: res.Id,
Key: StatusKey,
Status: status,
})
return err
}
func (r *linkReconciler) linkingFailed(ctx context.Context, rt controller.Runtime, res *pbresource.Resource) error {
newStatus := &pbresource.Status{
ObservedGeneration: res.Generation,
Conditions: []*pbresource.Condition{ConditionFailed},
}
return r.writeStatusIfNotEqual(ctx, rt, res, newStatus)
dataDir string
}
func hcpAccessLevelToConsul(level *gnmmod.HashicorpCloudGlobalNetworkManager20220215ClusterConsulAccessLevel) pbhcp.AccessLevel {
@ -128,12 +115,12 @@ func (r *linkReconciler) Reconcile(ctx context.Context, rt controller.Runtime, r
ObservedGeneration: res.Generation,
Conditions: []*pbresource.Condition{ConditionDisabled},
}
return r.writeStatusIfNotEqual(ctx, rt, res, newStatus)
return writeStatusIfNotEqual(ctx, rt, res, newStatus)
}
hcpClient, err := r.hcpClientFn(&link)
if err != nil {
rt.Logger.Error("error creating HCP Client", "error", err)
rt.Logger.Error("error creating HCP client", "error", err)
return err
}
@ -141,7 +128,8 @@ func (r *linkReconciler) Reconcile(ctx context.Context, rt controller.Runtime, r
cluster, err := hcpClient.GetCluster(ctx)
if err != nil {
rt.Logger.Error("error querying HCP for cluster", "error", err)
return r.linkingFailed(ctx, rt, res)
linkingFailed(ctx, rt, res, err)
return err
}
accessLevel := hcpAccessLevelToConsul(cluster.AccessLevel)
@ -170,12 +158,23 @@ func (r *linkReconciler) Reconcile(ctx context.Context, rt controller.Runtime, r
}
}
// Load the management token if access is not set to read-only. Read-only clusters
// will not have a management token provided by HCP.
if accessLevel != pbhcp.AccessLevel_ACCESS_LEVEL_GLOBAL_READ_ONLY {
_, err = bootstrap.LoadManagementToken(ctx, rt.Logger, hcpClient, r.dataDir)
if err != nil {
linkingFailed(ctx, rt, res, err)
return err
}
// TODO: Update the HCP manager with the loaded management token as part of CC-7044
}
newStatus = &pbresource.Status{
ObservedGeneration: res.Generation,
Conditions: []*pbresource.Condition{ConditionLinked(link.ResourceId)},
}
return r.writeStatusIfNotEqual(ctx, rt, res, newStatus)
return writeStatusIfNotEqual(ctx, rt, res, newStatus)
}
type linkInitializer struct {

146
internal/hcp/internal/controllers/link/controller_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/hashicorp/go-uuid"
gnmmod "github.com/hashicorp/hcp-sdk-go/clients/cloud-global-network-manager-service/preview/2022-02-15/models"
svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing"
@ -19,7 +20,6 @@ import (
"github.com/hashicorp/consul/agent/hcp/config"
"github.com/hashicorp/consul/internal/controller"
"github.com/hashicorp/consul/internal/hcp/internal/types"
"github.com/hashicorp/consul/internal/resource/resourcetest"
rtest "github.com/hashicorp/consul/internal/resource/resourcetest"
pbhcp "github.com/hashicorp/consul/proto-public/pbhcp/v2"
"github.com/hashicorp/consul/proto-public/pbresource"
@ -33,7 +33,6 @@ type controllerSuite struct {
client *rtest.Client
rt controller.Runtime
ctl linkReconciler
tenancies []*pbresource.Tenancy
}
@ -49,7 +48,7 @@ func mockHcpClientFn(t *testing.T) (*hcpclient.MockClient, HCPClientFn) {
func (suite *controllerSuite) SetupTest() {
suite.ctx = testutil.TestContext(suite.T())
suite.tenancies = resourcetest.TestTenancies()
suite.tenancies = rtest.TestTenancies()
client := svctest.NewResourceServiceBuilder().
WithRegisterFns(types.Register).
WithTenancies(suite.tenancies...).
@ -76,19 +75,35 @@ func (suite *controllerSuite) TestController_Ok() {
// Run the controller manager
mgr := controller.NewManager(suite.client, suite.rt.Logger)
mockClient, mockClientFn := mockHcpClientFn(suite.T())
readOnly := gnmmod.HashicorpCloudGlobalNetworkManager20220215ClusterConsulAccessLevelCONSULACCESSLEVELGLOBALREADONLY
readWrite := gnmmod.HashicorpCloudGlobalNetworkManager20220215ClusterConsulAccessLevelCONSULACCESSLEVELGLOBALREADWRITE
mockClient.EXPECT().GetCluster(mock.Anything).Return(&hcpclient.Cluster{
HCPPortalURL: "http://test.com",
AccessLevel: &readOnly,
AccessLevel: &readWrite,
}, nil)
mgr.Register(LinkController(false, false, mockClientFn, config.CloudConfig{}))
token, err := uuid.GenerateUUID()
require.NoError(suite.T(), err)
mockClient.EXPECT().FetchBootstrap(mock.Anything).
Return(&hcpclient.BootstrapConfig{
ManagementToken: token,
ConsulConfig: "{}",
}, nil).Once()
dataDir := testutil.TempDir(suite.T(), "test-link-controller")
mgr.Register(LinkController(
false,
false,
mockClientFn,
config.CloudConfig{},
dataDir,
))
mgr.SetRaftLeader(true)
go mgr.Run(suite.ctx)
linkData := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: types.GenerateTestResourceID(suite.T()),
}
link := rtest.Resource(pbhcp.LinkType, "global").
@ -102,7 +117,7 @@ func (suite *controllerSuite) TestController_Ok() {
updatedLinkResource := suite.client.WaitForNewVersion(suite.T(), link.Id, link.Version)
require.NoError(suite.T(), updatedLinkResource.Data.UnmarshalTo(&updatedLink))
require.Equal(suite.T(), "http://test.com", updatedLink.HcpClusterUrl)
require.Equal(suite.T(), pbhcp.AccessLevel_ACCESS_LEVEL_GLOBAL_READ_ONLY, updatedLink.AccessLevel)
require.Equal(suite.T(), pbhcp.AccessLevel_ACCESS_LEVEL_GLOBAL_READ_WRITE, updatedLink.AccessLevel)
}
func (suite *controllerSuite) TestController_Initialize() {
@ -110,19 +125,27 @@ func (suite *controllerSuite) TestController_Initialize() {
mgr := controller.NewManager(suite.client, suite.rt.Logger)
mockClient, mockClientFn := mockHcpClientFn(suite.T())
readWrite := gnmmod.HashicorpCloudGlobalNetworkManager20220215ClusterConsulAccessLevelCONSULACCESSLEVELGLOBALREADWRITE
readOnly := gnmmod.HashicorpCloudGlobalNetworkManager20220215ClusterConsulAccessLevelCONSULACCESSLEVELGLOBALREADONLY
mockClient.EXPECT().GetCluster(mock.Anything).Return(&hcpclient.Cluster{
HCPPortalURL: "http://test.com",
AccessLevel: &readWrite,
AccessLevel: &readOnly,
}, nil)
cloudCfg := config.CloudConfig{
ClientID: "client-id-abc",
ClientSecret: "client-secret-abc",
ResourceID: "resource-id-abc",
ResourceID: types.GenerateTestResourceID(suite.T()),
}
mgr.Register(LinkController(false, false, mockClientFn, cloudCfg))
dataDir := testutil.TempDir(suite.T(), "test-link-controller")
mgr.Register(LinkController(
false,
false,
mockClientFn,
cloudCfg,
dataDir,
))
mgr.SetRaftLeader(true)
go mgr.Run(suite.ctx)
@ -152,14 +175,21 @@ func (suite *controllerSuite) TestControllerResourceApisEnabled_LinkDisabled() {
// Run the controller manager
mgr := controller.NewManager(suite.client, suite.rt.Logger)
_, mockClientFunc := mockHcpClientFn(suite.T())
mgr.Register(LinkController(true, false, mockClientFunc, config.CloudConfig{}))
dataDir := testutil.TempDir(suite.T(), "test-link-controller")
mgr.Register(LinkController(
true,
false,
mockClientFunc,
config.CloudConfig{},
dataDir,
))
mgr.SetRaftLeader(true)
go mgr.Run(suite.ctx)
linkData := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: types.GenerateTestResourceID(suite.T()),
}
link := rtest.Resource(pbhcp.LinkType, "global").
WithData(suite.T(), linkData).
@ -178,14 +208,31 @@ func (suite *controllerSuite) TestControllerResourceApisEnabledWithOverride_Link
HCPPortalURL: "http://test.com",
}, nil)
mgr.Register(LinkController(true, true, mockClientFunc, config.CloudConfig{}))
token, err := uuid.GenerateUUID()
require.NoError(suite.T(), err)
mockClient.EXPECT().FetchBootstrap(mock.Anything).
Return(&hcpclient.BootstrapConfig{
ManagementToken: token,
ConsulConfig: "{}",
}, nil).Once()
dataDir := testutil.TempDir(suite.T(), "test-link-controller")
mgr.Register(LinkController(
true,
true,
mockClientFunc,
config.CloudConfig{},
dataDir,
))
mgr.SetRaftLeader(true)
go mgr.Run(suite.ctx)
linkData := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: types.GenerateTestResourceID(suite.T()),
}
link := rtest.Resource(pbhcp.LinkType, "global").
WithData(suite.T(), linkData).
@ -197,27 +244,58 @@ func (suite *controllerSuite) TestControllerResourceApisEnabledWithOverride_Link
}
func (suite *controllerSuite) TestController_GetClusterError() {
// Run the controller manager
mgr := controller.NewManager(suite.client, suite.rt.Logger)
mockClient, mockClientFunc := mockHcpClientFn(suite.T())
mockClient.EXPECT().GetCluster(mock.Anything).Return(nil, fmt.Errorf("error"))
mgr.Register(LinkController(true, true, mockClientFunc, config.CloudConfig{}))
mgr.SetRaftLeader(true)
go mgr.Run(suite.ctx)
linkData := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
type testCase struct {
expectErr error
expectCondition *pbresource.Condition
}
tt := map[string]testCase{
"unexpected": {
expectErr: fmt.Errorf("error"),
expectCondition: ConditionFailed,
},
"unauthorized": {
expectErr: hcpclient.ErrUnauthorized,
expectCondition: ConditionUnauthorized,
},
"forbidden": {
expectErr: hcpclient.ErrForbidden,
expectCondition: ConditionForbidden,
},
}
link := rtest.Resource(pbhcp.LinkType, "global").
WithData(suite.T(), linkData).
Write(suite.T(), suite.client)
suite.T().Cleanup(suite.deleteResourceFunc(link.Id))
for name, tc := range tt {
suite.T().Run(name, func(t *testing.T) {
// Run the controller manager
mgr := controller.NewManager(suite.client, suite.rt.Logger)
mockClient, mockClientFunc := mockHcpClientFn(suite.T())
mockClient.EXPECT().GetCluster(mock.Anything).Return(nil, tc.expectErr)
suite.client.WaitForStatusCondition(suite.T(), link.Id, StatusKey, ConditionFailed)
dataDir := testutil.TempDir(suite.T(), "test-link-controller")
mgr.Register(LinkController(
true,
true,
mockClientFunc,
config.CloudConfig{},
dataDir,
))
mgr.SetRaftLeader(true)
go mgr.Run(suite.ctx)
linkData := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: types.GenerateTestResourceID(suite.T()),
}
link := rtest.Resource(pbhcp.LinkType, "global").
WithData(suite.T(), linkData).
Write(suite.T(), suite.client)
suite.T().Cleanup(suite.deleteResourceFunc(link.Id))
suite.client.WaitForStatusCondition(suite.T(), link.Id, StatusKey, tc.expectCondition)
})
}
}
func Test_hcpAccessModeToConsul(t *testing.T) {

59
internal/hcp/internal/controllers/link/status.go
View File

@ -4,8 +4,13 @@
package link
import (
"context"
"errors"
"fmt"
"github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/internal/controller"
"github.com/hashicorp/consul/internal/resource"
"github.com/hashicorp/consul/proto-public/pbresource"
)
@ -16,10 +21,14 @@ const (
LinkedReason = "SUCCESS"
FailedReason = "FAILED"
DisabledReasonV2ResourcesUnsupported = "DISABLED_V2_RESOURCES_UNSUPPORTED"
UnauthorizedReason = "UNAUTHORIZED"
ForbiddenReason = "FORBIDDEN"
LinkedMessageFormat = "Successfully linked to cluster '%s'"
FailedMessage = "Failed to link to HCP"
FailedMessage = "Failed to link to HCP due to unexpected error"
DisabledResourceAPIsEnabledMessage = "Link is disabled because resource-apis are enabled"
UnauthorizedMessage = "Access denied, check client_id and client_secret"
ForbiddenMessage = "Access denied, check the resource_id"
)
var (
@ -35,6 +44,18 @@ var (
Reason: FailedReason,
Message: FailedMessage,
}
ConditionUnauthorized = &pbresource.Condition{
Type: StatusLinked,
State: pbresource.Condition_STATE_FALSE,
Reason: UnauthorizedReason,
Message: UnauthorizedMessage,
}
ConditionForbidden = &pbresource.Condition{
Type: StatusLinked,
State: pbresource.Condition_STATE_FALSE,
Reason: ForbiddenReason,
Message: ForbiddenMessage,
}
)
func ConditionLinked(resourceId string) *pbresource.Condition {
@ -45,3 +66,39 @@ func ConditionLinked(resourceId string) *pbresource.Condition {
Message: fmt.Sprintf(LinkedMessageFormat, resourceId),
}
}
func writeStatusIfNotEqual(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, status *pbresource.Status) error {
if resource.EqualStatus(res.Status[StatusKey], status, false) {
return nil
}
_, err := rt.Client.WriteStatus(ctx, &pbresource.WriteStatusRequest{
Id: res.Id,
Key: StatusKey,
Status: status,
})
return err
}
func linkingFailed(ctx context.Context, rt controller.Runtime, res *pbresource.Resource, err error) error {
var condition *pbresource.Condition
switch {
case errors.Is(err, client.ErrUnauthorized):
condition = ConditionUnauthorized
case errors.Is(err, client.ErrForbidden):
condition = ConditionForbidden
default:
condition = ConditionFailed
}
newStatus := &pbresource.Status{
ObservedGeneration: res.Generation,
Conditions: []*pbresource.Condition{condition},
}
writeErr := writeStatusIfNotEqual(ctx, rt, res, newStatus)
if writeErr != nil {
rt.Logger.Error("error writing status", "error", writeErr)
return writeErr
}
return nil
}

4
internal/hcp/internal/controllers/register.go
View File

@ -4,7 +4,6 @@
package controllers
import (
hcpclient "github.com/hashicorp/consul/agent/hcp/client"
"github.com/hashicorp/consul/agent/hcp/config"
"github.com/hashicorp/consul/internal/controller"
"github.com/hashicorp/consul/internal/hcp/internal/controllers/link"
@ -14,7 +13,7 @@ type Dependencies struct {
CloudConfig config.CloudConfig
ResourceApisEnabled bool
HCPAllowV2ResourceApis bool
HCPClient hcpclient.Client
DataDir string
}
func Register(mgr *controller.Manager, deps Dependencies) {
@ -23,5 +22,6 @@ func Register(mgr *controller.Manager, deps Dependencies) {
deps.HCPAllowV2ResourceApis,
link.DefaultHCPClientFn,
deps.CloudConfig,
deps.DataDir,
))
}

14
internal/hcp/internal/types/link.go
View File

@ -9,6 +9,7 @@ import (
"github.com/hashicorp/consul/internal/resource"
pbhcp "github.com/hashicorp/consul/proto-public/pbhcp/v2"
"github.com/hashicorp/go-multierror"
hcpresource "github.com/hashicorp/hcp-sdk-go/resource"
)
type DecodedLink = resource.DecodedResource[*pbhcp.Link]
@ -20,7 +21,8 @@ const (
)
var (
linkConfigurationNameError = errors.New("only a single Link resource is allowed and it must be named global")
errLinkConfigurationName = errors.New("only a single Link resource is allowed and it must be named global")
errInvalidHCPResourceID = errors.New("could not parse, invalid format")
)
func RegisterLink(r resource.Registry) {
@ -40,7 +42,7 @@ func validateLink(res *DecodedLink) error {
if res.Id.Name != LinkName {
err = multierror.Append(err, resource.ErrInvalidField{
Name: "name",
Wrapped: linkConfigurationNameError,
Wrapped: errLinkConfigurationName,
})
}
@ -63,6 +65,14 @@ func validateLink(res *DecodedLink) error {
Name: "resource_id",
Wrapped: resource.ErrMissing,
})
} else {
_, parseErr := hcpresource.FromString(res.Data.ResourceId)
if parseErr != nil {
err = multierror.Append(err, resource.ErrInvalidField{
Name: "resource_id",
Wrapped: errInvalidHCPResourceID,
})
}
}
return err

33
internal/hcp/internal/types/link_test.go
View File

@ -35,7 +35,7 @@ func TestValidateLink_Ok(t *testing.T) {
data := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: GenerateTestResourceID(t),
}
res := createCloudLinkResource(t, data)
@ -60,7 +60,7 @@ func TestValidateLink_InvalidName(t *testing.T) {
data := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: GenerateTestResourceID(t),
}
res := createCloudLinkResource(t, data)
@ -70,7 +70,7 @@ func TestValidateLink_InvalidName(t *testing.T) {
expected := resource.ErrInvalidField{
Name: "name",
Wrapped: linkConfigurationNameError,
Wrapped: errLinkConfigurationName,
}
var actual resource.ErrInvalidField
@ -82,7 +82,7 @@ func TestValidateLink_MissingClientId(t *testing.T) {
data := &pbhcp.Link{
ClientId: "",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: GenerateTestResourceID(t),
}
res := createCloudLinkResource(t, data)
@ -103,7 +103,7 @@ func TestValidateLink_MissingClientSecret(t *testing.T) {
data := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "",
ResourceId: "abc",
ResourceId: GenerateTestResourceID(t),
}
res := createCloudLinkResource(t, data)
@ -141,6 +141,27 @@ func TestValidateLink_MissingResourceId(t *testing.T) {
require.Equal(t, expected, actual)
}
func TestValidateLink_InvalidResourceId(t *testing.T) {
data := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
}
res := createCloudLinkResource(t, data)
err := ValidateLink(res)
expected := resource.ErrInvalidField{
Name: "resource_id",
Wrapped: errInvalidHCPResourceID,
}
var actual resource.ErrInvalidField
require.ErrorAs(t, err, &actual)
require.Equal(t, expected, actual)
}
// Currently, we have no specific ACLs configured so the default `operator` permissions are required
func TestLinkACLs(t *testing.T) {
registry := resource.NewRegistry()
@ -149,7 +170,7 @@ func TestLinkACLs(t *testing.T) {
data := &pbhcp.Link{
ClientId: "abc",
ClientSecret: "abc",
ResourceId: "abc",
ResourceId: GenerateTestResourceID(t),
}
link := createCloudLinkResource(t, data)

23
internal/hcp/internal/types/testing.go Normal file
View File

@ -0,0 +1,23 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package types
import (
"fmt"
"testing"
"github.com/hashicorp/go-uuid"
"github.com/stretchr/testify/require"
)
func GenerateTestResourceID(t *testing.T) string {
orgID, err := uuid.GenerateUUID()
require.NoError(t, err)
projectID, err := uuid.GenerateUUID()
require.NoError(t, err)
template := "organization/%s/project/%s/hashicorp.consul.global-network-manager.cluster/test-cluster"
return fmt.Sprintf(template, orgID, projectID)
}