CA provider doc updates and Vault provider minor update (#17831)

Update CA provider docs

Clarify that providers can differ between
primary and secondary datacenters

Provide a comparison chart for consul vs
vault CA providers

Loosen Vault CA provider validation for RootPKIPath

Update Vault CA provider documentation

.changelog/17831.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+ca: Vault CA provider config no longer requires root_pki_path for secondary datacenters
+```

agent/config/builder.go

@@ -1473,7 +1473,7 @@ func (b *builder) validate(rt RuntimeConfig) error {
 				return err
 			}
 		case structs.VaultCAProvider:
-			if _, err := ca.ParseVaultCAConfig(rt.ConnectCAConfig); err != nil {
+			if _, err := ca.ParseVaultCAConfig(rt.ConnectCAConfig, rt.PrimaryDatacenter == rt.Datacenter); err != nil {
 				return err
 			}
 		case structs.AWSCAProvider:

agent/connect/ca/provider_test.go

@@ -113,7 +113,7 @@ func TestStructs_CAConfiguration_MsgpackEncodeDecode(t *testing.T) {
 				TLSSkipVerify:          true,
 			},
 			parseFunc: func(t *testing.T, raw map[string]interface{}) interface{} {
-				config, err := ParseVaultCAConfig(raw)
+				config, err := ParseVaultCAConfig(raw, true)
 				require.NoError(t, err)
 				return config
 			},

agent/connect/ca/provider_vault.go

@@ -101,7 +101,7 @@ func vaultTLSConfig(config *structs.VaultCAProviderConfig) *vaultapi.TLSConfig {
 // Configure sets up the provider using the given configuration.
 // Configure supports being called multiple times to re-configure the provider.
 func (v *VaultProvider) Configure(cfg ProviderConfig) error {
-	config, err := ParseVaultCAConfig(cfg.RawConfig)
+	config, err := ParseVaultCAConfig(cfg.RawConfig, v.isPrimary)
 	if err != nil {
 		return err
 	}
@@ -192,11 +192,11 @@ func (v *VaultProvider) Configure(cfg ProviderConfig) error {
 }
 
 func (v *VaultProvider) ValidateConfigUpdate(prevRaw, nextRaw map[string]interface{}) error {
-	prev, err := ParseVaultCAConfig(prevRaw)
+	prev, err := ParseVaultCAConfig(prevRaw, v.isPrimary)
 	if err != nil {
 		return fmt.Errorf("failed to parse existing CA config: %w", err)
 	}
-	next, err := ParseVaultCAConfig(nextRaw)
+	next, err := ParseVaultCAConfig(nextRaw, v.isPrimary)
 	if err != nil {
 		return fmt.Errorf("failed to parse new CA config: %w", err)
 	}
@@ -800,7 +800,7 @@ func (v *VaultProvider) Cleanup(providerTypeChange bool, otherConfig map[string]
 	v.Stop()
 
 	if !providerTypeChange {
-		newConfig, err := ParseVaultCAConfig(otherConfig)
+		newConfig, err := ParseVaultCAConfig(otherConfig, v.isPrimary)
 		if err != nil {
 			return err
 		}
@@ -900,7 +900,7 @@ func (v *VaultProvider) autotidyIssuers(path string) (bool, string) {
 	return tidySet, errStr
 }
 
-func ParseVaultCAConfig(raw map[string]interface{}) (*structs.VaultCAProviderConfig, error) {
+func ParseVaultCAConfig(raw map[string]interface{}, isPrimary bool) (*structs.VaultCAProviderConfig, error) {
 	config := structs.VaultCAProviderConfig{
 		CommonCAProviderConfig: defaultCommonConfig(),
 	}
@@ -931,10 +931,10 @@ func ParseVaultCAConfig(raw map[string]interface{}) (*structs.VaultCAProviderCon
 		return nil, fmt.Errorf("only one of Vault token or Vault auth method can be provided, but not both")
 	}
 
-	if config.RootPKIPath == "" {
+	if isPrimary && config.RootPKIPath == "" {
 		return nil, fmt.Errorf("must provide a valid path to a root PKI backend")
 	}
-	if !strings.HasSuffix(config.RootPKIPath, "/") {
+	if config.RootPKIPath != "" && !strings.HasSuffix(config.RootPKIPath, "/") {
 		config.RootPKIPath += "/"
 	}
 

agent/connect/ca/provider_vault_test.go

@@ -60,6 +60,7 @@ func TestVaultCAProvider_ParseVaultCAConfig(t *testing.T) {
 	cases := map[string]struct {
 		rawConfig map[string]interface{}
 		expConfig *structs.VaultCAProviderConfig
+		isPrimary bool
 		expError  string
 	}{
 		"no token and no auth method provided": {
@@ -70,15 +71,26 @@ func TestVaultCAProvider_ParseVaultCAConfig(t *testing.T) {
 			rawConfig: map[string]interface{}{"Token": "test", "AuthMethod": map[string]interface{}{"Type": "test"}},
 			expError:  "only one of Vault token or Vault auth method can be provided, but not both",
 		},
-		"no root PKI path": {
-			rawConfig: map[string]interface{}{"Token": "test"},
+		"primary no root PKI path": {
+			rawConfig: map[string]interface{}{"Token": "test", "IntermediatePKIPath": "test"},
+			isPrimary: true,
 			expError:  "must provide a valid path to a root PKI backend",
 		},
+		"secondary no root PKI path": {
+			rawConfig: map[string]interface{}{"Token": "test", "IntermediatePKIPath": "test"},
+			isPrimary: false,
+			expConfig: &structs.VaultCAProviderConfig{
+				CommonCAProviderConfig: defaultCommonConfig(),
+				Token:                  "test",
+				IntermediatePKIPath:    "test/",
+			},
+		},
 		"no root intermediate path": {
 			rawConfig: map[string]interface{}{"Token": "test", "RootPKIPath": "test"},
 			expError:  "must provide a valid path for the intermediate PKI backend",
 		},
 		"adds a slash to RootPKIPath and IntermediatePKIPath": {
+			isPrimary: true,
 			rawConfig: map[string]interface{}{"Token": "test", "RootPKIPath": "test", "IntermediatePKIPath": "test"},
 			expConfig: &structs.VaultCAProviderConfig{
 				CommonCAProviderConfig: defaultCommonConfig(),
@@ -91,7 +103,7 @@ func TestVaultCAProvider_ParseVaultCAConfig(t *testing.T) {
 
 	for name, c := range cases {
 		t.Run(name, func(t *testing.T) {
-			config, err := ParseVaultCAConfig(c.rawConfig)
+			config, err := ParseVaultCAConfig(c.rawConfig, c.isPrimary)
 			if c.expError != "" {
 				require.EqualError(t, err, c.expError)
 			} else {

website/content/docs/connect/ca/index.mdx

@@ -21,7 +21,7 @@ support for using
 [Vault as a CA](/consul/docs/connect/ca/vault). With Vault, the root certificate
 and private key material remain with the Vault cluster.
 
-### CA and Certificate relationship
+## CA and Certificate relationship
 
 This diagram shows the relationship between the CA certificates in a Consul primary datacenter and a
 secondary Consul datacenter.
@@ -34,9 +34,22 @@ services.
 - the Leaf Cert Client Agent is created by auto-encrypt and auto-config. It is used by
 client agents for HTTP API TLS, and for mTLS for RPC requests to servers.
 
-Any secondary datacenters receive an intermediate certificate, signed by the Primary Root
-CA, which is used as the CA certificate to sign leaf certificates in the secondary
-datacenter.
+Any secondary datacenters use their CA provider to generate an intermediate certificate
+signing request (CSR) to be signed by the primary root CA. They receive an intermediate
+CA certificate, which is used to sign leaf certificates in the secondary datacenter.
+
+You can use different providers across primary and secondary datacenters.
+For example, an operator may use a Vault CA provider for extra security in the primary
+datacenter but choose to use the built-in CA provider in the secondary datacenter, which
+may not have a reachable Vault cluster. The following table compares the built-in and Vault providers.
+
+## CA Provider Comparison
+
+|            | Consul built-in                    | Vault                                                                             |
+|------------|------------------------------------|-----------------------------------------------------------------------------------|
+| Security   | CA private keys are stored on disk | CA private keys are stored in Vault and are never exposed to Consul server agents |
+| Resiliency | No dependency on external systems. If Consul is available, it can sign certificates | Dependent on Vault availability  |
+| Latency    | Consul signs certificates locally  | A network call to Vault is required to sign certificates                          |
 
 ## CA Bootstrapping
 

website/content/docs/connect/ca/vault.mdx

@@ -7,7 +7,7 @@ description: >-
 
 # Vault as a Service Mesh Certificate Authority
 
-You can configure Consul to use [Vault](https://www.vaultproject.io/) as the certificate authority (CA) so that Vault can manage and sign certificates distributed to services in the mesh. 
+You can configure Consul to use [Vault](/vault) as the certificate authority (CA) so that Vault can manage and sign certificates distributed to services in the mesh.
 The Vault CA provider uses the [Vault PKI secrets engine](/vault/docs/secrets/pki) to generate and sign certificates.
 This page describes how configure the Vault CA provider.
 
@@ -15,11 +15,19 @@ This page describes how configure the Vault CA provider.
 
 ## Requirements
 
+- Vault 0.10.3 or higher
+
+~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem.
+
+## Recommendations
+
 - Refer to [Service Mesh Certificate Authority Overview](/consul/docs/connect/ca) for important background information about how Consul manages certificates with configurable CA providers.
 
-- Vault 0.10.3 to 1.10.x.
+- For best performance and resiliency, every datacenter should have a Vault cluster local to its Consul cluster.
 
-~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem.
+- If your Consul datacenters are WAN-federated and the secondary datacenter uses Vault Enterprise
+  [performance secondaries](/vault/docs/enterprise/replication#performance-replication), we recommend
+  configuring [`local`](/vault/docs/enterprise/replication#local) mounts for their [`intermediate_pki_path`](/consul/docs/connect/ca/vault#intermediatepkipath).
 
 ## Enable Vault as the CA
 
@@ -104,7 +112,8 @@ The key after the slash refers to the corresponding option name in the agent con
    Only the authentication related fields (for example, JWT's `path` and `role`) are supported. The optional management fields (for example: `remove_jwt_after_reading`) are not supported.
 
 - `RootPKIPath` / `root_pki_path` (`string: <required>`) - The path to
-  a PKI secrets engine for the root certificate.
+  a PKI secrets engine for the root certificate. Required for primary
+  datacenters. Secondary datacenters do not use this path.
 
   If the path does not
   exist, Consul will mount a new PKI secrets engine at the specified path with the
@@ -114,9 +123,6 @@ The key after the slash refers to the corresponding option name in the agent con
   the root certificate TTL was set to 8760 hour, or 1 year, and was not configurable.
   The root certificate will expire at the end of the specified period.
 
-  When WAN Federation is enabled, each secondary datacenter must use the same Vault cluster and share the same `root_pki_path`
-  with the primary datacenter.
-
   To use an intermediate certificate as the primary CA in Consul, initialize the
   `RootPKIPath` in Vault with a PEM bundle. The first certificate in the bundle
   must be the intermediate certificate that Consul will use as the primary CA.