CA provider doc updates and Vault provider minor update (#17831)

Update CA provider docs

Clarify that providers can differ between
primary and secondary datacenters

Provide a comparison chart for consul vs
vault CA providers

Loosen Vault CA provider validation for RootPKIPath

Update Vault CA provider documentation

.changelog/17831.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+ca: Vault CA provider config no longer requires root_pki_path for secondary datacenters
+```

agent/config/builder.go

@@ -1473,7 +1473,7 @@ func (b *builder) validate(rt RuntimeConfig) error {
 				return err
 			}
 		case structs.VaultCAProvider:
-			if _, err := ca.ParseVaultCAConfig(rt.ConnectCAConfig); err != nil {
+			if _, err := ca.ParseVaultCAConfig(rt.ConnectCAConfig, rt.PrimaryDatacenter == rt.Datacenter); err != nil {
 				return err
 			}
 		case structs.AWSCAProvider:

agent/connect/ca/provider_test.go

@@ -113,7 +113,7 @@ func TestStructs_CAConfiguration_MsgpackEncodeDecode(t *testing.T) {
 				TLSSkipVerify:          true,
 			},
 			parseFunc: func(t *testing.T, raw map[string]interface{}) interface{} {
-				config, err := ParseVaultCAConfig(raw)
+				config, err := ParseVaultCAConfig(raw, true)
 				require.NoError(t, err)
 				return config
 			},

agent/connect/ca/provider_vault.go

@@ -101,7 +101,7 @@ func vaultTLSConfig(config *structs.VaultCAProviderConfig) *vaultapi.TLSConfig {
 // Configure sets up the provider using the given configuration.
 // Configure supports being called multiple times to re-configure the provider.
 func (v *VaultProvider) Configure(cfg ProviderConfig) error {
-	config, err := ParseVaultCAConfig(cfg.RawConfig)
+	config, err := ParseVaultCAConfig(cfg.RawConfig, v.isPrimary)
 	if err != nil {
 		return err
 	}
@@ -192,11 +192,11 @@ func (v *VaultProvider) Configure(cfg ProviderConfig) error {
 }
 
 func (v *VaultProvider) ValidateConfigUpdate(prevRaw, nextRaw map[string]interface{}) error {
-	prev, err := ParseVaultCAConfig(prevRaw)
+	prev, err := ParseVaultCAConfig(prevRaw, v.isPrimary)
 	if err != nil {
 		return fmt.Errorf("failed to parse existing CA config: %w", err)
 	}
-	next, err := ParseVaultCAConfig(nextRaw)
+	next, err := ParseVaultCAConfig(nextRaw, v.isPrimary)
 	if err != nil {
 		return fmt.Errorf("failed to parse new CA config: %w", err)
 	}
@@ -800,7 +800,7 @@ func (v *VaultProvider) Cleanup(providerTypeChange bool, otherConfig map[string]
 	v.Stop()
 
 	if !providerTypeChange {
-		newConfig, err := ParseVaultCAConfig(otherConfig)
+		newConfig, err := ParseVaultCAConfig(otherConfig, v.isPrimary)
 		if err != nil {
 			return err
 		}
@@ -900,7 +900,7 @@ func (v *VaultProvider) autotidyIssuers(path string) (bool, string) {
 	return tidySet, errStr
 }
 
-func ParseVaultCAConfig(raw map[string]interface{}) (*structs.VaultCAProviderConfig, error) {
+func ParseVaultCAConfig(raw map[string]interface{}, isPrimary bool) (*structs.VaultCAProviderConfig, error) {
 	config := structs.VaultCAProviderConfig{
 		CommonCAProviderConfig: defaultCommonConfig(),
 	}
@@ -931,10 +931,10 @@ func ParseVaultCAConfig(raw map[string]interface{}) (*structs.VaultCAProviderCon
 		return nil, fmt.Errorf("only one of Vault token or Vault auth method can be provided, but not both")
 	}
 
-	if config.RootPKIPath == "" {
+	if isPrimary && config.RootPKIPath == "" {
 		return nil, fmt.Errorf("must provide a valid path to a root PKI backend")
 	}
-	if !strings.HasSuffix(config.RootPKIPath, "/") {
+	if config.RootPKIPath != "" && !strings.HasSuffix(config.RootPKIPath, "/") {
 		config.RootPKIPath += "/"
 	}
 

agent/connect/ca/provider_vault_test.go

@@ -60,6 +60,7 @@ func TestVaultCAProvider_ParseVaultCAConfig(t *testing.T) {
 	cases := map[string]struct {
 		rawConfig map[string]interface{}
 		expConfig *structs.VaultCAProviderConfig
+		isPrimary bool
 		expError  string
 	}{
 		"no token and no auth method provided": {
@@ -70,15 +71,26 @@ func TestVaultCAProvider_ParseVaultCAConfig(t *testing.T) {
 			rawConfig: map[string]interface{}{"Token": "test", "AuthMethod": map[string]interface{}{"Type": "test"}},
 			expError:  "only one of Vault token or Vault auth method can be provided, but not both",
 		},
-		"no root PKI path": {
-			rawConfig: map[string]interface{}{"Token": "test"},
+		"primary no root PKI path": {
+			rawConfig: map[string]interface{}{"Token": "test", "IntermediatePKIPath": "test"},
+			isPrimary: true,
 			expError:  "must provide a valid path to a root PKI backend",
 		},
+		"secondary no root PKI path": {
+			rawConfig: map[string]interface{}{"Token": "test", "IntermediatePKIPath": "test"},
+			isPrimary: false,
+			expConfig: &structs.VaultCAProviderConfig{
+				CommonCAProviderConfig: defaultCommonConfig(),
+				Token:                  "test",
+				IntermediatePKIPath:    "test/",
+			},
+		},
 		"no root intermediate path": {
 			rawConfig: map[string]interface{}{"Token": "test", "RootPKIPath": "test"},
 			expError:  "must provide a valid path for the intermediate PKI backend",
 		},
 		"adds a slash to RootPKIPath and IntermediatePKIPath": {
+			isPrimary: true,
 			rawConfig: map[string]interface{}{"Token": "test", "RootPKIPath": "test", "IntermediatePKIPath": "test"},
 			expConfig: &structs.VaultCAProviderConfig{
 				CommonCAProviderConfig: defaultCommonConfig(),
@@ -91,7 +103,7 @@ func TestVaultCAProvider_ParseVaultCAConfig(t *testing.T) {
 
 	for name, c := range cases {
 		t.Run(name, func(t *testing.T) {
-			config, err := ParseVaultCAConfig(c.rawConfig)
+			config, err := ParseVaultCAConfig(c.rawConfig, c.isPrimary)
 			if c.expError != "" {
 				require.EqualError(t, err, c.expError)
 			} else {

website/content/docs/connect/ca/index.mdx

@@ -21,7 +21,7 @@ support for using
 [Vault as a CA](/consul/docs/connect/ca/vault). With Vault, the root certificate
 and private key material remain with the Vault cluster.
 
-### CA and Certificate relationship
+## CA and Certificate relationship
 
 This diagram shows the relationship between the CA certificates in a Consul primary datacenter and a
 secondary Consul datacenter.
@@ -34,9 +34,22 @@ services.
 - the Leaf Cert Client Agent is created by auto-encrypt and auto-config. It is used by
 client agents for HTTP API TLS, and for mTLS for RPC requests to servers.
 
-Any secondary datacenters receive an intermediate certificate, signed by the Primary Root
-CA, which is used as the CA certificate to sign leaf certificates in the secondary
-datacenter.
+Any secondary datacenters use their CA provider to generate an intermediate certificate
+signing request (CSR) to be signed by the primary root CA. They receive an intermediate
+CA certificate, which is used to sign leaf certificates in the secondary datacenter.
+
+You can use different providers across primary and secondary datacenters.
+For example, an operator may use a Vault CA provider for extra security in the primary
+datacenter but choose to use the built-in CA provider in the secondary datacenter, which
+may not have a reachable Vault cluster. The following table compares the built-in and Vault providers.
+
+## CA Provider Comparison
+
+|            | Consul built-in                    | Vault                                                                             |
+|------------|------------------------------------|-----------------------------------------------------------------------------------|
+| Security   | CA private keys are stored on disk | CA private keys are stored in Vault and are never exposed to Consul server agents |
+| Resiliency | No dependency on external systems. If Consul is available, it can sign certificates | Dependent on Vault availability  |
+| Latency    | Consul signs certificates locally  | A network call to Vault is required to sign certificates                          |
 
 ## CA Bootstrapping
 

website/content/docs/connect/ca/vault.mdx

@@ -7,19 +7,27 @@ description: >-
 
 # Vault as a Service Mesh Certificate Authority
 
-You can configure Consul to use [Vault](https://www.vaultproject.io/) as the certificate authority (CA) so that Vault can manage and sign certificates distributed to services in the mesh. 
-The Vault CA provider uses the [Vault PKI secrets engine](/vault/docs/secrets/pki) to generate and sign certificates. 
+You can configure Consul to use [Vault](/vault) as the certificate authority (CA) so that Vault can manage and sign certificates distributed to services in the mesh.
+The Vault CA provider uses the [Vault PKI secrets engine](/vault/docs/secrets/pki) to generate and sign certificates.
 This page describes how configure the Vault CA provider.
 
 > **Tutorial:** Complete the [Vault as Consul Service Mesh Certification Authority](/consul/tutorials/vault-secure/vault-pki-consul-connect-ca) tutorial for hands-on guidance on how to configure Vault as the Consul service mesh certification authority.
 
 ## Requirements
 
+- Vault 0.10.3 or higher
+
+~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem.
+
+## Recommendations
+
 - Refer to [Service Mesh Certificate Authority Overview](/consul/docs/connect/ca) for important background information about how Consul manages certificates with configurable CA providers.
 
-- Vault 0.10.3 to 1.10.x.
+- For best performance and resiliency, every datacenter should have a Vault cluster local to its Consul cluster.
 
-~> **Compatibility note:** If you use Vault 1.11.0+ as Consul's service mesh CA, versions of Consul released before Dec 13, 2022 will develop an issue with Consul control plane or service mesh communication ([GH-15525](https://github.com/hashicorp/consul/pull/15525)). Use or upgrade to a [Consul version that includes the fix](https://support.hashicorp.com/hc/en-us/articles/11308460105491#01GMC24E6PPGXMRX8DMT4HZYTW) to avoid this problem.
+- If your Consul datacenters are WAN-federated and the secondary datacenter uses Vault Enterprise
+  [performance secondaries](/vault/docs/enterprise/replication#performance-replication), we recommend
+  configuring [`local`](/vault/docs/enterprise/replication#local) mounts for their [`intermediate_pki_path`](/consul/docs/connect/ca/vault#intermediatepkipath).
 
 ## Enable Vault as the CA
 
@@ -28,7 +36,7 @@ and including the required provider configuration options.
 You can provide the CA configuration in the server agents' configuration file
 or in the body of a `PUT` request to the
 [`/connect/ca/configuration`](/consul/api-docs/connect/ca#update-ca-configuration) API endpoint.
-Refer to the [Configuration Reference](#configuration-reference) for details about configuration options and for example use cases. 
+Refer to the [Configuration Reference](#configuration-reference) for details about configuration options and for example use cases.
 
 The following example shows the required configurations for a default implementation:
 
@@ -75,7 +83,7 @@ connect {
 You can specify the following configuration options.
 Note that a configuration option's name may differ between API calls and the agent configuration file.
 The first key refers to the option name for use in API calls.
-The key after the slash refers to the corresponding option name in the agent configuration file. 
+The key after the slash refers to the corresponding option name in the agent configuration file.
 
 - `Address` / `address` (`string: <required>`) - The address of the Vault
   server.
@@ -104,7 +112,8 @@ The key after the slash refers to the corresponding option name in the agent con
    Only the authentication related fields (for example, JWT's `path` and `role`) are supported. The optional management fields (for example: `remove_jwt_after_reading`) are not supported.
 
 - `RootPKIPath` / `root_pki_path` (`string: <required>`) - The path to
-  a PKI secrets engine for the root certificate.
+  a PKI secrets engine for the root certificate. Required for primary
+  datacenters. Secondary datacenters do not use this path.
 
   If the path does not
   exist, Consul will mount a new PKI secrets engine at the specified path with the
@@ -114,9 +123,6 @@ The key after the slash refers to the corresponding option name in the agent con
   the root certificate TTL was set to 8760 hour, or 1 year, and was not configurable.
   The root certificate will expire at the end of the specified period.
 
-  When WAN Federation is enabled, each secondary datacenter must use the same Vault cluster and share the same `root_pki_path`
-  with the primary datacenter.
-
   To use an intermediate certificate as the primary CA in Consul, initialize the
   `RootPKIPath` in Vault with a PEM bundle. The first certificate in the bundle
   must be the intermediate certificate that Consul will use as the primary CA.
@@ -242,7 +248,7 @@ Then, attach the following Vault ACL policy to the CA provider's
   path "/<root_pki_path>/" {
     capabilities = [ "read" ]
   }
-  
+
   path "/<root_pki_path>/root/sign-intermediate" {
     capabilities = [ "update" ]
   }
@@ -268,7 +274,7 @@ Then, attach the following Vault ACL policy to the CA provider's
     capabilities = [ "read" ]
   }
   ```
-  
+
   </CodeBlockConfig>
 
 #### Define a policy for Consul-managed PKI paths ((#consul-managed-pki-paths))
@@ -329,7 +335,7 @@ Then, attach the following Vault ACL policy to the CA provider's
     capabilities = [ "read" ]
   }
   ```
-  
+
   </CodeBlockConfig>
 
 #### Additional Vault ACL policies for sensitive operations
@@ -340,7 +346,7 @@ following CA provider configuration changes:
 - Changing the `RootPKIPath`
 
 Those configuration modifications trigger a root CA change that requires an
-extremely privileged root cross-sign operation. 
+extremely privileged root cross-sign operation.
 For that operation to succeed, the CA provider's [Vault token](#token) or
 [auth method](#authmethod) must contain the following rule: