diff --git a/.changelog/19881.txt b/.changelog/19881.txt new file mode 100644 index 0000000000..0164661749 --- /dev/null +++ b/.changelog/19881.txt @@ -0,0 +1,3 @@ +```release-note:bug +xds: Make TCP external service registered with terminating gateway reachable from peered cluster +``` diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 1fc1067215..e7e78ce55c 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -29,6 +29,7 @@ import ( envoy_tcp_proxy_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/hashicorp/consul/agent/xds/config" "github.com/hashicorp/consul/agent/xds/naming" "github.com/hashicorp/consul/agent/xds/platform" @@ -1178,29 +1179,9 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh } // Inject peering trust bundles if this service is exported to peered clusters. - if len(peerBundles) > 0 { - spiffeConfig, err := makeSpiffeValidatorConfig( - cfgSnap.Roots.TrustDomain, - cfgSnap.RootPEMs(), - peerBundles, - ) - if err != nil { - return nil, err - } - - typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext) - if !ok { - return nil, fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType) - } - - // makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA. - // We nil it out here since the local roots are included in the SPIFFE validator config. - typ.ValidationContext.TrustedCa = nil - typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{ - // The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib. - Name: "envoy.tls.cert_validator.spiffe", - TypedConfig: spiffeConfig, - } + err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext, peerBundles) + if err != nil { + return nil, err } return makeDownstreamTLSTransportSocket(&envoy_tls_v3.DownstreamTlsContext{ @@ -1209,6 +1190,32 @@ func createDownstreamTransportSocketForConnectTLS(cfgSnap *proxycfg.ConfigSnapsh }) } +func injectSpiffeValidatorConfigForPeers(cfgSnap *proxycfg.ConfigSnapshot, tlsContext *envoy_tls_v3.CommonTlsContext, peerBundles []*pbpeering.PeeringTrustBundle) error { + if len(peerBundles) == 0 { + return nil + } + + spiffeConfig, err := makeSpiffeValidatorConfig(cfgSnap.Roots.TrustDomain, cfgSnap.RootPEMs(), peerBundles) + if err != nil { + return err + } + + typ, ok := tlsContext.ValidationContextType.(*envoy_tls_v3.CommonTlsContext_ValidationContext) + if !ok { + return fmt.Errorf("unexpected type for TLS context validation: %T", tlsContext.ValidationContextType) + } + + // makeCommonTLSFromLead injects the local trust domain's CA root certs as the TrustedCA. + // We nil it out here since the local roots are included in the SPIFFE validator config. + typ.ValidationContext.TrustedCa = nil + typ.ValidationContext.CustomValidatorConfig = &envoy_core_v3.TypedExtensionConfig{ + // The typed config name is hard-coded because it is not available as a wellknown var in the control plane lib. + Name: "envoy.tls.cert_validator.spiffe", + TypedConfig: spiffeConfig, + } + return nil +} + // SPIFFECertValidatorConfig is used to validate certificates from trust domains other than our own. // With cluster peering we expect peered clusters to have independent certificate authorities. // This means that we cannot use a single set of root CA certificates to validate client certificates for mTLS, @@ -1752,6 +1759,15 @@ type terminatingGatewayFilterChainOpts struct { } func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg.ConfigSnapshot, tgtwyOpts terminatingGatewayFilterChainOpts) (*envoy_listener_v3.FilterChain, error) { + // We need to at least match the SNI and use the root PEMs from the local cluster; however, requests coming + // from peered clusters where the external service is exported to will have their own SNI and root PEMs. + sniMatches := []string{tgtwyOpts.cluster} + for _, bundle := range tgtwyOpts.peerTrustBundles { + svc := tgtwyOpts.service + sourceSNI := connect.PeeredServiceSNI(svc.Name, svc.NamespaceOrDefault(), svc.PartitionOrDefault(), bundle.PeerName, cfgSnap.Roots.TrustDomain) + sniMatches = append(sniMatches, sourceSNI) + } + tlsContext := &envoy_tls_v3.DownstreamTlsContext{ CommonTlsContext: makeCommonTLSContext( cfgSnap.TerminatingGateway.ServiceLeaves[tgtwyOpts.service], @@ -1760,13 +1776,19 @@ func (s *ResourceGenerator) makeFilterChainTerminatingGateway(cfgSnap *proxycfg. ), RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, } + + err := injectSpiffeValidatorConfigForPeers(cfgSnap, tlsContext.CommonTlsContext, tgtwyOpts.peerTrustBundles) + if err != nil { + return nil, err + } + transportSocket, err := makeDownstreamTLSTransportSocket(tlsContext) if err != nil { return nil, err } filterChain := &envoy_listener_v3.FilterChain{ - FilterChainMatch: makeSNIFilterChainMatch(tgtwyOpts.cluster), + FilterChainMatch: makeSNIFilterChainMatch(sniMatches...), Filters: make([]*envoy_listener_v3.Filter, 0, 3), TransportSocket: transportSocket, } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index fa2c51a35a..a6957d929d 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -29,7 +29,6 @@ import ( "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" - "github.com/hashicorp/consul/proto/private/pbpeering" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/types" ) @@ -2281,32 +2280,25 @@ func getTerminatingGatewayPeeringGoldenTestCases() []goldenTestCase { { name: "terminating-gateway-with-peer-trust-bundle", create: func(t testinf.T) *proxycfg.ConfigSnapshot { - roots, _ := proxycfg.TestCerts(t) + bundles := proxycfg.TestPeerTrustBundles(t) return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, []proxycfg.UpdateEvent{ { CorrelationID: "peer-trust-bundle:web", - Result: &pbpeering.TrustBundleListByServiceResponse{ - Bundles: []*pbpeering.PeeringTrustBundle{ - { - TrustDomain: "foo.bar.gov", - PeerName: "dc2", - Partition: "default", - RootPEMs: []string{ - roots.Roots[0].RootCert, - }, - ExportedPartition: "default", - CreateIndex: 0, - ModifyIndex: 0, - }, - }, - }, + Result: bundles, }, { CorrelationID: "service-intentions:web", Result: structs.SimplifiedIntentions{ { SourceName: "source", - SourcePeer: "dc2", + SourcePeer: bundles.Bundles[0].PeerName, + DestinationName: "web", + DestinationPartition: "default", + Action: structs.IntentionActionAllow, + }, + { + SourceName: "source", + SourcePeer: bundles.Bundles[1].PeerName, DestinationName: "web", DestinationPartition: "default", Action: structs.IntentionActionAllow, diff --git a/agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden b/agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden index feed47691f..e11a9f7244 100644 --- a/agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden +++ b/agent/xds/testdata/listeners/terminating-gateway-with-peer-trust-bundle.latest.golden @@ -163,7 +163,9 @@ { "filterChainMatch": { "serverNames": [ - "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul" + "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul", + "web.default.default.peer-a.external.11111111-2222-3333-4444-555555555555.consul", + "web.default.default.peer-b.external.11111111-2222-3333-4444-555555555555.consul" ] }, "filters": [ @@ -184,7 +186,16 @@ "authenticated": { "principalName": { "safeRegex": { - "regex": "^spiffe://foo.bar.gov/ns/default/dc/[^/]+/svc/source$" + "regex": "^spiffe://1c053652-8512-4373-90cf-5a7f6263a994.consul/ns/default/dc/[^/]+/svc/source$" + } + } + } + }, + { + "authenticated": { + "principalName": { + "safeRegex": { + "regex": "^spiffe://d89ac423-e95a-475d-94f2-1c557c57bf31.consul/ns/default/dc/[^/]+/svc/source$" } } } @@ -222,8 +233,31 @@ ], "tlsParams": {}, "validationContext": { - "trustedCa": { - "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + "customValidatorConfig": { + "name": "envoy.tls.cert_validator.spiffe", + "typedConfig": { + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig", + "trustDomains": [ + { + "name": "11111111-2222-3333-4444-555555555555.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICXDCCAgKgAwIBAgIICpZq70Z9LyUwCgYIKoZIzj0EAwIwFDESMBAGA1UEAxMJ\nVGVzdCBDQSAyMB4XDTE5MDMyMjEzNTgyNloXDTI5MDMyMjEzNTgyNlowFDESMBAG\nA1UEAxMJVGVzdCBDQSAyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIhywH1gx\nAsMwuF3ukAI5YL2jFxH6Usnma1HFSfVyxbXX1/uoZEYrj8yCAtdU2yoHETyd+Zx2\nThhRLP79pYegCaOCATwwggE4MA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTAD\nAQH/MGgGA1UdDgRhBF9kMToxMToxMTphYzoyYTpiYTo5NzpiMjozZjphYzo3Yjpi\nZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1ZTo0MTo2ZjpmMjo3\nMzo5NTo1ODowYzpkYjBqBgNVHSMEYzBhgF9kMToxMToxMTphYzoyYTpiYTo5Nzpi\nMjozZjphYzo3YjpiZDpkYTpiZTpiMTo4YTpmYzo5YTpiYTpiNTpiYzo4MzplNzo1\nZTo0MTo2ZjpmMjo3Mzo5NTo1ODowYzpkYjA/BgNVHREEODA2hjRzcGlmZmU6Ly8x\nMTExMTExMS0yMjIyLTMzMzMtNDQ0NC01NTU1NTU1NTU1NTUuY29uc3VsMAoGCCqG\nSM49BAMCA0gAMEUCICOY0i246rQHJt8o8Oya0D5PLL1FnmsQmQqIGCi31RwnAiEA\noR5f6Ku+cig2Il8T8LJujOp2/2A72QcHZA57B13y+8o=\n-----END CERTIFICATE-----\n" + } + }, + { + "name": "1c053652-8512-4373-90cf-5a7f6263a994.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICczCCAdwCCQC3BLnEmLCrSjANBgkqhkiG9w0BAQsFADB+MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQVoxEjAQBgNVBAcMCUZsYWdzdGFmZjEMMAoGA1UECgwDRm9v\nMRAwDgYDVQQLDAdleGFtcGxlMQ8wDQYDVQQDDAZwZWVyLWExHTAbBgkqhkiG9w0B\nCQEWDmZvb0BwZWVyLWEuY29tMB4XDTIyMDUyNjAxMDQ0NFoXDTIzMDUyNjAxMDQ0\nNFowfjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkFaMRIwEAYDVQQHDAlGbGFnc3Rh\nZmYxDDAKBgNVBAoMA0ZvbzEQMA4GA1UECwwHZXhhbXBsZTEPMA0GA1UEAwwGcGVl\nci1hMR0wGwYJKoZIhvcNAQkBFg5mb29AcGVlci1hLmNvbTCBnzANBgkqhkiG9w0B\nAQEFAAOBjQAwgYkCgYEA2zFYGTbXDAntT5pLTpZ2+VTiqx4J63VRJH1kdu11f0FV\nc2jl1pqCuYDbQXknDU0Pv1Q5y0+nSAihD2KqGS571r+vHQiPtKYPYRqPEe9FzAhR\n2KhWH6v/tk5DG1HqOjV9/zWRKB12gdFNZZqnw/e7NjLNq3wZ2UAwxXip5uJ8uwMC\nAwEAATANBgkqhkiG9w0BAQsFAAOBgQC/CJ9Syf4aL91wZizKTejwouRYoWv4gRAk\nyto45ZcNMHfJ0G2z+XAMl9ZbQsLgXmzAx4IM6y5Jckq8pKC4PEijCjlKTktLHlEy\n0ggmFxtNB1tid2NC8dOzcQ3l45+gDjDqdILhAvLDjlAIebdkqVqb2CfFNW/I2CQH\nZAuKN1aoKA==\n-----END CERTIFICATE-----\n" + } + }, + { + "name": "d89ac423-e95a-475d-94f2-1c557c57bf31.consul", + "trustBundle": { + "inlineString": "-----BEGIN CERTIFICATE-----\nMIICcTCCAdoCCQDyGxC08cD0BDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFkMQwwCgYDVQQKDANGb28x\nEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXItYjEdMBsGCSqGSIb3DQEJ\nARYOZm9vQHBlZXItYi5jb20wHhcNMjIwNTI2MDExNjE2WhcNMjMwNTI2MDExNjE2\nWjB9MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExETAPBgNVBAcMCENhcmxzYmFk\nMQwwCgYDVQQKDANGb28xEDAOBgNVBAsMB2V4YW1wbGUxDzANBgNVBAMMBnBlZXIt\nYjEdMBsGCSqGSIb3DQEJARYOZm9vQHBlZXItYi5jb20wgZ8wDQYJKoZIhvcNAQEB\nBQADgY0AMIGJAoGBAL4i5erdZ5vKk3mzW9Qt6Wvw/WN/IpMDlL0a28wz9oDCtMLN\ncD/XQB9yT5jUwb2s4mD1lCDZtee8MHeD8zygICozufWVB+u2KvMaoA50T9GMQD0E\nz/0nz/Z703I4q13VHeTpltmEpYcfxw/7nJ3leKA34+Nj3zteJ70iqvD/TNBBAgMB\nAAEwDQYJKoZIhvcNAQELBQADgYEAbL04gicH+EIznDNhZJEb1guMBtBBJ8kujPyU\nao8xhlUuorDTLwhLpkKsOhD8619oSS8KynjEBichidQRkwxIaze0a2mrGT+tGBMf\npVz6UeCkqpde6bSJ/ozEe/2seQzKqYvRT1oUjLwYvY7OIh2DzYibOAxh6fewYAmU\n5j5qNLc=\n-----END CERTIFICATE-----\n" + } + } + ] + } } } },